diff options
Diffstat (limited to 'org/gleam-ocda_lib.org')
| -rw-r--r-- | org/gleam-ocda_lib.org | 381 |
1 files changed, 381 insertions, 0 deletions
diff --git a/org/gleam-ocda_lib.org b/org/gleam-ocda_lib.org new file mode 100644 index 0000000..a359818 --- /dev/null +++ b/org/gleam-ocda_lib.org @@ -0,0 +1,381 @@ +-*- mode: org -*- +#+TITLE: gleam lib +#+DESCRIPTION: gleam lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* gleam :gleam: +** README.md +*** tangle + +#+HEADER: :tangle ../gleam/README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# ocda-lib, in Gleam + +The abstraction's model and both its loaders, as a Gleam library, with its +reader and the two programs that put it to the question. + +| | | +|------------------|-----------------------------------------------------------------------------------------| +| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | +| `ocda/load_ssp` | text -> model | +| `ocda/load_db` | sqlite -> model | +| `ocda/load` | what a path is, and loading it | +| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | +| `ocda/dump` | the model written out canonically, as an oracle | +| `read/main` | `ocda-read`, the reader | +| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | + +## Use + +```gleam +import ocda/load +import ocda/model + +case load.load(path, "en", False) { + Error(why) -> io.println_error(why) + Ok(doc) -> { + // and from here a writer neither knows nor cares which of the two + // files it was handed + list.each(model.section(doc, "body"), fn(obj) { ... }) + } +} +``` + +## The reader + +```sh +nix build +./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] + <file>.ssp|<file>.ocda.db|<file>.zip [lang] +``` + +`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers +had, over the library, so each reads every form this library reads; the +others are the D reader's. **Its output is the D `ocda-read`'s, byte for +byte**, in every mode both have. In `src/read/`, because Gleam compiles +nothing outside `src/`; a program depending on the library compiles it too, +and does not use it. + +A `.sisupod` it refuses, saying why: the zstd frame, below. + +## The two programs + +```sh +nix build +./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] +./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] +``` + +`agree` asks whether the two loaders give the same model. `dump` writes +the whole model out canonically, and is the stronger check of the two, +because it is **byte for byte what the D library's `ocda-dump` writes**. +So the four questions can all be asked with `diff`: + +| | | +|-----------------------------|-----------------------------| +| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | +| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | +| `.ssp`: D == Gleam | 36 of 36 | +| `.ocda.db`: D == Gleam | 36 of 36 | + +## Deliberately a mirror of the D library + +Field for field, name for name, including where that is not the most +Gleamish choice: + +- `text` is one `String` with the lines joined by newline, not a + `List(String)`. So an object whose only text line is blank is + indistinguishable from an object with no text, *in both languages, in the + same place*. A difference that existed in one and not the other would be + worse than a loss both share, because it would show up in every dump and + mean nothing. +- the properties are typed fields rather than a list of pairs. A writer + wants `obj.parent_ocn`, not a lookup that might miss. + +The point is that the two agree. Making them agree is much cheaper when +they are shaped alike, and the shape is the specification's, not either +language's. + +## What the agreement found + +Four disagreements, none of which a comparison of section counts or of a +handful of fields would have shown. All four are recorded in the D +library's history; the short version: + +1. `.identifier` and `.anchor` are omitted by the text form when they can + be worked out (the identifier is the ocn, the anchor is the segment). + Both loaders now derive them. Over one document that is 1,314 + identifiers. +2. `make` and `doc_has` are one metadata table in the database, + distinguished by a key prefix. Both loaders unprefix them. +3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` + as `\/` and every heading with a slash in it came back wrong. +4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, + which is one language. It is a fact about the file, so neither loader + puts it in the document. + +## Build + +```sh +nix build # sandboxed, no network +nix develop -c gleam test +``` + +Shipped as an Erlang shipment rather than an escript, because sqlite on the +BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. +The dependency hash in `flake.nix` pins what `manifest.toml` pins; change +one and the other must change with it. + +## Format + +Specified in **sisudoc-abstraction-format**: `ocda-model.md` for the model, +`ssp-format-specification.md` and `ocda-db-format-specification.md` for the +two serialisations. At format **2.x**. +#+END_SRC + +** gleam.toml +*** tangle + +#+HEADER: :tangle ../gleam/gleam.toml +#+HEADER: :noweb yes +#+BEGIN_SRC toml +name = "ocda_lib" +version = "0.1.0" +description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" +licences = ["AGPL-3.0-or-later"] + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +argv = ">= 1.1.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +sqlight = ">= 1.0.0 and < 2.0.0" +gleam_json = ">= 3.0.0 and < 4.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +#+END_SRC + +** gleam ocda_lib +*** tangle + +#+HEADER: :tangle ../gleam/src/ocda_lib.gleam +#+HEADER: :noweb yes +#+BEGIN_SRC org +////<<doc_header_including_copyright_and_license>> +<<ocda_lib>> +#+END_SRC + +*** gleam source + +#+NAME: ocda_lib +#+HEADER: :no-expand +#+BEGIN_SRC gleam +//// ocda-lib, in Gleam: the abstraction as a library, put to the question. +//// +//// A library has no binary, so this is the programs over it: the reader, +//// and the two that test its claims. +//// +//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] +//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] +//// ocda-lib read [options] <file> [lang] (also installed as ocda-read) +//// +//// `dump` writes the whole loaded model out canonically, byte for byte as +//// the D library's `ocda-dump` writes it, so that the two implementations +//// can be diffed. `agree` asks the same question the D library's +//// `ocda-agree` asks, within this language: does loading a document's +//// `.ssp` give what loading its `.ocda.db` gives? + +import argv +import gleam/int +import gleam/io +import gleam/list +import gleam/string +import ocda/dump +import ocda/load +import ocda/model.{type Document} +import read/main as reader + +pub fn main() { + case argv.load().arguments { + ["dump", "--unknown", path] -> dump_file(path, "", True) + ["dump", "--unknown", path, lang] -> dump_file(path, lang, True) + ["dump", path] -> dump_file(path, "", False) + ["dump", path, lang] -> dump_file(path, lang, False) + ["agree", ssp, db] -> agree(ssp, db, "") + ["agree", ssp, db, lang] -> agree(ssp, db, lang) + ["read", ..rest] -> + case reader.run(rest) { + 0 -> Nil + code -> halt(code) + } + _ -> usage() + } +} + +fn usage() -> Nil { + io.println_error( + "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] + ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] + ocda-lib read [--help] [options] <file> [lang]", + ) + halt(2) +} + +fn dump_file(path: String, lang: String, unknown: Bool) -> Nil { + case load.load(path, lang, False) { + Error(message) -> fail(message) + Ok(doc) -> + case unknown { + True -> io.print(dump.unknown(doc)) + False -> io.print(dump.document(doc)) + } + } +} + +/// The library's whole purpose is that a writer cannot tell which +/// serialisation it was handed. That is only true if the two loaders give +/// the same model, so this asks, and says where they part company rather +/// than only that they did. +fn agree(ssp_path: String, db_path: String, lang: String) -> Nil { + case load.load(ssp_path, "", False), load.load(db_path, lang, False) { + Error(message), _ -> fail(message) + _, Error(message) -> fail(message) + Ok(from_ssp), Ok(from_db) -> { + let differences = compare(from_ssp, from_db) + case differences { + [] -> io.println("agree: the two loaders give the same model") + _ -> { + list.each(list.take(differences, 12), io.println) + case list.length(differences) > 12 { + True -> io.println(" ... stopping after 12") + False -> Nil + } + io.println(int.to_string(list.length(differences)) <> " difference(s)") + halt(1) + } + } + } + } +} + +/// Compared through the dump rather than field by field, which is both +/// shorter and stricter: the dump carries every field, so nothing can +/// differ without showing here. +fn compare(from_ssp: Document, from_db: Document) -> List(String) { + let a = dump.document(from_ssp) + let b = dump.document(from_db) + case a == b { + True -> [] + False -> diff_lines(a, b) + } +} + +fn diff_lines(a: String, b: String) -> List(String) { + let la = split_lines(a) + let lb = split_lines(b) + case list.length(la) == list.length(lb) { + False -> [ + " differ: line count", + " .ssp: " <> int.to_string(list.length(la)), + " .ocda.db: " <> int.to_string(list.length(lb)), + ..pairwise(la, lb) + ] + True -> pairwise(la, lb) + } +} + +fn pairwise(la: List(String), lb: List(String)) -> List(String) { + list.zip(la, lb) + |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) }) + |> list.filter(fn(t) { t.1 != t.2 }) + |> list.map(fn(t) { + " differ: line " + <> int.to_string(t.0 + 1) + <> "\n .ssp: " + <> t.1 + <> "\n .ocda.db: " + <> t.2 + }) +} + +fn split_lines(text: String) -> List(String) { + case text { + "" -> [] + _ -> string.split(text, on: "\n") + } +} + +fn fail(message: String) -> Nil { + io.println_error("error: " <> message) + halt(2) +} + +/// Gleam has no standard exit, and the exit status is what a sweep script +/// reads. +@external(erlang, "erlang", "halt") +fn halt(code: Int) -> Nil +#+END_SRC + +** erlang +*** tangle + +#+HEADER: :tangle ../gleam/src/ocda_ffi.erl +#+HEADER: :noweb yes +#+BEGIN_SRC org +%%<<doc_header_including_copyright_and_license>> +<<erlang_ffi>> +#+END_SRC + +*** erlang source + +#+NAME: erlang_ffi +#+HEADER: :no-expand +#+BEGIN_SRC erlang +%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's +%% crypto has and the Gleam standard library does not. Upper case hex, as +%% the database stores it and the D library writes it. +-module(ocda_ffi). +-export([sha256_hex/1]). + +sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ |
