diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-02 15:01:56 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-02 21:39:24 -0400 |
| commit | afcdcd2e99b5d06f8b3dd9d43b5fd2033e46b9ed (patch) | |
| tree | 12bfcde907a47ba0a287c6e68e0f4c1381bf33e4 | |
| parent | track (diff) | |
track
| -rw-r--r-- | dlang/flake.nix | 1 | ||||
| -rw-r--r-- | dlang/src/ocda/dump.d | 49 | ||||
| -rw-r--r-- | dlang/src/ocda/images.d | 49 | ||||
| -rw-r--r-- | dlang/src/ocda/inventory.d | 49 | ||||
| -rw-r--r-- | dlang/src/ocda/model.d | 49 | ||||
| -rw-r--r-- | dlang/src/ocda/sqlite.d | 49 | ||||
| -rw-r--r-- | dlang/test/agree.d | 49 | ||||
| -rw-r--r-- | dlang/test/dump.d | 49 | ||||
| -rw-r--r-- | org/config.org | 830 | ||||
| -rw-r--r-- | org/config_git.org | 101 | ||||
| -rw-r--r-- | org/config_nix.org | 287 | ||||
| -rw-r--r-- | org/d-ocda-load.org | 953 | ||||
| -rw-r--r-- | org/d-ocda.org | 697 | ||||
| -rw-r--r-- | org/d-read.org | 18 | ||||
| -rw-r--r-- | org/d-test.org | 178 | ||||
| -rw-r--r-- | org/gleam-ocda-load.org | 1484 | ||||
| -rw-r--r-- | org/gleam-ocda-misc.org | 649 | ||||
| -rw-r--r-- | org/gleam-ocda-model_and_inventory.org | 494 | ||||
| -rw-r--r-- | org/gleam-ocda.org | 239 | ||||
| -rw-r--r-- | org/gleam-ocda_lib.org | 381 | ||||
| -rw-r--r-- | org/gleam-read.org (renamed from org/gleam-read-main.org) | 8 |
21 files changed, 2104 insertions, 4559 deletions
diff --git a/dlang/flake.nix b/dlang/flake.nix index 1596888..94b3e37 100644 --- a/dlang/flake.nix +++ b/dlang/flake.nix @@ -67,6 +67,7 @@ }; inherit ocda-read ocda-agree ocda-dump; }; + devShells.${system}.default = pkgs.mkShell { packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; }; diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d index 58d87ff..9b24cea 100644 --- a/dlang/src/ocda/dump.d +++ b/dlang/src/ocda/dump.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ module ocda.dump; /+ ↓ the model, written out canonically. diff --git a/dlang/src/ocda/images.d b/dlang/src/ocda/images.d index b3a9652..f6d2541 100644 --- a/dlang/src/ocda/images.d +++ b/dlang/src/ocda/images.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ module ocda.images; /+ ↓ an image's bytes, from whichever source the document came from. diff --git a/dlang/src/ocda/inventory.d b/dlang/src/ocda/inventory.d index dba2156..32b076b 100644 --- a/dlang/src/ocda/inventory.d +++ b/dlang/src/ocda/inventory.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ module ocda.inventory; /+ ↓ what a source holds, beyond the one document the model is loaded from. diff --git a/dlang/src/ocda/model.d b/dlang/src/ocda/model.d index 6c8fc37..6b6f40b 100644 --- a/dlang/src/ocda/model.d +++ b/dlang/src/ocda/model.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ module ocda.model; struct OcdaImage { diff --git a/dlang/src/ocda/sqlite.d b/dlang/src/ocda/sqlite.d index 6f3753c..fc6b2be 100644 --- a/dlang/src/ocda/sqlite.d +++ b/dlang/src/ocda/sqlite.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ module ocda.sqlite; /+ ↓ the sqlite3 C API, declared rather than depended upon. diff --git a/dlang/test/agree.d b/dlang/test/agree.d index bdf1b2e..92b1f91 100644 --- a/dlang/test/agree.d +++ b/dlang/test/agree.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ /++ Do the two loaders agree? diff --git a/dlang/test/dump.d b/dlang/test/dump.d index 17aa9bb..f11d456 100644 --- a/dlang/test/dump.d +++ b/dlang/test/dump.d @@ -1,3 +1,52 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ /++ Write a loaded abstraction out canonically. diff --git a/org/config.org b/org/config.org new file mode 100644 index 0000000..2c1294f --- /dev/null +++ b/org/config.org @@ -0,0 +1,830 @@ +-*- mode: org -*- +#+TITLE: configuration nix +#+DESCRIPTION: configuration nix +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: show3levels hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* ./ +** README.md + +#+HEADER: :tangle ../README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# sisudoc-ocda-lib + +The **object-centric document abstraction as a library**: the model, and +loaders for both of the forms it is written in. + +**One reader per language, in a directory named for the language.** + +| | | +|--------------------|---------| +| [`dlang/`](dlang/) | D | +| [`gleam/`](gleam/) | Gleam | +| `ruby/` | to come | + +## What it is for + +The readers in `sisudoc-ssp-reader` and `sisudoc-ocda-db-reader` each read +one serialisation and print. A **writer** needs neither: it needs the document. + +``` + .ssp ─┐ + ├─> the model ─> a writer + .ocda.db ─┘ +``` + +So this library holds the model and both loaders, and **a writer cannot +tell which serialisation it was handed** - exactly as spine's own writers +cannot, since spine 0.25.0. + +That is the whole of the idea. The writers are functions from the model to +bytes, and the reader is a program over the model. + +## The reader + +```sh +ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images|--verify --image-dir=<d>| + --languages|--files|--report-unknown|--validate] + <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] +``` + +One reader, in each language, where `sisudoc-ssp-reader` and +`sisudoc-ocda-db-reader` were two, each with a parser of its own. Built on +this library, **every mode reads every form and says the same thing of a +document's `.ssp`, `.ocda.db` and `.sisupod`**, 36 of 36 document-languages. +The D reader has every mode of both old readers; the Gleam one the modes the +Gleam readers had (the summary, `--tree`, `--languages`, `--files`, +`--report-unknown`), and in those **its output is the D one's, byte for +byte**, 36 of 36. + +It is in each language's `src/read/`, and not part of the library: a +program using the library does not build it (in D, by dub configuration; +in Gleam a dependant compiles it and does not use it, Gleam building +everything in `src/`). + +Writing it found one defect in the library: **the database loader could +not say what it had not understood**. It asks for columns by name, and so +never saw the ones it did not ask for, and reported that it had understood +everything. It now counts them on the conditions under which the `.ssp` +writes the same property, and the report is the same from either form. + +## The checks that matter + +The claim above is only true if the loaders agree, so the library asks. Two +programs, in each language: + +```sh +ocda-agree <doc>.ssp <doc>.ocda.db [lang] # do the two loaders agree? +ocda-dump <file>.ssp|<file>.ocda.db [lang] # the whole model, canonically +``` + +`agree` compares two models within one language. **`dump` is the stronger +of the two**, and the reason it exists: it writes the entire loaded model +out in a fully determined form, and the D and the Gleam library write it +*byte for byte alike*. So four questions can be asked, all of them with +`diff`: + +| | | +|-----------------------------|---------------------------------| +| D: `.ssp` == `.ocda.db` | **36 of 36** document-languages | +| Gleam: `.ssp` == `.ocda.db` | **36 of 36** | +| `.ssp`: D == Gleam | **36 of 36** | +| `.ocda.db`: D == Gleam | **36 of 36** | + +None of those held at first. + +## What the checks found + +### the ten-field check: a blank first text line + +An object whose **first text line is blank** lost it. The loader decided +whether to start a new line by asking whether the accumulated text was +empty, which is not the same question as whether a line has been seen. Of +the sample collection **seven documents have verse opening on a blank +line**, and in every one of them the blank was dropped. Eight of +thirty-six documents disagreed. + +Nothing else had caught it. The two `.ssp` readers had been compared on +*section counts*, which this does not change. It took loading the same +document two ways and comparing the objects. + +Fixed here and in `sisudoc-ssp-reader/dlang`, which carried the same code. + +### the whole-model dump: four more + +The ten-field check was enough for that and not enough for anything +subtler. Dumping the whole model found four further disagreements between +the two loaders: + +1. **`.identifier` and `.anchor`.** The `.ssp` emits a property only when it + carries information, so the identifier is absent when it is simply the + ocn and the anchor is absent when it is the segment name; the database + stores both unconditionally. Over one document that is **1,314 + identifiers and 69 anchors**. The loaders now derive them, which is what + the model means. +2. **`make` and `doc_has`.** The database has one metadata table where the + text form has three blocks, and tells them apart by a key prefix. The + loaders unprefix them, so a writer asks `make["footer"]` whichever + serialisation it was handed. +3. **The JSON columns were scanned rather than parsed.** Phobos writes `/` + as `\/`, so every heading containing a slash came back with a backslash + in it: **all 36 document-languages** differed on + `heading_ancestors_text` and nothing else had shown it. +4. **`translation.ocn_aligned`** is in the database and cannot be in a + `.ssp`, which is one language and cannot know whether the languages + share a numbering. It is a fact about the *file*, so the loaders leave + it out rather than letting a writer tell which serialisation it was + given. + +The first three are defects. The fourth is a real asymmetry between the two +serialisations, and naming it is the useful part. + +## What it reads, and what it does not + +| | | | +|-----------------------|------------------------------------------------------------------|-------------------| +| `.ssp` | yes | | +| `.ocda.db` | yes, any language of it | | +| `.sisupod` | yes, any language it carries | D only, see below | +| `.zip` pod | yes, any language it carries | | +| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work | | +| a url | no | | + +A pod carries the abstraction only when spine was asked for it: `--pod2` +puts it in, `--source` does not. A pod without one is refused with a message +saying which flag makes a pod this library can read, because that is a fact +about the pod rather than about the library. + +**Over the reference collection, a `.sisupod` gives the same model as the +`.ssp` beside it: 36 of 36 document-languages, byte identical dumps.** A +writer over the library therefore gained a third input form without being +touched, which is the point of the dispatch being in one place. + +### the one thing the Gleam library cannot do + +A `.sisupod` is **one zstd frame wrapping** the zip. The D library reads both +through a declared `libzstd`. The Gleam library reads the **zip and not the +frame**, because the only zstd on the BEAM is a NIF that clones zstd from +github in its own build and so cannot be built in a sandbox without patching +it. + +So a `.zip` pod is read in both languages, a `.sisupod` in D only, and the +Gleam refusal says why and says `zstd -d` rather than claiming the file is +malformed. It is a packaging limit, not anything about the format, and it is +the only place the two libraries differ in what they accept. + +`ocda/load` is the one place in each language that decides what a path is, +so a new input form is added once and every reader and writer gains it. + +## The format + +Specified in **sisudoc-abstraction-format**: `ocda-model.md` for the model +this library implements, and the two format specifications for the loaders. +Section 4 of `ocda-db-format-specification.md` gives the column by column +correspondence the `load_db` of each language follows. + +At format **2.x**. +#+END_SRC + +** git .gitignore :gitignore: + +#+HEADER: :tangle "../.gitignore" +#+BEGIN_SRC gitignore +# git ls-files --others --exclude-from=.git/info/exclude +# git ls-files --others --ignored --exclude-standard --directory +# git check-ignore -v flake.lock +# git clean -ix +# find . -name "*_" | xargs rm -rf +# find . -name "*.o" | xargs rm -rf + +# Deny everything, then whitelist what belongs in the repository. +# +# The practice is deliberate: a repository should hold what was decided to +# put in it, not whatever happened to be in the directory when someone ran +# "git add .". Build output, editor droppings, downloaded dependencies and +# exported binaries are numerous, are regenerated, and differ between +# machines; source is few and is chosen. So the default is no, and every +# yes below is a decision that can be read. +# +# To see what is being excluded, and why: +# git status --ignored +# git check-ignore -v <path> +# git ls-files --others --ignored --exclude-standard --directory + +* + +# ── the repository itself ──────────────────────────────────────────────── +!.gitignore +!README.md +!LICENSE + +# ── directories have to be walked into before a file inside can match ──── +!*/ +# but not these, whatever they contain +build/ +.build/ +.direnv/ +.gleam/ +bin/ +result +result-* + +# ── documentation and notes, in any language directory ─────────────────── +!**/*.md +!**/*.org + +# ── nix ────────────────────────────────────────────────────────────────── +!**/*.nix +!**/flake.lock +!**/.envrc + +# ── dlang ──────────────────────────────────────────────────────────────── +!**/*.d +!**/dub.json +!**/dub.sdl +!**/dub.selections.json +*.o +*.a +*.lst + +# ── gleam ──────────────────────────────────────────────────────────────── +!**/*.gleam +# Erlang, for what Gleam reaches through FFI (a sha256, from OTP crypto) +!**/src/*.erl +# gleam.toml is the manifest, manifest.toml is the lockfile: both belong +!**/*.toml +*.beam +*.ez +erl_crash.dump + +# ── ruby, when it arrives ──────────────────────────────────────────────── +!**/*.rb +!**/Gemfile +!**/Gemfile.lock +!**/*.gemspec + +# ── continuous integration ─────────────────────────────────────────────── +!**/*.yml +!**/*.yaml +#+END_SRC + +* ./dlang +** nix flake.nix :flake: + +#+HEADER: :tangle ../dlang/flake.nix +#+HEADER: :noweb yes +#+BEGIN_SRC nix +{ + description = "ocda-lib: the object-centric document abstraction as a D library"; + inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + /* A library, so the library itself is not a build product: a program + that uses it compiles its modules alongside its own, as the writer's + flake does, or takes it as a dub dependency. What is built here are + programs over it: the reader, and the two that put the library to + the question - the agreement check, which loads a document's .ssp + and its .ocda.db and compares the two models, and the dump, which + writes a loaded model out canonically so that an implementation in + another language can be diffed against this one. + . + `mains` is what is compiled besides the library: the checks are one + file each in test/, the reader is src/read/. + */ + mkTool = { tool, binary, what, mains, compiler, compilerBin }: + pkgs.stdenv.mkDerivation { + pname = "ocda-lib-${tool}"; + version = "0.1.0"; + src = self; + nativeBuildInputs = [ compiler ]; + buildInputs = [ pkgs.sqlite pkgs.zstd ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ${tool} $out/bin/${binary} + runHook postInstall + ''; + meta = { + description = what; + mainProgram = binary; + }; + }; + ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; + ocda-read = mkTool (ldc // { + tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; + what = "reads the abstraction in any of its forms and says what is in it"; + }); + ocda-agree = mkTool (ldc // { + tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; + what = "checks that the .ssp and .ocda.db loaders give one model"; + }); + ocda-dump = mkTool (ldc // { + tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; + what = "writes a loaded abstraction out canonically, as an oracle"; + }); + in { + packages.${system} = { + /* All three in one bin/, so that one build leaves every program in + one result link. Built separately into the same link, each build + replaced the one before, and a script looking for ocda-dump beside + ocda-agree found nothing. + */ + default = pkgs.symlinkJoin { + name = "ocda-lib-programs-0.1.0"; + paths = [ ocda-read ocda-agree ocda-dump ]; + }; + inherit ocda-read ocda-agree ocda-dump; + }; + + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; + }; + }; +} +#+END_SRC + +** README.md + +#+HEADER: :tangle ../dlang/README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# ocda-lib, in D + +The abstraction's model and both its loaders, as a dub library. + +## Modules + +| | | +|---|---| +| `ocda.model` | the document, the object, the eight sections. Imports nothing | +| `ocda.load_ssp` | text -> model | +| `ocda.load_db` | sqlite -> model | +| `ocda.load` | what a path is, and loading it | +| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries | +| `ocda.dump` | the model written out canonically, as an oracle | +| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon | +| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) | + +`ocda.model` importing nothing is the property that matters: a writer +depends on the model and on neither loader. + +## Use + +```json +"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } } +``` + +```d +import ocda.load; +auto doc = load("document.ocda.db", "de"); // or "document.ssp" +foreach (obj; doc.body) { ... } +``` + +Link `sqlite3`; the library declares the dozen functions it needs rather +than taking a binding from the registry, so there is nothing to fetch. + +## The reader + +```sh +nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl| + --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate] + <file>.ssp|<file>.ocda.db|<file>.sisupod [lang] +``` + +The library's reader, one program where `sisudoc-ssp-reader` and +`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp` +reader's, moved onto the library's model, so every mode reads every form +and **says the same thing of a document's `.ssp`, `.ocda.db` and +`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries +files; `--validate` checks a `.ssp` as text and refuses anything else. + +It lives in `src/read/` beside the library, and dub keeps it out of the +library: the configuration `library` excludes `src/read/*`, the +configuration `ocda-read` builds it. A program using the library by dub, or +by naming `src/ocda/*.d` as the writer's flake does, never compiles it. + +## The two checks + +```sh +nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang] +nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang] +``` + +`agree` loads both and compares ten fields of each object, naming the field +where they part company. `agree` found the blank-first-text-line defect and +was not enough for anything subtler. + +**`ocda-dump` is the stronger check**: the whole loaded model, every field, +written out in a form fully determined by the model, so that two +implementations can be compared with `diff`. It is an oracle, not an output +format; anything reading it to reconstruct a document has misunderstood it. +It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is +the point of having it. + +It found four disagreements `agree` could not see. They are in the +repository README and in the history. + +## Build + +```sh +nix build # all three programs, in one result/bin +nix build .#ocda-read # the reader +nix build .#ocda-agree # the agreement check +nix build .#ocda-dump # the canonical dump +nix develop # ldc, dub, dtools, sqlite, zstd +dub build # the library +dub build --config=ocda-read # the reader +``` +#+END_SRC + +** dub.json + +#+HEADER: :tangle ../dlang/dub.json +#+HEADER: :noweb yes +#+BEGIN_SRC json +{ + "authors": [ "Ralph Amissah" ], + "copyright": "Copyright (C) 2026 Ralph Amissah", + "name": "ocda-lib", + "version": "0.1.0", + "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations", + "homepage": "https://sisudoc.org", + "license": "AGPL-3.0+", + "sourcePaths": [ "./src" ], + "importPaths": [ "./src" ], + "libs": [ "sqlite3", "zstd" ], + "configurations": [ + { + "name": "library", + "targetType": "library", + "excludedSourceFiles": [ "src/read/*" ] + }, + { + "name": "ocda-read", + "targetType": "executable", + "targetName": "ocda-read", + "mainSourceFile": "src/read/main.d" + } + ] +} +#+END_SRC + +* ./gleam +** nix flake.nix :flake: + +#+HEADER: :tangle ../gleam/flake.nix +#+HEADER: :noweb yes +#+BEGIN_SRC nix +{ + description = "ocda-lib: the object-centric document abstraction as a Gleam library"; + inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + + name = "ocda_lib"; + + /* rebar3 with the port compiler already in it. + esqlite builds a NIF, and its rebar3 configuration asks for the + `pc` plugin, which rebar3 would otherwise fetch from Hex at build + time: impossible in a sandbox, and the build fails with + "missing_package pc". nixpkgs carries the plugin, so the answer is + a rebar3 that has it before it starts. + */ + rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { + plugins = [ pkgs.beamPackages.pc ]; + }; + + /* The Hex dependencies, as manifest.toml pins them. + . + There is no Gleam builder in nixpkgs and `gleam build` fetches from + Hex, so a sandboxed build cannot reach them. A fixed output + derivation may, because its result is pinned by hash. + . + When a dependency changes, manifest.toml changes and this hash must + change with it: build, take the hash nix reports, put it here. + */ + deps = pkgs.stdenv.mkDerivation { + pname = "${name}-deps"; + version = "0.1.0"; + src = ./.; + nativeBuildInputs = [ pkgs.gleam pkgs.cacert ]; + buildPhase = '' + export HOME=$TMPDIR + gleam deps download + ''; + installPhase = '' + mkdir -p $out + cp -r build/packages/* $out/ + ''; + dontFixup = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c="; + }; + in { + packages.${system} = { + default = self.packages.${system}.${name}; + /* Shipped as an Erlang shipment rather than as an escript, because + sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into + a shared object, and a NIF cannot be loaded from inside an + escript archive, the runtime needing a real file on disk. The + shipment keeps priv/ on disk, so it loads. + */ + ${name} = pkgs.stdenv.mkDerivation { + pname = "ocda-lib"; + version = "0.1.0"; + src = ./.; + nativeBuildInputs = [ + pkgs.gleam + pkgs.erlang + rebar3-pc + pkgs.makeWrapper + ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + # the dependencies, in place of the fetch gleam would do + mkdir -p build/packages + cp -r ${deps}/* build/packages/ + chmod -R u+w build/packages + gleam export erlang-shipment + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib $out/bin + cp -r build/erlang-shipment $out/lib/${name} + makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \ + --add-flags run \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} + # the reader under its own name, as the D flake installs it + makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \ + --add-flags "run read" \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} + runHook postInstall + ''; + meta = { + description = "the abstraction as a Gleam library, with its reader, dump and agreement check"; + mainProgram = "ocda-lib"; + }; + }; + }; + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ]; + }; + }; +} +#+END_SRC + +** README.md + +#+HEADER: :tangle ../gleam/README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# ocda-lib, in Gleam + +The abstraction's model and both its loaders, as a Gleam library, with its +reader and the two programs that put it to the question. + +| | | +|------------------|-----------------------------------------------------------------------------------------| +| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | +| `ocda/load_ssp` | text -> model | +| `ocda/load_db` | sqlite -> model | +| `ocda/load` | what a path is, and loading it | +| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | +| `ocda/dump` | the model written out canonically, as an oracle | +| `read/main` | `ocda-read`, the reader | +| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | + +## Use + +```gleam +import ocda/load +import ocda/model + +case load.load(path, "en", False) { + Error(why) -> io.println_error(why) + Ok(doc) -> { + // and from here a writer neither knows nor cares which of the two + // files it was handed + list.each(model.section(doc, "body"), fn(obj) { ... }) + } +} +``` + +## The reader + +```sh +nix build +./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] + <file>.ssp|<file>.ocda.db|<file>.zip [lang] +``` + +`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers +had, over the library, so each reads every form this library reads; the +others are the D reader's. **Its output is the D `ocda-read`'s, byte for +byte**, in every mode both have. In `src/read/`, because Gleam compiles +nothing outside `src/`; a program depending on the library compiles it too, +and does not use it. + +A `.sisupod` it refuses, saying why: the zstd frame, below. + +## The two programs + +```sh +nix build +./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] +./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] +``` + +`agree` asks whether the two loaders give the same model. `dump` writes +the whole model out canonically, and is the stronger check of the two, +because it is **byte for byte what the D library's `ocda-dump` writes**. +So the four questions can all be asked with `diff`: + +| | | +|-----------------------------|-----------------------------| +| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | +| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | +| `.ssp`: D == Gleam | 36 of 36 | +| `.ocda.db`: D == Gleam | 36 of 36 | + +## Deliberately a mirror of the D library + +Field for field, name for name, including where that is not the most +Gleamish choice: + +- `text` is one `String` with the lines joined by newline, not a + `List(String)`. So an object whose only text line is blank is + indistinguishable from an object with no text, *in both languages, in the + same place*. A difference that existed in one and not the other would be + worse than a loss both share, because it would show up in every dump and + mean nothing. +- the properties are typed fields rather than a list of pairs. A writer + wants `obj.parent_ocn`, not a lookup that might miss. + +The point is that the two agree. Making them agree is much cheaper when +they are shaped alike, and the shape is the specification's, not either +language's. + +## What the agreement found + +Four disagreements, none of which a comparison of section counts or of a +handful of fields would have shown. All four are recorded in the D +library's history; the short version: + +1. `.identifier` and `.anchor` are omitted by the text form when they can + be worked out (the identifier is the ocn, the anchor is the segment). + Both loaders now derive them. Over one document that is 1,314 + identifiers. +2. `make` and `doc_has` are one metadata table in the database, + distinguished by a key prefix. Both loaders unprefix them. +3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` + as `\/` and every heading with a slash in it came back wrong. +4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, + which is one language. It is a fact about the file, so neither loader + puts it in the document. + +## Build + +```sh +nix build # sandboxed, no network +nix develop -c gleam test +``` + +Shipped as an Erlang shipment rather than an escript, because sqlite on the +BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. +The dependency hash in `flake.nix` pins what `manifest.toml` pins; change +one and the other must change with it. + +## Format + +Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, +`ssp-format-specification.org` and `ocda-db-format-specification.org` for the +two serialisations. At format **2.x**. +#+END_SRC + +** gleam.toml + +#+HEADER: :tangle ../gleam/gleam.toml +#+HEADER: :noweb yes +#+BEGIN_SRC toml +name = "ocda_lib" +version = "0.1.0" +description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" +licences = ["AGPL-3.0-or-later"] + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +argv = ">= 1.1.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +sqlight = ">= 1.0.0 and < 2.0.0" +gleam_json = ">= 3.0.0 and < 4.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +#+END_SRC + +* versions GET +** dlang overlays +*** ldc + +#+NAME: ldc_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./nix-develop-dlang-shared.org:ldc-version()>> +#+END_SRC + +*** dmd + +#+NAME: dmd_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./nix-develop-dlang-shared.org:dmd-version()>> +#+END_SRC + +*** dub + +#+NAME: dub_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./nix-develop-dlang-shared.org:dub-version()>> +#+END_SRC + +*** dtools + +#+NAME: dtools_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./nix-develop-dlang-shared.org:dtools-version()>> +#+END_SRC + +** sha256 blank hash + +#+NAME: sha256-blank +#+BEGIN_SRC nix +<<./nix-develop-dlang-shared.org:sha256-blank()>> +#+END_SRC + +#+NAME: blank_hash +#+BEGIN_SRC nix +<<./nix-develop-dlang-shared.org:sha256-blank()>> +#+END_SRC + +* org includes +** spine project VERSION + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* nixpkgs source + +#+NAME: nixpkgs_unstable +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> +#+END_SRC + +* __END__ diff --git a/org/config_git.org b/org/config_git.org deleted file mode 100644 index 11cb940..0000000 --- a/org/config_git.org +++ /dev/null @@ -1,101 +0,0 @@ --*- mode: org -*- -#+TITLE: configuration git -#+DESCRIPTION: documents - structuring, various output representations & search -#+FILETAGS: :spine:hub: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* git -** .gitignore :gitignore: - -#+HEADER: :tangle "../.gitignore" -#+BEGIN_SRC gitignore -# git ls-files --others --exclude-from=.git/info/exclude -# git ls-files --others --ignored --exclude-standard --directory -# git check-ignore -v flake.lock -# git clean -ix -# find . -name "*_" | xargs rm -rf -# find . -name "*.o" | xargs rm -rf - -# Deny everything, then whitelist what belongs in the repository. -# -# The practice is deliberate: a repository should hold what was decided to -# put in it, not whatever happened to be in the directory when someone ran -# "git add .". Build output, editor droppings, downloaded dependencies and -# exported binaries are numerous, are regenerated, and differ between -# machines; source is few and is chosen. So the default is no, and every -# yes below is a decision that can be read. -# -# To see what is being excluded, and why: -# git status --ignored -# git check-ignore -v <path> -# git ls-files --others --ignored --exclude-standard --directory - -* - -# ── the repository itself ──────────────────────────────────────────────── -!.gitignore -!README.md -!LICENSE - -# ── directories have to be walked into before a file inside can match ──── -!*/ -# but not these, whatever they contain -build/ -.build/ -.direnv/ -.gleam/ -bin/ -result -result-* - -# ── documentation and notes, in any language directory ─────────────────── -!**/*.md -!**/*.org - -# ── nix ────────────────────────────────────────────────────────────────── -!**/*.nix -!**/flake.lock -!**/.envrc - -# ── dlang ──────────────────────────────────────────────────────────────── -!**/*.d -!**/dub.json -!**/dub.sdl -!**/dub.selections.json -*.o -*.a -*.lst - -# ── gleam ──────────────────────────────────────────────────────────────── -!**/*.gleam -# Erlang, for what Gleam reaches through FFI (a sha256, from OTP crypto) -!**/src/*.erl -# gleam.toml is the manifest, manifest.toml is the lockfile: both belong -!**/*.toml -*.beam -*.ez -erl_crash.dump - -# ── ruby, when it arrives ──────────────────────────────────────────────── -!**/*.rb -!**/Gemfile -!**/Gemfile.lock -!**/*.gemspec - -# ── continuous integration ─────────────────────────────────────────────── -!**/*.yml -!**/*.yaml -#+END_SRC diff --git a/org/config_nix.org b/org/config_nix.org deleted file mode 100644 index fcf53f3..0000000 --- a/org/config_nix.org +++ /dev/null @@ -1,287 +0,0 @@ --*- mode: org -*- -#+TITLE: configuration nix -#+DESCRIPTION: configuration nix -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* nix :nix: -** dlang flake.nix :flake: - -#+HEADER: :tangle ../dlang/flake.nix -#+HEADER: :noweb yes -#+BEGIN_SRC nix -{ - description = "ocda-lib: the object-centric document abstraction as a D library"; - inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; - outputs = { self, nixpkgs }: - let - system = "x86_64-linux"; # adjust for aarch64-linux / darwin - pkgs = import nixpkgs { inherit system; }; - /* A library, so the library itself is not a build product: a program - that uses it compiles its modules alongside its own, as the writer's - flake does, or takes it as a dub dependency. What is built here are - programs over it: the reader, and the two that put the library to - the question - the agreement check, which loads a document's .ssp - and its .ocda.db and compares the two models, and the dump, which - writes a loaded model out canonically so that an implementation in - another language can be diffed against this one. - . - `mains` is what is compiled besides the library: the checks are one - file each in test/, the reader is src/read/. - */ - mkTool = { tool, binary, what, mains, compiler, compilerBin }: - pkgs.stdenv.mkDerivation { - pname = "ocda-lib-${tool}"; - version = "0.1.0"; - src = self; - nativeBuildInputs = [ compiler ]; - buildInputs = [ pkgs.sqlite pkgs.zstd ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/bin - install -m755 ${tool} $out/bin/${binary} - runHook postInstall - ''; - meta = { - description = what; - mainProgram = binary; - }; - }; - ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; - ocda-read = mkTool (ldc // { - tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; - what = "reads the abstraction in any of its forms and says what is in it"; - }); - ocda-agree = mkTool (ldc // { - tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; - what = "checks that the .ssp and .ocda.db loaders give one model"; - }); - ocda-dump = mkTool (ldc // { - tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; - what = "writes a loaded abstraction out canonically, as an oracle"; - }); - in { - packages.${system} = { - /* All three in one bin/, so that one build leaves every program in - one result link. Built separately into the same link, each build - replaced the one before, and a script looking for ocda-dump beside - ocda-agree found nothing. - */ - default = pkgs.symlinkJoin { - name = "ocda-lib-programs-0.1.0"; - paths = [ ocda-read ocda-agree ocda-dump ]; - }; - inherit ocda-read ocda-agree ocda-dump; - }; - devShells.${system}.default = pkgs.mkShell { - packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; - }; - }; -} -#+END_SRC - -** gleam flake.nix :flake: - -#+HEADER: :tangle ../gleam/flake.nix -#+HEADER: :noweb yes -#+BEGIN_SRC nix -{ - description = "ocda-lib: the object-centric document abstraction as a Gleam library"; - inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; - outputs = { self, nixpkgs }: - let - system = "x86_64-linux"; # adjust for aarch64-linux / darwin - pkgs = import nixpkgs { inherit system; }; - - name = "ocda_lib"; - - /* rebar3 with the port compiler already in it. - esqlite builds a NIF, and its rebar3 configuration asks for the - `pc` plugin, which rebar3 would otherwise fetch from Hex at build - time: impossible in a sandbox, and the build fails with - "missing_package pc". nixpkgs carries the plugin, so the answer is - a rebar3 that has it before it starts. - */ - rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { - plugins = [ pkgs.beamPackages.pc ]; - }; - - /* The Hex dependencies, as manifest.toml pins them. - . - There is no Gleam builder in nixpkgs and `gleam build` fetches from - Hex, so a sandboxed build cannot reach them. A fixed output - derivation may, because its result is pinned by hash. - . - When a dependency changes, manifest.toml changes and this hash must - change with it: build, take the hash nix reports, put it here. - */ - deps = pkgs.stdenv.mkDerivation { - pname = "${name}-deps"; - version = "0.1.0"; - src = ./.; - nativeBuildInputs = [ pkgs.gleam pkgs.cacert ]; - buildPhase = '' - export HOME=$TMPDIR - gleam deps download - ''; - installPhase = '' - mkdir -p $out - cp -r build/packages/* $out/ - ''; - dontFixup = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c="; - }; - in { - packages.${system} = { - default = self.packages.${system}.${name}; - /* Shipped as an Erlang shipment rather than as an escript, because - sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into - a shared object, and a NIF cannot be loaded from inside an - escript archive, the runtime needing a real file on disk. The - shipment keeps priv/ on disk, so it loads. - */ - ${name} = pkgs.stdenv.mkDerivation { - pname = "ocda-lib"; - version = "0.1.0"; - src = ./.; - nativeBuildInputs = [ - pkgs.gleam - pkgs.erlang - rebar3-pc - pkgs.makeWrapper - ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - # the dependencies, in place of the fetch gleam would do - mkdir -p build/packages - cp -r ${deps}/* build/packages/ - chmod -R u+w build/packages - gleam export erlang-shipment - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/lib $out/bin - cp -r build/erlang-shipment $out/lib/${name} - makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \ - --add-flags run \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} - # the reader under its own name, as the D flake installs it - makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \ - --add-flags "run read" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} - runHook postInstall - ''; - meta = { - description = "the abstraction as a Gleam library, with its reader, dump and agreement check"; - mainProgram = "ocda-lib"; - }; - }; - }; - devShells.${system}.default = pkgs.mkShell { - packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ]; - }; - }; -} -#+END_SRC - -* versions GET -** dlang overlays -*** ldc - -#+NAME: ldc_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./nix-develop-dlang-shared.org:ldc-version()>> -#+END_SRC - -*** dmd - -#+NAME: dmd_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./nix-develop-dlang-shared.org:dmd-version()>> -#+END_SRC - -*** dub - -#+NAME: dub_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./nix-develop-dlang-shared.org:dub-version()>> -#+END_SRC - -*** dtools - -#+NAME: dtools_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./nix-develop-dlang-shared.org:dtools-version()>> -#+END_SRC - -** sha256 blank hash - -#+NAME: sha256-blank -#+BEGIN_SRC nix -<<./nix-develop-dlang-shared.org:sha256-blank()>> -#+END_SRC - -#+NAME: blank_hash -#+BEGIN_SRC nix -<<./nix-develop-dlang-shared.org:sha256-blank()>> -#+END_SRC - -* org includes -** spine project VERSION - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* nixpkgs source - -#+NAME: nixpkgs_unstable -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> -#+END_SRC - -* __END__ diff --git a/org/d-ocda-load.org b/org/d-ocda-load.org deleted file mode 100644 index 59762a8..0000000 --- a/org/d-ocda-load.org +++ /dev/null @@ -1,953 +0,0 @@ --*- mode: org -*- -#+TITLE: dlang load -#+DESCRIPTION: dlang lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* dlang :dlang: -** load dlang - -#+HEADER: :tangle "../dlang/src/ocda/load.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load; - -/+ ↓ what a path is, and loading it. - . - One place that decides, so that every reader and every writer accepts - the same inputs and a new input type is added once. This mirrors - spine's own ocda/abstraction/load.d, which names the sources it can - enter the pipeline from and tells them apart by suffix. - . - Deliberately *not* here: .sst and .ssm markup. Reading markup means - implementing the parser, which is spine's work and a far larger thing - than a reader or a sample writer needs. This library reads - abstractions. -+/ - -import std.algorithm : endsWith; -import ocda.model; -import ocda.load_ssp; -import ocda.load_db; -import ocda.load_pod; - -enum Source { - ssp, // .ssp, the abstraction as text - ocda_db, // .ocda.db, the abstraction as sqlite - pod, // .sisupod or .zip, an archive carrying a .ssp - unknown, -} - -Source sourceOf(string path) { - if (path.endsWith(".ocda.db")) { return Source.ocda_db; } - if (path.endsWith(".ssp")) { return Source.ssp; } - /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote - before, and every pod already published carries that name, so both - are read. What is inside decides how it is opened, not the name. - +/ - if (path.endsWith(".sisupod")) { return Source.pod; } - if (path.endsWith(".zip")) { return Source.pod; } - if (path.endsWith(".db")) { return Source.ocda_db; } - return Source.unknown; -} - -string sourceName(Source s) { - final switch (s) { - case Source.ssp: return ".ssp (the abstraction as text)"; - case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; - case Source.pod: return "a pod archive carrying a .ssp"; - case Source.unknown: return "not an abstraction this library reads"; - } -} - -class OcdaLoadError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ load whatever the path is into the model. - lang names which language to take from a .ocda.db, which holds - every language of its document, and from a pod, which may carry more - than one .ssp. A .ssp is one language by construction and ignores it. -+/ -OcdaDocument load(string path, string lang = "", bool any_version = false) { - final switch (sourceOf(path)) { - case Source.ssp: return parseSspFile(path); - case Source.ocda_db: return loadDbFile(path, lang, any_version); - case Source.pod: return loadPodFile(path, lang); - case Source.unknown: - throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) - ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); - } -} -#+END_SRC - -** load db dlang - -#+HEADER: :tangle "../dlang/src/ocda/load_db.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_db; - -/+ ↓ build the model from a .ocda.db. - . - The point of this module is that it returns *the same value* - ocda.load_ssp returns from the same document's .ssp. A writer is then - a function from the model to bytes, and cannot tell which - serialisation it was given, exactly as spine's own writers cannot. - . - Written from the sisudoc-abstraction-format specification, - ocda-db-format-specification.md, whose section 4 gives the column by - column correspondence this module implements. -+/ - -import std.array : join, split; -import std.json : JSONType, parseJSON; -import std.conv : to; -import std.algorithm : startsWith; -import std.string : strip; -import ocda.model; -import ocda.sqlite; - -/+ ↓ the schema major this loader knows (specification section 2) +/ -enum schema_major_read = "2"; - -class OcdaDbError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ every language the database holds, in manifest order +/ -struct DbLanguage { - int doc_id; - string lang; - string source_filename; - string ssp_digest; -} - -DbLanguage[] dbLanguages(ref Db db) { - DbLanguage[] out_; - foreach (r; db.rows( - "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') - FROM documents ORDER BY id")) { - out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); - } - return out_; -} - -/+ ↓ load one language of a database as the model. - lang == "" takes the first language the manifest gave. -+/ -OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { - auto db = Db.open(path); - scope(exit) { db.close(); } - - if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { - throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" - ~ " it does not look like a .ocda.db"); - } - - /+ ↓ specification sections 2 and 3.2: the rows describing the file rather - than a document in it are the only ones with doc_id null +/ - string schema_version = db.one( - "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); - if (schema_version.length > 0 && !any_version) { - auto major = schema_version.split(".")[0]; - if (major != schema_major_read) { - throw new OcdaDbError(path ~ " is schema v" ~ schema_version - ~ ", this loader knows v" ~ schema_major_read ~ ".x"); - } - } - - auto languages = dbLanguages(db); - if (languages.length == 0) { - throw new OcdaDbError(path ~ ": no documents rows"); - } - DbLanguage chosen = languages[0]; - if (lang.length > 0) { - bool found = false; - foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } - if (!found) { - string have; - foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } - throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); - } - } - - OcdaDocument doc; - doc.format_version = schema_version; - doc.source_filename = chosen.source_filename; - - /+ ↓ @source of the .ssp corresponds to the documents row and the - source.digest metadata (specification section 5.1 of the .ssp, and - section 7 here) +/ - doc.source["language"] = chosen.lang; - string languages_joined; - foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } - doc.source["languages"] = languages_joined; - auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " - ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); - if (digest.length > 0) { doc.source["digest"] = digest; } - - /+ ↓ @meta: every metadata row of this language that is not a source.* - property. source.* describes the artefact rather than the document - and lives in @source in the text form. - +/ - foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " - ~ chosen.doc_id.to!string ~ " ORDER BY key")) { - string key = r[0]; - if (key.startsWith("source.")) { continue; } - /+ ↓ translation.ocn_aligned says whether the languages of this - document share one numbering. A .ocda.db can know it because it - holds every language; a .ssp is one language and cannot. So it is - a fact about the *file* rather than about the document, and putting - it in the document's metadata would let a writer tell which - serialisation it was handed, which is the one thing this library - exists to prevent. Left out until the model has a place for - collection-wide facts. - +/ - if (key.startsWith("translation.")) { continue; } - /+ ↓ the database has one metadata table where the text form has three - blocks, and tells them apart by a key prefix. Unprefixing them - here is what makes the two loaders give one model: a writer asks - doc.make["footer"] whichever serialisation it was handed, and - never doc.metadata["make.footer"]. - +/ - if (key.startsWith("make.")) { - doc.make[key["make.".length .. $]] = r[1].strip; - } else if (key.startsWith("doc_has.")) { - doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; - } else { - doc.metadata[key] = r[1].strip; - } - } - - /+ ↓ 2.1 added the container range as two columns. A 2.0 database has - neither, and reads as having no range, which is what 2.0 meant. +/ - bool has_range = false; - foreach (c; db.rows("PRAGMA table_info(objects)")) { - if (c[1] == "ocn_range_first") { has_range = true; } - } - string range_cols = has_range - ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" - : "0, 0"; - - /+ ↓ the objects, in document order within each section +/ - foreach (r; db.rows( - "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), - coalesce(is_of_section,''), coalesce(is_of_type,''), - coalesce(heading_level,-1), coalesce(identifier,''), - coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), - coalesce(children,''), coalesce(ancestors,''), - coalesce(dummy_heading,0), coalesce(object_number_off,0), - coalesce(indent_base,0), coalesce(indent_hang,0), - coalesce(bullet,0), coalesce(lang,''), - coalesce(has_links,0), coalesce(has_notes_reg,0), - coalesce(has_notes_star,0), coalesce(has_images,0), - coalesce(segment,''), coalesce(segment_prev,''), - coalesce(segment_next,''), coalesce(anchor,''), - coalesce(segment_html_is,''), - coalesce(table_cols,0), coalesce(table_widths,''), - coalesce(table_aligns,''), coalesce(table_header,0), - coalesce(code_linenumbers,0), - coalesce(heading_ancestors_text,''), coalesce(sha256,''), - coalesce(text,''), id, coalesce(heading_lev_anchor,''), - coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ " - FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string - ~ " ORDER BY CASE section - WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 - WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 - WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 - WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { - OcdaObject obj; - obj.section = r[0]; - obj.ocn = r[2].to!int; - obj.is_a = r[3]; - obj.part = r[4]; - obj.own_section = (r[5] == r[0]) ? "" : r[5]; - obj.is_of_type = r[6]; - /+ ↓ the database holds the level as a number, the text form as the - letter or digit the markup wrote. Model section 4 is the mapping. +/ - obj.heading_level = levelName(r[7].to!int); - obj.identifier = r[8]; - obj.parent_ocn = r[9].to!int; - obj.last_descendant_ocn = r[10].to!int; - obj.children = jsonInts(r[11]); - obj.ancestors = jsonInts(r[12]); - obj.dummy_heading = (r[13] != "0"); - obj.ocn_off = (r[14] != "0"); - obj.indent_base = r[15].to!int; - obj.indent_hang = r[16].to!int; - obj.bullet = (r[17] != "0"); - obj.lang = r[18]; - obj.has_links = (r[19] != "0"); - obj.has_notes_reg = (r[20] != "0"); - obj.has_notes_star = (r[21] != "0"); - obj.has_images = (r[22] != "0"); - obj.segment = r[23]; - obj.segment_prev = r[24]; - obj.segment_next = r[25]; - obj.anchor = r[26]; - obj.segment_html_is = r[27]; - obj.heading_lev_anchor = r[37]; - obj.segment_epub = r[38]; - obj.segment_epub_is = r[39]; - obj.ocn_range_first = r[40].to!int; - obj.ocn_range_last = r[41].to!int; - obj.table_cols = r[28].to!int; - obj.table_widths = jsonWords(r[29]); - obj.table_aligns = jsonWords(r[30]); - obj.table_header = (r[31] != "0"); - obj.code_linenumbers = (r[32] != "0"); - obj.heading_ancestors_text = jsonJoined(r[33]); - obj.sha256 = r[34]; - obj.text = r[35]; - - /+ ↓ the four repeatable properties of the text form are child tables - here (specification section 4) +/ - string object_id = r[36]; - foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), - coalesce(width,0), coalesce(height,0), - coalesce(missing,0) - FROM object_images WHERE object_id = " ~ object_id - ~ " ORDER BY seq")) { - OcdaImage img; - img.name = i[0]; - img.sha256 = i[1]; - img.bytes = i[2].to!ulong; - img.width = i[3].to!int; - img.height = i[4].to!int; - img.missing = (i[5] != "0"); - obj.images ~= img; - } - foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " - ~ object_id ~ " ORDER BY seq")) { - obj.stow_link ~= l[0]; - } - foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " - ~ object_id ~ " ORDER BY seq")) { - obj.anchor_tags ~= a[0]; - } - - auto section = doc.sectionByName(r[0]); - if (section !is null) { *section ~= obj; } - } - countUnread(db, chosen.doc_id, doc); - return doc; -} - -/+ ↓ what the database holds and the model does not. - . - The text loader meets every property line and counts those it does not - keep, so a reader can say what was read and not understood. This - loader asks for columns by name, and would never see the ones it did - not ask for: it would report that it had understood everything, which - is the silence the count exists to prevent. So it counts them, on the - conditions under which the .ssp writes the same property (specification - section 6.3), and the report is the same from either form: checked, - 36 of 36. -+/ -private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { - string id = doc_id.to!string; - void count(string property, string where) { - auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id - ~ " AND " ~ where).to!size_t; - if (n > 0) { doc.unknown.property[property] = n; } - } - count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); - count("dom_status", "coalesce(dom_status,'') != ''"); - count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); - count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); - count("parent_lev", "coalesce(parent_lev,0) != 0"); - count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); - count("attrib", "coalesce(attrib,'') != ''"); - if (db.has("object_subtoc")) { - auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" - ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; - if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } - } -} - -/+ ↓ model section 4: the eight levels and their indices +/ -private string levelName(int index) { - switch (index) { - case 0: return "A"; - case 1: return "B"; - case 2: return "C"; - case 3: return "D"; - case 4: return "1"; - case 5: return "2"; - case 6: return "3"; - case 7: return "4"; - default: return ""; - } -} - -/+ ↓ the JSON columns, read as JSON. - . - The fixed width arrays are JSON in the database and space separated - (or "|" separated) in the text form. This was first written as a - character scan that threw away brackets and quotes, which is wrong the - moment a value holds an escape: Phobos' JSONValue.toString escapes - "/" as "\/", so every heading containing a slash came back with a - backslash in it. 36 of 36 documents differed on - heading_ancestors_text and nothing else had shown it. - . - So: parse. The values are small, there are few of them, and a real - parser is shorter than the scan it replaces. -+/ -private string[] jsonStrings(string v) { - if (v.length == 0) { return []; } - string[] out_; - try { - auto j = parseJSON(v); - if (j.type != JSONType.array) { return []; } - foreach (e; j.array) { - switch (e.type) { - case JSONType.string: out_ ~= e.str; break; - case JSONType.integer: out_ ~= e.integer.to!string; break; - case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; - case JSONType.float_: out_ ~= e.floating.to!string; break; - case JSONType.true_: out_ ~= "true"; break; - case JSONType.false_: out_ ~= "false"; break; - default: out_ ~= ""; break; - } - } - } catch (Exception) { - return []; - } - return out_; -} - -private int[] jsonInts(string v) { - int[] out_; - foreach (s; jsonStrings(v)) { - if (s.length == 0) { continue; } - try { out_ ~= s.to!int; } catch (Exception) {} - } - return out_; -} - -/+ ↓ space separated in the text form: table_widths, table_aligns +/ -private string jsonWords(string v) { - return jsonStrings(v).join(" "); -} - -/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text - form and a JSON array here. The empty slots are significant: they say - which levels the heading sits under, so the separators carry as much - as the text does. -+/ -private string jsonJoined(string v) { - auto parts = jsonStrings(v); - if (parts.length == 0) { return ""; } - return parts.join("|"); -} -#+END_SRC - -** load pod dlang - -#+HEADER: :tangle "../dlang/src/ocda/load_pod.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_pod; - -/+ ↓ the abstraction out of a pod archive. - . - A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same - archive unwrapped, which is what spine published before it wrapped them. - Either way, a pod built with --pod2 carries the abstraction inside it, - at media/abstraction/<uid>.<lang>.ssp, and that is what is read. - . - So this is a third input form and *not* a third model: the .ssp inside - is parsed by ocda.load_ssp and comes back as the same value. A writer - over the library gains .sisupod without being touched, which is the - whole reason the dispatch is in one place. - . - A pod built without --pod2 carries markup and no abstraction. That is - refused, and the refusal says which flag makes a pod this reader can - read, because "no abstraction in it" is a fact about the pod rather - than about the reader. -+/ - -import std.algorithm : endsWith, sort; -import std.array : array; -import std.file : read; -import std.string : indexOf; -import ocda.model; -import ocda.load_ssp; -import ocda.zip; -import ocda.zstd; - -/+ ↓ a limit on what a frame may expand to. - A reader of an artefact it did not make declines to be told how much - memory to allocate. 512 MiB is far above the largest pod in the - reference collection (4 MiB) and far below anything that matters. -+/ -enum size_t max_pod_size = 512 * 1024 * 1024; - -class OcdaPodError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ which languages of the abstraction the pod carries, in sorted order +/ -string[] podLanguages(string path) { - auto bytes = archiveBytes(path); - string[] out_; - foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { - string lang = langOfSspName(e.name); - if (lang.length > 0) { out_ ~= lang; } - } - return out_.sort.array; -} - -/+ ↓ load one language of the abstraction a pod carries. - lang == "" takes the only one there is, and where there is a choice it - says so rather than choosing, as the .ocda.db loader does. -+/ -OcdaDocument loadPodFile(string path, string lang = "") { - auto bytes = archiveBytes(path); - auto entries = zipEntries(bytes); - auto ssps = zipFindSuffix(entries, ".ssp"); - if (ssps.length == 0) { - throw new OcdaPodError(path ~ ": carries no .ssp." - ~ " A pod carries the abstraction only when spine was asked for it:" - ~ " spine --pod2 puts it in, spine --source does not."); - } - ZipEntry chosen = ssps[0]; - if (lang.length > 0) { - bool found = false; - foreach (e; ssps) { - if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } - } - if (!found) { - string have; - foreach (i, e; ssps) { - have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); - } - throw new OcdaPodError(path ~ ": no language '" ~ lang - ~ "'; it holds: " ~ have); - } - } else if (ssps.length > 1) { - string have; - foreach (i, e; ssps) { - have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); - } - throw new OcdaPodError(path ~ ": holds more than one language (" - ~ have ~ "); name the one wanted"); - } - auto ssp = zipRead(bytes, chosen); - return parseSsp(cast(string) ssp.idup); -} - -/+ ↓ the archive, unwrapped if it was wrapped. - The bytes decide, not the name: a .sisupod is a zstd frame and a .zip - is not, and a file misnamed either way still reads. -+/ -package ubyte[] archiveBytes(string path) { - auto bytes = cast(ubyte[]) read(path); - return zstdIsFrame(bytes) - ? zstdDecompress(bytes, max_pod_size) - : bytes; -} - -/+ ↓ <uid>.<lang>.ssp -> <lang> +/ -package string langOfSspName(string name) { - if (!name.endsWith(".ssp")) { return ""; } - string stem = name[0 .. $ - 4]; - auto dot = stem.lastIndexOf('.'); - return (dot > 0) ? stem[dot + 1 .. $] : ""; -} - -private ptrdiff_t lastIndexOf(string s, char c) { - for (ptrdiff_t i = s.length - 1; i >= 0; i--) { - if (s[i] == c) { return i; } - } - return -1; -} -#+END_SRC - -** load ssp dlang - -#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_ssp; - -import std.algorithm : splitter, startsWith; -import std.array : array, split; -import std.conv : to, ConvException; -import std.file : readText; -import std.stdio : writeln; -import std.string : strip, stripLeft, indexOf; -import ocda.model; - -OcdaDocument parseSspFile(string filename) { - string content = readText(filename); - return parseSsp(content); -} - -OcdaDocument parseSsp(string content) { - OcdaDocument doc; - OcdaObject current_obj; - bool in_object = false; - /+ ↓ whether a text line has been seen for the object being read. - Not "is the text non-empty": an object whose first text line is blank - (a verse opening on an empty line, which the sample set has seven of) - would then lose it, the next line overwriting rather than appending. - +/ - bool seen_text = false; - string current_section = ""; - string current_block = ""; // meta, make, doc_has, or section name - - foreach (line; content.splitter("\n")) { - /+ ↓ comment. The first line of a .ssp declares the format version, and - it is the one comment that carries meaning: a reader checks the - major part and refuses what it does not know rather than loading - an unfamiliar artefact half populated in silence. - +/ - if (line.length > 0 && line[0] == '%') { - enum _banner = "% SiSU Document Abstraction v"; - enum _source = "% Source: "; - if (doc.format_version.length == 0 && line.startsWith(_banner)) { - doc.format_version = line[_banner.length .. $].strip; - } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { - doc.source_filename = line[_source.length .. $].strip; - } - continue; - } - - /+ ↓ blank line - finalize current object +/ - if (line.length == 0 || line.strip.length == 0) { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = OcdaObject.init; - in_object = false; - seen_text = false; - } - continue; - } - - /+ ↓ section/block open +/ - if (line[0] == '@') { - auto brace_pos = line.indexOf('{'); - if (brace_pos > 0) { - string name = line[1 .. brace_pos].strip; - current_block = name; - if (!isPreambleBlock(name)) { - current_section = name; - /+ ↓ a block that is neither preamble nor a section this reader - knows: counted, so that --report-unknown can say so +/ - if (doc.sectionByName(name) is null) { - doc.unknown.block[name]++; - } - } - } - continue; - } - - /+ ↓ section/block close +/ - if (line[0] == '}') { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = OcdaObject.init; - in_object = false; - seen_text = false; - } - current_block = ""; - continue; - } - - /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ - if (isPreambleBlock(current_block)) { - string stripped = line.stripLeft; - auto colon_pos = stripped.indexOf(':'); - if (colon_pos > 0) { - string key = stripped[0 .. colon_pos].strip; - string value = (colon_pos + 1 < stripped.length) - ? stripped[colon_pos + 1 .. $].strip : ""; - if (current_block == "meta") { doc.metadata[key] = value; } - else if (current_block == "make") { doc.make[key] = value; } - else if (current_block == "source") { doc.source[key] = value; } - else { doc.doc_has[key] = value; } - } - continue; - } - - /+ ↓ object declaration: [N] type ... +/ - if (line[0] == '[') { - if (in_object) { - commitObject(doc, current_section, current_obj); - } - current_obj = OcdaObject.init; - current_obj.section = current_section; - in_object = true; - seen_text = false; - parseObjectDeclaration(line, current_obj); - continue; - } - - /+ ↓ property: .name: value +/ - if (line[0] == '.') { - if (in_object) { - parseProperty(line, current_obj, doc.unknown); - } - continue; - } - - /+ ↓ text content: | content +/ - if (line.length >= 1 && line[0] == '|') { - if (in_object) { - string text_content = (line.length > 2) ? line[2 .. $] : ""; - if (seen_text) { - current_obj.text ~= "\n" ~ text_content; - } else { - current_obj.text = text_content; - seen_text = true; - } - } - continue; - } - - /+ ↓ no known shape. Counted by its first character so that a reader - can be told the file held something this one did not expect - +/ - doc.unknown.line[line[0 .. 1].idup]++; - } - - /+ ↓ finalize last object +/ - if (in_object) { - commitObject(doc, current_section, current_obj); - } - - return doc; -} - -private void parseObjectDeclaration(string line, ref OcdaObject obj) { - /+ parse: [N] type :level identifier +/ - auto bracket_close = line.indexOf(']'); - if (bracket_close < 0) return; - try { - obj.ocn = line[1 .. bracket_close].strip.to!int; - } catch (ConvException) { - obj.ocn = 0; - } - string rest = (bracket_close + 1 < line.length) - ? line[bracket_close + 1 .. $].strip : ""; - if (rest.length == 0) return; - - auto parts = rest.split(" "); - if (parts.length == 0) return; - obj.is_a = parts[0]; - - if (obj.is_a == "heading" && parts.length >= 2) { - /+ parse :A, :B, :1, etc. +/ - string lev = parts[1]; - if (lev.length >= 2 && lev[0] == ':') { - obj.heading_level = lev[1 .. $]; - } - /+ optional identifier after level +/ - if (parts.length >= 3) { - obj.identifier = parts[2]; - } - } -} - -/+ ↓ the blocks that are document preamble rather than sections of objects +/ -private bool isPreambleBlock(string name) { - return name == "source" || name == "meta" || name == "make" || name == "doc_has"; -} - -private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { - /+ parse: .name: value +/ - auto colon_pos = line.indexOf(':'); - if (colon_pos < 1) return; - string name = line[1 .. colon_pos].strip; - string value = (colon_pos + 1 < line.length) - ? line[colon_pos + 1 .. $].strip : ""; - - switch (name) { - case "part": obj.part = value; break; - case "is_of_type": obj.is_of_type = value; break; - case "section": obj.own_section = value; break; - case "identifier": obj.identifier = value; break; - case "segment_html_is": obj.segment_html_is = value; break; - /+ ↓ the epub segments at heading levels 1 to 4 where the html - segments at level 4 alone, so the two are different files and - the abstraction records both +/ - case "segment_epub": obj.segment_epub = value; break; - case "segment_epub_is": obj.segment_epub_is = value; break; - case "table_aligns": obj.table_aligns = value; break; - case "stow_link": obj.stow_link ~= value; break; - case "anchor_tag": obj.anchor_tags ~= value; break; - case "parent": obj.parent_ocn = toIntSafe(value); break; - case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; - case "ocn_range": { - auto r = parseIntArray(value); - if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; } - break; - } - case "children": obj.children = parseIntArray(value); break; - case "ancestors": obj.ancestors = parseIntArray(value); break; - case "dummy": obj.dummy_heading = (value == "true"); break; - case "ocn_off": obj.ocn_off = (value == "true"); break; - case "indent": - auto parts = value.split(" "); - if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); - if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); - break; - case "bullet": obj.bullet = (value == "true"); break; - case "lang": obj.lang = value; break; - case "has": - foreach (flag; value.split(" ")) { - switch (flag) { - case "links": obj.has_links = true; break; - case "notes_reg": obj.has_notes_reg = true; break; - case "notes_star": obj.has_notes_star = true; break; - case "images": obj.has_images = true; break; - default: break; - } - } - break; - case "segment": obj.segment = value; break; - case "segment_prev": obj.segment_prev = value; break; - case "segment_next": obj.segment_next = value; break; - case "anchor": obj.anchor = value; break; - case "heading_lev_anchor": obj.heading_lev_anchor = value; break; - case "table_cols": obj.table_cols = toIntSafe(value); break; - case "table_widths": obj.table_widths = value; break; - case "table_header": obj.table_header = (value == "true"); break; - case "code_syntax": obj.code_syntax = value; break; - case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; - case "heading_ancestors_text": obj.heading_ancestors_text = value; break; - case "sha256": obj.sha256 = value; break; - case "image": obj.images ~= parseImageRecord(value); break; - /+ ↓ a property this reader has no use for, or has never heard of. - Ignored, which is the conforming behaviour within a major - version, but counted rather than passed over in silence - +/ - default: unknown.property[name]++; break; - } -} - -private OcdaImage parseImageRecord(string value) { - /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] - filename first, remaining fields order-independent; unrecognised - fields are kept rather than discarded +/ - OcdaImage img; - bool first = true; - foreach (field; value.split(" ")) { - if (field.length == 0) continue; - if (first) { - img.name = field; - first = false; - continue; - } - auto colon_pos = field.indexOf(':'); - if (colon_pos < 1) continue; - string key = field[0 .. colon_pos]; - string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; - switch (key) { - case "sha256": img.sha256 = val; break; - case "bytes": img.bytes = toUlongSafe(val); break; - case "px": - auto dims = val.split("x"); - if (dims.length == 2) { - img.width = toIntSafe(dims[0]); - img.height = toIntSafe(dims[1]); - } - break; - case "missing": img.missing = (val == "true"); break; - default: img.fields[key] = val; break; - } - } - return img; -} - -/+ ↓ two values the text form leaves out because they can be worked out. - . - The .ssp emits a property only when it carries information, so - .identifier is absent when it is simply the ocn, and .anchor is absent - when it is the segment name. The .ocda.db, being columns rather than - lines, stores both unconditionally. So a loader that takes the text - form literally gives a *different model* from the one the database - gives, and the difference is not small: in the reference collection - the identifier is omitted on all but a handful of objects. - . - The model is the thing both serialisations describe, so the loader - fills them in. Found by dumping both models and diffing them, which no - amount of comparing section counts would have shown. -+/ -private void deriveOmitted(ref OcdaObject obj) { - import std.conv : to; - if (obj.identifier.length == 0 && obj.ocn != 0) { - obj.identifier = obj.ocn.to!string; - } - if (obj.anchor.length == 0 && obj.segment.length > 0) { - obj.anchor = obj.segment; - } -} - -private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { - deriveOmitted(obj); - auto sec = doc.sectionByName(section); - if (sec !is null) { - ,*sec ~= obj; - } -} - -private int toIntSafe(string s) { - try { return s.strip.to!int; } - catch (ConvException) { return 0; } -} - -private ulong toUlongSafe(string s) { - try { return s.strip.to!ulong; } - catch (ConvException) { return 0; } -} - -private int[] parseIntArray(string s) { - int[] result; - foreach (part; s.split(" ")) { - auto v = toIntSafe(part); - result ~= v; - } - return result; -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/d-ocda.org b/org/d-ocda.org index 218a8cf..dd50bde 100644 --- a/org/d-ocda.org +++ b/org/d-ocda.org @@ -16,8 +16,164 @@ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * dlang +** model + +#+HEADER: :tangle "../dlang/src/ocda/model.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.model; + +struct OcdaImage { + string name; // image filename as referenced by the object + string sha256; // hex-encoded (uppercase) digest of the file + ulong bytes; // file size + int width; // file's own pixel width (not display width) + int height; // file's own pixel height + bool missing; // file absent or unreadable, no sha256/bytes + string[string] fields; // any further/unrecognised key:value fields +} + +struct OcdaObject { + int ocn; + string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb + string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb + string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) + string identifier; // heading: from the declaration line; otherwise .identifier + string part; // frontmatter, body, backmatter + string is_of_type; // para, block, comment: how the text behaves + string own_section; // .section, when it differs from the block it sits in + string segment_html_is;// the html segment this object sits in + string segment_epub; // the epub segment this heading opens + string segment_epub_is;// the epub segment this object sits in + string table_aligns; // l c r per column + string[] stow_link; // links extracted from the object + string[] anchor_tags; // named targets the markup placed on this object + int parent_ocn; + int last_descendant_ocn; + int ocn_range_first;// a container (a poem): the first and last + int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1) + int[] children; // direct child heading OCNs + int[] ancestors; // markedup_ancestors[8] + bool dummy_heading; + bool ocn_off; + int indent_base; + int indent_hang; + bool bullet; + string lang; + bool has_links; + bool has_notes_reg; + bool has_notes_star; + bool has_images; + string segment; + string segment_prev; + string segment_next; + string anchor; + string heading_lev_anchor; // a heading's own named target + int table_cols; + string table_widths; + bool table_header; + string code_syntax; + bool code_linenumbers; + string heading_ancestors_text; + string sha256; + OcdaImage[] images; // repeatable .image records, in order + string text; // content (lines joined with newline) + + /+ ↓ whether this object is rendered at all. + is_of_type "comment" is markup commentary: carried by the + abstraction so that nothing is lost, and not put into output. An + output that renders it leaks the author notes into the document, + which is what spine itself does not do. + +/ + bool renders() const { + return is_of_type != "comment"; + } + + /+ ↓ the section this object belongs to, which is the block it sits in + unless it says otherwise +/ + string sectionOf() const { + return (own_section.length > 0) ? own_section : section; + } +} + +/+ ↓ what the reader met and did not understand. + A reader that silently drops what it does not know cannot say whether + it understood the document, and cannot be used to check a + specification against an implementation: silence and understanding + look alike. So the parser counts what it ignored, and + --report-unknown prints it. ++/ +struct OcdaUnknown { + size_t[string] property; // .name -> times seen + size_t[string] block; // @name -> times seen + size_t[string] line; // first char -> times seen, lines of no known shape + + bool any() const { + return property.length > 0 || block.length > 0 || line.length > 0; + } +} + +struct OcdaDocument { + string format_version; // as declared on line 1, "" when absent + string source_filename; // from the "% Source:" line + string[string] source; // @source: language, languages, digest + OcdaUnknown unknown; // what was met and not understood + string[string] metadata; // key -> value + string[string] make; // key -> value + string[string] doc_has; // key -> value + OcdaObject[] head; + OcdaObject[] toc; + OcdaObject[] body; + OcdaObject[] endnotes; + OcdaObject[] glossary; + OcdaObject[] bibliography; + OcdaObject[] bookindex; + OcdaObject[] blurb; + OcdaObject[] tail; + + /+ ↓ the major part of the declared version, "" when none was declared +/ + string formatVersionMajor() const { + import std.string : indexOf; + if (format_version.length == 0) { return ""; } + auto dot = format_version.indexOf('.'); + return (dot > 0) ? format_version[0 .. dot] : format_version; + } + + string meta(string key) { + if (auto p = key in metadata) return *p; + return ""; + } + + OcdaObject[]* sectionByName(string name) { + switch (name) { + case "head": return &head; + case "toc": return &toc; + case "body": return &body; + case "endnotes": return &endnotes; + case "glossary": return &glossary; + case "bibliography": return &bibliography; + case "bookindex": return &bookindex; + case "blurb": return &blurb; + case "tail": return &tail; + default: return null; + } + } +} + +/+ ↓ the format major version this reader knows. + The .ssp and the .ocda.db carry one version between them, declared in + band: the .ssp on its first line, the database in its schema.version + row. The minor part is raised when a property is added, and a reader of + the newer format still reads the older one, so only the major part is + checked here. Raise this when the reader has been brought to a new + major, not before: the point of the gate is to refuse what has not been + looked at. ++/ +enum ssp_format_major_read = "2"; +#+END_SRC + ** load -*** load dlang +*** load #+HEADER: :tangle "../dlang/src/ocda/load.d" #+BEGIN_SRC d @@ -93,7 +249,7 @@ OcdaDocument load(string path, string lang = "", bool any_version = false) { } #+END_SRC -*** load db dlang +*** load db #+HEADER: :tangle "../dlang/src/ocda/load_db.d" #+BEGIN_SRC d @@ -455,7 +611,7 @@ private string jsonJoined(string v) { } #+END_SRC -*** load pod dlang +*** load pod #+HEADER: :tangle "../dlang/src/ocda/load_pod.d" #+BEGIN_SRC d @@ -577,7 +733,7 @@ private ptrdiff_t lastIndexOf(string s, char c) { } #+END_SRC -* load ssp dlang +*** load ssp #+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" #+BEGIN_SRC d @@ -926,6 +1082,366 @@ private int[] parseIntArray(string s) { } #+END_SRC +** inventory + +#+HEADER: :tangle "../dlang/src/ocda/inventory.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} +#+END_SRC + +** images + +#+HEADER: :tangle "../dlang/src/ocda/images.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.images; + +/+ ↓ an image's bytes, from whichever source the document came from. + . + The model *names* images, with digest, byte size and pixel size; it does + not hold them. That is right - an abstraction of a document is not a + bundle of its assets - but a writer of any format that shows images + needs the bytes, so something has to know where they are. This is that + something, and it is in the library rather than in a writer because the + answer depends on the *input form* and not on the output. + . + Three answers, and one refusal: + . + .ocda.db the files table, role 'image', by name + .sisupod the archive entry whose path ends in /image/<name> + .zip the same + .ssp beside it: a pod puts the abstraction in + media/abstraction/ and its images in media/image/, so the + path is documented rather than guessed + . + A .ssp that is not in a pod has nowhere to look, and that is reported as + absent rather than as an error: a reader that cannot find an image + should say so and carry on, because the document is still readable. ++/ + +import std.algorithm : endsWith; +import std.array : array; +import std.file : exists, isFile, read; +import std.path : buildPath, dirName; +import ocda.load; +import ocda.sqlite; +import ocda.zip; +import ocda.zstd; + +/+ ↓ the bytes, or an empty slice when the source has no such image +/ +ubyte[] imageBytes(string source, string name) { + final switch (sourceOf(source)) { + case Source.ocda_db: return fromDb(source, name); + case Source.pod: return fromArchive(source, name); + case Source.ssp: return fromBeside(source, name); + case Source.unknown: return []; + } +} + +/+ ↓ can this source give images at all? + A writer asks before it decides whether to emit them, because an <img> + pointing at a file it did not include is worse than no image. ++/ +bool canGiveImages(string source) { + final switch (sourceOf(source)) { + case Source.ocda_db: return true; + case Source.pod: return true; + case Source.ssp: return imageDirBeside(source).length > 0; + case Source.unknown: return false; + } +} + +private ubyte[] fromDb(string path, string name) { + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + /+ ↓ the name is from the abstraction, not from a caller, and the quote + doubling is what keeps it a value rather than syntax +/ + return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '" + ~ sqlQuote(name) ~ "'"); +} + +private ubyte[] fromArchive(string path, string name) { + auto bytes = cast(ubyte[]) read(path); + if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); } + auto entries = zipEntries(bytes); + foreach (e; entries) { + if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); } + } + return []; +} + +private ubyte[] fromBeside(string path, string name) { + string dir = imageDirBeside(path); + if (dir.length == 0) { return []; } + string file = buildPath(dir, name); + return file.exists && file.isFile ? cast(ubyte[]) read(file) : []; +} + +/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/ +private string imageDirBeside(string path) { + string dir = path.dirName; + if (!dir.endsWith("abstraction")) { return ""; } + string image_dir = buildPath(dir.dirName, "image"); + return image_dir.exists ? image_dir : ""; +} + +/+ ↓ a single quote doubled is how sql says a single quote +/ +private string sqlQuote(string s) { + string out_; + foreach (c; s) { + out_ ~= (c == '\'') ? "''" : [c]; + } + return out_; +} +#+END_SRC + +** sqlite + +#+HEADER: :tangle "../dlang/src/ocda/sqlite.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.sqlite; + +/+ ↓ the sqlite3 C API, declared rather than depended upon. + . + A binding from the dub registry would be a dependency to fetch, and + the point of this reader is that it builds in a sandbox with no + network. The dozen functions a reader needs are declared here and the + library is linked directly, which costs a page of extern(C) and saves + a dependency. ++/ + +import std.conv : to; +import std.string : fromStringz, toStringz; + +extern (C) { + struct sqlite3; + struct sqlite3_stmt; + + int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs); + int sqlite3_close(sqlite3* db); + int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte, + sqlite3_stmt** stmt, const(char)** tail); + int sqlite3_step(sqlite3_stmt* stmt); + int sqlite3_finalize(sqlite3_stmt* stmt); + int sqlite3_column_count(sqlite3_stmt* stmt); + const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col); + const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col); + int sqlite3_column_int(sqlite3_stmt* stmt, int col); + int sqlite3_column_type(sqlite3_stmt* stmt, int col); + const(char)* sqlite3_errmsg(sqlite3* db); + const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col); + int sqlite3_column_bytes(sqlite3_stmt* stmt, int col); +} + +enum SQLITE_OK = 0; +enum SQLITE_ROW = 100; +enum SQLITE_DONE = 101; +enum SQLITE_NULL = 5; +enum SQLITE_OPEN_READONLY = 0x00000001; + +class SqliteError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ a database, opened read only. + Read only is deliberate: a reader has no business writing, and a + request that opens a database read-write can create an empty one where + a path is mistyped. ++/ +struct Db { + private sqlite3* _handle; + + static Db open(string path) { + sqlite3* handle; + auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null); + if (rc != SQLITE_OK) { + string msg = (handle is null) + ? "cannot open " ~ path + : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup; + if (handle !is null) { sqlite3_close(handle); } + throw new SqliteError(msg); + } + Db db; + db._handle = handle; + return db; + } + + void close() { + if (_handle !is null) { sqlite3_close(_handle); _handle = null; } + } + + /+ ↓ every row of a query, each row as its columns in order +/ + string[][] rows(string sql) { + sqlite3_stmt* stmt; + auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null); + if (rc != SQLITE_OK) { + throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql); + } + scope(exit) { sqlite3_finalize(stmt); } + string[][] out_rows; + int cols = sqlite3_column_count(stmt); + while (sqlite3_step(stmt) == SQLITE_ROW) { + string[] row; + foreach (c; 0 .. cols) { + row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL) + ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup; + } + out_rows ~= row; + } + return out_rows; + } + + /+ ↓ the first column of the first row, "" when there is none +/ + string one(string sql) { + auto r = rows(sql); + return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; + } + + /+ ↓ the first column of the first row as bytes, for a blob. + Separate from one(), because a blob is not text: an image read as text + stops at the first zero byte, and a png's ninth byte is zero. + +/ + ubyte[] blob(string sql) { + sqlite3_stmt* stmt; + if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) { + throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup + ~ " [" ~ sql ~ "]"); + } + scope(exit) { sqlite3_finalize(stmt); } + if (sqlite3_step(stmt) != SQLITE_ROW) { return []; } + int n = sqlite3_column_bytes(stmt, 0); + if (n <= 0) { return []; } + auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0); + if (p is null) { return []; } + return p[0 .. n].dup; + } + + /+ ↓ does a table or view of this name exist +/ + bool has(string name) { + return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; + } +} +#+END_SRC + ** compression *** dlang read zip @@ -1135,6 +1651,179 @@ ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) { } #+END_SRC +** dump + +#+HEADER: :tangle "../dlang/src/ocda/dump.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.dump; + +/+ ↓ the model, written out canonically. + . + Not an output format: an *oracle*. Two libraries in two languages + implement the same model from the same specification, and the cheap + way to ask whether they agree is to have each write the whole loaded + document as text and diff the two. That is only useful if the text is + fully determined by the model, so: + . + - every line is "<key> <value>", the value running to end of line, + and only one free-form value per line; + - a field at its default is *omitted*, so the output says what the + document has rather than what the struct declares; + - dictionaries are emitted in sorted key order, because neither + language's hash map has an order worth relying on; + - sections come in the fixed order of the format, not + alphabetically. + . + Anything that reads this to reconstruct a document has misunderstood + it. Load the .ssp or the .ocda.db. ++/ + +import std.algorithm : sort, splitter; +import std.array : appender, array, join; +import std.conv : to; +import ocda.model; + +/+ ↓ the nine sections, in format order +/ +static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + +string dumpDocument(ref OcdaDocument doc) { + auto out_ = appender!string; + + void kv(string key, string value) { + if (value.length == 0) { return; } + out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; + } + void kn(string key, long value) { + if (value == 0) { return; } + kv(key, value.to!string); + } + void kb(string key, bool value) { + if (!value) { return; } + kv(key, "1"); + } + void dict(string prefix, string[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key]; out_ ~= "\n"; + } + } + + kv("version", doc.format_version); + kv("source_filename", doc.source_filename); + dict("source", doc.source); + dict("meta", doc.metadata); + dict("make", doc.make); + dict("doc_has", doc.doc_has); + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null || (*sec).length == 0) { continue; } + kv("section", name ~ " " ~ (*sec).length.to!string); + } + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null) { continue; } + foreach (i, obj; *sec) { + out_ ~= "object "; out_ ~= name; out_ ~= " "; + out_ ~= i.to!string; out_ ~= "\n"; + kn("ocn", obj.ocn); + kv("is_a", obj.is_a); + kv("own_section", obj.own_section); + kv("level", obj.heading_level); + kv("identifier", obj.identifier); + kv("part", obj.part); + kv("is_of_type", obj.is_of_type); + kv("segment_html_is", obj.segment_html_is); + kv("segment_epub", obj.segment_epub); + kv("segment_epub_is", obj.segment_epub_is); + kn("parent", obj.parent_ocn); + kn("last_descendant", obj.last_descendant_ocn); + kn("ocn_range_first", obj.ocn_range_first); + kn("ocn_range_last", obj.ocn_range_last); + kv("children", ints(obj.children)); + kv("ancestors", ints(obj.ancestors)); + kb("dummy", obj.dummy_heading); + kb("ocn_off", obj.ocn_off); + kn("indent_base", obj.indent_base); + kn("indent_hang", obj.indent_hang); + kb("bullet", obj.bullet); + kv("lang", obj.lang); + kb("has_links", obj.has_links); + kb("has_notes_reg", obj.has_notes_reg); + kb("has_notes_star", obj.has_notes_star); + kb("has_images", obj.has_images); + kv("segment", obj.segment); + kv("segment_prev", obj.segment_prev); + kv("segment_next", obj.segment_next); + kv("anchor", obj.anchor); + kv("heading_lev_anchor", obj.heading_lev_anchor); + kn("table_cols", obj.table_cols); + kv("table_widths", obj.table_widths); + kv("table_aligns", obj.table_aligns); + kb("table_header", obj.table_header); + kv("code_syntax", obj.code_syntax); + kb("code_linenumbers", obj.code_linenumbers); + kv("heading_ancestors_text", obj.heading_ancestors_text); + kv("sha256", obj.sha256); + foreach (url; obj.stow_link) { kv("stow_link", url); } + foreach (a; obj.anchor_tags) { kv("anchor_tag", a); } + foreach (img; obj.images) { + /+ ↓ the name last, because a filename may hold a space where + none of the other fields can +/ + kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " + ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " + ~ (img.missing ? "1" : "0") ~ " " ~ img.name); + foreach (key; img.fields.keys.dup.sort) { + kv("image_field", key ~ " " ~ img.fields[key]); + } + } + /+ ↓ text, when there is any. An object whose only text line is + blank holds "" here, indistinguishable from no text at all, + because the model joins the lines into one string. Both + libraries lose the same distinction in the same place, which is + what makes them comparable; it is recorded rather than hidden. + +/ + if (obj.text.length > 0) { + auto lines = obj.text.splitter("\n").array; + kv("text", lines.length.to!string); + foreach (line; lines) { + out_ ~= "| "; out_ ~= line; out_ ~= "\n"; + } + } + } + } + return out_.data; +} + +/+ ↓ what the reader met and did not understand, sorted. Separate from the + document dump: it describes the reader, not the document, and the + .ocda.db loader has nothing to report where the .ssp loader does. ++/ +string dumpUnknown(ref OcdaDocument doc) { + auto out_ = appender!string; + void counts(string prefix, size_t[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key].to!string; out_ ~= "\n"; + } + } + counts("property", doc.unknown.property); + counts("block", doc.unknown.block); + counts("line", doc.unknown.line); + return out_.data; +} + +private string ints(int[] xs) { + if (xs.length == 0) { return ""; } + string[] parts; + foreach (x; xs) { parts ~= x.to!string; } + return parts.join(","); +} +#+END_SRC + * org includes ** project version diff --git a/org/d-read.org b/org/d-read.org index c9f0334..44bb5a0 100644 --- a/org/d-read.org +++ b/org/d-read.org @@ -17,7 +17,7 @@ * dlang :dlang: ** main & tree -*** dlang read main +*** read main #+HEADER: :tangle "../dlang/src/read/main.d" #+BEGIN_SRC d @@ -252,7 +252,7 @@ void printFiles(string path) { } #+END_SRC -*** dlang read tree +*** read tree #+HEADER: :tangle "../dlang/src/read/tree.d" #+BEGIN_SRC d @@ -329,7 +329,7 @@ private string stripInternalMarkup(string text) { #+END_SRC ** check -*** dlang read verify +*** read verify #+HEADER: :tangle "../dlang/src/read/verify.d" #+BEGIN_SRC d @@ -548,7 +548,7 @@ private int beInt(const(ubyte)[] b) { } #+END_SRC -*** dlang read validate +*** read validate #+HEADER: :tangle "../dlang/src/read/validate.d" #+BEGIN_SRC d @@ -820,7 +820,7 @@ void reportValidation(string filename, SspFinding[] found) { } #+END_SRC -*** dlang read report unknown +*** read report unknown #+HEADER: :tangle "../dlang/src/read/report_unknown.d" #+BEGIN_SRC d @@ -869,7 +869,7 @@ void reportUnknown(ref OcdaDocument doc) { #+END_SRC ** misc -*** dlang read html +*** read html #+HEADER: :tangle "../dlang/src/read/html.d" #+BEGIN_SRC d @@ -1264,7 +1264,7 @@ private string[] htmlTail() { } #+END_SRC -*** dlang read images +*** read images #+HEADER: :tangle "../dlang/src/read/images.d" #+BEGIN_SRC d @@ -1386,7 +1386,7 @@ package immutable string[] sectionNames = [ #+END_SRC ** llm -*** dlang read llm text +*** read llm text #+HEADER: :tangle "../dlang/src/read/llm_text.d" #+BEGIN_SRC d @@ -1575,7 +1575,7 @@ string stripMarkup(string text) { } #+END_SRC -*** dlang read llm jsonl +*** read llm jsonl #+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" #+BEGIN_SRC d diff --git a/org/d-test.org b/org/d-test.org new file mode 100644 index 0000000..d159819 --- /dev/null +++ b/org/d-test.org @@ -0,0 +1,178 @@ +-*- mode: org -*- +#+TITLE: dlang load +#+DESCRIPTION: dlang lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* dlang :dlang: +** tests +*** test agree + +#+HEADER: :tangle "../dlang/test/agree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + Do the two loaders agree? + + The library's whole purpose is that a writer cannot tell which + serialisation it was handed. That is only true if loading a document's + .ssp and loading its .ocda.db give the same model, so this asks. + + Usage: agree <doc>.ssp <doc>.ocda.db [lang] ++/ +import std.conv : to; +import std.stdio; +import ocda.model; +import ocda.load; + +int main(string[] args) { + if (args.length < 3) { + stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]"); + return 2; + } + string lang = (args.length > 3) ? args[3] : ""; + OcdaDocument a, b; + try { + a = load(args[1]); + b = load(args[2], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + + int differences = 0; + void differ(string what, string from_ssp, string from_db) { + writeln(" differ: ", what); + writeln(" .ssp: ", from_ssp); + writeln(" .ocda.db: ", from_db); + differences++; + } + + if (a.format_version != b.format_version) { + differ("format_version", a.format_version, b.format_version); + } + if (a.source_filename != b.source_filename) { + differ("source_filename", a.source_filename, b.source_filename); + } + + /+ ↓ the sections, object by object. Position within a section is the + identity to compare on: an ocn is not unique and 0 is shared +/ + static immutable sections = ["head", "toc", "body", "endnotes", "glossary", + "bibliography", "bookindex", "blurb", "tail"]; + foreach (name; sections) { + auto sa = a.sectionByName(name); + auto sb = b.sectionByName(name); + if (sa is null || sb is null) { continue; } + if ((*sa).length != (*sb).length) { + differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string); + continue; + } + foreach (i, oa; *sa) { + auto ob = (*sb)[i]; + string at = name ~ "[" ~ i.to!string ~ "]"; + if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); } + if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); } + if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); } + if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); } + if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); } + if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); } + if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); } + if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); } + if (oa.ancestors != ob.ancestors) { + differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string); + } + if (oa.images.length != ob.images.length) { + differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string); + } + if (differences > 12) { writeln(" ... stopping after 12"); return 1; } + } + } + + if (differences == 0) { + writeln("agree: the two loaders give the same model"); + return 0; + } + writeln(differences, " difference(s)"); + return 1; +} +#+END_SRC + +*** test dump + +#+HEADER: :tangle "../dlang/test/dump.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + Write a loaded abstraction out canonically. + + The oracle for everything that claims to implement this model: load a + .ssp or a .ocda.db and print the model, deterministically, so that two + implementations can be compared with diff. + + Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang] ++/ +import std.stdio; +import ocda.model; +import ocda.load; +import ocda.dump; + +int main(string[] args) { + bool unknown = false; + string[] rest; + foreach (a; args[1 .. $]) { + if (a == "--unknown") { unknown = true; } else { rest ~= a; } + } + if (rest.length < 1) { + stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]"); + return 2; + } + string lang = (rest.length > 1) ? rest[1] : ""; + OcdaDocument doc; + try { + doc = load(rest[0], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); + return 0; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/gleam-ocda-load.org b/org/gleam-ocda-load.org deleted file mode 100644 index f3f111f..0000000 --- a/org/gleam-ocda-load.org +++ /dev/null @@ -1,1484 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** load gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/load.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<load>> -#+END_SRC - -*** gleam source - -#+NAME: load -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// What a path is, and loading it. -//// -//// One place that decides, so that every reader and every writer accepts -//// the same inputs and a new input form is added once. This mirrors -//// spine's own `ocda/abstraction/load.d`, which names the sources it can -//// enter the pipeline from and tells them apart by suffix. -//// -//// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means -//// implementing the parser, which is spine's work and a far larger thing -//// than a reader or a sample writer needs. This library reads -//// abstractions. - -import gleam/list -import gleam/string -import ocda/load_db -import ocda/load_pod -import ocda/load_ssp -import ocda/model.{type Document} - -pub type Source { - /// `.ssp`, the abstraction as text - Ssp - /// `.ocda.db`, the abstraction as sqlite - OcdaDb - /// `.sisupod` or `.zip`, an archive carrying a `.ssp` - Pod - Unknown -} - -/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote -/// before, and every pod already published carries that name, so both are -/// recognised. What is inside decides how it is opened, not the name. -const suffixes = [ - #(".ocda.db", OcdaDb), - #(".ssp", Ssp), - #(".sisupod", Pod), - #(".zip", Pod), - #(".db", OcdaDb), -] - -pub fn source_of(path: String) -> Source { - case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) { - Ok(#(_, source)) -> source - Error(_) -> Unknown - } -} - -pub fn source_name(source: Source) -> String { - case source { - Ssp -> ".ssp (the abstraction as text)" - OcdaDb -> ".ocda.db (the abstraction as sqlite)" - Pod -> "a pod archive carrying a .ssp" - Unknown -> "not an abstraction this library reads" - } -} - -/// Load whatever the path is into the model. -/// -/// `lang` names which language to take from a `.ocda.db`, which holds -/// every language of its document, and from a pod, which may carry more than -/// one `.ssp`. A `.ssp` is one language by construction and ignores it. -pub fn load( - path: String, - lang: String, - any_version: Bool, -) -> Result(Document, String) { - case source_of(path) { - Ssp -> load_ssp.load_file(path) - OcdaDb -> load_db.load_file(path, lang, any_version) - Pod -> load_pod.load_file(path, lang) - Unknown -> - Error( - path - <> ": " - <> source_name(Unknown) - <> " (expected .ssp, .ocda.db, .sisupod or .zip)", - ) - } -} -#+END_SRC - -** load pod -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/load_pod.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<load_pod>> -#+END_SRC - -*** gleam source - -#+NAME: load_pod -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// The abstraction out of a pod archive. -//// -//// A pod built with `--pod2` carries the abstraction inside it, at -//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is -//// a third input form and *not* a third model: the `.ssp` inside is parsed by -//// `ocda/load_ssp` and comes back as the same value. A writer over the -//// library gains pods without being touched, which is the whole reason the -//// dispatch is in one place. -//// -//// ## The one thing this cannot do -//// -//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip -//// and **not the frame**, because there is no zstd on the BEAM without a NIF, -//// and the one that exists, `ezstd`, clones zstd from github in its build, so -//// it cannot be built in a sandbox without patching it. The D library has -//// both, through a declared `libzstd`. -//// -//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says -//// why rather than pretending the file is malformed. It is the only place the -//// two libraries differ in what they accept, and it is a packaging limit -//// rather than anything about the format. - -import gleam/list -import gleam/result -import gleam/string -import ocda/load_ssp -import ocda/model.{type Document} -import ocda/zip -import simplifile - -/// The magic number of a zstd frame, little endian 0xFD2FB528. -const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>> - -/// Which languages of the abstraction the pod carries, sorted. -pub fn languages(path: String) -> Result(List(String), String) { - use bytes <- result.try(archive_bytes(path)) - use entries <- result.try(zip.entries(bytes)) - zip.find_suffix(entries, ".ssp") - |> list.filter_map(fn(e) { lang_of(e.name) }) - |> list.sort(string.compare) - |> Ok -} - -/// Load one language of the abstraction a pod carries. -/// -/// `lang` == "" takes the only one there is, and where there is a choice it -/// says so rather than choosing, as the `.ocda.db` loader does. -pub fn load_file(path: String, lang: String) -> Result(Document, String) { - use bytes <- result.try(archive_bytes(path)) - use entries <- result.try(zip.entries(bytes)) - let ssps = zip.find_suffix(entries, ".ssp") - use chosen <- result.try(case ssps, lang { - [], _ -> - Error( - path - <> ": carries no .ssp. A pod carries the abstraction only when spine" - <> " was asked for it: spine --pod2 puts it in, spine --source does not.", - ) - [only], "" -> Ok(only) - _, "" -> - Error( - path - <> ": holds more than one language (" - <> string.join(langs(ssps), " ") - <> "); name the one wanted", - ) - _, wanted -> - case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) { - Ok(found) -> Ok(found) - Error(_) -> - Error( - path - <> ": no language '" - <> wanted - <> "'; it holds: " - <> string.join(langs(ssps), " "), - ) - } - }) - use content <- result.try(zip.read(bytes, chosen)) - case bit_array_to_string(content) { - Ok(text) -> Ok(load_ssp.parse(text)) - Error(_) -> Error(chosen.name <> ": not valid UTF-8") - } -} - -fn langs(entries: List(zip.Entry)) -> List(String) { - list.filter_map(entries, fn(e) { lang_of(e.name) }) -} - -/// `<uid>.<lang>.ssp` -> `<lang>` -pub fn lang_of(name: String) -> Result(String, Nil) { - case string.ends_with(name, ".ssp") { - False -> Error(Nil) - True -> { - let stem = string.drop_end(name, 4) - case string.split(stem, on: ".") |> list.last { - Ok(lang) if lang != stem -> Ok(lang) - _ -> Error(Nil) - } - } - } -} - -/// The bytes decide, not the name: a file misnamed either way is treated by -/// what it holds. -pub fn archive_bytes(path: String) -> Result(BitArray, String) { - case simplifile.read_bits(path) { - Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) - Ok(bytes) -> - case is_zstd_frame(bytes) { - False -> Ok(bytes) - True -> - Error( - path - <> " is a zstd frame wrapping the archive, and this library has no" - <> " zstd: the only one on the BEAM is a NIF that fetches zstd in" - <> " its own build, so it cannot be built sandboxed. Unwrap it" - <> " first (zstd -d) and read the archive, or use the D library," - <> " which reads both.", - ) - } - } -} - -fn is_zstd_frame(bytes: BitArray) -> Bool { - case bytes { - <<head:bytes-size(4), _:bits>> -> head == zstd_magic - _ -> False - } -} - -@external(erlang, "unicode", "characters_to_binary") -fn bit_array_to_string_ffi(b: BitArray) -> String - -pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { - Ok(bit_array_to_string_ffi(b)) -} -#+END_SRC - -** load ssp -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/load_ssp.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<load_ssp>> -#+END_SRC - -*** gleam source - -#+NAME: load_ssp -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// Loading a `.ssp` into the model. -//// -//// The format is line oriented and every line is one of eight shapes told -//// apart by its first character, so the loader is a fold over the lines -//// carrying the state the shapes need. -//// -//// Written against `ssp-format-specification.org`. Where it and the D -//// library differ the D library is the bug, or this is: they are meant to -//// give the same model, and `ocda-dump` says whether they do. - -import gleam/dict.{type Dict} -import gleam/int -import gleam/list -import gleam/option -import gleam/result -import gleam/string -import ocda/model.{type Document, type Image, type Object, Image, Object} -import simplifile - -pub fn load_file(path: String) -> Result(Document, String) { - case simplifile.read(path) { - Ok(content) -> Ok(parse(content)) - Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) - } -} - -type State { - State( - document: Document, - /// the block currently open, "" when none - block: String, - /// the section currently open, "" when in the preamble - section: String, - /// objects of the current section, newest first - objects: List(Object), - /// the object being read, if any - current: Result(Object, Nil), - /// text lines of the object being read, newest first - text: List(String), - /// whether a text line has been seen for the object being read. - /// Not "is the text non-empty": an object whose first text line is - /// blank, which the reference collection has seven of, would then lose - /// it. - seen_text: Bool, - ) -} - -pub fn parse(content: String) -> Document { - content - |> string.split(on: "\n") - |> list.fold(from: initial(), with: line) - |> finish -} - -fn initial() -> State { - State( - document: model.new_document(), - block: "", - section: "", - objects: [], - current: Error(Nil), - text: [], - seen_text: False, - ) -} - -fn line(state: State, raw: String) -> State { - case raw { - "% SiSU Document Abstraction v" <> version -> - case state.document.format_version { - "" -> - State( - ..state, - document: model.Document( - ..state.document, - format_version: string.trim(version), - ), - ) - _ -> state - } - "% Source: " <> filename -> - case state.document.source_filename { - "" -> - State( - ..state, - document: model.Document( - ..state.document, - source_filename: string.trim(filename), - ), - ) - _ -> state - } - "%" <> _ -> state - "@" <> rest -> open_block(state, rest) - "}" <> _ -> close_block(state) - "[" <> rest -> declaration(state, rest) - "." <> rest -> property(state, rest) - "|" <> rest -> text_line(state, rest) - _ -> - case string.trim(raw) { - // a blank line ends the object being read - "" -> commit(state) - _ -> - case model.is_preamble_block(state.block) { - True -> preamble_pair(state, raw) - False -> unknown_line(state, raw) - } - } - } -} - -fn open_block(state: State, rest: String) -> State { - let name = case string.split_once(rest, "{") { - Ok(#(name, _)) -> string.trim(name) - Error(_) -> string.trim(rest) - } - case model.is_preamble_block(name) { - True -> State(..flush(state), block: name) - False -> { - let state = flush(state) - let document = case model.is_section(name) { - True -> state.document - // a block that is neither preamble nor one of the nine sections: - // counted, so that a reader can say the file held something it did - // not expect - False -> - model.Document( - ..state.document, - unknown: model.Unknown( - ..state.document.unknown, - block: bump(state.document.unknown.block, name), - ), - ) - } - State(..state, block: name, section: name, document: document) - } - } -} - -fn close_block(state: State) -> State { - State(..flush(state), block: "") -} - -/// Commit the object being read, then put the section's objects into the -/// document and close the section. -fn flush(state: State) -> State { - let state = commit(state) - case state.section, state.objects { - "", _ -> state - _, [] -> State(..state, section: "", objects: []) - name, objects -> - State( - ..state, - section: "", - objects: [], - document: model.Document( - ..state.document, - sections: add_section( - state.document.sections, - name, - list.reverse(objects), - ), - ), - ) - } -} - -fn preamble_pair(state: State, raw: String) -> State { - case string.split_once(string.trim_start(raw), ":") { - Error(_) -> state - Ok(#(key, value)) -> { - let key = string.trim(key) - let value = string.trim(value) - let document = state.document - let document = case state.block { - "meta" -> - model.Document( - ..document, - metadata: dict.insert(document.metadata, key, value), - ) - "make" -> - model.Document(..document, make: dict.insert(document.make, key, value)) - "source" -> - model.Document( - ..document, - source: dict.insert(document.source, key, value), - ) - _ -> - model.Document( - ..document, - doc_has: dict.insert(document.doc_has, key, value), - ) - } - State(..state, document: document) - } - } -} - -/// `[<ocn>] <kind>`, and for a heading `[<ocn>] heading :<level> -/// [<identifier>]`. -fn declaration(state: State, rest: String) -> State { - let state = commit(state) - case string.split_once(rest, "]") { - Error(_) -> state - Ok(#(ocn_text, tail)) -> { - let ocn = parse_int(string.trim(ocn_text)) - let object = - Object(..model.new_object(), ocn: ocn, section: state.section) - let object = case string.split(string.trim(tail), on: " ") { - ["heading", level, ..identifier] -> - Object( - ..object, - is_a: "heading", - heading_level: string.drop_start(level, 1), - identifier: string.join(identifier, " "), - ) - [""] -> object - [kind, ..] -> Object(..object, is_a: kind) - [] -> object - } - State(..state, current: Ok(object), text: [], seen_text: False) - } - } -} - -fn property(state: State, rest: String) -> State { - case state.current, string.split_once(rest, ":") { - Ok(object), Ok(#(name, raw_value)) -> { - let name = string.trim(name) - let value = string.trim(raw_value) - let #(object, known) = apply_property(object, name, value) - let document = case known { - True -> state.document - False -> - model.Document( - ..state.document, - unknown: model.Unknown( - ..state.document.unknown, - property: bump(state.document.unknown.property, name), - ), - ) - } - State(..state, current: Ok(object), document: document) - } - _, _ -> state - } -} - -/// The properties this library reads, and what each sets. -/// -/// A property it has no use for is counted rather than passed over: -/// ignoring one is conforming within a major version, doing so in silence -/// is what stops a reader being a check on the specification. -fn apply_property( - object: Object, - name: String, - value: String, -) -> #(Object, Bool) { - case name { - "part" -> #(Object(..object, part: value), True) - "is_of_type" -> #(Object(..object, is_of_type: value), True) - "section" -> #(Object(..object, own_section: value), True) - "identifier" -> #(Object(..object, identifier: value), True) - "segment_html_is" -> #(Object(..object, segment_html_is: value), True) - "segment_epub" -> #(Object(..object, segment_epub: value), True) - "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True) - "table_aligns" -> #(Object(..object, table_aligns: value), True) - "stow_link" -> #( - Object(..object, stow_link: list.append(object.stow_link, [value])), - True, - ) - "anchor_tag" -> #( - Object(..object, anchor_tags: list.append(object.anchor_tags, [value])), - True, - ) - "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True) - "last_descendant" -> #( - Object(..object, last_descendant_ocn: parse_int(value)), - True, - ) - "ocn_range" -> - case numbers(value) { - [first, last] -> #( - Object(..object, ocn_range_first: first, ocn_range_last: last), - True, - ) - _ -> #(object, True) - } - "children" -> #(Object(..object, children: numbers(value)), True) - "ancestors" -> #(Object(..object, ancestors: numbers(value)), True) - "dummy" -> #(Object(..object, dummy_heading: value == "true"), True) - "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True) - "indent" -> { - let parts = string.split(value, on: " ") - let base = case parts { - [first, ..] -> parse_int(first) - [] -> 0 - } - let hang = case parts { - [_, second, ..] -> parse_int(second) - _ -> 0 - } - #(Object(..object, indent_base: base, indent_hang: hang), True) - } - "bullet" -> #(Object(..object, bullet: value == "true"), True) - "lang" -> #(Object(..object, lang: value), True) - "has" -> #(has_flags(object, value), True) - "segment" -> #(Object(..object, segment: value), True) - "segment_prev" -> #(Object(..object, segment_prev: value), True) - "segment_next" -> #(Object(..object, segment_next: value), True) - "anchor" -> #(Object(..object, anchor: value), True) - "heading_lev_anchor" -> #( - Object(..object, heading_lev_anchor: value), - True, - ) - "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True) - "table_widths" -> #(Object(..object, table_widths: value), True) - "table_header" -> #(Object(..object, table_header: value == "true"), True) - "code_syntax" -> #(Object(..object, code_syntax: value), True) - "code_linenumbers" -> #( - Object(..object, code_linenumbers: value == "true"), - True, - ) - "heading_ancestors_text" -> #( - Object(..object, heading_ancestors_text: value), - True, - ) - "sha256" -> #(Object(..object, sha256: value), True) - "image" -> #( - Object(..object, images: list.append(object.images, [image(value)])), - True, - ) - _ -> #(object, False) - } -} - -fn has_flags(object: Object, value: String) -> Object { - value - |> string.split(on: " ") - |> list.fold(from: object, with: fn(obj, flag) { - case flag { - "links" -> Object(..obj, has_links: True) - "notes_reg" -> Object(..obj, has_notes_reg: True) - "notes_star" -> Object(..obj, has_notes_star: True) - "images" -> Object(..obj, has_images: True) - _ -> obj - } - }) -} - -/// `<filename> sha256:<HEX> bytes:<n> [px:<w>x<h>] [missing:true] [...]` -/// -/// The filename comes first, the rest are order-independent, and a field -/// this library does not recognise is kept rather than discarded. -fn image(value: String) -> Image { - case string.split(value, on: " ") { - [] -> model.new_image() - [name, ..fields] -> - list.fold(fields, model.new_image(), fn(img, field) { - case string.split_once(field, ":") { - Error(_) -> img - Ok(#(key, val)) -> - case key { - "sha256" -> Image(..img, sha256: val) - "bytes" -> Image(..img, bytes: parse_int(val)) - "px" -> - case string.split_once(val, "x") { - Ok(#(w, h)) -> - Image(..img, width: parse_int(w), height: parse_int(h)) - Error(_) -> img - } - "missing" -> Image(..img, missing: val == "true") - _ -> Image(..img, fields: dict.insert(img.fields, key, val)) - } - } - }) - |> fn(img) { Image(..img, name: name) } - } -} - -/// `| <text>`, and `| ` alone for a blank line. Never a bare `|`: the -/// emitter writes the space. -fn text_line(state: State, rest: String) -> State { - case state.current { - Error(_) -> state - Ok(_) -> { - let content = string.drop_start(rest, 1) - State(..state, text: [content, ..state.text], seen_text: True) - } - } -} - -fn unknown_line(state: State, raw: String) -> State { - let first = string.slice(raw, 0, 1) - State( - ..state, - document: model.Document( - ..state.document, - unknown: model.Unknown( - ..state.document.unknown, - line: bump(state.document.unknown.line, first), - ), - ), - ) -} - -fn commit(state: State) -> State { - case state.current { - Error(_) -> state - Ok(object) -> { - let text = case state.seen_text { - True -> string.join(list.reverse(state.text), "\n") - False -> "" - } - let object = derive_omitted(Object(..object, text: text)) - let objects = case model.is_section(state.section) { - True -> [object, ..state.objects] - False -> state.objects - } - State( - ..state, - current: Error(Nil), - text: [], - seen_text: False, - objects: objects, - ) - } - } -} - -/// Two values the text form leaves out because they can be worked out. -/// -/// `.identifier` is absent when it is simply the ocn, `.anchor` when it is -/// the segment name. The `.ocda.db`, being columns rather than lines, -/// stores both unconditionally. So a loader that takes the text form -/// literally gives a different model from the one the database gives, and -/// over one document that is 1,314 identifiers and 69 anchors. -fn derive_omitted(object: Object) -> Object { - let object = case object.identifier, object.ocn { - "", 0 -> object - "", ocn -> Object(..object, identifier: int.to_string(ocn)) - _, _ -> object - } - case object.anchor, object.segment { - "", "" -> object - "", segment -> Object(..object, anchor: segment) - _, _ -> object - } -} - -fn finish(state: State) -> Document { - flush(state).document -} - -fn add_section( - sections: Dict(String, List(Object)), - name: String, - objects: List(Object), -) -> Dict(String, List(Object)) { - dict.upsert(sections, name, fn(existing) { - case existing { - option.Some(previous) -> list.append(previous, objects) - option.None -> objects - } - }) -} - -fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) { - dict.upsert(counts, key, fn(existing) { - case existing { - option.Some(n) -> n + 1 - option.None -> 1 - } - }) -} - -fn parse_int(value: String) -> Int { - value |> string.trim |> int.parse |> result.unwrap(0) -} - -fn numbers(value: String) -> List(Int) { - value - |> string.split(on: " ") - |> list.filter(fn(s) { s != "" }) - |> list.map(parse_int) -} -#+END_SRC - -** load db -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/load_db.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<load_db>> -#+END_SRC - -*** gleam source - -#+NAME: load_db -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// Loading a `.ocda.db` into the model. -//// -//// The point of this module is that it returns *the same value* -//// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is -//// then a function from the model to bytes and cannot tell which -//// serialisation it was given, exactly as spine's own writers cannot. -//// -//// Written against `ocda-db-format-specification.org`, whose section 4 -//// gives the column by column correspondence this module implements. - -import gleam/dict -import gleam/dynamic/decode -import gleam/float -import gleam/int -import gleam/json -import gleam/list -import gleam/option -import gleam/result -import gleam/string -import ocda/model.{type Document, Image, Object} -import sqlight - -/// The schema major version this loader knows. -pub const schema_major_read = "2" - -/// One language of the database, as the manifest gives it. -pub type Language { - Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String) -} - -/// Load one language of a database as the model. `lang` == "" takes the -/// first language the manifest gives. -pub fn load_file( - path: String, - lang: String, - any_version: Bool, -) -> Result(Document, String) { - // read only: a loader has no business writing, and a read-write open - // *creates* an empty database where a path is mistyped, which then reads - // as a document with nothing in it, a failure that looks like an answer - use conn <- with_connection("file:" <> path <> "?mode=ro", path) - - use _ <- result.try(case - has(conn, "documents") && has(conn, "objects") && has(conn, "metadata") - { - True -> Ok(Nil) - False -> - Error( - path - <> ": no documents/objects/metadata tables; it does not look like a .ocda.db", - ) - }) - - let schema_version = - one(conn, "select value from metadata where doc_id is null - and key = 'schema.version'") - use _ <- result.try(case schema_version, any_version { - "", _ -> Ok(Nil) - _, True -> Ok(Nil) - v, False -> - case major_of(v) == schema_major_read { - True -> Ok(Nil) - False -> - Error( - path - <> " is schema v" - <> v - <> ", this loader knows v" - <> schema_major_read - <> ".x", - ) - } - }) - - let languages = languages(conn) - use chosen <- result.try(case languages, lang { - [], _ -> Error(path <> ": no documents rows") - [first, ..], "" -> Ok(first) - _, wanted -> - case list.find(languages, fn(l) { l.lang == wanted }) { - Ok(found) -> Ok(found) - Error(_) -> - Error( - path - <> ": no language '" - <> wanted - <> "'; it holds: " - <> string.join(list.map(languages, fn(l) { l.lang }), " "), - ) - } - }) - - let source = - dict.new() - |> dict.insert("language", chosen.lang) - |> dict.insert( - "languages", - string.join(list.map(languages, fn(l) { l.lang }), " "), - ) - let digest = meta_value(conn, chosen.doc_id, "source.digest") - let source = case digest { - "" -> source - d -> dict.insert(source, "digest", d) - } - - let document = - model.Document( - ..model.new_document(), - format_version: schema_version, - source_filename: chosen.source_filename, - source: source, - ) - let document = metadata(conn, chosen.doc_id, document) - let document = objects(conn, chosen.doc_id, document) - let document = count_unread(conn, chosen.doc_id, document) - Ok(document) -} - -fn with_connection( - uri: String, - path: String, - then: fn(sqlight.Connection) -> Result(a, String), -) -> Result(a, String) { - case sqlight.open(uri) { - Error(e) -> Error(path <> ": " <> e.message) - Ok(conn) -> { - let outcome = then(conn) - let _ = sqlight.close(conn) - outcome - } - } -} - -/// Every language the database holds, in manifest order. -pub fn languages(conn: sqlight.Connection) -> List(Language) { - let d = { - use id <- decode.field(0, decode.int) - use lang <- decode.field(1, decode.string) - use source <- decode.field(2, decode.string) - use digest <- decode.field(3, decode.string) - decode.success(Language(id, lang, source, digest)) - } - sqlight.query( - "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') - from documents order by id", - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) -} - -/// The three preamble blocks of the text form, out of the one metadata -/// table the database keeps them in. -fn metadata( - conn: sqlight.Connection, - doc_id: Int, - document: Document, -) -> Document { - let d = { - use key <- decode.field(0, decode.string) - use value <- decode.field(1, decode.string) - decode.success(#(key, value)) - } - let rows = - sqlight.query( - "select key, value from metadata where doc_id = " - <> int.to_string(doc_id) - <> " order by key", - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) - - list.fold(rows, document, fn(doc, row) { - let #(key, raw) = row - let value = string.trim(raw) - case key { - // source.* describes the artefact rather than the document, and is - // @source in the text form - "source." <> _ -> doc - // translation.ocn_aligned says whether the languages of this document - // share one numbering. A .ocda.db can know it because it holds every - // language; a .ssp is one language and cannot. It is a fact about the - // file, not the document, and carrying it here would let a writer - // tell which serialisation it was handed. - "translation." <> _ -> doc - // the database tells the three blocks apart by a key prefix. - // Unprefixing them is what makes the two loaders give one model: a - // writer asks make["footer"], never metadata["make.footer"]. - "make." <> rest -> - model.Document(..doc, make: dict.insert(doc.make, rest, value)) - "doc_has." <> rest -> - model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value)) - _ -> - model.Document(..doc, metadata: dict.insert(doc.metadata, key, value)) - } - }) -} - -/// The objects, in document order within each section, the sections in the -/// order the format gives rather than the alphabetical one sqlite would. -fn objects( - conn: sqlight.Connection, - doc_id: Int, - document: Document, -) -> Document { - let d = { - use section <- decode.field(0, decode.string) - use ocn <- decode.field(2, decode.int) - use is_a <- decode.field(3, decode.string) - use part <- decode.field(4, decode.string) - use is_of_section <- decode.field(5, decode.string) - use is_of_type <- decode.field(6, decode.string) - use heading_level <- decode.field(7, decode.int) - use identifier <- decode.field(8, decode.string) - use parent <- decode.field(9, decode.int) - use last_descendant <- decode.field(10, decode.int) - use children <- decode.field(11, decode.string) - use ancestors <- decode.field(12, decode.string) - use dummy <- decode.field(13, decode.int) - use ocn_off <- decode.field(14, decode.int) - use indent_base <- decode.field(15, decode.int) - use indent_hang <- decode.field(16, decode.int) - use bullet <- decode.field(17, decode.int) - use lang <- decode.field(18, decode.string) - use has_links <- decode.field(19, decode.int) - use has_notes_reg <- decode.field(20, decode.int) - use has_notes_star <- decode.field(21, decode.int) - use has_images <- decode.field(22, decode.int) - use segment <- decode.field(23, decode.string) - use segment_prev <- decode.field(24, decode.string) - use segment_next <- decode.field(25, decode.string) - use anchor <- decode.field(26, decode.string) - use segment_html_is <- decode.field(27, decode.string) - use table_cols <- decode.field(28, decode.int) - use table_widths <- decode.field(29, decode.string) - use table_aligns <- decode.field(30, decode.string) - use table_header <- decode.field(31, decode.int) - use code_linenumbers <- decode.field(32, decode.int) - use hat <- decode.field(33, decode.string) - use sha256 <- decode.field(34, decode.string) - use text <- decode.field(35, decode.string) - use id <- decode.field(36, decode.int) - use hla <- decode.field(37, decode.string) - use seg_epub <- decode.field(38, decode.string) - use seg_epub_is <- decode.field(39, decode.string) - use range_first <- decode.field(40, decode.int) - use range_last <- decode.field(41, decode.int) - decode.success(#( - id, - Object( - ocn: ocn, - is_a: is_a, - section: section, - heading_level: level_name(heading_level), - identifier: identifier, - part: part, - is_of_type: is_of_type, - own_section: case is_of_section == section { - True -> "" - False -> is_of_section - }, - segment_html_is: segment_html_is, - table_aligns: json_words(table_aligns), - stow_link: [], - anchor_tags: [], - parent_ocn: parent, - last_descendant_ocn: last_descendant, - ocn_range_first: range_first, - ocn_range_last: range_last, - children: json_ints(children), - ancestors: json_ints(ancestors), - dummy_heading: dummy != 0, - ocn_off: ocn_off != 0, - indent_base: indent_base, - indent_hang: indent_hang, - bullet: bullet != 0, - lang: lang, - has_links: has_links != 0, - has_notes_reg: has_notes_reg != 0, - has_notes_star: has_notes_star != 0, - has_images: has_images != 0, - segment: segment, - segment_prev: segment_prev, - segment_next: segment_next, - anchor: anchor, - heading_lev_anchor: hla, - segment_epub: seg_epub, - segment_epub_is: seg_epub_is, - table_cols: table_cols, - table_widths: json_words(table_widths), - table_header: table_header != 0, - code_syntax: "", - code_linenumbers: code_linenumbers != 0, - heading_ancestors_text: json_joined(hat), - sha256: sha256, - images: [], - text: text, - ), - )) - } - let rows = - sqlight.query( - object_sql(doc_id, has_range_columns(conn)), - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) - - // Built newest-first per section and reversed once at the end. Appending - // to the tail of each section instead would be quadratic, and the largest - // document in the reference collection has some thousands of objects in - // one section. - let sections = - list.fold(rows, dict.new(), fn(acc, row) { - let #(id, object) = row - // the four repeatable properties of the text form are child tables - // here, one query each - let object = - Object( - ..object, - images: images(conn, id), - stow_link: links(conn, id), - anchor_tags: anchors(conn, id), - ) - case model.is_section(object.section) { - False -> acc - True -> - dict.upsert(acc, object.section, fn(existing) { - case existing { - option.Some(previous) -> [object, ..previous] - option.None -> [object] - } - }) - } - }) - |> dict.map_values(fn(_, objects) { list.reverse(objects) }) - model.Document(..document, sections: sections) -} - -/// 2.1 added the container range as two columns. A 2.0 database has -/// neither, and reads as having no range, which is what 2.0 meant. -fn has_range_columns(conn: sqlight.Connection) -> Bool { - let d = decode.field(0, decode.int, decode.success) - case - sqlight.query( - "select count(*) from pragma_table_info('objects') - where name = 'ocn_range_first'", - on: conn, - with: [], - expecting: d, - ) - { - Ok([n]) -> n > 0 - _ -> False - } -} - -fn object_sql(doc_id: Int, has_range: Bool) -> String { - let range_cols = case has_range { - True -> "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" - False -> "0, 0" - } - "select section, seq, ocn, is_a, coalesce(is_of_part,''), - coalesce(is_of_section,''), coalesce(is_of_type,''), - coalesce(heading_level,-1), coalesce(identifier,''), - coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), - coalesce(children,''), coalesce(ancestors,''), - coalesce(dummy_heading,0), coalesce(object_number_off,0), - coalesce(indent_base,0), coalesce(indent_hang,0), - coalesce(bullet,0), coalesce(lang,''), - coalesce(has_links,0), coalesce(has_notes_reg,0), - coalesce(has_notes_star,0), coalesce(has_images,0), - coalesce(segment,''), coalesce(segment_prev,''), - coalesce(segment_next,''), coalesce(anchor,''), - coalesce(segment_html_is,''), - coalesce(table_cols,0), coalesce(table_widths,''), - coalesce(table_aligns,''), coalesce(table_header,0), - coalesce(code_linenumbers,0), - coalesce(heading_ancestors_text,''), coalesce(sha256,''), - coalesce(text,''), id, coalesce(heading_lev_anchor,''), - coalesce(segment_epub,''), coalesce(segment_epub_is,''), " <> range_cols <> " - from objects where doc_id = " <> int.to_string(doc_id) <> " - order by case section - when 'head' then 1 when 'toc' then 2 when 'body' then 3 - when 'endnotes' then 4 when 'glossary' then 5 - when 'bibliography' then 6 when 'bookindex' then 7 - when 'blurb' then 8 when 'tail' then 9 else 10 end, seq" -} - -fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) { - let d = { - use name <- decode.field(0, decode.string) - use sha256 <- decode.field(1, decode.string) - use bytes <- decode.field(2, decode.int) - use width <- decode.field(3, decode.int) - use height <- decode.field(4, decode.int) - use missing <- decode.field(5, decode.int) - decode.success(Image( - name: name, - sha256: sha256, - bytes: bytes, - width: width, - height: height, - missing: missing != 0, - fields: dict.new(), - )) - } - sqlight.query( - "select name, coalesce(sha256,''), coalesce(bytes,0), - coalesce(width,0), coalesce(height,0), coalesce(missing,0) - from object_images where object_id = " - <> int.to_string(object_id) - <> " order by seq", - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) -} - -fn links(conn: sqlight.Connection, object_id: Int) -> List(String) { - let d = { - use url <- decode.field(0, decode.string) - decode.success(url) - } - sqlight.query( - "select url from object_links where object_id = " - <> int.to_string(object_id) - <> " order by seq", - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) -} - -fn anchors(conn: sqlight.Connection, object_id: Int) -> List(String) { - let d = { - use anchor <- decode.field(0, decode.string) - decode.success(anchor) - } - sqlight.query( - "select anchor from object_anchors where object_id = " - <> int.to_string(object_id) - <> " order by seq", - on: conn, - with: [], - expecting: d, - ) - |> result.unwrap([]) -} - -fn one(conn: sqlight.Connection, sql: String) -> String { - let d = { - use v <- decode.field(0, decode.string) - decode.success(v) - } - case sqlight.query(sql, on: conn, with: [], expecting: d) { - Ok([v, ..]) -> v - _ -> "" - } -} - -fn has(conn: sqlight.Connection, name: String) -> Bool { - one( - conn, - "select cast(count(*) as text) from sqlite_master where name = '" - <> name - <> "'", - ) - != "0" -} - -fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String { - one( - conn, - "select value from metadata where doc_id = " - <> int.to_string(doc_id) - <> " and key = '" - <> key - <> "'", - ) -} - -fn major_of(version: String) -> String { - case string.split_once(version, ".") { - Ok(#(major, _)) -> major - Error(_) -> version - } -} - -/// The eight heading levels and their indices. -fn level_name(index: Int) -> String { - case index { - 0 -> "A" - 1 -> "B" - 2 -> "C" - 3 -> "D" - 4 -> "1" - 5 -> "2" - 6 -> "3" - 7 -> "4" - _ -> "" - } -} - -/// The JSON columns, read as JSON. -/// -/// Not as a character scan that strips brackets and quotes: that is wrong -/// the moment a value holds an escape, and spine's encoder writes "/" as -/// "\/", so every heading containing a slash comes back with a backslash in -/// it. The D library had exactly that bug and all 36 document-languages -/// differed on heading_ancestors_text because of it. -fn json_strings(value: String) -> List(String) { - case value { - "" -> [] - v -> - case json.parse(v, decode.list(of: loose_string())) { - Ok(items) -> items - Error(_) -> [] - } - } -} - -/// An element that may be a string or a number, which is what these columns -/// hold: children and ancestors are int arrays, table_aligns and -/// heading_ancestors_text string arrays, and table_widths an int array -/// except where the columns divide evenly, when it is [33.3333,...]. One -/// table in the reference collection does that, and without the float -/// branch the widths came back as three empty strings: the fallback turned -/// a value it could not read into a value that looked read. -fn loose_string() -> decode.Decoder(String) { - decode.one_of(decode.string, [ - decode.int |> decode.map(int.to_string), - decode.float |> decode.map(float.to_string), - decode.success(""), - ]) -} - -fn json_ints(value: String) -> List(Int) { - value - |> json_strings - |> list.filter(fn(s) { s != "" }) - |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) }) -} - -/// Space separated in the text form: table_widths, table_aligns. -fn json_words(value: String) -> String { - value |> json_strings |> string.join(" ") -} - -/// heading_ancestors_text is eight slots, joined with "|" in the text form -/// and a JSON array here. The empty slots are significant: they say which -/// levels the heading sits under. -fn json_joined(value: String) -> String { - case json_strings(value) { - [] -> "" - parts -> string.join(parts, "|") - } -} - -/// What the database holds and the model does not. -/// -/// The text loader meets every property line and counts those it does not -/// keep, so a reader can say what was read and not understood. This loader -/// asks for columns by name, and would never see the ones it did not ask -/// for: it would report that it had understood everything, which is the -/// silence the count exists to prevent. So it counts them, on the -/// conditions under which the `.ssp` writes the same property -/// (specification section 6.3), and the report is the same from either -/// form, as the D loader's is. -fn count_unread( - conn: sqlight.Connection, - doc_id: Int, - document: Document, -) -> Document { - let id = int.to_string(doc_id) - let counts = [ - #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"), - #("dom_status", "coalesce(dom_status,'') != ''"), - #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"), - #("heading_lev_collapsed", "heading_lev_collapsed is not null"), - #("parent_lev", "coalesce(parent_lev,0) != 0"), - #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"), - #("attrib", "coalesce(attrib,'') != ''"), - ] - let property = - list.fold(counts, document.unknown.property, fn(acc, c) { - let n = - count( - conn, - "select count(*) from objects where doc_id = " <> id <> " and " <> c.1, - ) - case n > 0 { - True -> dict.insert(acc, c.0, n) - False -> acc - } - }) - let property = case has(conn, "object_subtoc") { - False -> property - True -> { - let n = - count( - conn, - "select count(*) from object_subtoc s join objects o - on o.id = s.object_id where o.doc_id = " <> id, - ) - case n > 0 { - True -> dict.insert(property, "lev4_subtoc", n) - False -> property - } - } - } - model.Document( - ..document, - unknown: model.Unknown(..document.unknown, property: property), - ) -} - -fn count(conn: sqlight.Connection, sql: String) -> Int { - let d = decode.field(0, decode.int, decode.success) - case sqlight.query(sql, on: conn, with: [], expecting: d) { - Ok([n, ..]) -> n - _ -> 0 - } -} - -/// Every language the database at `path` holds, in manifest order. -pub fn languages_of_file(path: String) -> Result(List(Language), String) { - use conn <- with_connection("file:" <> path <> "?mode=ro", path) - Ok(languages(conn)) -} - -/// What the database at `path` carries besides the abstraction, by role -/// and storage: role, compression, count, original bytes, stored bytes. -pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { - use conn <- with_connection("file:" <> path <> "?mode=ro", path) - case has(conn, "files") { - False -> Ok([]) - True -> { - let d = { - use role <- decode.field(0, decode.string) - use compression <- decode.field(1, decode.string) - use n <- decode.field(2, decode.int) - use bytes <- decode.field(3, decode.int) - use stored <- decode.field(4, decode.int) - decode.success([ - role, - compression, - int.to_string(n), - int.to_string(bytes), - int.to_string(stored), - ]) - } - sqlight.query( - "select role, compression, count(*), sum(bytes), sum(length(data)) - from files group by role, compression order by role", - on: conn, - with: [], - expecting: d, - ) - |> result.map_error(fn(e) { path <> ": " <> e.message }) - } - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-ocda-misc.org b/org/gleam-ocda-misc.org deleted file mode 100644 index cc32dcb..0000000 --- a/org/gleam-ocda-misc.org +++ /dev/null @@ -1,649 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** zip gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/zip.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<zip>> -#+END_SRC - -*** gleam source - -#+NAME: zip -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// Enough of the zip format to find one file in a pod and read it. -//// -//// Not a zip library. A pod archive holds a handful of entries, spine writes -//// them **stored** (uncompressed) so that the outer zstd frame does the -//// compressing once over the whole thing, and what is wanted from it here is -//// one named entry. -//// -//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could -//// inflate it, but a library that says it read an entry it in fact skipped is -//// worse than one that says it cannot, and spine's own archives do not need -//// it. - -import gleam/bit_array -import gleam/int -import gleam/list -import gleam/result -import gleam/string - -pub type Entry { - Entry( - name: String, - /// 0 stored, 8 deflate - method: Int, - compressed: Int, - uncompressed: Int, - local_offset: Int, - ) -} - -const eocd_sig = 0x06054b50 - -const cd_sig = 0x02014b50 - -const local_sig = 0x04034b50 - -/// The entries, from the central directory at the end of the file. -pub fn entries(bytes: BitArray) -> Result(List(Entry), String) { - let size = bit_array.byte_size(bytes) - use _ <- result.try(case size >= 22 { - True -> Ok(Nil) - False -> Error("too short to be a zip archive") - }) - use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size))) - use count <- result.try(u16(bytes, eocd + 10)) - use offset <- result.try(u32(bytes, eocd + 16)) - read_central(bytes, offset, count, []) -} - -/// The end of central directory record is last, and may be followed by a -/// comment of up to 65535 bytes, so it is searched for backwards. -fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) { - case at < lowest { - True -> - Error( - "no end of central directory record; it does not look like a zip archive", - ) - False -> - case u32(bytes, at) { - Ok(sig) if sig == eocd_sig -> Ok(at) - _ -> find_eocd(bytes, at - 1, lowest) - } - } -} - -fn lowest(size: Int) -> Int { - case size > 22 + 65_535 { - True -> size - 22 - 65_535 - False -> 0 - } -} - -fn read_central( - bytes: BitArray, - at: Int, - remaining: Int, - acc: List(Entry), -) -> Result(List(Entry), String) { - case remaining { - 0 -> Ok(list.reverse(acc)) - _ -> { - use sig <- result.try(u32(bytes, at)) - use _ <- result.try(case sig == cd_sig { - True -> Ok(Nil) - False -> - Error( - "central directory entry " - <> int.to_string(list.length(acc)) - <> " is malformed", - ) - }) - use method <- result.try(u16(bytes, at + 10)) - use compressed <- result.try(u32(bytes, at + 20)) - use uncompressed <- result.try(u32(bytes, at + 24)) - use name_len <- result.try(u16(bytes, at + 28)) - use extra_len <- result.try(u16(bytes, at + 30)) - use cmnt_len <- result.try(u16(bytes, at + 32)) - use local_offset <- result.try(u32(bytes, at + 42)) - use name <- result.try(text(bytes, at + 46, name_len)) - let entry = - Entry( - name: name, - method: method, - compressed: compressed, - uncompressed: uncompressed, - local_offset: local_offset, - ) - read_central( - bytes, - at + 46 + name_len + extra_len + cmnt_len, - remaining - 1, - [entry, ..acc], - ) - } - } -} - -/// One entry's bytes, from its local header. -pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) { - use sig <- result.try(u32(bytes, entry.local_offset)) - use _ <- result.try(case sig == local_sig { - True -> Ok(Nil) - False -> Error(entry.name <> ": local header is malformed") - }) - use name_len <- result.try(u16(bytes, entry.local_offset + 26)) - use extra_len <- result.try(u16(bytes, entry.local_offset + 28)) - use _ <- result.try(case entry.method { - 0 -> Ok(Nil) - m -> - Error( - entry.name - <> ": compression method " - <> int.to_string(m) - <> "; this reader takes only stored entries, which is what spine writes", - ) - }) - let start = entry.local_offset + 30 + name_len + extra_len - case bit_array.slice(bytes, start, entry.compressed) { - Ok(slice) -> Ok(slice) - Error(_) -> Error(entry.name <> ": runs past the end of the archive") - } -} - -/// The entries whose names end this way, in archive order. Used to find -/// media/abstraction/<uid>.<lang>.ssp without knowing the uid. -pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) { - list.filter(entries, fn(e) { string.ends_with(e.name, suffix) }) -} - -fn u16(bytes: BitArray, at: Int) -> Result(Int, String) { - case bit_array.slice(bytes, at, 2) { - Ok(<<v:little-size(16)>>) -> Ok(v) - _ -> Error("read past the end of the archive at " <> int.to_string(at)) - } -} - -fn u32(bytes: BitArray, at: Int) -> Result(Int, String) { - case bit_array.slice(bytes, at, 4) { - Ok(<<v:little-size(32)>>) -> Ok(v) - _ -> Error("read past the end of the archive at " <> int.to_string(at)) - } -} - -fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { - case bit_array.slice(bytes, at, len) { - Ok(slice) -> - bit_array.to_string(slice) - |> result.replace_error("an entry name is not valid UTF-8") - Error(_) -> Error("an entry name runs past the end of the archive") - } -} -#+END_SRC - -** imapges gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/images.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<images>> -#+END_SRC - -*** gleam source - -#+NAME: images -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// An image's bytes, from whichever source the document came from. -//// -//// The model *names* images, with digest, byte size and pixel size; it does -//// not hold them. That is right - an abstraction of a document is not a -//// bundle of its assets - but a writer of any format that shows images needs -//// the bytes, so something has to know where they are. This is that -//// something, and it is in the library rather than in a writer because the -//// answer depends on the *input form* and not on the output. -//// -//// | `.ocda.db` | the `files` table, role 'image', by name | -//// | `.zip` | the archive entry whose path ends in `/image/<name>` | -//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot | -//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` | - -import gleam/bit_array -import gleam/dynamic/decode -import gleam/list -import gleam/result -import gleam/string -import ocda/load -import ocda/zip -import simplifile -import sqlight - -/// The bytes, or an error saying why not. -pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) { - case load.source_of(source) { - load.OcdaDb -> from_db(source, name) - load.Pod -> from_archive(source, name) - load.Ssp -> from_beside(source, name) - load.Unknown -> Error(source <> ": not a source this library reads") - } -} - -/// Can this source give images at all? -/// -/// A writer asks before it decides whether to emit them, because an `<img>` -/// pointing at a file it did not include is worse than no image. -pub fn can_give_images(source: String) -> Bool { - case load.source_of(source) { - load.OcdaDb -> True - load.Pod -> zip_readable(source) - load.Ssp -> image_dir_beside(source) != "" - load.Unknown -> False - } -} - -fn zip_readable(path: String) -> Bool { - case simplifile.read_bits(path) { - Error(_) -> False - Ok(bytes) -> - case zip.entries(bytes) { - Ok(_) -> True - Error(_) -> False - } - } -} - -fn from_db(path: String, name: String) -> Result(BitArray, String) { - case sqlight.open("file:" <> path <> "?mode=ro") { - Error(e) -> Error(path <> ": " <> e.message) - Ok(conn) -> { - let out = - sqlight.query( - "select data from files where role = 'image' and name = ?1", - on: conn, - with: [sqlight.text(name)], - expecting: blob(), - ) - |> result.replace_error(path <> ": could not read " <> name) - let _ = sqlight.close(conn) - case out { - Ok([data, ..]) -> Ok(data) - Ok([]) -> Error(name <> ": not in this database") - Error(e) -> Error(e) - } - } - } -} - -/// A blob column, read as bytes rather than as text: an image read as text -/// stops at the first zero byte, and a png's ninth byte is zero. -fn blob() -> decode.Decoder(BitArray) { - use v <- decode.field(0, decode.bit_array) - decode.success(v) -} - -fn from_archive(path: String, name: String) -> Result(BitArray, String) { - use bytes <- result.try( - simplifile.read_bits(path) - |> result.replace_error(path <> ": cannot read"), - ) - use entries <- result.try(zip.entries(bytes)) - case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) { - Error(_) -> Error(name <> ": not in this archive") - Ok(entry) -> zip.read(bytes, entry) - } -} - -fn from_beside(path: String, name: String) -> Result(BitArray, String) { - case image_dir_beside(path) { - "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it") - dir -> - simplifile.read_bits(dir <> "/" <> name) - |> result.replace_error(dir <> "/" <> name <> ": cannot read") - } -} - -/// `media/abstraction/<x>.ssp` -> `media/image/` -fn image_dir_beside(path: String) -> String { - let parts = string.split(path, on: "/") - let dir = case list.reverse(parts) { - [_file, ..rest] -> string.join(list.reverse(rest), "/") - [] -> "" - } - case string.ends_with(dir, "abstraction") { - False -> "" - True -> { - let parent = case list.reverse(string.split(dir, on: "/")) { - [_, ..rest] -> string.join(list.reverse(rest), "/") - [] -> "" - } - let image_dir = parent <> "/image" - case simplifile.is_directory(image_dir) { - Ok(True) -> image_dir - _ -> "" - } - } - } -} - -/// Unused here, kept so that a caller can size a read before making it. -pub fn byte_size(b: BitArray) -> Int { - bit_array.byte_size(b) -} -#+END_SRC - -** dump gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/dump.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<dump>> -#+END_SRC - -*** gleam source - -#+NAME: dump -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// The model, written out canonically. -//// -//// Not an output format: an *oracle*. Two libraries in two languages -//// implement the same model from the same specification, and the cheap way -//// to ask whether they agree is to have each write the whole loaded -//// document as text and diff the two. Byte for byte identical to the D -//// library's `ocda/dump.d`, which is the whole point of it. -//// -//// The rules, which both implementations follow: -//// -//// - every line is `<key> <value>`, the value running to end of line, -//// and only one free-form value per line; -//// - a field at its default is *omitted*, so the output says what the -//// document has rather than what the type declares; -//// - dictionaries come in sorted key order, because neither language's -//// map has an order worth relying on; -//// - sections come in the fixed order of the format. -//// -//// Anything that reads this to reconstruct a document has misunderstood -//// it. Load the `.ssp` or the `.ocda.db`. - -import gleam/dict.{type Dict} -import gleam/int -import gleam/list -import gleam/string -import ocda/model.{type Document, type Object} - -pub fn document(doc: Document) -> String { - let lines = - [] - |> kv("version", doc.format_version) - |> kv("source_filename", doc.source_filename) - |> entries("source", doc.source) - |> entries("meta", doc.metadata) - |> entries("make", doc.make) - |> entries("doc_has", doc.doc_has) - |> counts(doc) - |> objects(doc) - lines |> list.reverse |> join -} - -/// What the loader met and did not understand, sorted. -/// -/// Separate from the document dump: it describes the loader, not the -/// document, and the `.ocda.db` loader has nothing to report where the -/// `.ssp` loader does. -pub fn unknown(doc: Document) -> String { - [] - |> numbered("property", doc.unknown.property) - |> numbered("block", doc.unknown.block) - |> numbered("line", doc.unknown.line) - |> list.reverse - |> join -} - -fn join(lines: List(String)) -> String { - case lines { - [] -> "" - _ -> string.join(lines, "\n") <> "\n" - } -} - -/// Accumulated newest-first and reversed once, so that nothing appends to -/// the tail of a list that grows to a few hundred thousand lines. -fn kv(acc: List(String), key: String, value: String) -> List(String) { - case value { - "" -> acc - v -> [key <> " " <> v, ..acc] - } -} - -fn kn(acc: List(String), key: String, value: Int) -> List(String) { - case value { - 0 -> acc - n -> [key <> " " <> int.to_string(n), ..acc] - } -} - -fn kb(acc: List(String), key: String, value: Bool) -> List(String) { - case value { - False -> acc - True -> [key <> " 1", ..acc] - } -} - -fn entries( - acc: List(String), - prefix: String, - d: Dict(String, String), -) -> List(String) { - d - |> dict.to_list - |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) - |> list.fold(acc, fn(lines, pair) { - [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines] - }) -} - -fn numbered( - acc: List(String), - prefix: String, - d: Dict(String, Int), -) -> List(String) { - d - |> dict.to_list - |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) - |> list.fold(acc, fn(lines, pair) { - [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines] - }) -} - -fn counts(acc: List(String), doc: Document) -> List(String) { - list.fold(model.section_names, acc, fn(lines, name) { - case model.section(doc, name) { - [] -> lines - objects -> - [ - "section " <> name <> " " <> int.to_string(list.length(objects)), - ..lines - ] - } - }) -} - -fn objects(acc: List(String), doc: Document) -> List(String) { - list.fold(model.section_names, acc, fn(lines, name) { - model.section(doc, name) - |> list.index_fold(lines, fn(l, obj, i) { - object(l, name, i, obj) - }) - }) -} - -fn object( - acc: List(String), - section: String, - index: Int, - obj: Object, -) -> List(String) { - let acc = ["object " <> section <> " " <> int.to_string(index), ..acc] - acc - |> kn("ocn", obj.ocn) - |> kv("is_a", obj.is_a) - |> kv("own_section", obj.own_section) - |> kv("level", obj.heading_level) - |> kv("identifier", obj.identifier) - |> kv("part", obj.part) - |> kv("is_of_type", obj.is_of_type) - |> kv("segment_html_is", obj.segment_html_is) - |> kv("segment_epub", obj.segment_epub) - |> kv("segment_epub_is", obj.segment_epub_is) - |> kn("parent", obj.parent_ocn) - |> kn("last_descendant", obj.last_descendant_ocn) - |> kn("ocn_range_first", obj.ocn_range_first) - |> kn("ocn_range_last", obj.ocn_range_last) - |> kv("children", ints(obj.children)) - |> kv("ancestors", ints(obj.ancestors)) - |> kb("dummy", obj.dummy_heading) - |> kb("ocn_off", obj.ocn_off) - |> kn("indent_base", obj.indent_base) - |> kn("indent_hang", obj.indent_hang) - |> kb("bullet", obj.bullet) - |> kv("lang", obj.lang) - |> kb("has_links", obj.has_links) - |> kb("has_notes_reg", obj.has_notes_reg) - |> kb("has_notes_star", obj.has_notes_star) - |> kb("has_images", obj.has_images) - |> kv("segment", obj.segment) - |> kv("segment_prev", obj.segment_prev) - |> kv("segment_next", obj.segment_next) - |> kv("anchor", obj.anchor) - |> kv("heading_lev_anchor", obj.heading_lev_anchor) - |> kn("table_cols", obj.table_cols) - |> kv("table_widths", obj.table_widths) - |> kv("table_aligns", obj.table_aligns) - |> kb("table_header", obj.table_header) - |> kv("code_syntax", obj.code_syntax) - |> kb("code_linenumbers", obj.code_linenumbers) - |> kv("heading_ancestors_text", obj.heading_ancestors_text) - |> kv("sha256", obj.sha256) - |> links(obj) - |> anchor_tags(obj) - |> images(obj) - |> text(obj) -} - -fn links(acc: List(String), obj: Object) -> List(String) { - list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) }) -} - -fn anchor_tags(acc: List(String), obj: Object) -> List(String) { - list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) }) -} - -fn images(acc: List(String), obj: Object) -> List(String) { - list.fold(obj.images, acc, fn(lines, img) { - // the name last, because a filename may hold a space where none of the - // other fields can - let lines = - kv( - lines, - "image", - img.sha256 - <> " " - <> int.to_string(img.bytes) - <> " " - <> int.to_string(img.width) - <> "x" - <> int.to_string(img.height) - <> " " - <> case img.missing { - True -> "1" - False -> "0" - } - <> " " - <> img.name, - ) - img.fields - |> dict.to_list - |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) - |> list.fold(lines, fn(l, pair) { - kv(l, "image_field", pair.0 <> " " <> pair.1) - }) - }) -} - -/// Text, when there is any. -/// -/// An object whose only text line is blank holds "" here, indistinguishable -/// from no text at all, because the model joins the lines into one string. -/// Both libraries lose the same distinction in the same place, which is -/// what makes them comparable; it is recorded rather than hidden. -fn text(acc: List(String), obj: Object) -> List(String) { - case obj.text { - "" -> acc - t -> { - let lines = string.split(t, on: "\n") - let acc = kv(acc, "text", int.to_string(list.length(lines))) - list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] }) - } - } -} - -fn ints(xs: List(Int)) -> String { - case xs { - [] -> "" - _ -> xs |> list.map(int.to_string) |> string.join(",") - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-ocda-model_and_inventory.org b/org/gleam-ocda-model_and_inventory.org deleted file mode 100644 index 0e091a4..0000000 --- a/org/gleam-ocda-model_and_inventory.org +++ /dev/null @@ -1,494 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** model gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/model.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<model>> -#+END_SRC - -#+NAME: model -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// The object-centric document abstraction, as Gleam types. -//// -//// A deliberate mirror of the D library's `ocda.model`, field for field. -//// The two are meant to agree, and the cheapest way to make that testable -//// is for them to hold the same things under the same names: the dump of a -//// loaded document is then identical by construction rather than by -//// negotiation. -//// -//// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for -//// the model, the two format specifications for what each serialisation -//// says about it. - -import gleam/dict.{type Dict} -import gleam/list -import gleam/string - -/// The format major version this library knows. -/// -/// The `.ssp` and the `.ocda.db` carry one version between them, declared -/// in band. The minor part rises when a property is added and a reader of -/// the newer format still reads the older, so only the major is checked. -pub const format_major_read = "2" - -/// The nine sections, in the order the format gives them, which is not the -/// order a sorted key list or a `group by` would give. -pub const section_names = [ - "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", - "blurb", "tail", -] - -/// The four blocks that are document preamble rather than sections of -/// objects. -pub const preamble_blocks = ["source", "meta", "make", "doc_has"] - -/// A repeatable `.image` record: the file as the abstraction found it. -pub type Image { - Image( - /// filename as the object refers to it - name: String, - /// hex, uppercase - sha256: String, - bytes: Int, - /// the file's own pixel dimensions, not any display width - width: Int, - height: Int, - /// absent or unreadable, so no digest and no size - missing: Bool, - /// any further key:value fields, kept rather than dropped - fields: Dict(String, String), - ) -} - -pub fn new_image() -> Image { - Image( - name: "", - sha256: "", - bytes: 0, - width: 0, - height: 0, - missing: False, - fields: dict.new(), - ) -} - -/// One object of the abstraction. -/// -/// `text` is one string with the lines joined by newline, as in the D -/// library and for the same reason: an object whose only text line is -/// blank is then indistinguishable from an object with no text, in both -/// libraries, in the same place. A difference that exists in one language -/// and not the other would be worse than a loss both share. -pub type Object { - Object( - ocn: Int, - /// one of the sixteen kinds - is_a: String, - /// the section this object was found in - section: String, - /// headings: A B C D 1 2 3 4 - heading_level: String, - identifier: String, - part: String, - /// how the text behaves: para, block, comment - is_of_type: String, - /// the object's own section, when it differs from the block it sits in - own_section: String, - segment_html_is: String, - segment_epub: String, - segment_epub_is: String, - table_aligns: String, - stow_link: List(String), - anchor_tags: List(String), - parent_ocn: Int, - last_descendant_ocn: Int, - ocn_range_first: Int, - ocn_range_last: Int, - children: List(Int), - ancestors: List(Int), - dummy_heading: Bool, - ocn_off: Bool, - indent_base: Int, - indent_hang: Int, - bullet: Bool, - lang: String, - has_links: Bool, - has_notes_reg: Bool, - has_notes_star: Bool, - has_images: Bool, - segment: String, - segment_prev: String, - segment_next: String, - anchor: String, - heading_lev_anchor: String, - table_cols: Int, - table_widths: String, - table_header: Bool, - code_syntax: String, - code_linenumbers: Bool, - heading_ancestors_text: String, - sha256: String, - images: List(Image), - text: String, - ) -} - -pub fn new_object() -> Object { - Object( - ocn: 0, - is_a: "", - section: "", - heading_level: "", - identifier: "", - part: "", - is_of_type: "", - own_section: "", - segment_html_is: "", - segment_epub: "", - segment_epub_is: "", - table_aligns: "", - stow_link: [], - anchor_tags: [], - parent_ocn: 0, - last_descendant_ocn: 0, - ocn_range_first: 0, - ocn_range_last: 0, - children: [], - ancestors: [], - dummy_heading: False, - ocn_off: False, - indent_base: 0, - indent_hang: 0, - bullet: False, - lang: "", - has_links: False, - has_notes_reg: False, - has_notes_star: False, - has_images: False, - segment: "", - segment_prev: "", - segment_next: "", - anchor: "", - heading_lev_anchor: "", - table_cols: 0, - table_widths: "", - table_header: False, - code_syntax: "", - code_linenumbers: False, - heading_ancestors_text: "", - sha256: "", - images: [], - text: "", - ) -} - -/// Whether this object is rendered at all. -/// -/// `is_of_type` "comment" is markup commentary: carried by the abstraction -/// so that nothing is lost, and not put into output. An output that -/// renders it leaks the author's notes into the document, which is what -/// spine itself does not do. -pub fn renders(object: Object) -> Bool { - object.is_of_type != "comment" -} - -/// The section this object belongs to: the block it sits in, unless it -/// says otherwise. -pub fn section_of(object: Object) -> String { - case object.own_section { - "" -> object.section - own -> own - } -} - -/// What the loader met and did not understand. -/// -/// A loader that silently drops what it does not know cannot say whether -/// it understood the document, and so cannot be used to check a -/// specification against an implementation: silence and understanding look -/// alike. -pub type Unknown { - Unknown( - /// `.name` -> times seen - property: Dict(String, Int), - /// `@name` -> times seen - block: Dict(String, Int), - /// first character -> times seen, for lines of no known shape - line: Dict(String, Int), - ) -} - -pub fn new_unknown() -> Unknown { - Unknown(property: dict.new(), block: dict.new(), line: dict.new()) -} - -pub fn any_unknown(unknown: Unknown) -> Bool { - dict.size(unknown.property) > 0 - || dict.size(unknown.block) > 0 - || dict.size(unknown.line) > 0 -} - -pub type Document { - Document( - /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when - /// absent - format_version: String, - source_filename: String, - /// language, languages, digest - source: Dict(String, String), - unknown: Unknown, - metadata: Dict(String, String), - make: Dict(String, String), - doc_has: Dict(String, String), - /// section name -> objects, in document order - sections: Dict(String, List(Object)), - ) -} - -pub fn new_document() -> Document { - Document( - format_version: "", - source_filename: "", - source: dict.new(), - unknown: new_unknown(), - metadata: dict.new(), - make: dict.new(), - doc_has: dict.new(), - sections: dict.new(), - ) -} - -/// The major part of the declared version, "" when none was declared. -pub fn version_major(document: Document) -> String { - case string.split_once(document.format_version, ".") { - Ok(#(major, _)) -> major - Error(_) -> document.format_version - } -} - -pub fn meta(document: Document, key: String) -> String { - case dict.get(document.metadata, key) { - Ok(value) -> value - Error(_) -> "" - } -} - -/// The objects of a named section, empty when the document has none. -pub fn section(document: Document, name: String) -> List(Object) { - case dict.get(document.sections, name) { - Ok(objects) -> objects - Error(_) -> [] - } -} - -/// Is this one of the nine sections of the format? -pub fn is_section(name: String) -> Bool { - list.contains(section_names, name) -} - -/// Is this one of the four preamble blocks? -pub fn is_preamble_block(name: String) -> Bool { - list.contains(preamble_blocks, name) -} -#+END_SRC - -** inventory gleam -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/inventory.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<inventory>> -#+END_SRC - -#+NAME: inventory -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// What a source holds, beyond the one document the model is loaded from. -//// -//// The model is one language of one document. Two questions a reader asks -//// are about the file instead: which languages it holds, and, for a -//// database, what else it carries. They are here rather than in a reader -//// because the answer depends on the input form, which is the library's to -//// know, and so that a reader asking them does not learn which form it was -//// handed. A mirror of the D library's `ocda.inventory`. -//// -//// `languages` answers for every form, and answers the same: the languages -//// in the order the manifest gave them (which every `.ssp` records, in -//// `@source languages`), each with the markup file it was made from and the -//// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a -//// pod it is taken from the bytes, which are the same bytes. -//// -//// `carried` is the database's: a `.ocda.db` holds the markup, images, -//// configuration and catalogues it was built from, a `.ssp` holds none of -//// them, and a pod's are its archive entries rather than anything the -//// abstraction says. - -import gleam/bit_array -import gleam/dict -import gleam/list -import gleam/result -import gleam/string -import ocda/load -import ocda/load_db -import ocda/load_pod -import ocda/load_ssp -import ocda/zip -import simplifile - -pub type Language { - Language(lang: String, source_filename: String, ssp_digest: String) -} - -pub type Carried { - Carried( - role: String, - compression: String, - count: String, - bytes: String, - stored: String, - ) -} - -pub fn languages(path: String) -> Result(List(Language), String) { - case load.source_of(path) { - load.OcdaDb -> { - use held <- result.try(load_db.languages_of_file(path)) - Ok( - list.map(held, fn(l) { - Language(l.lang, l.source_filename, l.ssp_digest) - }), - ) - } - load.Ssp -> { - use bytes <- result.try( - simplifile.read_bits(path) - |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }), - ) - use text <- result.try( - bit_array.to_string(bytes) - |> result.replace_error(path <> ": not valid UTF-8"), - ) - let doc = load_ssp.parse(text) - Ok([ - Language( - source_value(doc.source, "language"), - doc.source_filename, - sha256_hex(bytes), - ), - ]) - } - load.Pod -> { - use archive <- result.try(load_pod.archive_bytes(path)) - use entries <- result.try(zip.entries(archive)) - use found <- result.try( - zip.find_suffix(entries, ".ssp") - |> list.try_map(fn(e) { - use bytes <- result.try(zip.read(archive, e)) - use text <- result.try( - bit_array.to_string(bytes) - |> result.replace_error(e.name <> ": not valid UTF-8"), - ) - let doc = load_ssp.parse(text) - let lang = result.unwrap(load_pod.lang_of(e.name), "") - Ok(#( - Language(lang, doc.source_filename, sha256_hex(bytes)), - string.split(source_value(doc.source, "languages"), " "), - )) - }), - ) - let order = case found { - [#(_, o), ..] -> list.filter(o, fn(l) { l != "" }) - [] -> [] - } - let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list - // the manifest's order, and anything it did not name after it - let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) }) - let rest = - list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) }) - |> list.map(fn(f) { f.0 }) - Ok(list.append(named, rest)) - } - load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown)) - } -} - -/// Only a `.ocda.db` carries files. -pub fn is_carrier(path: String) -> Bool { - load.source_of(path) == load.OcdaDb -} - -pub fn carried(path: String) -> Result(List(Carried), String) { - case is_carrier(path) { - False -> Ok([]) - True -> { - use rows <- result.try(load_db.carried_of_file(path)) - Ok( - list.filter_map(rows, fn(r) { - case r { - [role, compression, n, bytes, stored] -> - Ok(Carried(role, compression, n, bytes, stored)) - _ -> Error(Nil) - } - }), - ) - } - } -} - -fn source_value(source: dict.Dict(String, String), key: String) -> String { - result.unwrap(dict.get(source, key), "") -} - -@external(erlang, "ocda_ffi", "sha256_hex") -fn sha256_hex(bytes: BitArray) -> String -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-ocda.org b/org/gleam-ocda.org index a7cac9a..2cb2e20 100644 --- a/org/gleam-ocda.org +++ b/org/gleam-ocda.org @@ -6,9 +6,9 @@ #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t @@ -16,159 +16,7 @@ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * gleam :gleam: -** README.md -*** tangle - -#+HEADER: :tangle ../gleam/README.md -#+HEADER: :noweb yes -#+BEGIN_SRC md -# ocda-lib, in Gleam - -The abstraction's model and both its loaders, as a Gleam library, with its -reader and the two programs that put it to the question. - -| | | -|------------------|-----------------------------------------------------------------------------------------| -| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | -| `ocda/load_ssp` | text -> model | -| `ocda/load_db` | sqlite -> model | -| `ocda/load` | what a path is, and loading it | -| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | -| `ocda/dump` | the model written out canonically, as an oracle | -| `read/main` | `ocda-read`, the reader | -| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | - -## Use - -```gleam -import ocda/load -import ocda/model - -case load.load(path, "en", False) { - Error(why) -> io.println_error(why) - Ok(doc) -> { - // and from here a writer neither knows nor cares which of the two - // files it was handed - list.each(model.section(doc, "body"), fn(obj) { ... }) - } -} -``` - -## The reader - -```sh -nix build -./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] - <file>.ssp|<file>.ocda.db|<file>.zip [lang] -``` - -`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers -had, over the library, so each reads every form this library reads; the -others are the D reader's. **Its output is the D `ocda-read`'s, byte for -byte**, in every mode both have. In `src/read/`, because Gleam compiles -nothing outside `src/`; a program depending on the library compiles it too, -and does not use it. - -A `.sisupod` it refuses, saying why: the zstd frame, below. - -## The two programs - -```sh -nix build -./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] -./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] -``` - -`agree` asks whether the two loaders give the same model. `dump` writes -the whole model out canonically, and is the stronger check of the two, -because it is **byte for byte what the D library's `ocda-dump` writes**. -So the four questions can all be asked with `diff`: - -| | | -|-----------------------------|-----------------------------| -| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | -| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | -| `.ssp`: D == Gleam | 36 of 36 | -| `.ocda.db`: D == Gleam | 36 of 36 | - -## Deliberately a mirror of the D library - -Field for field, name for name, including where that is not the most -Gleamish choice: - -- `text` is one `String` with the lines joined by newline, not a - `List(String)`. So an object whose only text line is blank is - indistinguishable from an object with no text, *in both languages, in the - same place*. A difference that existed in one and not the other would be - worse than a loss both share, because it would show up in every dump and - mean nothing. -- the properties are typed fields rather than a list of pairs. A writer - wants `obj.parent_ocn`, not a lookup that might miss. - -The point is that the two agree. Making them agree is much cheaper when -they are shaped alike, and the shape is the specification's, not either -language's. - -## What the agreement found - -Four disagreements, none of which a comparison of section counts or of a -handful of fields would have shown. All four are recorded in the D -library's history; the short version: - -1. `.identifier` and `.anchor` are omitted by the text form when they can - be worked out (the identifier is the ocn, the anchor is the segment). - Both loaders now derive them. Over one document that is 1,314 - identifiers. -2. `make` and `doc_has` are one metadata table in the database, - distinguished by a key prefix. Both loaders unprefix them. -3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` - as `\/` and every heading with a slash in it came back wrong. -4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, - which is one language. It is a fact about the file, so neither loader - puts it in the document. - -## Build - -```sh -nix build # sandboxed, no network -nix develop -c gleam test -``` - -Shipped as an Erlang shipment rather than an escript, because sqlite on the -BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. -The dependency hash in `flake.nix` pins what `manifest.toml` pins; change -one and the other must change with it. - -## Format - -Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, -`ssp-format-specification.org` and `ocda-db-format-specification.org` for the -two serialisations. At format **2.x**. -#+END_SRC - -** gleam.toml -*** tangle - -#+HEADER: :tangle ../gleam/gleam.toml -#+HEADER: :noweb yes -#+BEGIN_SRC toml -name = "ocda_lib" -version = "0.1.0" -description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" -licences = ["AGPL-3.0-or-later"] - -[dependencies] -gleam_stdlib = ">= 1.0.0 and < 2.0.0" -argv = ">= 1.1.0 and < 2.0.0" -simplifile = ">= 2.7.0 and < 3.0.0" -sqlight = ">= 1.0.0 and < 2.0.0" -gleam_json = ">= 3.0.0 and < 4.0.0" - -[dev-dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" -#+END_SRC - -** gleam ocda_lib +** ocda lib *** tangle #+HEADER: :tangle ../gleam/src/ocda_lib.gleam @@ -178,7 +26,7 @@ gleeunit = ">= 1.0.0 and < 2.0.0" <<ocda_lib>> #+END_SRC -*** gleam source +*** source #+NAME: ocda_lib #+HEADER: :no-expand @@ -329,31 +177,7 @@ fn fail(message: String) -> Nil { fn halt(code: Int) -> Nil #+END_SRC -** erlang -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda_ffi.erl -#+HEADER: :noweb yes -#+BEGIN_SRC org -%%<<doc_header_including_copyright_and_license>> -<<erlang_ffi>> -#+END_SRC - -*** erlang source - -#+NAME: erlang_ffi -#+HEADER: :no-expand -#+BEGIN_SRC erlang -%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's -%% crypto has and the Gleam standard library does not. Upper case hex, as -%% the database stores it and the D library writes it. --module(ocda_ffi). --export([sha256_hex/1]). - -sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). -#+END_SRC - -** model gleam +** model *** tangle #+HEADER: :tangle ../gleam/src/ocda/model.gleam @@ -363,6 +187,8 @@ sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). <<model>> #+END_SRC +*** source + #+NAME: model #+HEADER: :no-expand #+BEGIN_SRC gleam @@ -650,7 +476,7 @@ pub fn is_preamble_block(name: String) -> Bool { } #+END_SRC -** inventory gleam +** inventory *** tangle #+HEADER: :tangle ../gleam/src/ocda/inventory.gleam @@ -660,6 +486,8 @@ pub fn is_preamble_block(name: String) -> Bool { <<inventory>> #+END_SRC +*** source + #+NAME: inventory #+HEADER: :no-expand #+BEGIN_SRC gleam @@ -803,7 +631,7 @@ fn source_value(source: dict.Dict(String, String), key: String) -> String { fn sha256_hex(bytes: BitArray) -> String #+END_SRC -** load gleam +** load *** tangle #+HEADER: :tangle ../gleam/src/ocda/load.gleam @@ -813,7 +641,7 @@ fn sha256_hex(bytes: BitArray) -> String <<load>> #+END_SRC -*** gleam source +*** source #+NAME: load #+HEADER: :no-expand @@ -909,7 +737,7 @@ pub fn load( <<load_pod>> #+END_SRC -*** gleam source +*** source #+NAME: load_pod #+HEADER: :no-expand @@ -1064,7 +892,7 @@ pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { <<load_ssp>> #+END_SRC -*** gleam source +*** source #+NAME: load_ssp #+HEADER: :no-expand @@ -1567,7 +1395,7 @@ fn numbers(value: String) -> List(Int) { <<load_db>> #+END_SRC -*** gleam source +*** source #+NAME: load_db #+HEADER: :no-expand @@ -2243,7 +2071,7 @@ pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { } #+END_SRC -** zip gleam +** zip *** tangle #+HEADER: :tangle ../gleam/src/ocda/zip.gleam @@ -2253,7 +2081,7 @@ pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { <<zip>> #+END_SRC -*** gleam source +*** source #+NAME: zip #+HEADER: :no-expand @@ -2430,7 +2258,7 @@ fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { } #+END_SRC -** imapges gleam +** imapges *** tangle #+HEADER: :tangle ../gleam/src/ocda/images.gleam @@ -2440,7 +2268,7 @@ fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { <<images>> #+END_SRC -*** gleam source +*** source #+NAME: images #+HEADER: :no-expand @@ -2582,7 +2410,7 @@ pub fn byte_size(b: BitArray) -> Int { } #+END_SRC -** dump gleam +** dump *** tangle #+HEADER: :tangle ../gleam/src/ocda/dump.gleam @@ -2592,7 +2420,7 @@ pub fn byte_size(b: BitArray) -> Int { <<dump>> #+END_SRC -*** gleam source +*** source #+NAME: dump #+HEADER: :no-expand @@ -2848,6 +2676,31 @@ fn ints(xs: List(Int)) -> String { } #+END_SRC +* erlang +** erlang +*** tangle + +#+HEADER: :tangle ../gleam/src/ocda_ffi.erl +#+HEADER: :noweb yes +#+BEGIN_SRC org +%%<<doc_header_including_copyright_and_license>> +<<erlang_ffi>> +#+END_SRC + +*** erlang source + +#+NAME: erlang_ffi +#+HEADER: :no-expand +#+BEGIN_SRC erlang +%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's +%% crypto has and the Gleam standard library does not. Upper case hex, as +%% the database stores it and the D library writes it. +-module(ocda_ffi). +-export([sha256_hex/1]). + +sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). +#+END_SRC + * org includes ** project version diff --git a/org/gleam-ocda_lib.org b/org/gleam-ocda_lib.org deleted file mode 100644 index e19715b..0000000 --- a/org/gleam-ocda_lib.org +++ /dev/null @@ -1,381 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** README.md -*** tangle - -#+HEADER: :tangle ../gleam/README.md -#+HEADER: :noweb yes -#+BEGIN_SRC md -# ocda-lib, in Gleam - -The abstraction's model and both its loaders, as a Gleam library, with its -reader and the two programs that put it to the question. - -| | | -|------------------|-----------------------------------------------------------------------------------------| -| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | -| `ocda/load_ssp` | text -> model | -| `ocda/load_db` | sqlite -> model | -| `ocda/load` | what a path is, and loading it | -| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | -| `ocda/dump` | the model written out canonically, as an oracle | -| `read/main` | `ocda-read`, the reader | -| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | - -## Use - -```gleam -import ocda/load -import ocda/model - -case load.load(path, "en", False) { - Error(why) -> io.println_error(why) - Ok(doc) -> { - // and from here a writer neither knows nor cares which of the two - // files it was handed - list.each(model.section(doc, "body"), fn(obj) { ... }) - } -} -``` - -## The reader - -```sh -nix build -./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] - <file>.ssp|<file>.ocda.db|<file>.zip [lang] -``` - -`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers -had, over the library, so each reads every form this library reads; the -others are the D reader's. **Its output is the D `ocda-read`'s, byte for -byte**, in every mode both have. In `src/read/`, because Gleam compiles -nothing outside `src/`; a program depending on the library compiles it too, -and does not use it. - -A `.sisupod` it refuses, saying why: the zstd frame, below. - -## The two programs - -```sh -nix build -./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] -./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] -``` - -`agree` asks whether the two loaders give the same model. `dump` writes -the whole model out canonically, and is the stronger check of the two, -because it is **byte for byte what the D library's `ocda-dump` writes**. -So the four questions can all be asked with `diff`: - -| | | -|-----------------------------|-----------------------------| -| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | -| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | -| `.ssp`: D == Gleam | 36 of 36 | -| `.ocda.db`: D == Gleam | 36 of 36 | - -## Deliberately a mirror of the D library - -Field for field, name for name, including where that is not the most -Gleamish choice: - -- `text` is one `String` with the lines joined by newline, not a - `List(String)`. So an object whose only text line is blank is - indistinguishable from an object with no text, *in both languages, in the - same place*. A difference that existed in one and not the other would be - worse than a loss both share, because it would show up in every dump and - mean nothing. -- the properties are typed fields rather than a list of pairs. A writer - wants `obj.parent_ocn`, not a lookup that might miss. - -The point is that the two agree. Making them agree is much cheaper when -they are shaped alike, and the shape is the specification's, not either -language's. - -## What the agreement found - -Four disagreements, none of which a comparison of section counts or of a -handful of fields would have shown. All four are recorded in the D -library's history; the short version: - -1. `.identifier` and `.anchor` are omitted by the text form when they can - be worked out (the identifier is the ocn, the anchor is the segment). - Both loaders now derive them. Over one document that is 1,314 - identifiers. -2. `make` and `doc_has` are one metadata table in the database, - distinguished by a key prefix. Both loaders unprefix them. -3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` - as `\/` and every heading with a slash in it came back wrong. -4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, - which is one language. It is a fact about the file, so neither loader - puts it in the document. - -## Build - -```sh -nix build # sandboxed, no network -nix develop -c gleam test -``` - -Shipped as an Erlang shipment rather than an escript, because sqlite on the -BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. -The dependency hash in `flake.nix` pins what `manifest.toml` pins; change -one and the other must change with it. - -## Format - -Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, -`ssp-format-specification.org` and `ocda-db-format-specification.org` for the -two serialisations. At format **2.x**. -#+END_SRC - -** gleam.toml -*** tangle - -#+HEADER: :tangle ../gleam/gleam.toml -#+HEADER: :noweb yes -#+BEGIN_SRC toml -name = "ocda_lib" -version = "0.1.0" -description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" -licences = ["AGPL-3.0-or-later"] - -[dependencies] -gleam_stdlib = ">= 1.0.0 and < 2.0.0" -argv = ">= 1.1.0 and < 2.0.0" -simplifile = ">= 2.7.0 and < 3.0.0" -sqlight = ">= 1.0.0 and < 2.0.0" -gleam_json = ">= 3.0.0 and < 4.0.0" - -[dev-dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" -#+END_SRC - -** gleam ocda_lib -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda_lib.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<ocda_lib>> -#+END_SRC - -*** gleam source - -#+NAME: ocda_lib -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// ocda-lib, in Gleam: the abstraction as a library, put to the question. -//// -//// A library has no binary, so this is the programs over it: the reader, -//// and the two that test its claims. -//// -//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] -//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] -//// ocda-lib read [options] <file> [lang] (also installed as ocda-read) -//// -//// `dump` writes the whole loaded model out canonically, byte for byte as -//// the D library's `ocda-dump` writes it, so that the two implementations -//// can be diffed. `agree` asks the same question the D library's -//// `ocda-agree` asks, within this language: does loading a document's -//// `.ssp` give what loading its `.ocda.db` gives? - -import argv -import gleam/int -import gleam/io -import gleam/list -import gleam/string -import ocda/dump -import ocda/load -import ocda/model.{type Document} -import read/main as reader - -pub fn main() { - case argv.load().arguments { - ["dump", "--unknown", path] -> dump_file(path, "", True) - ["dump", "--unknown", path, lang] -> dump_file(path, lang, True) - ["dump", path] -> dump_file(path, "", False) - ["dump", path, lang] -> dump_file(path, lang, False) - ["agree", ssp, db] -> agree(ssp, db, "") - ["agree", ssp, db, lang] -> agree(ssp, db, lang) - ["read", ..rest] -> - case reader.run(rest) { - 0 -> Nil - code -> halt(code) - } - _ -> usage() - } -} - -fn usage() -> Nil { - io.println_error( - "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] - ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] - ocda-lib read [--help] [options] <file> [lang]", - ) - halt(2) -} - -fn dump_file(path: String, lang: String, unknown: Bool) -> Nil { - case load.load(path, lang, False) { - Error(message) -> fail(message) - Ok(doc) -> - case unknown { - True -> io.print(dump.unknown(doc)) - False -> io.print(dump.document(doc)) - } - } -} - -/// The library's whole purpose is that a writer cannot tell which -/// serialisation it was handed. That is only true if the two loaders give -/// the same model, so this asks, and says where they part company rather -/// than only that they did. -fn agree(ssp_path: String, db_path: String, lang: String) -> Nil { - case load.load(ssp_path, "", False), load.load(db_path, lang, False) { - Error(message), _ -> fail(message) - _, Error(message) -> fail(message) - Ok(from_ssp), Ok(from_db) -> { - let differences = compare(from_ssp, from_db) - case differences { - [] -> io.println("agree: the two loaders give the same model") - _ -> { - list.each(list.take(differences, 12), io.println) - case list.length(differences) > 12 { - True -> io.println(" ... stopping after 12") - False -> Nil - } - io.println(int.to_string(list.length(differences)) <> " difference(s)") - halt(1) - } - } - } - } -} - -/// Compared through the dump rather than field by field, which is both -/// shorter and stricter: the dump carries every field, so nothing can -/// differ without showing here. -fn compare(from_ssp: Document, from_db: Document) -> List(String) { - let a = dump.document(from_ssp) - let b = dump.document(from_db) - case a == b { - True -> [] - False -> diff_lines(a, b) - } -} - -fn diff_lines(a: String, b: String) -> List(String) { - let la = split_lines(a) - let lb = split_lines(b) - case list.length(la) == list.length(lb) { - False -> [ - " differ: line count", - " .ssp: " <> int.to_string(list.length(la)), - " .ocda.db: " <> int.to_string(list.length(lb)), - ..pairwise(la, lb) - ] - True -> pairwise(la, lb) - } -} - -fn pairwise(la: List(String), lb: List(String)) -> List(String) { - list.zip(la, lb) - |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) }) - |> list.filter(fn(t) { t.1 != t.2 }) - |> list.map(fn(t) { - " differ: line " - <> int.to_string(t.0 + 1) - <> "\n .ssp: " - <> t.1 - <> "\n .ocda.db: " - <> t.2 - }) -} - -fn split_lines(text: String) -> List(String) { - case text { - "" -> [] - _ -> string.split(text, on: "\n") - } -} - -fn fail(message: String) -> Nil { - io.println_error("error: " <> message) - halt(2) -} - -/// Gleam has no standard exit, and the exit status is what a sweep script -/// reads. -@external(erlang, "erlang", "halt") -fn halt(code: Int) -> Nil -#+END_SRC - -** erlang -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda_ffi.erl -#+HEADER: :noweb yes -#+BEGIN_SRC org -%%<<doc_header_including_copyright_and_license>> -<<erlang_ffi>> -#+END_SRC - -*** erlang source - -#+NAME: erlang_ffi -#+HEADER: :no-expand -#+BEGIN_SRC erlang -%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's -%% crypto has and the Gleam standard library does not. Upper case hex, as -%% the database stores it and the D library writes it. --module(ocda_ffi). --export([sha256_hex/1]). - -sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-read-main.org b/org/gleam-read.org index f968fd1..05fa2d5 100644 --- a/org/gleam-read-main.org +++ b/org/gleam-read.org @@ -6,9 +6,9 @@ #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t @@ -16,7 +16,7 @@ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * gleam :gleam: -** gleam +** main *** tangle #+HEADER: :tangle ../gleam/src/read/main.gleam @@ -26,7 +26,7 @@ <<read>> #+END_SRC -*** gleam source +*** source #+NAME: read #+HEADER: :no-expand |
