summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-10-02 15:01:56 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-10-02 21:39:24 -0400
commitafcdcd2e99b5d06f8b3dd9d43b5fd2033e46b9ed (patch)
tree12bfcde907a47ba0a287c6e68e0f4c1381bf33e4
parenttrack (diff)
track
-rw-r--r--dlang/flake.nix1
-rw-r--r--dlang/src/ocda/dump.d49
-rw-r--r--dlang/src/ocda/images.d49
-rw-r--r--dlang/src/ocda/inventory.d49
-rw-r--r--dlang/src/ocda/model.d49
-rw-r--r--dlang/src/ocda/sqlite.d49
-rw-r--r--dlang/test/agree.d49
-rw-r--r--dlang/test/dump.d49
-rw-r--r--org/config.org830
-rw-r--r--org/config_git.org101
-rw-r--r--org/config_nix.org287
-rw-r--r--org/d-ocda-load.org953
-rw-r--r--org/d-ocda.org697
-rw-r--r--org/d-read.org18
-rw-r--r--org/d-test.org178
-rw-r--r--org/gleam-ocda-load.org1484
-rw-r--r--org/gleam-ocda-misc.org649
-rw-r--r--org/gleam-ocda-model_and_inventory.org494
-rw-r--r--org/gleam-ocda.org239
-rw-r--r--org/gleam-ocda_lib.org381
-rw-r--r--org/gleam-read.org (renamed from org/gleam-read-main.org)8
21 files changed, 2104 insertions, 4559 deletions
diff --git a/dlang/flake.nix b/dlang/flake.nix
index 1596888..94b3e37 100644
--- a/dlang/flake.nix
+++ b/dlang/flake.nix
@@ -67,6 +67,7 @@
};
inherit ocda-read ocda-agree ocda-dump;
};
+
devShells.${system}.default = pkgs.mkShell {
packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
};
diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d
index 58d87ff..9b24cea 100644
--- a/dlang/src/ocda/dump.d
+++ b/dlang/src/ocda/dump.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
module ocda.dump;
/+ ↓ the model, written out canonically.
diff --git a/dlang/src/ocda/images.d b/dlang/src/ocda/images.d
index b3a9652..f6d2541 100644
--- a/dlang/src/ocda/images.d
+++ b/dlang/src/ocda/images.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
module ocda.images;
/+ ↓ an image's bytes, from whichever source the document came from.
diff --git a/dlang/src/ocda/inventory.d b/dlang/src/ocda/inventory.d
index dba2156..32b076b 100644
--- a/dlang/src/ocda/inventory.d
+++ b/dlang/src/ocda/inventory.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
module ocda.inventory;
/+ ↓ what a source holds, beyond the one document the model is loaded from.
diff --git a/dlang/src/ocda/model.d b/dlang/src/ocda/model.d
index 6c8fc37..6b6f40b 100644
--- a/dlang/src/ocda/model.d
+++ b/dlang/src/ocda/model.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
module ocda.model;
struct OcdaImage {
diff --git a/dlang/src/ocda/sqlite.d b/dlang/src/ocda/sqlite.d
index 6f3753c..fc6b2be 100644
--- a/dlang/src/ocda/sqlite.d
+++ b/dlang/src/ocda/sqlite.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
module ocda.sqlite;
/+ ↓ the sqlite3 C API, declared rather than depended upon.
diff --git a/dlang/test/agree.d b/dlang/test/agree.d
index bdf1b2e..92b1f91 100644
--- a/dlang/test/agree.d
+++ b/dlang/test/agree.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
/++
Do the two loaders agree?
diff --git a/dlang/test/dump.d b/dlang/test/dump.d
index 17aa9bb..f11d456 100644
--- a/dlang/test/dump.d
+++ b/dlang/test/dump.d
@@ -1,3 +1,52 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+ - Description: documents, structuring, processing, publishing, search
+ - static content generator
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later:
+
+ Spine (SiSU), a framework for document structuring, publishing and
+ search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU AFERO General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at these locations:
+ [https://www.fsf.org/licensing/licenses/agpl.html]
+ [https://www.gnu.org/licenses/agpl.html]
+
+ - Spine (by Doc Reform, related to SiSU) uses standard:
+ - docReform markup syntax
+ - standard SiSU markup syntax with modified headers and minor modifications
+ - docReform object numbering
+ - standard SiSU object citation numbering & system
+
+ - Homepages:
+ [https://www.sisudoc.org]
+ [https://www.doc-reform.org]
+
+ - Git
+ [https://git.sisudoc.org/]
+
++/
/++
Write a loaded abstraction out canonically.
diff --git a/org/config.org b/org/config.org
new file mode 100644
index 0000000..2c1294f
--- /dev/null
+++ b/org/config.org
@@ -0,0 +1,830 @@
+-*- mode: org -*-
+#+TITLE: configuration nix
+#+DESCRIPTION: configuration nix
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: show3levels hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* ./
+** README.md
+
+#+HEADER: :tangle ../README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# sisudoc-ocda-lib
+
+The **object-centric document abstraction as a library**: the model, and
+loaders for both of the forms it is written in.
+
+**One reader per language, in a directory named for the language.**
+
+| | |
+|--------------------|---------|
+| [`dlang/`](dlang/) | D |
+| [`gleam/`](gleam/) | Gleam |
+| `ruby/` | to come |
+
+## What it is for
+
+The readers in `sisudoc-ssp-reader` and `sisudoc-ocda-db-reader` each read
+one serialisation and print. A **writer** needs neither: it needs the document.
+
+```
+ .ssp ─┐
+ ├─> the model ─> a writer
+ .ocda.db ─┘
+```
+
+So this library holds the model and both loaders, and **a writer cannot
+tell which serialisation it was handed** - exactly as spine's own writers
+cannot, since spine 0.25.0.
+
+That is the whole of the idea. The writers are functions from the model to
+bytes, and the reader is a program over the model.
+
+## The reader
+
+```sh
+ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images|--verify --image-dir=<d>|
+ --languages|--files|--report-unknown|--validate]
+ <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]
+```
+
+One reader, in each language, where `sisudoc-ssp-reader` and
+`sisudoc-ocda-db-reader` were two, each with a parser of its own. Built on
+this library, **every mode reads every form and says the same thing of a
+document's `.ssp`, `.ocda.db` and `.sisupod`**, 36 of 36 document-languages.
+The D reader has every mode of both old readers; the Gleam one the modes the
+Gleam readers had (the summary, `--tree`, `--languages`, `--files`,
+`--report-unknown`), and in those **its output is the D one's, byte for
+byte**, 36 of 36.
+
+It is in each language's `src/read/`, and not part of the library: a
+program using the library does not build it (in D, by dub configuration;
+in Gleam a dependant compiles it and does not use it, Gleam building
+everything in `src/`).
+
+Writing it found one defect in the library: **the database loader could
+not say what it had not understood**. It asks for columns by name, and so
+never saw the ones it did not ask for, and reported that it had understood
+everything. It now counts them on the conditions under which the `.ssp`
+writes the same property, and the report is the same from either form.
+
+## The checks that matter
+
+The claim above is only true if the loaders agree, so the library asks. Two
+programs, in each language:
+
+```sh
+ocda-agree <doc>.ssp <doc>.ocda.db [lang] # do the two loaders agree?
+ocda-dump <file>.ssp|<file>.ocda.db [lang] # the whole model, canonically
+```
+
+`agree` compares two models within one language. **`dump` is the stronger
+of the two**, and the reason it exists: it writes the entire loaded model
+out in a fully determined form, and the D and the Gleam library write it
+*byte for byte alike*. So four questions can be asked, all of them with
+`diff`:
+
+| | |
+|-----------------------------|---------------------------------|
+| D: `.ssp` == `.ocda.db` | **36 of 36** document-languages |
+| Gleam: `.ssp` == `.ocda.db` | **36 of 36** |
+| `.ssp`: D == Gleam | **36 of 36** |
+| `.ocda.db`: D == Gleam | **36 of 36** |
+
+None of those held at first.
+
+## What the checks found
+
+### the ten-field check: a blank first text line
+
+An object whose **first text line is blank** lost it. The loader decided
+whether to start a new line by asking whether the accumulated text was
+empty, which is not the same question as whether a line has been seen. Of
+the sample collection **seven documents have verse opening on a blank
+line**, and in every one of them the blank was dropped. Eight of
+thirty-six documents disagreed.
+
+Nothing else had caught it. The two `.ssp` readers had been compared on
+*section counts*, which this does not change. It took loading the same
+document two ways and comparing the objects.
+
+Fixed here and in `sisudoc-ssp-reader/dlang`, which carried the same code.
+
+### the whole-model dump: four more
+
+The ten-field check was enough for that and not enough for anything
+subtler. Dumping the whole model found four further disagreements between
+the two loaders:
+
+1. **`.identifier` and `.anchor`.** The `.ssp` emits a property only when it
+ carries information, so the identifier is absent when it is simply the
+ ocn and the anchor is absent when it is the segment name; the database
+ stores both unconditionally. Over one document that is **1,314
+ identifiers and 69 anchors**. The loaders now derive them, which is what
+ the model means.
+2. **`make` and `doc_has`.** The database has one metadata table where the
+ text form has three blocks, and tells them apart by a key prefix. The
+ loaders unprefix them, so a writer asks `make["footer"]` whichever
+ serialisation it was handed.
+3. **The JSON columns were scanned rather than parsed.** Phobos writes `/`
+ as `\/`, so every heading containing a slash came back with a backslash
+ in it: **all 36 document-languages** differed on
+ `heading_ancestors_text` and nothing else had shown it.
+4. **`translation.ocn_aligned`** is in the database and cannot be in a
+ `.ssp`, which is one language and cannot know whether the languages
+ share a numbering. It is a fact about the *file*, so the loaders leave
+ it out rather than letting a writer tell which serialisation it was
+ given.
+
+The first three are defects. The fourth is a real asymmetry between the two
+serialisations, and naming it is the useful part.
+
+## What it reads, and what it does not
+
+| | | |
+|-----------------------|------------------------------------------------------------------|-------------------|
+| `.ssp` | yes | |
+| `.ocda.db` | yes, any language of it | |
+| `.sisupod` | yes, any language it carries | D only, see below |
+| `.zip` pod | yes, any language it carries | |
+| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work | |
+| a url | no | |
+
+A pod carries the abstraction only when spine was asked for it: `--pod2`
+puts it in, `--source` does not. A pod without one is refused with a message
+saying which flag makes a pod this library can read, because that is a fact
+about the pod rather than about the library.
+
+**Over the reference collection, a `.sisupod` gives the same model as the
+`.ssp` beside it: 36 of 36 document-languages, byte identical dumps.** A
+writer over the library therefore gained a third input form without being
+touched, which is the point of the dispatch being in one place.
+
+### the one thing the Gleam library cannot do
+
+A `.sisupod` is **one zstd frame wrapping** the zip. The D library reads both
+through a declared `libzstd`. The Gleam library reads the **zip and not the
+frame**, because the only zstd on the BEAM is a NIF that clones zstd from
+github in its own build and so cannot be built in a sandbox without patching
+it.
+
+So a `.zip` pod is read in both languages, a `.sisupod` in D only, and the
+Gleam refusal says why and says `zstd -d` rather than claiming the file is
+malformed. It is a packaging limit, not anything about the format, and it is
+the only place the two libraries differ in what they accept.
+
+`ocda/load` is the one place in each language that decides what a path is,
+so a new input form is added once and every reader and writer gains it.
+
+## The format
+
+Specified in **sisudoc-abstraction-format**: `ocda-model.md` for the model
+this library implements, and the two format specifications for the loaders.
+Section 4 of `ocda-db-format-specification.md` gives the column by column
+correspondence the `load_db` of each language follows.
+
+At format **2.x**.
+#+END_SRC
+
+** git .gitignore :gitignore:
+
+#+HEADER: :tangle "../.gitignore"
+#+BEGIN_SRC gitignore
+# git ls-files --others --exclude-from=.git/info/exclude
+# git ls-files --others --ignored --exclude-standard --directory
+# git check-ignore -v flake.lock
+# git clean -ix
+# find . -name "*_" | xargs rm -rf
+# find . -name "*.o" | xargs rm -rf
+
+# Deny everything, then whitelist what belongs in the repository.
+#
+# The practice is deliberate: a repository should hold what was decided to
+# put in it, not whatever happened to be in the directory when someone ran
+# "git add .". Build output, editor droppings, downloaded dependencies and
+# exported binaries are numerous, are regenerated, and differ between
+# machines; source is few and is chosen. So the default is no, and every
+# yes below is a decision that can be read.
+#
+# To see what is being excluded, and why:
+# git status --ignored
+# git check-ignore -v <path>
+# git ls-files --others --ignored --exclude-standard --directory
+
+*
+
+# ── the repository itself ────────────────────────────────────────────────
+!.gitignore
+!README.md
+!LICENSE
+
+# ── directories have to be walked into before a file inside can match ────
+!*/
+# but not these, whatever they contain
+build/
+.build/
+.direnv/
+.gleam/
+bin/
+result
+result-*
+
+# ── documentation and notes, in any language directory ───────────────────
+!**/*.md
+!**/*.org
+
+# ── nix ──────────────────────────────────────────────────────────────────
+!**/*.nix
+!**/flake.lock
+!**/.envrc
+
+# ── dlang ────────────────────────────────────────────────────────────────
+!**/*.d
+!**/dub.json
+!**/dub.sdl
+!**/dub.selections.json
+*.o
+*.a
+*.lst
+
+# ── gleam ────────────────────────────────────────────────────────────────
+!**/*.gleam
+# Erlang, for what Gleam reaches through FFI (a sha256, from OTP crypto)
+!**/src/*.erl
+# gleam.toml is the manifest, manifest.toml is the lockfile: both belong
+!**/*.toml
+*.beam
+*.ez
+erl_crash.dump
+
+# ── ruby, when it arrives ────────────────────────────────────────────────
+!**/*.rb
+!**/Gemfile
+!**/Gemfile.lock
+!**/*.gemspec
+
+# ── continuous integration ───────────────────────────────────────────────
+!**/*.yml
+!**/*.yaml
+#+END_SRC
+
+* ./dlang
+** nix flake.nix :flake:
+
+#+HEADER: :tangle ../dlang/flake.nix
+#+HEADER: :noweb yes
+#+BEGIN_SRC nix
+{
+ description = "ocda-lib: the object-centric document abstraction as a D library";
+ inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+ outputs = { self, nixpkgs }:
+ let
+ system = "x86_64-linux"; # adjust for aarch64-linux / darwin
+ pkgs = import nixpkgs { inherit system; };
+ /* A library, so the library itself is not a build product: a program
+ that uses it compiles its modules alongside its own, as the writer's
+ flake does, or takes it as a dub dependency. What is built here are
+ programs over it: the reader, and the two that put the library to
+ the question - the agreement check, which loads a document's .ssp
+ and its .ocda.db and compares the two models, and the dump, which
+ writes a loaded model out canonically so that an implementation in
+ another language can be diffed against this one.
+ .
+ `mains` is what is compiled besides the library: the checks are one
+ file each in test/, the reader is src/read/.
+ */
+ mkTool = { tool, binary, what, mains, compiler, compilerBin }:
+ pkgs.stdenv.mkDerivation {
+ pname = "ocda-lib-${tool}";
+ version = "0.1.0";
+ src = self;
+ nativeBuildInputs = [ compiler ];
+ buildInputs = [ pkgs.sqlite pkgs.zstd ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/bin
+ install -m755 ${tool} $out/bin/${binary}
+ runHook postInstall
+ '';
+ meta = {
+ description = what;
+ mainProgram = binary;
+ };
+ };
+ ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
+ ocda-read = mkTool (ldc // {
+ tool = "read"; binary = "ocda-read"; mains = "src/read/*.d";
+ what = "reads the abstraction in any of its forms and says what is in it";
+ });
+ ocda-agree = mkTool (ldc // {
+ tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d";
+ what = "checks that the .ssp and .ocda.db loaders give one model";
+ });
+ ocda-dump = mkTool (ldc // {
+ tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d";
+ what = "writes a loaded abstraction out canonically, as an oracle";
+ });
+ in {
+ packages.${system} = {
+ /* All three in one bin/, so that one build leaves every program in
+ one result link. Built separately into the same link, each build
+ replaced the one before, and a script looking for ocda-dump beside
+ ocda-agree found nothing.
+ */
+ default = pkgs.symlinkJoin {
+ name = "ocda-lib-programs-0.1.0";
+ paths = [ ocda-read ocda-agree ocda-dump ];
+ };
+ inherit ocda-read ocda-agree ocda-dump;
+ };
+
+ devShells.${system}.default = pkgs.mkShell {
+ packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
+ };
+ };
+}
+#+END_SRC
+
+** README.md
+
+#+HEADER: :tangle ../dlang/README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# ocda-lib, in D
+
+The abstraction's model and both its loaders, as a dub library.
+
+## Modules
+
+| | |
+|---|---|
+| `ocda.model` | the document, the object, the eight sections. Imports nothing |
+| `ocda.load_ssp` | text -> model |
+| `ocda.load_db` | sqlite -> model |
+| `ocda.load` | what a path is, and loading it |
+| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries |
+| `ocda.dump` | the model written out canonically, as an oracle |
+| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon |
+| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) |
+
+`ocda.model` importing nothing is the property that matters: a writer
+depends on the model and on neither loader.
+
+## Use
+
+```json
+"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } }
+```
+
+```d
+import ocda.load;
+auto doc = load("document.ocda.db", "de"); // or "document.ssp"
+foreach (obj; doc.body) { ... }
+```
+
+Link `sqlite3`; the library declares the dozen functions it needs rather
+than taking a binding from the registry, so there is nothing to fetch.
+
+## The reader
+
+```sh
+nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl|
+ --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate]
+ <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]
+```
+
+The library's reader, one program where `sisudoc-ssp-reader` and
+`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp`
+reader's, moved onto the library's model, so every mode reads every form
+and **says the same thing of a document's `.ssp`, `.ocda.db` and
+`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries
+files; `--validate` checks a `.ssp` as text and refuses anything else.
+
+It lives in `src/read/` beside the library, and dub keeps it out of the
+library: the configuration `library` excludes `src/read/*`, the
+configuration `ocda-read` builds it. A program using the library by dub, or
+by naming `src/ocda/*.d` as the writer's flake does, never compiles it.
+
+## The two checks
+
+```sh
+nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang]
+nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang]
+```
+
+`agree` loads both and compares ten fields of each object, naming the field
+where they part company. `agree` found the blank-first-text-line defect and
+was not enough for anything subtler.
+
+**`ocda-dump` is the stronger check**: the whole loaded model, every field,
+written out in a form fully determined by the model, so that two
+implementations can be compared with `diff`. It is an oracle, not an output
+format; anything reading it to reconstruct a document has misunderstood it.
+It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is
+the point of having it.
+
+It found four disagreements `agree` could not see. They are in the
+repository README and in the history.
+
+## Build
+
+```sh
+nix build # all three programs, in one result/bin
+nix build .#ocda-read # the reader
+nix build .#ocda-agree # the agreement check
+nix build .#ocda-dump # the canonical dump
+nix develop # ldc, dub, dtools, sqlite, zstd
+dub build # the library
+dub build --config=ocda-read # the reader
+```
+#+END_SRC
+
+** dub.json
+
+#+HEADER: :tangle ../dlang/dub.json
+#+HEADER: :noweb yes
+#+BEGIN_SRC json
+{
+ "authors": [ "Ralph Amissah" ],
+ "copyright": "Copyright (C) 2026 Ralph Amissah",
+ "name": "ocda-lib",
+ "version": "0.1.0",
+ "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations",
+ "homepage": "https://sisudoc.org",
+ "license": "AGPL-3.0+",
+ "sourcePaths": [ "./src" ],
+ "importPaths": [ "./src" ],
+ "libs": [ "sqlite3", "zstd" ],
+ "configurations": [
+ {
+ "name": "library",
+ "targetType": "library",
+ "excludedSourceFiles": [ "src/read/*" ]
+ },
+ {
+ "name": "ocda-read",
+ "targetType": "executable",
+ "targetName": "ocda-read",
+ "mainSourceFile": "src/read/main.d"
+ }
+ ]
+}
+#+END_SRC
+
+* ./gleam
+** nix flake.nix :flake:
+
+#+HEADER: :tangle ../gleam/flake.nix
+#+HEADER: :noweb yes
+#+BEGIN_SRC nix
+{
+ description = "ocda-lib: the object-centric document abstraction as a Gleam library";
+ inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+ outputs = { self, nixpkgs }:
+ let
+ system = "x86_64-linux"; # adjust for aarch64-linux / darwin
+ pkgs = import nixpkgs { inherit system; };
+
+ name = "ocda_lib";
+
+ /* rebar3 with the port compiler already in it.
+ esqlite builds a NIF, and its rebar3 configuration asks for the
+ `pc` plugin, which rebar3 would otherwise fetch from Hex at build
+ time: impossible in a sandbox, and the build fails with
+ "missing_package pc". nixpkgs carries the plugin, so the answer is
+ a rebar3 that has it before it starts.
+ */
+ rebar3-pc = pkgs.beamPackages.rebar3WithPlugins {
+ plugins = [ pkgs.beamPackages.pc ];
+ };
+
+ /* The Hex dependencies, as manifest.toml pins them.
+ .
+ There is no Gleam builder in nixpkgs and `gleam build` fetches from
+ Hex, so a sandboxed build cannot reach them. A fixed output
+ derivation may, because its result is pinned by hash.
+ .
+ When a dependency changes, manifest.toml changes and this hash must
+ change with it: build, take the hash nix reports, put it here.
+ */
+ deps = pkgs.stdenv.mkDerivation {
+ pname = "${name}-deps";
+ version = "0.1.0";
+ src = ./.;
+ nativeBuildInputs = [ pkgs.gleam pkgs.cacert ];
+ buildPhase = ''
+ export HOME=$TMPDIR
+ gleam deps download
+ '';
+ installPhase = ''
+ mkdir -p $out
+ cp -r build/packages/* $out/
+ '';
+ dontFixup = true;
+ outputHashMode = "recursive";
+ outputHashAlgo = "sha256";
+ outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c=";
+ };
+ in {
+ packages.${system} = {
+ default = self.packages.${system}.${name};
+ /* Shipped as an Erlang shipment rather than as an escript, because
+ sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into
+ a shared object, and a NIF cannot be loaded from inside an
+ escript archive, the runtime needing a real file on disk. The
+ shipment keeps priv/ on disk, so it loads.
+ */
+ ${name} = pkgs.stdenv.mkDerivation {
+ pname = "ocda-lib";
+ version = "0.1.0";
+ src = ./.;
+ nativeBuildInputs = [
+ pkgs.gleam
+ pkgs.erlang
+ rebar3-pc
+ pkgs.makeWrapper
+ ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ # the dependencies, in place of the fetch gleam would do
+ mkdir -p build/packages
+ cp -r ${deps}/* build/packages/
+ chmod -R u+w build/packages
+ gleam export erlang-shipment
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/lib $out/bin
+ cp -r build/erlang-shipment $out/lib/${name}
+ makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \
+ --add-flags run \
+ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
+ # the reader under its own name, as the D flake installs it
+ makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \
+ --add-flags "run read" \
+ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
+ runHook postInstall
+ '';
+ meta = {
+ description = "the abstraction as a Gleam library, with its reader, dump and agreement check";
+ mainProgram = "ocda-lib";
+ };
+ };
+ };
+ devShells.${system}.default = pkgs.mkShell {
+ packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ];
+ };
+ };
+}
+#+END_SRC
+
+** README.md
+
+#+HEADER: :tangle ../gleam/README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# ocda-lib, in Gleam
+
+The abstraction's model and both its loaders, as a Gleam library, with its
+reader and the two programs that put it to the question.
+
+| | |
+|------------------|-----------------------------------------------------------------------------------------|
+| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
+| `ocda/load_ssp` | text -> model |
+| `ocda/load_db` | sqlite -> model |
+| `ocda/load` | what a path is, and loading it |
+| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
+| `ocda/dump` | the model written out canonically, as an oracle |
+| `read/main` | `ocda-read`, the reader |
+| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
+
+## Use
+
+```gleam
+import ocda/load
+import ocda/model
+
+case load.load(path, "en", False) {
+ Error(why) -> io.println_error(why)
+ Ok(doc) -> {
+ // and from here a writer neither knows nor cares which of the two
+ // files it was handed
+ list.each(model.section(doc, "body"), fn(obj) { ... })
+ }
+}
+```
+
+## The reader
+
+```sh
+nix build
+./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
+ <file>.ssp|<file>.ocda.db|<file>.zip [lang]
+```
+
+`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
+had, over the library, so each reads every form this library reads; the
+others are the D reader's. **Its output is the D `ocda-read`'s, byte for
+byte**, in every mode both have. In `src/read/`, because Gleam compiles
+nothing outside `src/`; a program depending on the library compiles it too,
+and does not use it.
+
+A `.sisupod` it refuses, saying why: the zstd frame, below.
+
+## The two programs
+
+```sh
+nix build
+./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+```
+
+`agree` asks whether the two loaders give the same model. `dump` writes
+the whole model out canonically, and is the stronger check of the two,
+because it is **byte for byte what the D library's `ocda-dump` writes**.
+So the four questions can all be asked with `diff`:
+
+| | |
+|-----------------------------|-----------------------------|
+| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
+| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
+| `.ssp`: D == Gleam | 36 of 36 |
+| `.ocda.db`: D == Gleam | 36 of 36 |
+
+## Deliberately a mirror of the D library
+
+Field for field, name for name, including where that is not the most
+Gleamish choice:
+
+- `text` is one `String` with the lines joined by newline, not a
+ `List(String)`. So an object whose only text line is blank is
+ indistinguishable from an object with no text, *in both languages, in the
+ same place*. A difference that existed in one and not the other would be
+ worse than a loss both share, because it would show up in every dump and
+ mean nothing.
+- the properties are typed fields rather than a list of pairs. A writer
+ wants `obj.parent_ocn`, not a lookup that might miss.
+
+The point is that the two agree. Making them agree is much cheaper when
+they are shaped alike, and the shape is the specification's, not either
+language's.
+
+## What the agreement found
+
+Four disagreements, none of which a comparison of section counts or of a
+handful of fields would have shown. All four are recorded in the D
+library's history; the short version:
+
+1. `.identifier` and `.anchor` are omitted by the text form when they can
+ be worked out (the identifier is the ocn, the anchor is the segment).
+ Both loaders now derive them. Over one document that is 1,314
+ identifiers.
+2. `make` and `doc_has` are one metadata table in the database,
+ distinguished by a key prefix. Both loaders unprefix them.
+3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
+ as `\/` and every heading with a slash in it came back wrong.
+4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
+ which is one language. It is a fact about the file, so neither loader
+ puts it in the document.
+
+## Build
+
+```sh
+nix build # sandboxed, no network
+nix develop -c gleam test
+```
+
+Shipped as an Erlang shipment rather than an escript, because sqlite on the
+BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
+The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
+one and the other must change with it.
+
+## Format
+
+Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
+`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
+two serialisations. At format **2.x**.
+#+END_SRC
+
+** gleam.toml
+
+#+HEADER: :tangle ../gleam/gleam.toml
+#+HEADER: :noweb yes
+#+BEGIN_SRC toml
+name = "ocda_lib"
+version = "0.1.0"
+description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
+licences = ["AGPL-3.0-or-later"]
+
+[dependencies]
+gleam_stdlib = ">= 1.0.0 and < 2.0.0"
+argv = ">= 1.1.0 and < 2.0.0"
+simplifile = ">= 2.7.0 and < 3.0.0"
+sqlight = ">= 1.0.0 and < 2.0.0"
+gleam_json = ">= 3.0.0 and < 4.0.0"
+
+[dev-dependencies]
+gleeunit = ">= 1.0.0 and < 2.0.0"
+#+END_SRC
+
+* versions GET
+** dlang overlays
+*** ldc
+
+#+NAME: ldc_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./nix-develop-dlang-shared.org:ldc-version()>>
+#+END_SRC
+
+*** dmd
+
+#+NAME: dmd_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./nix-develop-dlang-shared.org:dmd-version()>>
+#+END_SRC
+
+*** dub
+
+#+NAME: dub_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./nix-develop-dlang-shared.org:dub-version()>>
+#+END_SRC
+
+*** dtools
+
+#+NAME: dtools_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./nix-develop-dlang-shared.org:dtools-version()>>
+#+END_SRC
+
+** sha256 blank hash
+
+#+NAME: sha256-blank
+#+BEGIN_SRC nix
+<<./nix-develop-dlang-shared.org:sha256-blank()>>
+#+END_SRC
+
+#+NAME: blank_hash
+#+BEGIN_SRC nix
+<<./nix-develop-dlang-shared.org:sha256-blank()>>
+#+END_SRC
+
+* org includes
+** spine project VERSION
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* nixpkgs source
+
+#+NAME: nixpkgs_unstable
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>>
+#+END_SRC
+
+* __END__
diff --git a/org/config_git.org b/org/config_git.org
deleted file mode 100644
index 11cb940..0000000
--- a/org/config_git.org
+++ /dev/null
@@ -1,101 +0,0 @@
--*- mode: org -*-
-#+TITLE: configuration git
-#+DESCRIPTION: documents - structuring, various output representations & search
-#+FILETAGS: :spine:hub:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-
-* git
-** .gitignore :gitignore:
-
-#+HEADER: :tangle "../.gitignore"
-#+BEGIN_SRC gitignore
-# git ls-files --others --exclude-from=.git/info/exclude
-# git ls-files --others --ignored --exclude-standard --directory
-# git check-ignore -v flake.lock
-# git clean -ix
-# find . -name "*_" | xargs rm -rf
-# find . -name "*.o" | xargs rm -rf
-
-# Deny everything, then whitelist what belongs in the repository.
-#
-# The practice is deliberate: a repository should hold what was decided to
-# put in it, not whatever happened to be in the directory when someone ran
-# "git add .". Build output, editor droppings, downloaded dependencies and
-# exported binaries are numerous, are regenerated, and differ between
-# machines; source is few and is chosen. So the default is no, and every
-# yes below is a decision that can be read.
-#
-# To see what is being excluded, and why:
-# git status --ignored
-# git check-ignore -v <path>
-# git ls-files --others --ignored --exclude-standard --directory
-
-*
-
-# ── the repository itself ────────────────────────────────────────────────
-!.gitignore
-!README.md
-!LICENSE
-
-# ── directories have to be walked into before a file inside can match ────
-!*/
-# but not these, whatever they contain
-build/
-.build/
-.direnv/
-.gleam/
-bin/
-result
-result-*
-
-# ── documentation and notes, in any language directory ───────────────────
-!**/*.md
-!**/*.org
-
-# ── nix ──────────────────────────────────────────────────────────────────
-!**/*.nix
-!**/flake.lock
-!**/.envrc
-
-# ── dlang ────────────────────────────────────────────────────────────────
-!**/*.d
-!**/dub.json
-!**/dub.sdl
-!**/dub.selections.json
-*.o
-*.a
-*.lst
-
-# ── gleam ────────────────────────────────────────────────────────────────
-!**/*.gleam
-# Erlang, for what Gleam reaches through FFI (a sha256, from OTP crypto)
-!**/src/*.erl
-# gleam.toml is the manifest, manifest.toml is the lockfile: both belong
-!**/*.toml
-*.beam
-*.ez
-erl_crash.dump
-
-# ── ruby, when it arrives ────────────────────────────────────────────────
-!**/*.rb
-!**/Gemfile
-!**/Gemfile.lock
-!**/*.gemspec
-
-# ── continuous integration ───────────────────────────────────────────────
-!**/*.yml
-!**/*.yaml
-#+END_SRC
diff --git a/org/config_nix.org b/org/config_nix.org
deleted file mode 100644
index fcf53f3..0000000
--- a/org/config_nix.org
+++ /dev/null
@@ -1,287 +0,0 @@
--*- mode: org -*-
-#+TITLE: configuration nix
-#+DESCRIPTION: configuration nix
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* nix :nix:
-** dlang flake.nix :flake:
-
-#+HEADER: :tangle ../dlang/flake.nix
-#+HEADER: :noweb yes
-#+BEGIN_SRC nix
-{
- description = "ocda-lib: the object-centric document abstraction as a D library";
- inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
- outputs = { self, nixpkgs }:
- let
- system = "x86_64-linux"; # adjust for aarch64-linux / darwin
- pkgs = import nixpkgs { inherit system; };
- /* A library, so the library itself is not a build product: a program
- that uses it compiles its modules alongside its own, as the writer's
- flake does, or takes it as a dub dependency. What is built here are
- programs over it: the reader, and the two that put the library to
- the question - the agreement check, which loads a document's .ssp
- and its .ocda.db and compares the two models, and the dump, which
- writes a loaded model out canonically so that an implementation in
- another language can be diffed against this one.
- .
- `mains` is what is compiled besides the library: the checks are one
- file each in test/, the reader is src/read/.
- */
- mkTool = { tool, binary, what, mains, compiler, compilerBin }:
- pkgs.stdenv.mkDerivation {
- pname = "ocda-lib-${tool}";
- version = "0.1.0";
- src = self;
- nativeBuildInputs = [ compiler ];
- buildInputs = [ pkgs.sqlite pkgs.zstd ];
- buildPhase = ''
- runHook preBuild
- export HOME=$TMPDIR
- ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd
- runHook postBuild
- '';
- installPhase = ''
- runHook preInstall
- mkdir -p $out/bin
- install -m755 ${tool} $out/bin/${binary}
- runHook postInstall
- '';
- meta = {
- description = what;
- mainProgram = binary;
- };
- };
- ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
- ocda-read = mkTool (ldc // {
- tool = "read"; binary = "ocda-read"; mains = "src/read/*.d";
- what = "reads the abstraction in any of its forms and says what is in it";
- });
- ocda-agree = mkTool (ldc // {
- tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d";
- what = "checks that the .ssp and .ocda.db loaders give one model";
- });
- ocda-dump = mkTool (ldc // {
- tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d";
- what = "writes a loaded abstraction out canonically, as an oracle";
- });
- in {
- packages.${system} = {
- /* All three in one bin/, so that one build leaves every program in
- one result link. Built separately into the same link, each build
- replaced the one before, and a script looking for ocda-dump beside
- ocda-agree found nothing.
- */
- default = pkgs.symlinkJoin {
- name = "ocda-lib-programs-0.1.0";
- paths = [ ocda-read ocda-agree ocda-dump ];
- };
- inherit ocda-read ocda-agree ocda-dump;
- };
- devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
- };
- };
-}
-#+END_SRC
-
-** gleam flake.nix :flake:
-
-#+HEADER: :tangle ../gleam/flake.nix
-#+HEADER: :noweb yes
-#+BEGIN_SRC nix
-{
- description = "ocda-lib: the object-centric document abstraction as a Gleam library";
- inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
- outputs = { self, nixpkgs }:
- let
- system = "x86_64-linux"; # adjust for aarch64-linux / darwin
- pkgs = import nixpkgs { inherit system; };
-
- name = "ocda_lib";
-
- /* rebar3 with the port compiler already in it.
- esqlite builds a NIF, and its rebar3 configuration asks for the
- `pc` plugin, which rebar3 would otherwise fetch from Hex at build
- time: impossible in a sandbox, and the build fails with
- "missing_package pc". nixpkgs carries the plugin, so the answer is
- a rebar3 that has it before it starts.
- */
- rebar3-pc = pkgs.beamPackages.rebar3WithPlugins {
- plugins = [ pkgs.beamPackages.pc ];
- };
-
- /* The Hex dependencies, as manifest.toml pins them.
- .
- There is no Gleam builder in nixpkgs and `gleam build` fetches from
- Hex, so a sandboxed build cannot reach them. A fixed output
- derivation may, because its result is pinned by hash.
- .
- When a dependency changes, manifest.toml changes and this hash must
- change with it: build, take the hash nix reports, put it here.
- */
- deps = pkgs.stdenv.mkDerivation {
- pname = "${name}-deps";
- version = "0.1.0";
- src = ./.;
- nativeBuildInputs = [ pkgs.gleam pkgs.cacert ];
- buildPhase = ''
- export HOME=$TMPDIR
- gleam deps download
- '';
- installPhase = ''
- mkdir -p $out
- cp -r build/packages/* $out/
- '';
- dontFixup = true;
- outputHashMode = "recursive";
- outputHashAlgo = "sha256";
- outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c=";
- };
- in {
- packages.${system} = {
- default = self.packages.${system}.${name};
- /* Shipped as an Erlang shipment rather than as an escript, because
- sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into
- a shared object, and a NIF cannot be loaded from inside an
- escript archive, the runtime needing a real file on disk. The
- shipment keeps priv/ on disk, so it loads.
- */
- ${name} = pkgs.stdenv.mkDerivation {
- pname = "ocda-lib";
- version = "0.1.0";
- src = ./.;
- nativeBuildInputs = [
- pkgs.gleam
- pkgs.erlang
- rebar3-pc
- pkgs.makeWrapper
- ];
- buildPhase = ''
- runHook preBuild
- export HOME=$TMPDIR
- # the dependencies, in place of the fetch gleam would do
- mkdir -p build/packages
- cp -r ${deps}/* build/packages/
- chmod -R u+w build/packages
- gleam export erlang-shipment
- runHook postBuild
- '';
- installPhase = ''
- runHook preInstall
- mkdir -p $out/lib $out/bin
- cp -r build/erlang-shipment $out/lib/${name}
- makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \
- --add-flags run \
- --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
- # the reader under its own name, as the D flake installs it
- makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \
- --add-flags "run read" \
- --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
- runHook postInstall
- '';
- meta = {
- description = "the abstraction as a Gleam library, with its reader, dump and agreement check";
- mainProgram = "ocda-lib";
- };
- };
- };
- devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ];
- };
- };
-}
-#+END_SRC
-
-* versions GET
-** dlang overlays
-*** ldc
-
-#+NAME: ldc_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./nix-develop-dlang-shared.org:ldc-version()>>
-#+END_SRC
-
-*** dmd
-
-#+NAME: dmd_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./nix-develop-dlang-shared.org:dmd-version()>>
-#+END_SRC
-
-*** dub
-
-#+NAME: dub_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./nix-develop-dlang-shared.org:dub-version()>>
-#+END_SRC
-
-*** dtools
-
-#+NAME: dtools_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./nix-develop-dlang-shared.org:dtools-version()>>
-#+END_SRC
-
-** sha256 blank hash
-
-#+NAME: sha256-blank
-#+BEGIN_SRC nix
-<<./nix-develop-dlang-shared.org:sha256-blank()>>
-#+END_SRC
-
-#+NAME: blank_hash
-#+BEGIN_SRC nix
-<<./nix-develop-dlang-shared.org:sha256-blank()>>
-#+END_SRC
-
-* org includes
-** spine project VERSION
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* nixpkgs source
-
-#+NAME: nixpkgs_unstable
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>>
-#+END_SRC
-
-* __END__
diff --git a/org/d-ocda-load.org b/org/d-ocda-load.org
deleted file mode 100644
index 59762a8..0000000
--- a/org/d-ocda-load.org
+++ /dev/null
@@ -1,953 +0,0 @@
--*- mode: org -*-
-#+TITLE: dlang load
-#+DESCRIPTION: dlang lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* dlang :dlang:
-** load dlang
-
-#+HEADER: :tangle "../dlang/src/ocda/load.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load;
-
-/+ ↓ what a path is, and loading it.
- .
- One place that decides, so that every reader and every writer accepts
- the same inputs and a new input type is added once. This mirrors
- spine's own ocda/abstraction/load.d, which names the sources it can
- enter the pipeline from and tells them apart by suffix.
- .
- Deliberately *not* here: .sst and .ssm markup. Reading markup means
- implementing the parser, which is spine's work and a far larger thing
- than a reader or a sample writer needs. This library reads
- abstractions.
-+/
-
-import std.algorithm : endsWith;
-import ocda.model;
-import ocda.load_ssp;
-import ocda.load_db;
-import ocda.load_pod;
-
-enum Source {
- ssp, // .ssp, the abstraction as text
- ocda_db, // .ocda.db, the abstraction as sqlite
- pod, // .sisupod or .zip, an archive carrying a .ssp
- unknown,
-}
-
-Source sourceOf(string path) {
- if (path.endsWith(".ocda.db")) { return Source.ocda_db; }
- if (path.endsWith(".ssp")) { return Source.ssp; }
- /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote
- before, and every pod already published carries that name, so both
- are read. What is inside decides how it is opened, not the name.
- +/
- if (path.endsWith(".sisupod")) { return Source.pod; }
- if (path.endsWith(".zip")) { return Source.pod; }
- if (path.endsWith(".db")) { return Source.ocda_db; }
- return Source.unknown;
-}
-
-string sourceName(Source s) {
- final switch (s) {
- case Source.ssp: return ".ssp (the abstraction as text)";
- case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)";
- case Source.pod: return "a pod archive carrying a .ssp";
- case Source.unknown: return "not an abstraction this library reads";
- }
-}
-
-class OcdaLoadError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ load whatever the path is into the model.
- lang names which language to take from a .ocda.db, which holds
- every language of its document, and from a pod, which may carry more
- than one .ssp. A .ssp is one language by construction and ignores it.
-+/
-OcdaDocument load(string path, string lang = "", bool any_version = false) {
- final switch (sourceOf(path)) {
- case Source.ssp: return parseSspFile(path);
- case Source.ocda_db: return loadDbFile(path, lang, any_version);
- case Source.pod: return loadPodFile(path, lang);
- case Source.unknown:
- throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)
- ~ " (expected .ssp, .ocda.db, .sisupod or .zip)");
- }
-}
-#+END_SRC
-
-** load db dlang
-
-#+HEADER: :tangle "../dlang/src/ocda/load_db.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_db;
-
-/+ ↓ build the model from a .ocda.db.
- .
- The point of this module is that it returns *the same value*
- ocda.load_ssp returns from the same document's .ssp. A writer is then
- a function from the model to bytes, and cannot tell which
- serialisation it was given, exactly as spine's own writers cannot.
- .
- Written from the sisudoc-abstraction-format specification,
- ocda-db-format-specification.md, whose section 4 gives the column by
- column correspondence this module implements.
-+/
-
-import std.array : join, split;
-import std.json : JSONType, parseJSON;
-import std.conv : to;
-import std.algorithm : startsWith;
-import std.string : strip;
-import ocda.model;
-import ocda.sqlite;
-
-/+ ↓ the schema major this loader knows (specification section 2) +/
-enum schema_major_read = "2";
-
-class OcdaDbError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ every language the database holds, in manifest order +/
-struct DbLanguage {
- int doc_id;
- string lang;
- string source_filename;
- string ssp_digest;
-}
-
-DbLanguage[] dbLanguages(ref Db db) {
- DbLanguage[] out_;
- foreach (r; db.rows(
- "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
- FROM documents ORDER BY id")) {
- out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]);
- }
- return out_;
-}
-
-/+ ↓ load one language of a database as the model.
- lang == "" takes the first language the manifest gave.
-+/
-OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) {
- auto db = Db.open(path);
- scope(exit) { db.close(); }
-
- if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) {
- throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;"
- ~ " it does not look like a .ocda.db");
- }
-
- /+ ↓ specification sections 2 and 3.2: the rows describing the file rather
- than a document in it are the only ones with doc_id null +/
- string schema_version = db.one(
- "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'");
- if (schema_version.length > 0 && !any_version) {
- auto major = schema_version.split(".")[0];
- if (major != schema_major_read) {
- throw new OcdaDbError(path ~ " is schema v" ~ schema_version
- ~ ", this loader knows v" ~ schema_major_read ~ ".x");
- }
- }
-
- auto languages = dbLanguages(db);
- if (languages.length == 0) {
- throw new OcdaDbError(path ~ ": no documents rows");
- }
- DbLanguage chosen = languages[0];
- if (lang.length > 0) {
- bool found = false;
- foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } }
- if (!found) {
- string have;
- foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; }
- throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have);
- }
- }
-
- OcdaDocument doc;
- doc.format_version = schema_version;
- doc.source_filename = chosen.source_filename;
-
- /+ ↓ @source of the .ssp corresponds to the documents row and the
- source.digest metadata (specification section 5.1 of the .ssp, and
- section 7 here) +/
- doc.source["language"] = chosen.lang;
- string languages_joined;
- foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; }
- doc.source["languages"] = languages_joined;
- auto digest = db.one("SELECT value FROM metadata WHERE doc_id = "
- ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'");
- if (digest.length > 0) { doc.source["digest"] = digest; }
-
- /+ ↓ @meta: every metadata row of this language that is not a source.*
- property. source.* describes the artefact rather than the document
- and lives in @source in the text form.
- +/
- foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
- ~ chosen.doc_id.to!string ~ " ORDER BY key")) {
- string key = r[0];
- if (key.startsWith("source.")) { continue; }
- /+ ↓ translation.ocn_aligned says whether the languages of this
- document share one numbering. A .ocda.db can know it because it
- holds every language; a .ssp is one language and cannot. So it is
- a fact about the *file* rather than about the document, and putting
- it in the document's metadata would let a writer tell which
- serialisation it was handed, which is the one thing this library
- exists to prevent. Left out until the model has a place for
- collection-wide facts.
- +/
- if (key.startsWith("translation.")) { continue; }
- /+ ↓ the database has one metadata table where the text form has three
- blocks, and tells them apart by a key prefix. Unprefixing them
- here is what makes the two loaders give one model: a writer asks
- doc.make["footer"] whichever serialisation it was handed, and
- never doc.metadata["make.footer"].
- +/
- if (key.startsWith("make.")) {
- doc.make[key["make.".length .. $]] = r[1].strip;
- } else if (key.startsWith("doc_has.")) {
- doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
- } else {
- doc.metadata[key] = r[1].strip;
- }
- }
-
- /+ ↓ 2.1 added the container range as two columns. A 2.0 database has
- neither, and reads as having no range, which is what 2.0 meant. +/
- bool has_range = false;
- foreach (c; db.rows("PRAGMA table_info(objects)")) {
- if (c[1] == "ocn_range_first") { has_range = true; }
- }
- string range_cols = has_range
- ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
- : "0, 0";
-
- /+ ↓ the objects, in document order within each section +/
- foreach (r; db.rows(
- "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''),
- coalesce(is_of_section,''), coalesce(is_of_type,''),
- coalesce(heading_level,-1), coalesce(identifier,''),
- coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
- coalesce(children,''), coalesce(ancestors,''),
- coalesce(dummy_heading,0), coalesce(object_number_off,0),
- coalesce(indent_base,0), coalesce(indent_hang,0),
- coalesce(bullet,0), coalesce(lang,''),
- coalesce(has_links,0), coalesce(has_notes_reg,0),
- coalesce(has_notes_star,0), coalesce(has_images,0),
- coalesce(segment,''), coalesce(segment_prev,''),
- coalesce(segment_next,''), coalesce(anchor,''),
- coalesce(segment_html_is,''),
- coalesce(table_cols,0), coalesce(table_widths,''),
- coalesce(table_aligns,''), coalesce(table_header,0),
- coalesce(code_linenumbers,0),
- coalesce(heading_ancestors_text,''), coalesce(sha256,''),
- coalesce(text,''), id, coalesce(heading_lev_anchor,''),
- coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ "
- FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
- ~ " ORDER BY CASE section
- WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
- WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5
- WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7
- WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) {
- OcdaObject obj;
- obj.section = r[0];
- obj.ocn = r[2].to!int;
- obj.is_a = r[3];
- obj.part = r[4];
- obj.own_section = (r[5] == r[0]) ? "" : r[5];
- obj.is_of_type = r[6];
- /+ ↓ the database holds the level as a number, the text form as the
- letter or digit the markup wrote. Model section 4 is the mapping. +/
- obj.heading_level = levelName(r[7].to!int);
- obj.identifier = r[8];
- obj.parent_ocn = r[9].to!int;
- obj.last_descendant_ocn = r[10].to!int;
- obj.children = jsonInts(r[11]);
- obj.ancestors = jsonInts(r[12]);
- obj.dummy_heading = (r[13] != "0");
- obj.ocn_off = (r[14] != "0");
- obj.indent_base = r[15].to!int;
- obj.indent_hang = r[16].to!int;
- obj.bullet = (r[17] != "0");
- obj.lang = r[18];
- obj.has_links = (r[19] != "0");
- obj.has_notes_reg = (r[20] != "0");
- obj.has_notes_star = (r[21] != "0");
- obj.has_images = (r[22] != "0");
- obj.segment = r[23];
- obj.segment_prev = r[24];
- obj.segment_next = r[25];
- obj.anchor = r[26];
- obj.segment_html_is = r[27];
- obj.heading_lev_anchor = r[37];
- obj.segment_epub = r[38];
- obj.segment_epub_is = r[39];
- obj.ocn_range_first = r[40].to!int;
- obj.ocn_range_last = r[41].to!int;
- obj.table_cols = r[28].to!int;
- obj.table_widths = jsonWords(r[29]);
- obj.table_aligns = jsonWords(r[30]);
- obj.table_header = (r[31] != "0");
- obj.code_linenumbers = (r[32] != "0");
- obj.heading_ancestors_text = jsonJoined(r[33]);
- obj.sha256 = r[34];
- obj.text = r[35];
-
- /+ ↓ the four repeatable properties of the text form are child tables
- here (specification section 4) +/
- string object_id = r[36];
- foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0),
- coalesce(width,0), coalesce(height,0),
- coalesce(missing,0)
- FROM object_images WHERE object_id = " ~ object_id
- ~ " ORDER BY seq")) {
- OcdaImage img;
- img.name = i[0];
- img.sha256 = i[1];
- img.bytes = i[2].to!ulong;
- img.width = i[3].to!int;
- img.height = i[4].to!int;
- img.missing = (i[5] != "0");
- obj.images ~= img;
- }
- foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = "
- ~ object_id ~ " ORDER BY seq")) {
- obj.stow_link ~= l[0];
- }
- foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = "
- ~ object_id ~ " ORDER BY seq")) {
- obj.anchor_tags ~= a[0];
- }
-
- auto section = doc.sectionByName(r[0]);
- if (section !is null) { *section ~= obj; }
- }
- countUnread(db, chosen.doc_id, doc);
- return doc;
-}
-
-/+ ↓ what the database holds and the model does not.
- .
- The text loader meets every property line and counts those it does not
- keep, so a reader can say what was read and not understood. This
- loader asks for columns by name, and would never see the ones it did
- not ask for: it would report that it had understood everything, which
- is the silence the count exists to prevent. So it counts them, on the
- conditions under which the .ssp writes the same property (specification
- section 6.3), and the report is the same from either form: checked,
- 36 of 36.
-+/
-private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) {
- string id = doc_id.to!string;
- void count(string property, string where) {
- auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id
- ~ " AND " ~ where).to!size_t;
- if (n > 0) { doc.unknown.property[property] = n; }
- }
- count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''");
- count("dom_status", "coalesce(dom_status,'') != ''");
- count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''");
- count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL");
- count("parent_lev", "coalesce(parent_lev,0) != 0");
- count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''");
- count("attrib", "coalesce(attrib,'') != ''");
- if (db.has("object_subtoc")) {
- auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o"
- ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t;
- if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; }
- }
-}
-
-/+ ↓ model section 4: the eight levels and their indices +/
-private string levelName(int index) {
- switch (index) {
- case 0: return "A";
- case 1: return "B";
- case 2: return "C";
- case 3: return "D";
- case 4: return "1";
- case 5: return "2";
- case 6: return "3";
- case 7: return "4";
- default: return "";
- }
-}
-
-/+ ↓ the JSON columns, read as JSON.
- .
- The fixed width arrays are JSON in the database and space separated
- (or "|" separated) in the text form. This was first written as a
- character scan that threw away brackets and quotes, which is wrong the
- moment a value holds an escape: Phobos' JSONValue.toString escapes
- "/" as "\/", so every heading containing a slash came back with a
- backslash in it. 36 of 36 documents differed on
- heading_ancestors_text and nothing else had shown it.
- .
- So: parse. The values are small, there are few of them, and a real
- parser is shorter than the scan it replaces.
-+/
-private string[] jsonStrings(string v) {
- if (v.length == 0) { return []; }
- string[] out_;
- try {
- auto j = parseJSON(v);
- if (j.type != JSONType.array) { return []; }
- foreach (e; j.array) {
- switch (e.type) {
- case JSONType.string: out_ ~= e.str; break;
- case JSONType.integer: out_ ~= e.integer.to!string; break;
- case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
- case JSONType.float_: out_ ~= e.floating.to!string; break;
- case JSONType.true_: out_ ~= "true"; break;
- case JSONType.false_: out_ ~= "false"; break;
- default: out_ ~= ""; break;
- }
- }
- } catch (Exception) {
- return [];
- }
- return out_;
-}
-
-private int[] jsonInts(string v) {
- int[] out_;
- foreach (s; jsonStrings(v)) {
- if (s.length == 0) { continue; }
- try { out_ ~= s.to!int; } catch (Exception) {}
- }
- return out_;
-}
-
-/+ ↓ space separated in the text form: table_widths, table_aligns +/
-private string jsonWords(string v) {
- return jsonStrings(v).join(" ");
-}
-
-/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
- form and a JSON array here. The empty slots are significant: they say
- which levels the heading sits under, so the separators carry as much
- as the text does.
-+/
-private string jsonJoined(string v) {
- auto parts = jsonStrings(v);
- if (parts.length == 0) { return ""; }
- return parts.join("|");
-}
-#+END_SRC
-
-** load pod dlang
-
-#+HEADER: :tangle "../dlang/src/ocda/load_pod.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_pod;
-
-/+ ↓ the abstraction out of a pod archive.
- .
- A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same
- archive unwrapped, which is what spine published before it wrapped them.
- Either way, a pod built with --pod2 carries the abstraction inside it,
- at media/abstraction/<uid>.<lang>.ssp, and that is what is read.
- .
- So this is a third input form and *not* a third model: the .ssp inside
- is parsed by ocda.load_ssp and comes back as the same value. A writer
- over the library gains .sisupod without being touched, which is the
- whole reason the dispatch is in one place.
- .
- A pod built without --pod2 carries markup and no abstraction. That is
- refused, and the refusal says which flag makes a pod this reader can
- read, because "no abstraction in it" is a fact about the pod rather
- than about the reader.
-+/
-
-import std.algorithm : endsWith, sort;
-import std.array : array;
-import std.file : read;
-import std.string : indexOf;
-import ocda.model;
-import ocda.load_ssp;
-import ocda.zip;
-import ocda.zstd;
-
-/+ ↓ a limit on what a frame may expand to.
- A reader of an artefact it did not make declines to be told how much
- memory to allocate. 512 MiB is far above the largest pod in the
- reference collection (4 MiB) and far below anything that matters.
-+/
-enum size_t max_pod_size = 512 * 1024 * 1024;
-
-class OcdaPodError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ which languages of the abstraction the pod carries, in sorted order +/
-string[] podLanguages(string path) {
- auto bytes = archiveBytes(path);
- string[] out_;
- foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) {
- string lang = langOfSspName(e.name);
- if (lang.length > 0) { out_ ~= lang; }
- }
- return out_.sort.array;
-}
-
-/+ ↓ load one language of the abstraction a pod carries.
- lang == "" takes the only one there is, and where there is a choice it
- says so rather than choosing, as the .ocda.db loader does.
-+/
-OcdaDocument loadPodFile(string path, string lang = "") {
- auto bytes = archiveBytes(path);
- auto entries = zipEntries(bytes);
- auto ssps = zipFindSuffix(entries, ".ssp");
- if (ssps.length == 0) {
- throw new OcdaPodError(path ~ ": carries no .ssp."
- ~ " A pod carries the abstraction only when spine was asked for it:"
- ~ " spine --pod2 puts it in, spine --source does not.");
- }
- ZipEntry chosen = ssps[0];
- if (lang.length > 0) {
- bool found = false;
- foreach (e; ssps) {
- if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; }
- }
- if (!found) {
- string have;
- foreach (i, e; ssps) {
- have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
- }
- throw new OcdaPodError(path ~ ": no language '" ~ lang
- ~ "'; it holds: " ~ have);
- }
- } else if (ssps.length > 1) {
- string have;
- foreach (i, e; ssps) {
- have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
- }
- throw new OcdaPodError(path ~ ": holds more than one language ("
- ~ have ~ "); name the one wanted");
- }
- auto ssp = zipRead(bytes, chosen);
- return parseSsp(cast(string) ssp.idup);
-}
-
-/+ ↓ the archive, unwrapped if it was wrapped.
- The bytes decide, not the name: a .sisupod is a zstd frame and a .zip
- is not, and a file misnamed either way still reads.
-+/
-package ubyte[] archiveBytes(string path) {
- auto bytes = cast(ubyte[]) read(path);
- return zstdIsFrame(bytes)
- ? zstdDecompress(bytes, max_pod_size)
- : bytes;
-}
-
-/+ ↓ <uid>.<lang>.ssp -> <lang> +/
-package string langOfSspName(string name) {
- if (!name.endsWith(".ssp")) { return ""; }
- string stem = name[0 .. $ - 4];
- auto dot = stem.lastIndexOf('.');
- return (dot > 0) ? stem[dot + 1 .. $] : "";
-}
-
-private ptrdiff_t lastIndexOf(string s, char c) {
- for (ptrdiff_t i = s.length - 1; i >= 0; i--) {
- if (s[i] == c) { return i; }
- }
- return -1;
-}
-#+END_SRC
-
-** load ssp dlang
-
-#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_ssp;
-
-import std.algorithm : splitter, startsWith;
-import std.array : array, split;
-import std.conv : to, ConvException;
-import std.file : readText;
-import std.stdio : writeln;
-import std.string : strip, stripLeft, indexOf;
-import ocda.model;
-
-OcdaDocument parseSspFile(string filename) {
- string content = readText(filename);
- return parseSsp(content);
-}
-
-OcdaDocument parseSsp(string content) {
- OcdaDocument doc;
- OcdaObject current_obj;
- bool in_object = false;
- /+ ↓ whether a text line has been seen for the object being read.
- Not "is the text non-empty": an object whose first text line is blank
- (a verse opening on an empty line, which the sample set has seven of)
- would then lose it, the next line overwriting rather than appending.
- +/
- bool seen_text = false;
- string current_section = "";
- string current_block = ""; // meta, make, doc_has, or section name
-
- foreach (line; content.splitter("\n")) {
- /+ ↓ comment. The first line of a .ssp declares the format version, and
- it is the one comment that carries meaning: a reader checks the
- major part and refuses what it does not know rather than loading
- an unfamiliar artefact half populated in silence.
- +/
- if (line.length > 0 && line[0] == '%') {
- enum _banner = "% SiSU Document Abstraction v";
- enum _source = "% Source: ";
- if (doc.format_version.length == 0 && line.startsWith(_banner)) {
- doc.format_version = line[_banner.length .. $].strip;
- } else if (doc.source_filename.length == 0 && line.startsWith(_source)) {
- doc.source_filename = line[_source.length .. $].strip;
- }
- continue;
- }
-
- /+ ↓ blank line - finalize current object +/
- if (line.length == 0 || line.strip.length == 0) {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- current_obj = OcdaObject.init;
- in_object = false;
- seen_text = false;
- }
- continue;
- }
-
- /+ ↓ section/block open +/
- if (line[0] == '@') {
- auto brace_pos = line.indexOf('{');
- if (brace_pos > 0) {
- string name = line[1 .. brace_pos].strip;
- current_block = name;
- if (!isPreambleBlock(name)) {
- current_section = name;
- /+ ↓ a block that is neither preamble nor a section this reader
- knows: counted, so that --report-unknown can say so +/
- if (doc.sectionByName(name) is null) {
- doc.unknown.block[name]++;
- }
- }
- }
- continue;
- }
-
- /+ ↓ section/block close +/
- if (line[0] == '}') {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- current_obj = OcdaObject.init;
- in_object = false;
- seen_text = false;
- }
- current_block = "";
- continue;
- }
-
- /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/
- if (isPreambleBlock(current_block)) {
- string stripped = line.stripLeft;
- auto colon_pos = stripped.indexOf(':');
- if (colon_pos > 0) {
- string key = stripped[0 .. colon_pos].strip;
- string value = (colon_pos + 1 < stripped.length)
- ? stripped[colon_pos + 1 .. $].strip : "";
- if (current_block == "meta") { doc.metadata[key] = value; }
- else if (current_block == "make") { doc.make[key] = value; }
- else if (current_block == "source") { doc.source[key] = value; }
- else { doc.doc_has[key] = value; }
- }
- continue;
- }
-
- /+ ↓ object declaration: [N] type ... +/
- if (line[0] == '[') {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- }
- current_obj = OcdaObject.init;
- current_obj.section = current_section;
- in_object = true;
- seen_text = false;
- parseObjectDeclaration(line, current_obj);
- continue;
- }
-
- /+ ↓ property: .name: value +/
- if (line[0] == '.') {
- if (in_object) {
- parseProperty(line, current_obj, doc.unknown);
- }
- continue;
- }
-
- /+ ↓ text content: | content +/
- if (line.length >= 1 && line[0] == '|') {
- if (in_object) {
- string text_content = (line.length > 2) ? line[2 .. $] : "";
- if (seen_text) {
- current_obj.text ~= "\n" ~ text_content;
- } else {
- current_obj.text = text_content;
- seen_text = true;
- }
- }
- continue;
- }
-
- /+ ↓ no known shape. Counted by its first character so that a reader
- can be told the file held something this one did not expect
- +/
- doc.unknown.line[line[0 .. 1].idup]++;
- }
-
- /+ ↓ finalize last object +/
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- }
-
- return doc;
-}
-
-private void parseObjectDeclaration(string line, ref OcdaObject obj) {
- /+ parse: [N] type :level identifier +/
- auto bracket_close = line.indexOf(']');
- if (bracket_close < 0) return;
- try {
- obj.ocn = line[1 .. bracket_close].strip.to!int;
- } catch (ConvException) {
- obj.ocn = 0;
- }
- string rest = (bracket_close + 1 < line.length)
- ? line[bracket_close + 1 .. $].strip : "";
- if (rest.length == 0) return;
-
- auto parts = rest.split(" ");
- if (parts.length == 0) return;
- obj.is_a = parts[0];
-
- if (obj.is_a == "heading" && parts.length >= 2) {
- /+ parse :A, :B, :1, etc. +/
- string lev = parts[1];
- if (lev.length >= 2 && lev[0] == ':') {
- obj.heading_level = lev[1 .. $];
- }
- /+ optional identifier after level +/
- if (parts.length >= 3) {
- obj.identifier = parts[2];
- }
- }
-}
-
-/+ ↓ the blocks that are document preamble rather than sections of objects +/
-private bool isPreambleBlock(string name) {
- return name == "source" || name == "meta" || name == "make" || name == "doc_has";
-}
-
-private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) {
- /+ parse: .name: value +/
- auto colon_pos = line.indexOf(':');
- if (colon_pos < 1) return;
- string name = line[1 .. colon_pos].strip;
- string value = (colon_pos + 1 < line.length)
- ? line[colon_pos + 1 .. $].strip : "";
-
- switch (name) {
- case "part": obj.part = value; break;
- case "is_of_type": obj.is_of_type = value; break;
- case "section": obj.own_section = value; break;
- case "identifier": obj.identifier = value; break;
- case "segment_html_is": obj.segment_html_is = value; break;
- /+ ↓ the epub segments at heading levels 1 to 4 where the html
- segments at level 4 alone, so the two are different files and
- the abstraction records both +/
- case "segment_epub": obj.segment_epub = value; break;
- case "segment_epub_is": obj.segment_epub_is = value; break;
- case "table_aligns": obj.table_aligns = value; break;
- case "stow_link": obj.stow_link ~= value; break;
- case "anchor_tag": obj.anchor_tags ~= value; break;
- case "parent": obj.parent_ocn = toIntSafe(value); break;
- case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break;
- case "ocn_range": {
- auto r = parseIntArray(value);
- if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; }
- break;
- }
- case "children": obj.children = parseIntArray(value); break;
- case "ancestors": obj.ancestors = parseIntArray(value); break;
- case "dummy": obj.dummy_heading = (value == "true"); break;
- case "ocn_off": obj.ocn_off = (value == "true"); break;
- case "indent":
- auto parts = value.split(" ");
- if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]);
- if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]);
- break;
- case "bullet": obj.bullet = (value == "true"); break;
- case "lang": obj.lang = value; break;
- case "has":
- foreach (flag; value.split(" ")) {
- switch (flag) {
- case "links": obj.has_links = true; break;
- case "notes_reg": obj.has_notes_reg = true; break;
- case "notes_star": obj.has_notes_star = true; break;
- case "images": obj.has_images = true; break;
- default: break;
- }
- }
- break;
- case "segment": obj.segment = value; break;
- case "segment_prev": obj.segment_prev = value; break;
- case "segment_next": obj.segment_next = value; break;
- case "anchor": obj.anchor = value; break;
- case "heading_lev_anchor": obj.heading_lev_anchor = value; break;
- case "table_cols": obj.table_cols = toIntSafe(value); break;
- case "table_widths": obj.table_widths = value; break;
- case "table_header": obj.table_header = (value == "true"); break;
- case "code_syntax": obj.code_syntax = value; break;
- case "code_linenumbers": obj.code_linenumbers = (value == "true"); break;
- case "heading_ancestors_text": obj.heading_ancestors_text = value; break;
- case "sha256": obj.sha256 = value; break;
- case "image": obj.images ~= parseImageRecord(value); break;
- /+ ↓ a property this reader has no use for, or has never heard of.
- Ignored, which is the conforming behaviour within a major
- version, but counted rather than passed over in silence
- +/
- default: unknown.property[name]++; break;
- }
-}
-
-private OcdaImage parseImageRecord(string value) {
- /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...]
- filename first, remaining fields order-independent; unrecognised
- fields are kept rather than discarded +/
- OcdaImage img;
- bool first = true;
- foreach (field; value.split(" ")) {
- if (field.length == 0) continue;
- if (first) {
- img.name = field;
- first = false;
- continue;
- }
- auto colon_pos = field.indexOf(':');
- if (colon_pos < 1) continue;
- string key = field[0 .. colon_pos];
- string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : "";
- switch (key) {
- case "sha256": img.sha256 = val; break;
- case "bytes": img.bytes = toUlongSafe(val); break;
- case "px":
- auto dims = val.split("x");
- if (dims.length == 2) {
- img.width = toIntSafe(dims[0]);
- img.height = toIntSafe(dims[1]);
- }
- break;
- case "missing": img.missing = (val == "true"); break;
- default: img.fields[key] = val; break;
- }
- }
- return img;
-}
-
-/+ ↓ two values the text form leaves out because they can be worked out.
- .
- The .ssp emits a property only when it carries information, so
- .identifier is absent when it is simply the ocn, and .anchor is absent
- when it is the segment name. The .ocda.db, being columns rather than
- lines, stores both unconditionally. So a loader that takes the text
- form literally gives a *different model* from the one the database
- gives, and the difference is not small: in the reference collection
- the identifier is omitted on all but a handful of objects.
- .
- The model is the thing both serialisations describe, so the loader
- fills them in. Found by dumping both models and diffing them, which no
- amount of comparing section counts would have shown.
-+/
-private void deriveOmitted(ref OcdaObject obj) {
- import std.conv : to;
- if (obj.identifier.length == 0 && obj.ocn != 0) {
- obj.identifier = obj.ocn.to!string;
- }
- if (obj.anchor.length == 0 && obj.segment.length > 0) {
- obj.anchor = obj.segment;
- }
-}
-
-private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) {
- deriveOmitted(obj);
- auto sec = doc.sectionByName(section);
- if (sec !is null) {
- ,*sec ~= obj;
- }
-}
-
-private int toIntSafe(string s) {
- try { return s.strip.to!int; }
- catch (ConvException) { return 0; }
-}
-
-private ulong toUlongSafe(string s) {
- try { return s.strip.to!ulong; }
- catch (ConvException) { return 0; }
-}
-
-private int[] parseIntArray(string s) {
- int[] result;
- foreach (part; s.split(" ")) {
- auto v = toIntSafe(part);
- result ~= v;
- }
- return result;
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/d-ocda.org b/org/d-ocda.org
index 218a8cf..dd50bde 100644
--- a/org/d-ocda.org
+++ b/org/d-ocda.org
@@ -16,8 +16,164 @@
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
* dlang
+** model
+
+#+HEADER: :tangle "../dlang/src/ocda/model.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.model;
+
+struct OcdaImage {
+ string name; // image filename as referenced by the object
+ string sha256; // hex-encoded (uppercase) digest of the file
+ ulong bytes; // file size
+ int width; // file's own pixel width (not display width)
+ int height; // file's own pixel height
+ bool missing; // file absent or unreadable, no sha256/bytes
+ string[string] fields; // any further/unrecognised key:value fields
+}
+
+struct OcdaObject {
+ int ocn;
+ string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb
+ string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb
+ string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings)
+ string identifier; // heading: from the declaration line; otherwise .identifier
+ string part; // frontmatter, body, backmatter
+ string is_of_type; // para, block, comment: how the text behaves
+ string own_section; // .section, when it differs from the block it sits in
+ string segment_html_is;// the html segment this object sits in
+ string segment_epub; // the epub segment this heading opens
+ string segment_epub_is;// the epub segment this object sits in
+ string table_aligns; // l c r per column
+ string[] stow_link; // links extracted from the object
+ string[] anchor_tags; // named targets the markup placed on this object
+ int parent_ocn;
+ int last_descendant_ocn;
+ int ocn_range_first;// a container (a poem): the first and last
+ int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1)
+ int[] children; // direct child heading OCNs
+ int[] ancestors; // markedup_ancestors[8]
+ bool dummy_heading;
+ bool ocn_off;
+ int indent_base;
+ int indent_hang;
+ bool bullet;
+ string lang;
+ bool has_links;
+ bool has_notes_reg;
+ bool has_notes_star;
+ bool has_images;
+ string segment;
+ string segment_prev;
+ string segment_next;
+ string anchor;
+ string heading_lev_anchor; // a heading's own named target
+ int table_cols;
+ string table_widths;
+ bool table_header;
+ string code_syntax;
+ bool code_linenumbers;
+ string heading_ancestors_text;
+ string sha256;
+ OcdaImage[] images; // repeatable .image records, in order
+ string text; // content (lines joined with newline)
+
+ /+ ↓ whether this object is rendered at all.
+ is_of_type "comment" is markup commentary: carried by the
+ abstraction so that nothing is lost, and not put into output. An
+ output that renders it leaks the author notes into the document,
+ which is what spine itself does not do.
+ +/
+ bool renders() const {
+ return is_of_type != "comment";
+ }
+
+ /+ ↓ the section this object belongs to, which is the block it sits in
+ unless it says otherwise +/
+ string sectionOf() const {
+ return (own_section.length > 0) ? own_section : section;
+ }
+}
+
+/+ ↓ what the reader met and did not understand.
+ A reader that silently drops what it does not know cannot say whether
+ it understood the document, and cannot be used to check a
+ specification against an implementation: silence and understanding
+ look alike. So the parser counts what it ignored, and
+ --report-unknown prints it.
++/
+struct OcdaUnknown {
+ size_t[string] property; // .name -> times seen
+ size_t[string] block; // @name -> times seen
+ size_t[string] line; // first char -> times seen, lines of no known shape
+
+ bool any() const {
+ return property.length > 0 || block.length > 0 || line.length > 0;
+ }
+}
+
+struct OcdaDocument {
+ string format_version; // as declared on line 1, "" when absent
+ string source_filename; // from the "% Source:" line
+ string[string] source; // @source: language, languages, digest
+ OcdaUnknown unknown; // what was met and not understood
+ string[string] metadata; // key -> value
+ string[string] make; // key -> value
+ string[string] doc_has; // key -> value
+ OcdaObject[] head;
+ OcdaObject[] toc;
+ OcdaObject[] body;
+ OcdaObject[] endnotes;
+ OcdaObject[] glossary;
+ OcdaObject[] bibliography;
+ OcdaObject[] bookindex;
+ OcdaObject[] blurb;
+ OcdaObject[] tail;
+
+ /+ ↓ the major part of the declared version, "" when none was declared +/
+ string formatVersionMajor() const {
+ import std.string : indexOf;
+ if (format_version.length == 0) { return ""; }
+ auto dot = format_version.indexOf('.');
+ return (dot > 0) ? format_version[0 .. dot] : format_version;
+ }
+
+ string meta(string key) {
+ if (auto p = key in metadata) return *p;
+ return "";
+ }
+
+ OcdaObject[]* sectionByName(string name) {
+ switch (name) {
+ case "head": return &head;
+ case "toc": return &toc;
+ case "body": return &body;
+ case "endnotes": return &endnotes;
+ case "glossary": return &glossary;
+ case "bibliography": return &bibliography;
+ case "bookindex": return &bookindex;
+ case "blurb": return &blurb;
+ case "tail": return &tail;
+ default: return null;
+ }
+ }
+}
+
+/+ ↓ the format major version this reader knows.
+ The .ssp and the .ocda.db carry one version between them, declared in
+ band: the .ssp on its first line, the database in its schema.version
+ row. The minor part is raised when a property is added, and a reader of
+ the newer format still reads the older one, so only the major part is
+ checked here. Raise this when the reader has been brought to a new
+ major, not before: the point of the gate is to refuse what has not been
+ looked at.
++/
+enum ssp_format_major_read = "2";
+#+END_SRC
+
** load
-*** load dlang
+*** load
#+HEADER: :tangle "../dlang/src/ocda/load.d"
#+BEGIN_SRC d
@@ -93,7 +249,7 @@ OcdaDocument load(string path, string lang = "", bool any_version = false) {
}
#+END_SRC
-*** load db dlang
+*** load db
#+HEADER: :tangle "../dlang/src/ocda/load_db.d"
#+BEGIN_SRC d
@@ -455,7 +611,7 @@ private string jsonJoined(string v) {
}
#+END_SRC
-*** load pod dlang
+*** load pod
#+HEADER: :tangle "../dlang/src/ocda/load_pod.d"
#+BEGIN_SRC d
@@ -577,7 +733,7 @@ private ptrdiff_t lastIndexOf(string s, char c) {
}
#+END_SRC
-* load ssp dlang
+*** load ssp
#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d"
#+BEGIN_SRC d
@@ -926,6 +1082,366 @@ private int[] parseIntArray(string s) {
}
#+END_SRC
+** inventory
+
+#+HEADER: :tangle "../dlang/src/ocda/inventory.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.inventory;
+
+/+ ↓ what a source holds, beyond the one document the model is loaded from.
+ .
+ The model is one language of one document. Two questions a reader asks
+ are about the file instead: which languages it holds, and, for a
+ database, what else it carries. They are here rather than in a reader
+ because the answer depends on the input form, which is the library's
+ to know, and so that a reader asking them does not learn which form it
+ was handed.
+ .
+ languages() answers for every form, and answers the same: the
+ languages in the order the manifest gave them (which every .ssp
+ records, in @source languages), each with the markup file it was made
+ from and the sha256 of its .ssp. The database stores that digest; for a
+ .ssp and a pod it is taken from the bytes, which are the same bytes.
+ .
+ carried() is the database's: a .ocda.db holds the markup, images,
+ configuration and catalogues it was built from, a .ssp holds none of
+ them, and a pod's are its archive entries rather than anything the
+ abstraction says. So it answers for a database and says so otherwise.
++/
+
+import std.algorithm : endsWith, countUntil;
+import std.array : array, split;
+import std.conv : to;
+import std.digest : toHexString, LetterCase;
+import std.digest.sha : sha256Of;
+import std.file : read;
+import std.string : strip;
+import ocda.model;
+import ocda.load;
+import ocda.load_ssp;
+import ocda.load_db;
+import ocda.load_pod;
+import ocda.sqlite;
+import ocda.zip;
+
+struct OcdaLanguage {
+ string lang;
+ string source_filename; // the markup file the abstraction was made from
+ string ssp_digest; // sha256 of that language's .ssp, upper case hex
+}
+
+struct OcdaCarried {
+ string role; // source, image, conf, manifest, tools
+ string compression; // how it is stored: none, zstd
+ string count;
+ string bytes; // the files' own size, whatever the storage
+ string stored; // what they take in the database
+}
+
+OcdaLanguage[] languages(string path) {
+ final switch (sourceOf(path)) {
+ case Source.ocda_db:
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ OcdaLanguage[] out_;
+ foreach (l; dbLanguages(db)) {
+ out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest);
+ }
+ return out_;
+ case Source.ssp:
+ auto bytes = cast(ubyte[]) read(path);
+ auto doc = parseSsp(cast(string) bytes.idup);
+ return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ];
+ case Source.pod:
+ auto archive = archiveBytes(path);
+ OcdaLanguage[string] found;
+ string[] order;
+ foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) {
+ auto bytes = zipRead(archive, e);
+ auto doc = parseSsp(cast(string) bytes.idup);
+ string lang = langOfSspName(e.name);
+ found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes));
+ if (order.length == 0) {
+ order = sourceValue(doc, "languages").split;
+ }
+ }
+ /+ ↓ the manifest's order, and anything it did not name after it +/
+ OcdaLanguage[] out_;
+ foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } }
+ foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } }
+ return out_;
+ case Source.unknown:
+ throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown));
+ }
+}
+
+/+ ↓ what a database carries, by role and storage; empty for any other form,
+ which isCarrier() says in advance +/
+bool isCarrier(string path) {
+ return sourceOf(path) == Source.ocda_db;
+}
+
+OcdaCarried[] carried(string path) {
+ if (!isCarrier(path)) { return []; }
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ if (!db.has("files")) { return []; }
+ OcdaCarried[] out_;
+ foreach (r; db.rows(
+ "SELECT role, compression, count(*), sum(bytes), sum(length(data))
+ FROM files GROUP BY role, compression ORDER BY role")) {
+ out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]);
+ }
+ return out_;
+}
+
+private string sourceValue(ref OcdaDocument doc, string key) {
+ if (auto p = key in doc.source) { return *p; }
+ return "";
+}
+
+private string hexOf(const(ubyte)[] bytes) {
+ return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup;
+}
+#+END_SRC
+
+** images
+
+#+HEADER: :tangle "../dlang/src/ocda/images.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.images;
+
+/+ ↓ an image's bytes, from whichever source the document came from.
+ .
+ The model *names* images, with digest, byte size and pixel size; it does
+ not hold them. That is right - an abstraction of a document is not a
+ bundle of its assets - but a writer of any format that shows images
+ needs the bytes, so something has to know where they are. This is that
+ something, and it is in the library rather than in a writer because the
+ answer depends on the *input form* and not on the output.
+ .
+ Three answers, and one refusal:
+ .
+ .ocda.db the files table, role 'image', by name
+ .sisupod the archive entry whose path ends in /image/<name>
+ .zip the same
+ .ssp beside it: a pod puts the abstraction in
+ media/abstraction/ and its images in media/image/, so the
+ path is documented rather than guessed
+ .
+ A .ssp that is not in a pod has nowhere to look, and that is reported as
+ absent rather than as an error: a reader that cannot find an image
+ should say so and carry on, because the document is still readable.
++/
+
+import std.algorithm : endsWith;
+import std.array : array;
+import std.file : exists, isFile, read;
+import std.path : buildPath, dirName;
+import ocda.load;
+import ocda.sqlite;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ the bytes, or an empty slice when the source has no such image +/
+ubyte[] imageBytes(string source, string name) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return fromDb(source, name);
+ case Source.pod: return fromArchive(source, name);
+ case Source.ssp: return fromBeside(source, name);
+ case Source.unknown: return [];
+ }
+}
+
+/+ ↓ can this source give images at all?
+ A writer asks before it decides whether to emit them, because an <img>
+ pointing at a file it did not include is worse than no image.
++/
+bool canGiveImages(string source) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return true;
+ case Source.pod: return true;
+ case Source.ssp: return imageDirBeside(source).length > 0;
+ case Source.unknown: return false;
+ }
+}
+
+private ubyte[] fromDb(string path, string name) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ if (!db.has("files")) { return []; }
+ /+ ↓ the name is from the abstraction, not from a caller, and the quote
+ doubling is what keeps it a value rather than syntax +/
+ return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '"
+ ~ sqlQuote(name) ~ "'");
+}
+
+private ubyte[] fromArchive(string path, string name) {
+ auto bytes = cast(ubyte[]) read(path);
+ if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); }
+ auto entries = zipEntries(bytes);
+ foreach (e; entries) {
+ if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); }
+ }
+ return [];
+}
+
+private ubyte[] fromBeside(string path, string name) {
+ string dir = imageDirBeside(path);
+ if (dir.length == 0) { return []; }
+ string file = buildPath(dir, name);
+ return file.exists && file.isFile ? cast(ubyte[]) read(file) : [];
+}
+
+/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/
+private string imageDirBeside(string path) {
+ string dir = path.dirName;
+ if (!dir.endsWith("abstraction")) { return ""; }
+ string image_dir = buildPath(dir.dirName, "image");
+ return image_dir.exists ? image_dir : "";
+}
+
+/+ ↓ a single quote doubled is how sql says a single quote +/
+private string sqlQuote(string s) {
+ string out_;
+ foreach (c; s) {
+ out_ ~= (c == '\'') ? "''" : [c];
+ }
+ return out_;
+}
+#+END_SRC
+
+** sqlite
+
+#+HEADER: :tangle "../dlang/src/ocda/sqlite.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.sqlite;
+
+/+ ↓ the sqlite3 C API, declared rather than depended upon.
+ .
+ A binding from the dub registry would be a dependency to fetch, and
+ the point of this reader is that it builds in a sandbox with no
+ network. The dozen functions a reader needs are declared here and the
+ library is linked directly, which costs a page of extern(C) and saves
+ a dependency.
++/
+
+import std.conv : to;
+import std.string : fromStringz, toStringz;
+
+extern (C) {
+ struct sqlite3;
+ struct sqlite3_stmt;
+
+ int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs);
+ int sqlite3_close(sqlite3* db);
+ int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte,
+ sqlite3_stmt** stmt, const(char)** tail);
+ int sqlite3_step(sqlite3_stmt* stmt);
+ int sqlite3_finalize(sqlite3_stmt* stmt);
+ int sqlite3_column_count(sqlite3_stmt* stmt);
+ const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col);
+ const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_int(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_type(sqlite3_stmt* stmt, int col);
+ const(char)* sqlite3_errmsg(sqlite3* db);
+ const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_bytes(sqlite3_stmt* stmt, int col);
+}
+
+enum SQLITE_OK = 0;
+enum SQLITE_ROW = 100;
+enum SQLITE_DONE = 101;
+enum SQLITE_NULL = 5;
+enum SQLITE_OPEN_READONLY = 0x00000001;
+
+class SqliteError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ a database, opened read only.
+ Read only is deliberate: a reader has no business writing, and a
+ request that opens a database read-write can create an empty one where
+ a path is mistyped.
++/
+struct Db {
+ private sqlite3* _handle;
+
+ static Db open(string path) {
+ sqlite3* handle;
+ auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null);
+ if (rc != SQLITE_OK) {
+ string msg = (handle is null)
+ ? "cannot open " ~ path
+ : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup;
+ if (handle !is null) { sqlite3_close(handle); }
+ throw new SqliteError(msg);
+ }
+ Db db;
+ db._handle = handle;
+ return db;
+ }
+
+ void close() {
+ if (_handle !is null) { sqlite3_close(_handle); _handle = null; }
+ }
+
+ /+ ↓ every row of a query, each row as its columns in order +/
+ string[][] rows(string sql) {
+ sqlite3_stmt* stmt;
+ auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null);
+ if (rc != SQLITE_OK) {
+ throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql);
+ }
+ scope(exit) { sqlite3_finalize(stmt); }
+ string[][] out_rows;
+ int cols = sqlite3_column_count(stmt);
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ string[] row;
+ foreach (c; 0 .. cols) {
+ row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL)
+ ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup;
+ }
+ out_rows ~= row;
+ }
+ return out_rows;
+ }
+
+ /+ ↓ the first column of the first row, "" when there is none +/
+ string one(string sql) {
+ auto r = rows(sql);
+ return (r.length > 0 && r[0].length > 0) ? r[0][0] : "";
+ }
+
+ /+ ↓ the first column of the first row as bytes, for a blob.
+ Separate from one(), because a blob is not text: an image read as text
+ stops at the first zero byte, and a png's ninth byte is zero.
+ +/
+ ubyte[] blob(string sql) {
+ sqlite3_stmt* stmt;
+ if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) {
+ throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup
+ ~ " [" ~ sql ~ "]");
+ }
+ scope(exit) { sqlite3_finalize(stmt); }
+ if (sqlite3_step(stmt) != SQLITE_ROW) { return []; }
+ int n = sqlite3_column_bytes(stmt, 0);
+ if (n <= 0) { return []; }
+ auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0);
+ if (p is null) { return []; }
+ return p[0 .. n].dup;
+ }
+
+ /+ ↓ does a table or view of this name exist +/
+ bool has(string name) {
+ return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0";
+ }
+}
+#+END_SRC
+
** compression
*** dlang read zip
@@ -1135,6 +1651,179 @@ ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) {
}
#+END_SRC
+** dump
+
+#+HEADER: :tangle "../dlang/src/ocda/dump.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.dump;
+
+/+ ↓ the model, written out canonically.
+ .
+ Not an output format: an *oracle*. Two libraries in two languages
+ implement the same model from the same specification, and the cheap
+ way to ask whether they agree is to have each write the whole loaded
+ document as text and diff the two. That is only useful if the text is
+ fully determined by the model, so:
+ .
+ - every line is "<key> <value>", the value running to end of line,
+ and only one free-form value per line;
+ - a field at its default is *omitted*, so the output says what the
+ document has rather than what the struct declares;
+ - dictionaries are emitted in sorted key order, because neither
+ language's hash map has an order worth relying on;
+ - sections come in the fixed order of the format, not
+ alphabetically.
+ .
+ Anything that reads this to reconstruct a document has misunderstood
+ it. Load the .ssp or the .ocda.db.
++/
+
+import std.algorithm : sort, splitter;
+import std.array : appender, array, join;
+import std.conv : to;
+import ocda.model;
+
+/+ ↓ the nine sections, in format order +/
+static immutable string[] dump_sections = ["head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb", "tail"];
+
+string dumpDocument(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+
+ void kv(string key, string value) {
+ if (value.length == 0) { return; }
+ out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n";
+ }
+ void kn(string key, long value) {
+ if (value == 0) { return; }
+ kv(key, value.to!string);
+ }
+ void kb(string key, bool value) {
+ if (!value) { return; }
+ kv(key, "1");
+ }
+ void dict(string prefix, string[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key]; out_ ~= "\n";
+ }
+ }
+
+ kv("version", doc.format_version);
+ kv("source_filename", doc.source_filename);
+ dict("source", doc.source);
+ dict("meta", doc.metadata);
+ dict("make", doc.make);
+ dict("doc_has", doc.doc_has);
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null || (*sec).length == 0) { continue; }
+ kv("section", name ~ " " ~ (*sec).length.to!string);
+ }
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null) { continue; }
+ foreach (i, obj; *sec) {
+ out_ ~= "object "; out_ ~= name; out_ ~= " ";
+ out_ ~= i.to!string; out_ ~= "\n";
+ kn("ocn", obj.ocn);
+ kv("is_a", obj.is_a);
+ kv("own_section", obj.own_section);
+ kv("level", obj.heading_level);
+ kv("identifier", obj.identifier);
+ kv("part", obj.part);
+ kv("is_of_type", obj.is_of_type);
+ kv("segment_html_is", obj.segment_html_is);
+ kv("segment_epub", obj.segment_epub);
+ kv("segment_epub_is", obj.segment_epub_is);
+ kn("parent", obj.parent_ocn);
+ kn("last_descendant", obj.last_descendant_ocn);
+ kn("ocn_range_first", obj.ocn_range_first);
+ kn("ocn_range_last", obj.ocn_range_last);
+ kv("children", ints(obj.children));
+ kv("ancestors", ints(obj.ancestors));
+ kb("dummy", obj.dummy_heading);
+ kb("ocn_off", obj.ocn_off);
+ kn("indent_base", obj.indent_base);
+ kn("indent_hang", obj.indent_hang);
+ kb("bullet", obj.bullet);
+ kv("lang", obj.lang);
+ kb("has_links", obj.has_links);
+ kb("has_notes_reg", obj.has_notes_reg);
+ kb("has_notes_star", obj.has_notes_star);
+ kb("has_images", obj.has_images);
+ kv("segment", obj.segment);
+ kv("segment_prev", obj.segment_prev);
+ kv("segment_next", obj.segment_next);
+ kv("anchor", obj.anchor);
+ kv("heading_lev_anchor", obj.heading_lev_anchor);
+ kn("table_cols", obj.table_cols);
+ kv("table_widths", obj.table_widths);
+ kv("table_aligns", obj.table_aligns);
+ kb("table_header", obj.table_header);
+ kv("code_syntax", obj.code_syntax);
+ kb("code_linenumbers", obj.code_linenumbers);
+ kv("heading_ancestors_text", obj.heading_ancestors_text);
+ kv("sha256", obj.sha256);
+ foreach (url; obj.stow_link) { kv("stow_link", url); }
+ foreach (a; obj.anchor_tags) { kv("anchor_tag", a); }
+ foreach (img; obj.images) {
+ /+ ↓ the name last, because a filename may hold a space where
+ none of the other fields can +/
+ kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " "
+ ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " "
+ ~ (img.missing ? "1" : "0") ~ " " ~ img.name);
+ foreach (key; img.fields.keys.dup.sort) {
+ kv("image_field", key ~ " " ~ img.fields[key]);
+ }
+ }
+ /+ ↓ text, when there is any. An object whose only text line is
+ blank holds "" here, indistinguishable from no text at all,
+ because the model joins the lines into one string. Both
+ libraries lose the same distinction in the same place, which is
+ what makes them comparable; it is recorded rather than hidden.
+ +/
+ if (obj.text.length > 0) {
+ auto lines = obj.text.splitter("\n").array;
+ kv("text", lines.length.to!string);
+ foreach (line; lines) {
+ out_ ~= "| "; out_ ~= line; out_ ~= "\n";
+ }
+ }
+ }
+ }
+ return out_.data;
+}
+
+/+ ↓ what the reader met and did not understand, sorted. Separate from the
+ document dump: it describes the reader, not the document, and the
+ .ocda.db loader has nothing to report where the .ssp loader does.
++/
+string dumpUnknown(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+ void counts(string prefix, size_t[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key].to!string; out_ ~= "\n";
+ }
+ }
+ counts("property", doc.unknown.property);
+ counts("block", doc.unknown.block);
+ counts("line", doc.unknown.line);
+ return out_.data;
+}
+
+private string ints(int[] xs) {
+ if (xs.length == 0) { return ""; }
+ string[] parts;
+ foreach (x; xs) { parts ~= x.to!string; }
+ return parts.join(",");
+}
+#+END_SRC
+
* org includes
** project version
diff --git a/org/d-read.org b/org/d-read.org
index c9f0334..44bb5a0 100644
--- a/org/d-read.org
+++ b/org/d-read.org
@@ -17,7 +17,7 @@
* dlang :dlang:
** main & tree
-*** dlang read main
+*** read main
#+HEADER: :tangle "../dlang/src/read/main.d"
#+BEGIN_SRC d
@@ -252,7 +252,7 @@ void printFiles(string path) {
}
#+END_SRC
-*** dlang read tree
+*** read tree
#+HEADER: :tangle "../dlang/src/read/tree.d"
#+BEGIN_SRC d
@@ -329,7 +329,7 @@ private string stripInternalMarkup(string text) {
#+END_SRC
** check
-*** dlang read verify
+*** read verify
#+HEADER: :tangle "../dlang/src/read/verify.d"
#+BEGIN_SRC d
@@ -548,7 +548,7 @@ private int beInt(const(ubyte)[] b) {
}
#+END_SRC
-*** dlang read validate
+*** read validate
#+HEADER: :tangle "../dlang/src/read/validate.d"
#+BEGIN_SRC d
@@ -820,7 +820,7 @@ void reportValidation(string filename, SspFinding[] found) {
}
#+END_SRC
-*** dlang read report unknown
+*** read report unknown
#+HEADER: :tangle "../dlang/src/read/report_unknown.d"
#+BEGIN_SRC d
@@ -869,7 +869,7 @@ void reportUnknown(ref OcdaDocument doc) {
#+END_SRC
** misc
-*** dlang read html
+*** read html
#+HEADER: :tangle "../dlang/src/read/html.d"
#+BEGIN_SRC d
@@ -1264,7 +1264,7 @@ private string[] htmlTail() {
}
#+END_SRC
-*** dlang read images
+*** read images
#+HEADER: :tangle "../dlang/src/read/images.d"
#+BEGIN_SRC d
@@ -1386,7 +1386,7 @@ package immutable string[] sectionNames = [
#+END_SRC
** llm
-*** dlang read llm text
+*** read llm text
#+HEADER: :tangle "../dlang/src/read/llm_text.d"
#+BEGIN_SRC d
@@ -1575,7 +1575,7 @@ string stripMarkup(string text) {
}
#+END_SRC
-*** dlang read llm jsonl
+*** read llm jsonl
#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d"
#+BEGIN_SRC d
diff --git a/org/d-test.org b/org/d-test.org
new file mode 100644
index 0000000..d159819
--- /dev/null
+++ b/org/d-test.org
@@ -0,0 +1,178 @@
+-*- mode: org -*-
+#+TITLE: dlang load
+#+DESCRIPTION: dlang lib
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* dlang :dlang:
+** tests
+*** test agree
+
+#+HEADER: :tangle "../dlang/test/agree.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ Do the two loaders agree?
+
+ The library's whole purpose is that a writer cannot tell which
+ serialisation it was handed. That is only true if loading a document's
+ .ssp and loading its .ocda.db give the same model, so this asks.
+
+ Usage: agree <doc>.ssp <doc>.ocda.db [lang]
++/
+import std.conv : to;
+import std.stdio;
+import ocda.model;
+import ocda.load;
+
+int main(string[] args) {
+ if (args.length < 3) {
+ stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]");
+ return 2;
+ }
+ string lang = (args.length > 3) ? args[3] : "";
+ OcdaDocument a, b;
+ try {
+ a = load(args[1]);
+ b = load(args[2], lang);
+ } catch (Exception e) {
+ stderr.writeln("error: ", e.msg);
+ return 2;
+ }
+
+ int differences = 0;
+ void differ(string what, string from_ssp, string from_db) {
+ writeln(" differ: ", what);
+ writeln(" .ssp: ", from_ssp);
+ writeln(" .ocda.db: ", from_db);
+ differences++;
+ }
+
+ if (a.format_version != b.format_version) {
+ differ("format_version", a.format_version, b.format_version);
+ }
+ if (a.source_filename != b.source_filename) {
+ differ("source_filename", a.source_filename, b.source_filename);
+ }
+
+ /+ ↓ the sections, object by object. Position within a section is the
+ identity to compare on: an ocn is not unique and 0 is shared +/
+ static immutable sections = ["head", "toc", "body", "endnotes", "glossary",
+ "bibliography", "bookindex", "blurb", "tail"];
+ foreach (name; sections) {
+ auto sa = a.sectionByName(name);
+ auto sb = b.sectionByName(name);
+ if (sa is null || sb is null) { continue; }
+ if ((*sa).length != (*sb).length) {
+ differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string);
+ continue;
+ }
+ foreach (i, oa; *sa) {
+ auto ob = (*sb)[i];
+ string at = name ~ "[" ~ i.to!string ~ "]";
+ if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); }
+ if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); }
+ if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); }
+ if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); }
+ if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); }
+ if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); }
+ if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); }
+ if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); }
+ if (oa.ancestors != ob.ancestors) {
+ differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string);
+ }
+ if (oa.images.length != ob.images.length) {
+ differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string);
+ }
+ if (differences > 12) { writeln(" ... stopping after 12"); return 1; }
+ }
+ }
+
+ if (differences == 0) {
+ writeln("agree: the two loaders give the same model");
+ return 0;
+ }
+ writeln(differences, " difference(s)");
+ return 1;
+}
+#+END_SRC
+
+*** test dump
+
+#+HEADER: :tangle "../dlang/test/dump.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ Write a loaded abstraction out canonically.
+
+ The oracle for everything that claims to implement this model: load a
+ .ssp or a .ocda.db and print the model, deterministically, so that two
+ implementations can be compared with diff.
+
+ Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
++/
+import std.stdio;
+import ocda.model;
+import ocda.load;
+import ocda.dump;
+
+int main(string[] args) {
+ bool unknown = false;
+ string[] rest;
+ foreach (a; args[1 .. $]) {
+ if (a == "--unknown") { unknown = true; } else { rest ~= a; }
+ }
+ if (rest.length < 1) {
+ stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]");
+ return 2;
+ }
+ string lang = (rest.length > 1) ? rest[1] : "";
+ OcdaDocument doc;
+ try {
+ doc = load(rest[0], lang);
+ } catch (Exception e) {
+ stderr.writeln("error: ", e.msg);
+ return 2;
+ }
+ write(unknown ? dumpUnknown(doc) : dumpDocument(doc));
+ return 0;
+}
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* __END__
diff --git a/org/gleam-ocda-load.org b/org/gleam-ocda-load.org
deleted file mode 100644
index f3f111f..0000000
--- a/org/gleam-ocda-load.org
+++ /dev/null
@@ -1,1484 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** load gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/load.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<load>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: load
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// What a path is, and loading it.
-////
-//// One place that decides, so that every reader and every writer accepts
-//// the same inputs and a new input form is added once. This mirrors
-//// spine's own `ocda/abstraction/load.d`, which names the sources it can
-//// enter the pipeline from and tells them apart by suffix.
-////
-//// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means
-//// implementing the parser, which is spine's work and a far larger thing
-//// than a reader or a sample writer needs. This library reads
-//// abstractions.
-
-import gleam/list
-import gleam/string
-import ocda/load_db
-import ocda/load_pod
-import ocda/load_ssp
-import ocda/model.{type Document}
-
-pub type Source {
- /// `.ssp`, the abstraction as text
- Ssp
- /// `.ocda.db`, the abstraction as sqlite
- OcdaDb
- /// `.sisupod` or `.zip`, an archive carrying a `.ssp`
- Pod
- Unknown
-}
-
-/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote
-/// before, and every pod already published carries that name, so both are
-/// recognised. What is inside decides how it is opened, not the name.
-const suffixes = [
- #(".ocda.db", OcdaDb),
- #(".ssp", Ssp),
- #(".sisupod", Pod),
- #(".zip", Pod),
- #(".db", OcdaDb),
-]
-
-pub fn source_of(path: String) -> Source {
- case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) {
- Ok(#(_, source)) -> source
- Error(_) -> Unknown
- }
-}
-
-pub fn source_name(source: Source) -> String {
- case source {
- Ssp -> ".ssp (the abstraction as text)"
- OcdaDb -> ".ocda.db (the abstraction as sqlite)"
- Pod -> "a pod archive carrying a .ssp"
- Unknown -> "not an abstraction this library reads"
- }
-}
-
-/// Load whatever the path is into the model.
-///
-/// `lang` names which language to take from a `.ocda.db`, which holds
-/// every language of its document, and from a pod, which may carry more than
-/// one `.ssp`. A `.ssp` is one language by construction and ignores it.
-pub fn load(
- path: String,
- lang: String,
- any_version: Bool,
-) -> Result(Document, String) {
- case source_of(path) {
- Ssp -> load_ssp.load_file(path)
- OcdaDb -> load_db.load_file(path, lang, any_version)
- Pod -> load_pod.load_file(path, lang)
- Unknown ->
- Error(
- path
- <> ": "
- <> source_name(Unknown)
- <> " (expected .ssp, .ocda.db, .sisupod or .zip)",
- )
- }
-}
-#+END_SRC
-
-** load pod
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/load_pod.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<load_pod>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: load_pod
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// The abstraction out of a pod archive.
-////
-//// A pod built with `--pod2` carries the abstraction inside it, at
-//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is
-//// a third input form and *not* a third model: the `.ssp` inside is parsed by
-//// `ocda/load_ssp` and comes back as the same value. A writer over the
-//// library gains pods without being touched, which is the whole reason the
-//// dispatch is in one place.
-////
-//// ## The one thing this cannot do
-////
-//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip
-//// and **not the frame**, because there is no zstd on the BEAM without a NIF,
-//// and the one that exists, `ezstd`, clones zstd from github in its build, so
-//// it cannot be built in a sandbox without patching it. The D library has
-//// both, through a declared `libzstd`.
-////
-//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says
-//// why rather than pretending the file is malformed. It is the only place the
-//// two libraries differ in what they accept, and it is a packaging limit
-//// rather than anything about the format.
-
-import gleam/list
-import gleam/result
-import gleam/string
-import ocda/load_ssp
-import ocda/model.{type Document}
-import ocda/zip
-import simplifile
-
-/// The magic number of a zstd frame, little endian 0xFD2FB528.
-const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>>
-
-/// Which languages of the abstraction the pod carries, sorted.
-pub fn languages(path: String) -> Result(List(String), String) {
- use bytes <- result.try(archive_bytes(path))
- use entries <- result.try(zip.entries(bytes))
- zip.find_suffix(entries, ".ssp")
- |> list.filter_map(fn(e) { lang_of(e.name) })
- |> list.sort(string.compare)
- |> Ok
-}
-
-/// Load one language of the abstraction a pod carries.
-///
-/// `lang` == "" takes the only one there is, and where there is a choice it
-/// says so rather than choosing, as the `.ocda.db` loader does.
-pub fn load_file(path: String, lang: String) -> Result(Document, String) {
- use bytes <- result.try(archive_bytes(path))
- use entries <- result.try(zip.entries(bytes))
- let ssps = zip.find_suffix(entries, ".ssp")
- use chosen <- result.try(case ssps, lang {
- [], _ ->
- Error(
- path
- <> ": carries no .ssp. A pod carries the abstraction only when spine"
- <> " was asked for it: spine --pod2 puts it in, spine --source does not.",
- )
- [only], "" -> Ok(only)
- _, "" ->
- Error(
- path
- <> ": holds more than one language ("
- <> string.join(langs(ssps), " ")
- <> "); name the one wanted",
- )
- _, wanted ->
- case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) {
- Ok(found) -> Ok(found)
- Error(_) ->
- Error(
- path
- <> ": no language '"
- <> wanted
- <> "'; it holds: "
- <> string.join(langs(ssps), " "),
- )
- }
- })
- use content <- result.try(zip.read(bytes, chosen))
- case bit_array_to_string(content) {
- Ok(text) -> Ok(load_ssp.parse(text))
- Error(_) -> Error(chosen.name <> ": not valid UTF-8")
- }
-}
-
-fn langs(entries: List(zip.Entry)) -> List(String) {
- list.filter_map(entries, fn(e) { lang_of(e.name) })
-}
-
-/// `<uid>.<lang>.ssp` -> `<lang>`
-pub fn lang_of(name: String) -> Result(String, Nil) {
- case string.ends_with(name, ".ssp") {
- False -> Error(Nil)
- True -> {
- let stem = string.drop_end(name, 4)
- case string.split(stem, on: ".") |> list.last {
- Ok(lang) if lang != stem -> Ok(lang)
- _ -> Error(Nil)
- }
- }
- }
-}
-
-/// The bytes decide, not the name: a file misnamed either way is treated by
-/// what it holds.
-pub fn archive_bytes(path: String) -> Result(BitArray, String) {
- case simplifile.read_bits(path) {
- Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
- Ok(bytes) ->
- case is_zstd_frame(bytes) {
- False -> Ok(bytes)
- True ->
- Error(
- path
- <> " is a zstd frame wrapping the archive, and this library has no"
- <> " zstd: the only one on the BEAM is a NIF that fetches zstd in"
- <> " its own build, so it cannot be built sandboxed. Unwrap it"
- <> " first (zstd -d) and read the archive, or use the D library,"
- <> " which reads both.",
- )
- }
- }
-}
-
-fn is_zstd_frame(bytes: BitArray) -> Bool {
- case bytes {
- <<head:bytes-size(4), _:bits>> -> head == zstd_magic
- _ -> False
- }
-}
-
-@external(erlang, "unicode", "characters_to_binary")
-fn bit_array_to_string_ffi(b: BitArray) -> String
-
-pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
- Ok(bit_array_to_string_ffi(b))
-}
-#+END_SRC
-
-** load ssp
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/load_ssp.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<load_ssp>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: load_ssp
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// Loading a `.ssp` into the model.
-////
-//// The format is line oriented and every line is one of eight shapes told
-//// apart by its first character, so the loader is a fold over the lines
-//// carrying the state the shapes need.
-////
-//// Written against `ssp-format-specification.org`. Where it and the D
-//// library differ the D library is the bug, or this is: they are meant to
-//// give the same model, and `ocda-dump` says whether they do.
-
-import gleam/dict.{type Dict}
-import gleam/int
-import gleam/list
-import gleam/option
-import gleam/result
-import gleam/string
-import ocda/model.{type Document, type Image, type Object, Image, Object}
-import simplifile
-
-pub fn load_file(path: String) -> Result(Document, String) {
- case simplifile.read(path) {
- Ok(content) -> Ok(parse(content))
- Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
- }
-}
-
-type State {
- State(
- document: Document,
- /// the block currently open, "" when none
- block: String,
- /// the section currently open, "" when in the preamble
- section: String,
- /// objects of the current section, newest first
- objects: List(Object),
- /// the object being read, if any
- current: Result(Object, Nil),
- /// text lines of the object being read, newest first
- text: List(String),
- /// whether a text line has been seen for the object being read.
- /// Not "is the text non-empty": an object whose first text line is
- /// blank, which the reference collection has seven of, would then lose
- /// it.
- seen_text: Bool,
- )
-}
-
-pub fn parse(content: String) -> Document {
- content
- |> string.split(on: "\n")
- |> list.fold(from: initial(), with: line)
- |> finish
-}
-
-fn initial() -> State {
- State(
- document: model.new_document(),
- block: "",
- section: "",
- objects: [],
- current: Error(Nil),
- text: [],
- seen_text: False,
- )
-}
-
-fn line(state: State, raw: String) -> State {
- case raw {
- "% SiSU Document Abstraction v" <> version ->
- case state.document.format_version {
- "" ->
- State(
- ..state,
- document: model.Document(
- ..state.document,
- format_version: string.trim(version),
- ),
- )
- _ -> state
- }
- "% Source: " <> filename ->
- case state.document.source_filename {
- "" ->
- State(
- ..state,
- document: model.Document(
- ..state.document,
- source_filename: string.trim(filename),
- ),
- )
- _ -> state
- }
- "%" <> _ -> state
- "@" <> rest -> open_block(state, rest)
- "}" <> _ -> close_block(state)
- "[" <> rest -> declaration(state, rest)
- "." <> rest -> property(state, rest)
- "|" <> rest -> text_line(state, rest)
- _ ->
- case string.trim(raw) {
- // a blank line ends the object being read
- "" -> commit(state)
- _ ->
- case model.is_preamble_block(state.block) {
- True -> preamble_pair(state, raw)
- False -> unknown_line(state, raw)
- }
- }
- }
-}
-
-fn open_block(state: State, rest: String) -> State {
- let name = case string.split_once(rest, "{") {
- Ok(#(name, _)) -> string.trim(name)
- Error(_) -> string.trim(rest)
- }
- case model.is_preamble_block(name) {
- True -> State(..flush(state), block: name)
- False -> {
- let state = flush(state)
- let document = case model.is_section(name) {
- True -> state.document
- // a block that is neither preamble nor one of the nine sections:
- // counted, so that a reader can say the file held something it did
- // not expect
- False ->
- model.Document(
- ..state.document,
- unknown: model.Unknown(
- ..state.document.unknown,
- block: bump(state.document.unknown.block, name),
- ),
- )
- }
- State(..state, block: name, section: name, document: document)
- }
- }
-}
-
-fn close_block(state: State) -> State {
- State(..flush(state), block: "")
-}
-
-/// Commit the object being read, then put the section's objects into the
-/// document and close the section.
-fn flush(state: State) -> State {
- let state = commit(state)
- case state.section, state.objects {
- "", _ -> state
- _, [] -> State(..state, section: "", objects: [])
- name, objects ->
- State(
- ..state,
- section: "",
- objects: [],
- document: model.Document(
- ..state.document,
- sections: add_section(
- state.document.sections,
- name,
- list.reverse(objects),
- ),
- ),
- )
- }
-}
-
-fn preamble_pair(state: State, raw: String) -> State {
- case string.split_once(string.trim_start(raw), ":") {
- Error(_) -> state
- Ok(#(key, value)) -> {
- let key = string.trim(key)
- let value = string.trim(value)
- let document = state.document
- let document = case state.block {
- "meta" ->
- model.Document(
- ..document,
- metadata: dict.insert(document.metadata, key, value),
- )
- "make" ->
- model.Document(..document, make: dict.insert(document.make, key, value))
- "source" ->
- model.Document(
- ..document,
- source: dict.insert(document.source, key, value),
- )
- _ ->
- model.Document(
- ..document,
- doc_has: dict.insert(document.doc_has, key, value),
- )
- }
- State(..state, document: document)
- }
- }
-}
-
-/// `[<ocn>] <kind>`, and for a heading `[<ocn>] heading :<level>
-/// [<identifier>]`.
-fn declaration(state: State, rest: String) -> State {
- let state = commit(state)
- case string.split_once(rest, "]") {
- Error(_) -> state
- Ok(#(ocn_text, tail)) -> {
- let ocn = parse_int(string.trim(ocn_text))
- let object =
- Object(..model.new_object(), ocn: ocn, section: state.section)
- let object = case string.split(string.trim(tail), on: " ") {
- ["heading", level, ..identifier] ->
- Object(
- ..object,
- is_a: "heading",
- heading_level: string.drop_start(level, 1),
- identifier: string.join(identifier, " "),
- )
- [""] -> object
- [kind, ..] -> Object(..object, is_a: kind)
- [] -> object
- }
- State(..state, current: Ok(object), text: [], seen_text: False)
- }
- }
-}
-
-fn property(state: State, rest: String) -> State {
- case state.current, string.split_once(rest, ":") {
- Ok(object), Ok(#(name, raw_value)) -> {
- let name = string.trim(name)
- let value = string.trim(raw_value)
- let #(object, known) = apply_property(object, name, value)
- let document = case known {
- True -> state.document
- False ->
- model.Document(
- ..state.document,
- unknown: model.Unknown(
- ..state.document.unknown,
- property: bump(state.document.unknown.property, name),
- ),
- )
- }
- State(..state, current: Ok(object), document: document)
- }
- _, _ -> state
- }
-}
-
-/// The properties this library reads, and what each sets.
-///
-/// A property it has no use for is counted rather than passed over:
-/// ignoring one is conforming within a major version, doing so in silence
-/// is what stops a reader being a check on the specification.
-fn apply_property(
- object: Object,
- name: String,
- value: String,
-) -> #(Object, Bool) {
- case name {
- "part" -> #(Object(..object, part: value), True)
- "is_of_type" -> #(Object(..object, is_of_type: value), True)
- "section" -> #(Object(..object, own_section: value), True)
- "identifier" -> #(Object(..object, identifier: value), True)
- "segment_html_is" -> #(Object(..object, segment_html_is: value), True)
- "segment_epub" -> #(Object(..object, segment_epub: value), True)
- "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True)
- "table_aligns" -> #(Object(..object, table_aligns: value), True)
- "stow_link" -> #(
- Object(..object, stow_link: list.append(object.stow_link, [value])),
- True,
- )
- "anchor_tag" -> #(
- Object(..object, anchor_tags: list.append(object.anchor_tags, [value])),
- True,
- )
- "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True)
- "last_descendant" -> #(
- Object(..object, last_descendant_ocn: parse_int(value)),
- True,
- )
- "ocn_range" ->
- case numbers(value) {
- [first, last] -> #(
- Object(..object, ocn_range_first: first, ocn_range_last: last),
- True,
- )
- _ -> #(object, True)
- }
- "children" -> #(Object(..object, children: numbers(value)), True)
- "ancestors" -> #(Object(..object, ancestors: numbers(value)), True)
- "dummy" -> #(Object(..object, dummy_heading: value == "true"), True)
- "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True)
- "indent" -> {
- let parts = string.split(value, on: " ")
- let base = case parts {
- [first, ..] -> parse_int(first)
- [] -> 0
- }
- let hang = case parts {
- [_, second, ..] -> parse_int(second)
- _ -> 0
- }
- #(Object(..object, indent_base: base, indent_hang: hang), True)
- }
- "bullet" -> #(Object(..object, bullet: value == "true"), True)
- "lang" -> #(Object(..object, lang: value), True)
- "has" -> #(has_flags(object, value), True)
- "segment" -> #(Object(..object, segment: value), True)
- "segment_prev" -> #(Object(..object, segment_prev: value), True)
- "segment_next" -> #(Object(..object, segment_next: value), True)
- "anchor" -> #(Object(..object, anchor: value), True)
- "heading_lev_anchor" -> #(
- Object(..object, heading_lev_anchor: value),
- True,
- )
- "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True)
- "table_widths" -> #(Object(..object, table_widths: value), True)
- "table_header" -> #(Object(..object, table_header: value == "true"), True)
- "code_syntax" -> #(Object(..object, code_syntax: value), True)
- "code_linenumbers" -> #(
- Object(..object, code_linenumbers: value == "true"),
- True,
- )
- "heading_ancestors_text" -> #(
- Object(..object, heading_ancestors_text: value),
- True,
- )
- "sha256" -> #(Object(..object, sha256: value), True)
- "image" -> #(
- Object(..object, images: list.append(object.images, [image(value)])),
- True,
- )
- _ -> #(object, False)
- }
-}
-
-fn has_flags(object: Object, value: String) -> Object {
- value
- |> string.split(on: " ")
- |> list.fold(from: object, with: fn(obj, flag) {
- case flag {
- "links" -> Object(..obj, has_links: True)
- "notes_reg" -> Object(..obj, has_notes_reg: True)
- "notes_star" -> Object(..obj, has_notes_star: True)
- "images" -> Object(..obj, has_images: True)
- _ -> obj
- }
- })
-}
-
-/// `<filename> sha256:<HEX> bytes:<n> [px:<w>x<h>] [missing:true] [...]`
-///
-/// The filename comes first, the rest are order-independent, and a field
-/// this library does not recognise is kept rather than discarded.
-fn image(value: String) -> Image {
- case string.split(value, on: " ") {
- [] -> model.new_image()
- [name, ..fields] ->
- list.fold(fields, model.new_image(), fn(img, field) {
- case string.split_once(field, ":") {
- Error(_) -> img
- Ok(#(key, val)) ->
- case key {
- "sha256" -> Image(..img, sha256: val)
- "bytes" -> Image(..img, bytes: parse_int(val))
- "px" ->
- case string.split_once(val, "x") {
- Ok(#(w, h)) ->
- Image(..img, width: parse_int(w), height: parse_int(h))
- Error(_) -> img
- }
- "missing" -> Image(..img, missing: val == "true")
- _ -> Image(..img, fields: dict.insert(img.fields, key, val))
- }
- }
- })
- |> fn(img) { Image(..img, name: name) }
- }
-}
-
-/// `| <text>`, and `| ` alone for a blank line. Never a bare `|`: the
-/// emitter writes the space.
-fn text_line(state: State, rest: String) -> State {
- case state.current {
- Error(_) -> state
- Ok(_) -> {
- let content = string.drop_start(rest, 1)
- State(..state, text: [content, ..state.text], seen_text: True)
- }
- }
-}
-
-fn unknown_line(state: State, raw: String) -> State {
- let first = string.slice(raw, 0, 1)
- State(
- ..state,
- document: model.Document(
- ..state.document,
- unknown: model.Unknown(
- ..state.document.unknown,
- line: bump(state.document.unknown.line, first),
- ),
- ),
- )
-}
-
-fn commit(state: State) -> State {
- case state.current {
- Error(_) -> state
- Ok(object) -> {
- let text = case state.seen_text {
- True -> string.join(list.reverse(state.text), "\n")
- False -> ""
- }
- let object = derive_omitted(Object(..object, text: text))
- let objects = case model.is_section(state.section) {
- True -> [object, ..state.objects]
- False -> state.objects
- }
- State(
- ..state,
- current: Error(Nil),
- text: [],
- seen_text: False,
- objects: objects,
- )
- }
- }
-}
-
-/// Two values the text form leaves out because they can be worked out.
-///
-/// `.identifier` is absent when it is simply the ocn, `.anchor` when it is
-/// the segment name. The `.ocda.db`, being columns rather than lines,
-/// stores both unconditionally. So a loader that takes the text form
-/// literally gives a different model from the one the database gives, and
-/// over one document that is 1,314 identifiers and 69 anchors.
-fn derive_omitted(object: Object) -> Object {
- let object = case object.identifier, object.ocn {
- "", 0 -> object
- "", ocn -> Object(..object, identifier: int.to_string(ocn))
- _, _ -> object
- }
- case object.anchor, object.segment {
- "", "" -> object
- "", segment -> Object(..object, anchor: segment)
- _, _ -> object
- }
-}
-
-fn finish(state: State) -> Document {
- flush(state).document
-}
-
-fn add_section(
- sections: Dict(String, List(Object)),
- name: String,
- objects: List(Object),
-) -> Dict(String, List(Object)) {
- dict.upsert(sections, name, fn(existing) {
- case existing {
- option.Some(previous) -> list.append(previous, objects)
- option.None -> objects
- }
- })
-}
-
-fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) {
- dict.upsert(counts, key, fn(existing) {
- case existing {
- option.Some(n) -> n + 1
- option.None -> 1
- }
- })
-}
-
-fn parse_int(value: String) -> Int {
- value |> string.trim |> int.parse |> result.unwrap(0)
-}
-
-fn numbers(value: String) -> List(Int) {
- value
- |> string.split(on: " ")
- |> list.filter(fn(s) { s != "" })
- |> list.map(parse_int)
-}
-#+END_SRC
-
-** load db
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/load_db.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<load_db>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: load_db
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// Loading a `.ocda.db` into the model.
-////
-//// The point of this module is that it returns *the same value*
-//// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is
-//// then a function from the model to bytes and cannot tell which
-//// serialisation it was given, exactly as spine's own writers cannot.
-////
-//// Written against `ocda-db-format-specification.org`, whose section 4
-//// gives the column by column correspondence this module implements.
-
-import gleam/dict
-import gleam/dynamic/decode
-import gleam/float
-import gleam/int
-import gleam/json
-import gleam/list
-import gleam/option
-import gleam/result
-import gleam/string
-import ocda/model.{type Document, Image, Object}
-import sqlight
-
-/// The schema major version this loader knows.
-pub const schema_major_read = "2"
-
-/// One language of the database, as the manifest gives it.
-pub type Language {
- Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String)
-}
-
-/// Load one language of a database as the model. `lang` == "" takes the
-/// first language the manifest gives.
-pub fn load_file(
- path: String,
- lang: String,
- any_version: Bool,
-) -> Result(Document, String) {
- // read only: a loader has no business writing, and a read-write open
- // *creates* an empty database where a path is mistyped, which then reads
- // as a document with nothing in it, a failure that looks like an answer
- use conn <- with_connection("file:" <> path <> "?mode=ro", path)
-
- use _ <- result.try(case
- has(conn, "documents") && has(conn, "objects") && has(conn, "metadata")
- {
- True -> Ok(Nil)
- False ->
- Error(
- path
- <> ": no documents/objects/metadata tables; it does not look like a .ocda.db",
- )
- })
-
- let schema_version =
- one(conn, "select value from metadata where doc_id is null
- and key = 'schema.version'")
- use _ <- result.try(case schema_version, any_version {
- "", _ -> Ok(Nil)
- _, True -> Ok(Nil)
- v, False ->
- case major_of(v) == schema_major_read {
- True -> Ok(Nil)
- False ->
- Error(
- path
- <> " is schema v"
- <> v
- <> ", this loader knows v"
- <> schema_major_read
- <> ".x",
- )
- }
- })
-
- let languages = languages(conn)
- use chosen <- result.try(case languages, lang {
- [], _ -> Error(path <> ": no documents rows")
- [first, ..], "" -> Ok(first)
- _, wanted ->
- case list.find(languages, fn(l) { l.lang == wanted }) {
- Ok(found) -> Ok(found)
- Error(_) ->
- Error(
- path
- <> ": no language '"
- <> wanted
- <> "'; it holds: "
- <> string.join(list.map(languages, fn(l) { l.lang }), " "),
- )
- }
- })
-
- let source =
- dict.new()
- |> dict.insert("language", chosen.lang)
- |> dict.insert(
- "languages",
- string.join(list.map(languages, fn(l) { l.lang }), " "),
- )
- let digest = meta_value(conn, chosen.doc_id, "source.digest")
- let source = case digest {
- "" -> source
- d -> dict.insert(source, "digest", d)
- }
-
- let document =
- model.Document(
- ..model.new_document(),
- format_version: schema_version,
- source_filename: chosen.source_filename,
- source: source,
- )
- let document = metadata(conn, chosen.doc_id, document)
- let document = objects(conn, chosen.doc_id, document)
- let document = count_unread(conn, chosen.doc_id, document)
- Ok(document)
-}
-
-fn with_connection(
- uri: String,
- path: String,
- then: fn(sqlight.Connection) -> Result(a, String),
-) -> Result(a, String) {
- case sqlight.open(uri) {
- Error(e) -> Error(path <> ": " <> e.message)
- Ok(conn) -> {
- let outcome = then(conn)
- let _ = sqlight.close(conn)
- outcome
- }
- }
-}
-
-/// Every language the database holds, in manifest order.
-pub fn languages(conn: sqlight.Connection) -> List(Language) {
- let d = {
- use id <- decode.field(0, decode.int)
- use lang <- decode.field(1, decode.string)
- use source <- decode.field(2, decode.string)
- use digest <- decode.field(3, decode.string)
- decode.success(Language(id, lang, source, digest))
- }
- sqlight.query(
- "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
- from documents order by id",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-}
-
-/// The three preamble blocks of the text form, out of the one metadata
-/// table the database keeps them in.
-fn metadata(
- conn: sqlight.Connection,
- doc_id: Int,
- document: Document,
-) -> Document {
- let d = {
- use key <- decode.field(0, decode.string)
- use value <- decode.field(1, decode.string)
- decode.success(#(key, value))
- }
- let rows =
- sqlight.query(
- "select key, value from metadata where doc_id = "
- <> int.to_string(doc_id)
- <> " order by key",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-
- list.fold(rows, document, fn(doc, row) {
- let #(key, raw) = row
- let value = string.trim(raw)
- case key {
- // source.* describes the artefact rather than the document, and is
- // @source in the text form
- "source." <> _ -> doc
- // translation.ocn_aligned says whether the languages of this document
- // share one numbering. A .ocda.db can know it because it holds every
- // language; a .ssp is one language and cannot. It is a fact about the
- // file, not the document, and carrying it here would let a writer
- // tell which serialisation it was handed.
- "translation." <> _ -> doc
- // the database tells the three blocks apart by a key prefix.
- // Unprefixing them is what makes the two loaders give one model: a
- // writer asks make["footer"], never metadata["make.footer"].
- "make." <> rest ->
- model.Document(..doc, make: dict.insert(doc.make, rest, value))
- "doc_has." <> rest ->
- model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value))
- _ ->
- model.Document(..doc, metadata: dict.insert(doc.metadata, key, value))
- }
- })
-}
-
-/// The objects, in document order within each section, the sections in the
-/// order the format gives rather than the alphabetical one sqlite would.
-fn objects(
- conn: sqlight.Connection,
- doc_id: Int,
- document: Document,
-) -> Document {
- let d = {
- use section <- decode.field(0, decode.string)
- use ocn <- decode.field(2, decode.int)
- use is_a <- decode.field(3, decode.string)
- use part <- decode.field(4, decode.string)
- use is_of_section <- decode.field(5, decode.string)
- use is_of_type <- decode.field(6, decode.string)
- use heading_level <- decode.field(7, decode.int)
- use identifier <- decode.field(8, decode.string)
- use parent <- decode.field(9, decode.int)
- use last_descendant <- decode.field(10, decode.int)
- use children <- decode.field(11, decode.string)
- use ancestors <- decode.field(12, decode.string)
- use dummy <- decode.field(13, decode.int)
- use ocn_off <- decode.field(14, decode.int)
- use indent_base <- decode.field(15, decode.int)
- use indent_hang <- decode.field(16, decode.int)
- use bullet <- decode.field(17, decode.int)
- use lang <- decode.field(18, decode.string)
- use has_links <- decode.field(19, decode.int)
- use has_notes_reg <- decode.field(20, decode.int)
- use has_notes_star <- decode.field(21, decode.int)
- use has_images <- decode.field(22, decode.int)
- use segment <- decode.field(23, decode.string)
- use segment_prev <- decode.field(24, decode.string)
- use segment_next <- decode.field(25, decode.string)
- use anchor <- decode.field(26, decode.string)
- use segment_html_is <- decode.field(27, decode.string)
- use table_cols <- decode.field(28, decode.int)
- use table_widths <- decode.field(29, decode.string)
- use table_aligns <- decode.field(30, decode.string)
- use table_header <- decode.field(31, decode.int)
- use code_linenumbers <- decode.field(32, decode.int)
- use hat <- decode.field(33, decode.string)
- use sha256 <- decode.field(34, decode.string)
- use text <- decode.field(35, decode.string)
- use id <- decode.field(36, decode.int)
- use hla <- decode.field(37, decode.string)
- use seg_epub <- decode.field(38, decode.string)
- use seg_epub_is <- decode.field(39, decode.string)
- use range_first <- decode.field(40, decode.int)
- use range_last <- decode.field(41, decode.int)
- decode.success(#(
- id,
- Object(
- ocn: ocn,
- is_a: is_a,
- section: section,
- heading_level: level_name(heading_level),
- identifier: identifier,
- part: part,
- is_of_type: is_of_type,
- own_section: case is_of_section == section {
- True -> ""
- False -> is_of_section
- },
- segment_html_is: segment_html_is,
- table_aligns: json_words(table_aligns),
- stow_link: [],
- anchor_tags: [],
- parent_ocn: parent,
- last_descendant_ocn: last_descendant,
- ocn_range_first: range_first,
- ocn_range_last: range_last,
- children: json_ints(children),
- ancestors: json_ints(ancestors),
- dummy_heading: dummy != 0,
- ocn_off: ocn_off != 0,
- indent_base: indent_base,
- indent_hang: indent_hang,
- bullet: bullet != 0,
- lang: lang,
- has_links: has_links != 0,
- has_notes_reg: has_notes_reg != 0,
- has_notes_star: has_notes_star != 0,
- has_images: has_images != 0,
- segment: segment,
- segment_prev: segment_prev,
- segment_next: segment_next,
- anchor: anchor,
- heading_lev_anchor: hla,
- segment_epub: seg_epub,
- segment_epub_is: seg_epub_is,
- table_cols: table_cols,
- table_widths: json_words(table_widths),
- table_header: table_header != 0,
- code_syntax: "",
- code_linenumbers: code_linenumbers != 0,
- heading_ancestors_text: json_joined(hat),
- sha256: sha256,
- images: [],
- text: text,
- ),
- ))
- }
- let rows =
- sqlight.query(
- object_sql(doc_id, has_range_columns(conn)),
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-
- // Built newest-first per section and reversed once at the end. Appending
- // to the tail of each section instead would be quadratic, and the largest
- // document in the reference collection has some thousands of objects in
- // one section.
- let sections =
- list.fold(rows, dict.new(), fn(acc, row) {
- let #(id, object) = row
- // the four repeatable properties of the text form are child tables
- // here, one query each
- let object =
- Object(
- ..object,
- images: images(conn, id),
- stow_link: links(conn, id),
- anchor_tags: anchors(conn, id),
- )
- case model.is_section(object.section) {
- False -> acc
- True ->
- dict.upsert(acc, object.section, fn(existing) {
- case existing {
- option.Some(previous) -> [object, ..previous]
- option.None -> [object]
- }
- })
- }
- })
- |> dict.map_values(fn(_, objects) { list.reverse(objects) })
- model.Document(..document, sections: sections)
-}
-
-/// 2.1 added the container range as two columns. A 2.0 database has
-/// neither, and reads as having no range, which is what 2.0 meant.
-fn has_range_columns(conn: sqlight.Connection) -> Bool {
- let d = decode.field(0, decode.int, decode.success)
- case
- sqlight.query(
- "select count(*) from pragma_table_info('objects')
- where name = 'ocn_range_first'",
- on: conn,
- with: [],
- expecting: d,
- )
- {
- Ok([n]) -> n > 0
- _ -> False
- }
-}
-
-fn object_sql(doc_id: Int, has_range: Bool) -> String {
- let range_cols = case has_range {
- True -> "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
- False -> "0, 0"
- }
- "select section, seq, ocn, is_a, coalesce(is_of_part,''),
- coalesce(is_of_section,''), coalesce(is_of_type,''),
- coalesce(heading_level,-1), coalesce(identifier,''),
- coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
- coalesce(children,''), coalesce(ancestors,''),
- coalesce(dummy_heading,0), coalesce(object_number_off,0),
- coalesce(indent_base,0), coalesce(indent_hang,0),
- coalesce(bullet,0), coalesce(lang,''),
- coalesce(has_links,0), coalesce(has_notes_reg,0),
- coalesce(has_notes_star,0), coalesce(has_images,0),
- coalesce(segment,''), coalesce(segment_prev,''),
- coalesce(segment_next,''), coalesce(anchor,''),
- coalesce(segment_html_is,''),
- coalesce(table_cols,0), coalesce(table_widths,''),
- coalesce(table_aligns,''), coalesce(table_header,0),
- coalesce(code_linenumbers,0),
- coalesce(heading_ancestors_text,''), coalesce(sha256,''),
- coalesce(text,''), id, coalesce(heading_lev_anchor,''),
- coalesce(segment_epub,''), coalesce(segment_epub_is,''), " <> range_cols <> "
- from objects where doc_id = " <> int.to_string(doc_id) <> "
- order by case section
- when 'head' then 1 when 'toc' then 2 when 'body' then 3
- when 'endnotes' then 4 when 'glossary' then 5
- when 'bibliography' then 6 when 'bookindex' then 7
- when 'blurb' then 8 when 'tail' then 9 else 10 end, seq"
-}
-
-fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) {
- let d = {
- use name <- decode.field(0, decode.string)
- use sha256 <- decode.field(1, decode.string)
- use bytes <- decode.field(2, decode.int)
- use width <- decode.field(3, decode.int)
- use height <- decode.field(4, decode.int)
- use missing <- decode.field(5, decode.int)
- decode.success(Image(
- name: name,
- sha256: sha256,
- bytes: bytes,
- width: width,
- height: height,
- missing: missing != 0,
- fields: dict.new(),
- ))
- }
- sqlight.query(
- "select name, coalesce(sha256,''), coalesce(bytes,0),
- coalesce(width,0), coalesce(height,0), coalesce(missing,0)
- from object_images where object_id = "
- <> int.to_string(object_id)
- <> " order by seq",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-}
-
-fn links(conn: sqlight.Connection, object_id: Int) -> List(String) {
- let d = {
- use url <- decode.field(0, decode.string)
- decode.success(url)
- }
- sqlight.query(
- "select url from object_links where object_id = "
- <> int.to_string(object_id)
- <> " order by seq",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-}
-
-fn anchors(conn: sqlight.Connection, object_id: Int) -> List(String) {
- let d = {
- use anchor <- decode.field(0, decode.string)
- decode.success(anchor)
- }
- sqlight.query(
- "select anchor from object_anchors where object_id = "
- <> int.to_string(object_id)
- <> " order by seq",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.unwrap([])
-}
-
-fn one(conn: sqlight.Connection, sql: String) -> String {
- let d = {
- use v <- decode.field(0, decode.string)
- decode.success(v)
- }
- case sqlight.query(sql, on: conn, with: [], expecting: d) {
- Ok([v, ..]) -> v
- _ -> ""
- }
-}
-
-fn has(conn: sqlight.Connection, name: String) -> Bool {
- one(
- conn,
- "select cast(count(*) as text) from sqlite_master where name = '"
- <> name
- <> "'",
- )
- != "0"
-}
-
-fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String {
- one(
- conn,
- "select value from metadata where doc_id = "
- <> int.to_string(doc_id)
- <> " and key = '"
- <> key
- <> "'",
- )
-}
-
-fn major_of(version: String) -> String {
- case string.split_once(version, ".") {
- Ok(#(major, _)) -> major
- Error(_) -> version
- }
-}
-
-/// The eight heading levels and their indices.
-fn level_name(index: Int) -> String {
- case index {
- 0 -> "A"
- 1 -> "B"
- 2 -> "C"
- 3 -> "D"
- 4 -> "1"
- 5 -> "2"
- 6 -> "3"
- 7 -> "4"
- _ -> ""
- }
-}
-
-/// The JSON columns, read as JSON.
-///
-/// Not as a character scan that strips brackets and quotes: that is wrong
-/// the moment a value holds an escape, and spine's encoder writes "/" as
-/// "\/", so every heading containing a slash comes back with a backslash in
-/// it. The D library had exactly that bug and all 36 document-languages
-/// differed on heading_ancestors_text because of it.
-fn json_strings(value: String) -> List(String) {
- case value {
- "" -> []
- v ->
- case json.parse(v, decode.list(of: loose_string())) {
- Ok(items) -> items
- Error(_) -> []
- }
- }
-}
-
-/// An element that may be a string or a number, which is what these columns
-/// hold: children and ancestors are int arrays, table_aligns and
-/// heading_ancestors_text string arrays, and table_widths an int array
-/// except where the columns divide evenly, when it is [33.3333,...]. One
-/// table in the reference collection does that, and without the float
-/// branch the widths came back as three empty strings: the fallback turned
-/// a value it could not read into a value that looked read.
-fn loose_string() -> decode.Decoder(String) {
- decode.one_of(decode.string, [
- decode.int |> decode.map(int.to_string),
- decode.float |> decode.map(float.to_string),
- decode.success(""),
- ])
-}
-
-fn json_ints(value: String) -> List(Int) {
- value
- |> json_strings
- |> list.filter(fn(s) { s != "" })
- |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) })
-}
-
-/// Space separated in the text form: table_widths, table_aligns.
-fn json_words(value: String) -> String {
- value |> json_strings |> string.join(" ")
-}
-
-/// heading_ancestors_text is eight slots, joined with "|" in the text form
-/// and a JSON array here. The empty slots are significant: they say which
-/// levels the heading sits under.
-fn json_joined(value: String) -> String {
- case json_strings(value) {
- [] -> ""
- parts -> string.join(parts, "|")
- }
-}
-
-/// What the database holds and the model does not.
-///
-/// The text loader meets every property line and counts those it does not
-/// keep, so a reader can say what was read and not understood. This loader
-/// asks for columns by name, and would never see the ones it did not ask
-/// for: it would report that it had understood everything, which is the
-/// silence the count exists to prevent. So it counts them, on the
-/// conditions under which the `.ssp` writes the same property
-/// (specification section 6.3), and the report is the same from either
-/// form, as the D loader's is.
-fn count_unread(
- conn: sqlight.Connection,
- doc_id: Int,
- document: Document,
-) -> Document {
- let id = int.to_string(doc_id)
- let counts = [
- #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"),
- #("dom_status", "coalesce(dom_status,'') != ''"),
- #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"),
- #("heading_lev_collapsed", "heading_lev_collapsed is not null"),
- #("parent_lev", "coalesce(parent_lev,0) != 0"),
- #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"),
- #("attrib", "coalesce(attrib,'') != ''"),
- ]
- let property =
- list.fold(counts, document.unknown.property, fn(acc, c) {
- let n =
- count(
- conn,
- "select count(*) from objects where doc_id = " <> id <> " and " <> c.1,
- )
- case n > 0 {
- True -> dict.insert(acc, c.0, n)
- False -> acc
- }
- })
- let property = case has(conn, "object_subtoc") {
- False -> property
- True -> {
- let n =
- count(
- conn,
- "select count(*) from object_subtoc s join objects o
- on o.id = s.object_id where o.doc_id = " <> id,
- )
- case n > 0 {
- True -> dict.insert(property, "lev4_subtoc", n)
- False -> property
- }
- }
- }
- model.Document(
- ..document,
- unknown: model.Unknown(..document.unknown, property: property),
- )
-}
-
-fn count(conn: sqlight.Connection, sql: String) -> Int {
- let d = decode.field(0, decode.int, decode.success)
- case sqlight.query(sql, on: conn, with: [], expecting: d) {
- Ok([n, ..]) -> n
- _ -> 0
- }
-}
-
-/// Every language the database at `path` holds, in manifest order.
-pub fn languages_of_file(path: String) -> Result(List(Language), String) {
- use conn <- with_connection("file:" <> path <> "?mode=ro", path)
- Ok(languages(conn))
-}
-
-/// What the database at `path` carries besides the abstraction, by role
-/// and storage: role, compression, count, original bytes, stored bytes.
-pub fn carried_of_file(path: String) -> Result(List(List(String)), String) {
- use conn <- with_connection("file:" <> path <> "?mode=ro", path)
- case has(conn, "files") {
- False -> Ok([])
- True -> {
- let d = {
- use role <- decode.field(0, decode.string)
- use compression <- decode.field(1, decode.string)
- use n <- decode.field(2, decode.int)
- use bytes <- decode.field(3, decode.int)
- use stored <- decode.field(4, decode.int)
- decode.success([
- role,
- compression,
- int.to_string(n),
- int.to_string(bytes),
- int.to_string(stored),
- ])
- }
- sqlight.query(
- "select role, compression, count(*), sum(bytes), sum(length(data))
- from files group by role, compression order by role",
- on: conn,
- with: [],
- expecting: d,
- )
- |> result.map_error(fn(e) { path <> ": " <> e.message })
- }
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-ocda-misc.org b/org/gleam-ocda-misc.org
deleted file mode 100644
index cc32dcb..0000000
--- a/org/gleam-ocda-misc.org
+++ /dev/null
@@ -1,649 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** zip gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/zip.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<zip>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: zip
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// Enough of the zip format to find one file in a pod and read it.
-////
-//// Not a zip library. A pod archive holds a handful of entries, spine writes
-//// them **stored** (uncompressed) so that the outer zstd frame does the
-//// compressing once over the whole thing, and what is wanted from it here is
-//// one named entry.
-////
-//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could
-//// inflate it, but a library that says it read an entry it in fact skipped is
-//// worse than one that says it cannot, and spine's own archives do not need
-//// it.
-
-import gleam/bit_array
-import gleam/int
-import gleam/list
-import gleam/result
-import gleam/string
-
-pub type Entry {
- Entry(
- name: String,
- /// 0 stored, 8 deflate
- method: Int,
- compressed: Int,
- uncompressed: Int,
- local_offset: Int,
- )
-}
-
-const eocd_sig = 0x06054b50
-
-const cd_sig = 0x02014b50
-
-const local_sig = 0x04034b50
-
-/// The entries, from the central directory at the end of the file.
-pub fn entries(bytes: BitArray) -> Result(List(Entry), String) {
- let size = bit_array.byte_size(bytes)
- use _ <- result.try(case size >= 22 {
- True -> Ok(Nil)
- False -> Error("too short to be a zip archive")
- })
- use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size)))
- use count <- result.try(u16(bytes, eocd + 10))
- use offset <- result.try(u32(bytes, eocd + 16))
- read_central(bytes, offset, count, [])
-}
-
-/// The end of central directory record is last, and may be followed by a
-/// comment of up to 65535 bytes, so it is searched for backwards.
-fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) {
- case at < lowest {
- True ->
- Error(
- "no end of central directory record; it does not look like a zip archive",
- )
- False ->
- case u32(bytes, at) {
- Ok(sig) if sig == eocd_sig -> Ok(at)
- _ -> find_eocd(bytes, at - 1, lowest)
- }
- }
-}
-
-fn lowest(size: Int) -> Int {
- case size > 22 + 65_535 {
- True -> size - 22 - 65_535
- False -> 0
- }
-}
-
-fn read_central(
- bytes: BitArray,
- at: Int,
- remaining: Int,
- acc: List(Entry),
-) -> Result(List(Entry), String) {
- case remaining {
- 0 -> Ok(list.reverse(acc))
- _ -> {
- use sig <- result.try(u32(bytes, at))
- use _ <- result.try(case sig == cd_sig {
- True -> Ok(Nil)
- False ->
- Error(
- "central directory entry "
- <> int.to_string(list.length(acc))
- <> " is malformed",
- )
- })
- use method <- result.try(u16(bytes, at + 10))
- use compressed <- result.try(u32(bytes, at + 20))
- use uncompressed <- result.try(u32(bytes, at + 24))
- use name_len <- result.try(u16(bytes, at + 28))
- use extra_len <- result.try(u16(bytes, at + 30))
- use cmnt_len <- result.try(u16(bytes, at + 32))
- use local_offset <- result.try(u32(bytes, at + 42))
- use name <- result.try(text(bytes, at + 46, name_len))
- let entry =
- Entry(
- name: name,
- method: method,
- compressed: compressed,
- uncompressed: uncompressed,
- local_offset: local_offset,
- )
- read_central(
- bytes,
- at + 46 + name_len + extra_len + cmnt_len,
- remaining - 1,
- [entry, ..acc],
- )
- }
- }
-}
-
-/// One entry's bytes, from its local header.
-pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) {
- use sig <- result.try(u32(bytes, entry.local_offset))
- use _ <- result.try(case sig == local_sig {
- True -> Ok(Nil)
- False -> Error(entry.name <> ": local header is malformed")
- })
- use name_len <- result.try(u16(bytes, entry.local_offset + 26))
- use extra_len <- result.try(u16(bytes, entry.local_offset + 28))
- use _ <- result.try(case entry.method {
- 0 -> Ok(Nil)
- m ->
- Error(
- entry.name
- <> ": compression method "
- <> int.to_string(m)
- <> "; this reader takes only stored entries, which is what spine writes",
- )
- })
- let start = entry.local_offset + 30 + name_len + extra_len
- case bit_array.slice(bytes, start, entry.compressed) {
- Ok(slice) -> Ok(slice)
- Error(_) -> Error(entry.name <> ": runs past the end of the archive")
- }
-}
-
-/// The entries whose names end this way, in archive order. Used to find
-/// media/abstraction/<uid>.<lang>.ssp without knowing the uid.
-pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) {
- list.filter(entries, fn(e) { string.ends_with(e.name, suffix) })
-}
-
-fn u16(bytes: BitArray, at: Int) -> Result(Int, String) {
- case bit_array.slice(bytes, at, 2) {
- Ok(<<v:little-size(16)>>) -> Ok(v)
- _ -> Error("read past the end of the archive at " <> int.to_string(at))
- }
-}
-
-fn u32(bytes: BitArray, at: Int) -> Result(Int, String) {
- case bit_array.slice(bytes, at, 4) {
- Ok(<<v:little-size(32)>>) -> Ok(v)
- _ -> Error("read past the end of the archive at " <> int.to_string(at))
- }
-}
-
-fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
- case bit_array.slice(bytes, at, len) {
- Ok(slice) ->
- bit_array.to_string(slice)
- |> result.replace_error("an entry name is not valid UTF-8")
- Error(_) -> Error("an entry name runs past the end of the archive")
- }
-}
-#+END_SRC
-
-** imapges gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/images.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<images>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: images
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// An image's bytes, from whichever source the document came from.
-////
-//// The model *names* images, with digest, byte size and pixel size; it does
-//// not hold them. That is right - an abstraction of a document is not a
-//// bundle of its assets - but a writer of any format that shows images needs
-//// the bytes, so something has to know where they are. This is that
-//// something, and it is in the library rather than in a writer because the
-//// answer depends on the *input form* and not on the output.
-////
-//// | `.ocda.db` | the `files` table, role 'image', by name |
-//// | `.zip` | the archive entry whose path ends in `/image/<name>` |
-//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot |
-//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` |
-
-import gleam/bit_array
-import gleam/dynamic/decode
-import gleam/list
-import gleam/result
-import gleam/string
-import ocda/load
-import ocda/zip
-import simplifile
-import sqlight
-
-/// The bytes, or an error saying why not.
-pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) {
- case load.source_of(source) {
- load.OcdaDb -> from_db(source, name)
- load.Pod -> from_archive(source, name)
- load.Ssp -> from_beside(source, name)
- load.Unknown -> Error(source <> ": not a source this library reads")
- }
-}
-
-/// Can this source give images at all?
-///
-/// A writer asks before it decides whether to emit them, because an `<img>`
-/// pointing at a file it did not include is worse than no image.
-pub fn can_give_images(source: String) -> Bool {
- case load.source_of(source) {
- load.OcdaDb -> True
- load.Pod -> zip_readable(source)
- load.Ssp -> image_dir_beside(source) != ""
- load.Unknown -> False
- }
-}
-
-fn zip_readable(path: String) -> Bool {
- case simplifile.read_bits(path) {
- Error(_) -> False
- Ok(bytes) ->
- case zip.entries(bytes) {
- Ok(_) -> True
- Error(_) -> False
- }
- }
-}
-
-fn from_db(path: String, name: String) -> Result(BitArray, String) {
- case sqlight.open("file:" <> path <> "?mode=ro") {
- Error(e) -> Error(path <> ": " <> e.message)
- Ok(conn) -> {
- let out =
- sqlight.query(
- "select data from files where role = 'image' and name = ?1",
- on: conn,
- with: [sqlight.text(name)],
- expecting: blob(),
- )
- |> result.replace_error(path <> ": could not read " <> name)
- let _ = sqlight.close(conn)
- case out {
- Ok([data, ..]) -> Ok(data)
- Ok([]) -> Error(name <> ": not in this database")
- Error(e) -> Error(e)
- }
- }
- }
-}
-
-/// A blob column, read as bytes rather than as text: an image read as text
-/// stops at the first zero byte, and a png's ninth byte is zero.
-fn blob() -> decode.Decoder(BitArray) {
- use v <- decode.field(0, decode.bit_array)
- decode.success(v)
-}
-
-fn from_archive(path: String, name: String) -> Result(BitArray, String) {
- use bytes <- result.try(
- simplifile.read_bits(path)
- |> result.replace_error(path <> ": cannot read"),
- )
- use entries <- result.try(zip.entries(bytes))
- case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) {
- Error(_) -> Error(name <> ": not in this archive")
- Ok(entry) -> zip.read(bytes, entry)
- }
-}
-
-fn from_beside(path: String, name: String) -> Result(BitArray, String) {
- case image_dir_beside(path) {
- "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it")
- dir ->
- simplifile.read_bits(dir <> "/" <> name)
- |> result.replace_error(dir <> "/" <> name <> ": cannot read")
- }
-}
-
-/// `media/abstraction/<x>.ssp` -> `media/image/`
-fn image_dir_beside(path: String) -> String {
- let parts = string.split(path, on: "/")
- let dir = case list.reverse(parts) {
- [_file, ..rest] -> string.join(list.reverse(rest), "/")
- [] -> ""
- }
- case string.ends_with(dir, "abstraction") {
- False -> ""
- True -> {
- let parent = case list.reverse(string.split(dir, on: "/")) {
- [_, ..rest] -> string.join(list.reverse(rest), "/")
- [] -> ""
- }
- let image_dir = parent <> "/image"
- case simplifile.is_directory(image_dir) {
- Ok(True) -> image_dir
- _ -> ""
- }
- }
- }
-}
-
-/// Unused here, kept so that a caller can size a read before making it.
-pub fn byte_size(b: BitArray) -> Int {
- bit_array.byte_size(b)
-}
-#+END_SRC
-
-** dump gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/dump.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<dump>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: dump
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// The model, written out canonically.
-////
-//// Not an output format: an *oracle*. Two libraries in two languages
-//// implement the same model from the same specification, and the cheap way
-//// to ask whether they agree is to have each write the whole loaded
-//// document as text and diff the two. Byte for byte identical to the D
-//// library's `ocda/dump.d`, which is the whole point of it.
-////
-//// The rules, which both implementations follow:
-////
-//// - every line is `<key> <value>`, the value running to end of line,
-//// and only one free-form value per line;
-//// - a field at its default is *omitted*, so the output says what the
-//// document has rather than what the type declares;
-//// - dictionaries come in sorted key order, because neither language's
-//// map has an order worth relying on;
-//// - sections come in the fixed order of the format.
-////
-//// Anything that reads this to reconstruct a document has misunderstood
-//// it. Load the `.ssp` or the `.ocda.db`.
-
-import gleam/dict.{type Dict}
-import gleam/int
-import gleam/list
-import gleam/string
-import ocda/model.{type Document, type Object}
-
-pub fn document(doc: Document) -> String {
- let lines =
- []
- |> kv("version", doc.format_version)
- |> kv("source_filename", doc.source_filename)
- |> entries("source", doc.source)
- |> entries("meta", doc.metadata)
- |> entries("make", doc.make)
- |> entries("doc_has", doc.doc_has)
- |> counts(doc)
- |> objects(doc)
- lines |> list.reverse |> join
-}
-
-/// What the loader met and did not understand, sorted.
-///
-/// Separate from the document dump: it describes the loader, not the
-/// document, and the `.ocda.db` loader has nothing to report where the
-/// `.ssp` loader does.
-pub fn unknown(doc: Document) -> String {
- []
- |> numbered("property", doc.unknown.property)
- |> numbered("block", doc.unknown.block)
- |> numbered("line", doc.unknown.line)
- |> list.reverse
- |> join
-}
-
-fn join(lines: List(String)) -> String {
- case lines {
- [] -> ""
- _ -> string.join(lines, "\n") <> "\n"
- }
-}
-
-/// Accumulated newest-first and reversed once, so that nothing appends to
-/// the tail of a list that grows to a few hundred thousand lines.
-fn kv(acc: List(String), key: String, value: String) -> List(String) {
- case value {
- "" -> acc
- v -> [key <> " " <> v, ..acc]
- }
-}
-
-fn kn(acc: List(String), key: String, value: Int) -> List(String) {
- case value {
- 0 -> acc
- n -> [key <> " " <> int.to_string(n), ..acc]
- }
-}
-
-fn kb(acc: List(String), key: String, value: Bool) -> List(String) {
- case value {
- False -> acc
- True -> [key <> " 1", ..acc]
- }
-}
-
-fn entries(
- acc: List(String),
- prefix: String,
- d: Dict(String, String),
-) -> List(String) {
- d
- |> dict.to_list
- |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
- |> list.fold(acc, fn(lines, pair) {
- [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines]
- })
-}
-
-fn numbered(
- acc: List(String),
- prefix: String,
- d: Dict(String, Int),
-) -> List(String) {
- d
- |> dict.to_list
- |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
- |> list.fold(acc, fn(lines, pair) {
- [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines]
- })
-}
-
-fn counts(acc: List(String), doc: Document) -> List(String) {
- list.fold(model.section_names, acc, fn(lines, name) {
- case model.section(doc, name) {
- [] -> lines
- objects ->
- [
- "section " <> name <> " " <> int.to_string(list.length(objects)),
- ..lines
- ]
- }
- })
-}
-
-fn objects(acc: List(String), doc: Document) -> List(String) {
- list.fold(model.section_names, acc, fn(lines, name) {
- model.section(doc, name)
- |> list.index_fold(lines, fn(l, obj, i) {
- object(l, name, i, obj)
- })
- })
-}
-
-fn object(
- acc: List(String),
- section: String,
- index: Int,
- obj: Object,
-) -> List(String) {
- let acc = ["object " <> section <> " " <> int.to_string(index), ..acc]
- acc
- |> kn("ocn", obj.ocn)
- |> kv("is_a", obj.is_a)
- |> kv("own_section", obj.own_section)
- |> kv("level", obj.heading_level)
- |> kv("identifier", obj.identifier)
- |> kv("part", obj.part)
- |> kv("is_of_type", obj.is_of_type)
- |> kv("segment_html_is", obj.segment_html_is)
- |> kv("segment_epub", obj.segment_epub)
- |> kv("segment_epub_is", obj.segment_epub_is)
- |> kn("parent", obj.parent_ocn)
- |> kn("last_descendant", obj.last_descendant_ocn)
- |> kn("ocn_range_first", obj.ocn_range_first)
- |> kn("ocn_range_last", obj.ocn_range_last)
- |> kv("children", ints(obj.children))
- |> kv("ancestors", ints(obj.ancestors))
- |> kb("dummy", obj.dummy_heading)
- |> kb("ocn_off", obj.ocn_off)
- |> kn("indent_base", obj.indent_base)
- |> kn("indent_hang", obj.indent_hang)
- |> kb("bullet", obj.bullet)
- |> kv("lang", obj.lang)
- |> kb("has_links", obj.has_links)
- |> kb("has_notes_reg", obj.has_notes_reg)
- |> kb("has_notes_star", obj.has_notes_star)
- |> kb("has_images", obj.has_images)
- |> kv("segment", obj.segment)
- |> kv("segment_prev", obj.segment_prev)
- |> kv("segment_next", obj.segment_next)
- |> kv("anchor", obj.anchor)
- |> kv("heading_lev_anchor", obj.heading_lev_anchor)
- |> kn("table_cols", obj.table_cols)
- |> kv("table_widths", obj.table_widths)
- |> kv("table_aligns", obj.table_aligns)
- |> kb("table_header", obj.table_header)
- |> kv("code_syntax", obj.code_syntax)
- |> kb("code_linenumbers", obj.code_linenumbers)
- |> kv("heading_ancestors_text", obj.heading_ancestors_text)
- |> kv("sha256", obj.sha256)
- |> links(obj)
- |> anchor_tags(obj)
- |> images(obj)
- |> text(obj)
-}
-
-fn links(acc: List(String), obj: Object) -> List(String) {
- list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) })
-}
-
-fn anchor_tags(acc: List(String), obj: Object) -> List(String) {
- list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) })
-}
-
-fn images(acc: List(String), obj: Object) -> List(String) {
- list.fold(obj.images, acc, fn(lines, img) {
- // the name last, because a filename may hold a space where none of the
- // other fields can
- let lines =
- kv(
- lines,
- "image",
- img.sha256
- <> " "
- <> int.to_string(img.bytes)
- <> " "
- <> int.to_string(img.width)
- <> "x"
- <> int.to_string(img.height)
- <> " "
- <> case img.missing {
- True -> "1"
- False -> "0"
- }
- <> " "
- <> img.name,
- )
- img.fields
- |> dict.to_list
- |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
- |> list.fold(lines, fn(l, pair) {
- kv(l, "image_field", pair.0 <> " " <> pair.1)
- })
- })
-}
-
-/// Text, when there is any.
-///
-/// An object whose only text line is blank holds "" here, indistinguishable
-/// from no text at all, because the model joins the lines into one string.
-/// Both libraries lose the same distinction in the same place, which is
-/// what makes them comparable; it is recorded rather than hidden.
-fn text(acc: List(String), obj: Object) -> List(String) {
- case obj.text {
- "" -> acc
- t -> {
- let lines = string.split(t, on: "\n")
- let acc = kv(acc, "text", int.to_string(list.length(lines)))
- list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] })
- }
- }
-}
-
-fn ints(xs: List(Int)) -> String {
- case xs {
- [] -> ""
- _ -> xs |> list.map(int.to_string) |> string.join(",")
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-ocda-model_and_inventory.org b/org/gleam-ocda-model_and_inventory.org
deleted file mode 100644
index 0e091a4..0000000
--- a/org/gleam-ocda-model_and_inventory.org
+++ /dev/null
@@ -1,494 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** model gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/model.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<model>>
-#+END_SRC
-
-#+NAME: model
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// The object-centric document abstraction, as Gleam types.
-////
-//// A deliberate mirror of the D library's `ocda.model`, field for field.
-//// The two are meant to agree, and the cheapest way to make that testable
-//// is for them to hold the same things under the same names: the dump of a
-//// loaded document is then identical by construction rather than by
-//// negotiation.
-////
-//// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for
-//// the model, the two format specifications for what each serialisation
-//// says about it.
-
-import gleam/dict.{type Dict}
-import gleam/list
-import gleam/string
-
-/// The format major version this library knows.
-///
-/// The `.ssp` and the `.ocda.db` carry one version between them, declared
-/// in band. The minor part rises when a property is added and a reader of
-/// the newer format still reads the older, so only the major is checked.
-pub const format_major_read = "2"
-
-/// The nine sections, in the order the format gives them, which is not the
-/// order a sorted key list or a `group by` would give.
-pub const section_names = [
- "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
- "blurb", "tail",
-]
-
-/// The four blocks that are document preamble rather than sections of
-/// objects.
-pub const preamble_blocks = ["source", "meta", "make", "doc_has"]
-
-/// A repeatable `.image` record: the file as the abstraction found it.
-pub type Image {
- Image(
- /// filename as the object refers to it
- name: String,
- /// hex, uppercase
- sha256: String,
- bytes: Int,
- /// the file's own pixel dimensions, not any display width
- width: Int,
- height: Int,
- /// absent or unreadable, so no digest and no size
- missing: Bool,
- /// any further key:value fields, kept rather than dropped
- fields: Dict(String, String),
- )
-}
-
-pub fn new_image() -> Image {
- Image(
- name: "",
- sha256: "",
- bytes: 0,
- width: 0,
- height: 0,
- missing: False,
- fields: dict.new(),
- )
-}
-
-/// One object of the abstraction.
-///
-/// `text` is one string with the lines joined by newline, as in the D
-/// library and for the same reason: an object whose only text line is
-/// blank is then indistinguishable from an object with no text, in both
-/// libraries, in the same place. A difference that exists in one language
-/// and not the other would be worse than a loss both share.
-pub type Object {
- Object(
- ocn: Int,
- /// one of the sixteen kinds
- is_a: String,
- /// the section this object was found in
- section: String,
- /// headings: A B C D 1 2 3 4
- heading_level: String,
- identifier: String,
- part: String,
- /// how the text behaves: para, block, comment
- is_of_type: String,
- /// the object's own section, when it differs from the block it sits in
- own_section: String,
- segment_html_is: String,
- segment_epub: String,
- segment_epub_is: String,
- table_aligns: String,
- stow_link: List(String),
- anchor_tags: List(String),
- parent_ocn: Int,
- last_descendant_ocn: Int,
- ocn_range_first: Int,
- ocn_range_last: Int,
- children: List(Int),
- ancestors: List(Int),
- dummy_heading: Bool,
- ocn_off: Bool,
- indent_base: Int,
- indent_hang: Int,
- bullet: Bool,
- lang: String,
- has_links: Bool,
- has_notes_reg: Bool,
- has_notes_star: Bool,
- has_images: Bool,
- segment: String,
- segment_prev: String,
- segment_next: String,
- anchor: String,
- heading_lev_anchor: String,
- table_cols: Int,
- table_widths: String,
- table_header: Bool,
- code_syntax: String,
- code_linenumbers: Bool,
- heading_ancestors_text: String,
- sha256: String,
- images: List(Image),
- text: String,
- )
-}
-
-pub fn new_object() -> Object {
- Object(
- ocn: 0,
- is_a: "",
- section: "",
- heading_level: "",
- identifier: "",
- part: "",
- is_of_type: "",
- own_section: "",
- segment_html_is: "",
- segment_epub: "",
- segment_epub_is: "",
- table_aligns: "",
- stow_link: [],
- anchor_tags: [],
- parent_ocn: 0,
- last_descendant_ocn: 0,
- ocn_range_first: 0,
- ocn_range_last: 0,
- children: [],
- ancestors: [],
- dummy_heading: False,
- ocn_off: False,
- indent_base: 0,
- indent_hang: 0,
- bullet: False,
- lang: "",
- has_links: False,
- has_notes_reg: False,
- has_notes_star: False,
- has_images: False,
- segment: "",
- segment_prev: "",
- segment_next: "",
- anchor: "",
- heading_lev_anchor: "",
- table_cols: 0,
- table_widths: "",
- table_header: False,
- code_syntax: "",
- code_linenumbers: False,
- heading_ancestors_text: "",
- sha256: "",
- images: [],
- text: "",
- )
-}
-
-/// Whether this object is rendered at all.
-///
-/// `is_of_type` "comment" is markup commentary: carried by the abstraction
-/// so that nothing is lost, and not put into output. An output that
-/// renders it leaks the author's notes into the document, which is what
-/// spine itself does not do.
-pub fn renders(object: Object) -> Bool {
- object.is_of_type != "comment"
-}
-
-/// The section this object belongs to: the block it sits in, unless it
-/// says otherwise.
-pub fn section_of(object: Object) -> String {
- case object.own_section {
- "" -> object.section
- own -> own
- }
-}
-
-/// What the loader met and did not understand.
-///
-/// A loader that silently drops what it does not know cannot say whether
-/// it understood the document, and so cannot be used to check a
-/// specification against an implementation: silence and understanding look
-/// alike.
-pub type Unknown {
- Unknown(
- /// `.name` -> times seen
- property: Dict(String, Int),
- /// `@name` -> times seen
- block: Dict(String, Int),
- /// first character -> times seen, for lines of no known shape
- line: Dict(String, Int),
- )
-}
-
-pub fn new_unknown() -> Unknown {
- Unknown(property: dict.new(), block: dict.new(), line: dict.new())
-}
-
-pub fn any_unknown(unknown: Unknown) -> Bool {
- dict.size(unknown.property) > 0
- || dict.size(unknown.block) > 0
- || dict.size(unknown.line) > 0
-}
-
-pub type Document {
- Document(
- /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when
- /// absent
- format_version: String,
- source_filename: String,
- /// language, languages, digest
- source: Dict(String, String),
- unknown: Unknown,
- metadata: Dict(String, String),
- make: Dict(String, String),
- doc_has: Dict(String, String),
- /// section name -> objects, in document order
- sections: Dict(String, List(Object)),
- )
-}
-
-pub fn new_document() -> Document {
- Document(
- format_version: "",
- source_filename: "",
- source: dict.new(),
- unknown: new_unknown(),
- metadata: dict.new(),
- make: dict.new(),
- doc_has: dict.new(),
- sections: dict.new(),
- )
-}
-
-/// The major part of the declared version, "" when none was declared.
-pub fn version_major(document: Document) -> String {
- case string.split_once(document.format_version, ".") {
- Ok(#(major, _)) -> major
- Error(_) -> document.format_version
- }
-}
-
-pub fn meta(document: Document, key: String) -> String {
- case dict.get(document.metadata, key) {
- Ok(value) -> value
- Error(_) -> ""
- }
-}
-
-/// The objects of a named section, empty when the document has none.
-pub fn section(document: Document, name: String) -> List(Object) {
- case dict.get(document.sections, name) {
- Ok(objects) -> objects
- Error(_) -> []
- }
-}
-
-/// Is this one of the nine sections of the format?
-pub fn is_section(name: String) -> Bool {
- list.contains(section_names, name)
-}
-
-/// Is this one of the four preamble blocks?
-pub fn is_preamble_block(name: String) -> Bool {
- list.contains(preamble_blocks, name)
-}
-#+END_SRC
-
-** inventory gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda/inventory.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<inventory>>
-#+END_SRC
-
-#+NAME: inventory
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// What a source holds, beyond the one document the model is loaded from.
-////
-//// The model is one language of one document. Two questions a reader asks
-//// are about the file instead: which languages it holds, and, for a
-//// database, what else it carries. They are here rather than in a reader
-//// because the answer depends on the input form, which is the library's to
-//// know, and so that a reader asking them does not learn which form it was
-//// handed. A mirror of the D library's `ocda.inventory`.
-////
-//// `languages` answers for every form, and answers the same: the languages
-//// in the order the manifest gave them (which every `.ssp` records, in
-//// `@source languages`), each with the markup file it was made from and the
-//// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a
-//// pod it is taken from the bytes, which are the same bytes.
-////
-//// `carried` is the database's: a `.ocda.db` holds the markup, images,
-//// configuration and catalogues it was built from, a `.ssp` holds none of
-//// them, and a pod's are its archive entries rather than anything the
-//// abstraction says.
-
-import gleam/bit_array
-import gleam/dict
-import gleam/list
-import gleam/result
-import gleam/string
-import ocda/load
-import ocda/load_db
-import ocda/load_pod
-import ocda/load_ssp
-import ocda/zip
-import simplifile
-
-pub type Language {
- Language(lang: String, source_filename: String, ssp_digest: String)
-}
-
-pub type Carried {
- Carried(
- role: String,
- compression: String,
- count: String,
- bytes: String,
- stored: String,
- )
-}
-
-pub fn languages(path: String) -> Result(List(Language), String) {
- case load.source_of(path) {
- load.OcdaDb -> {
- use held <- result.try(load_db.languages_of_file(path))
- Ok(
- list.map(held, fn(l) {
- Language(l.lang, l.source_filename, l.ssp_digest)
- }),
- )
- }
- load.Ssp -> {
- use bytes <- result.try(
- simplifile.read_bits(path)
- |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }),
- )
- use text <- result.try(
- bit_array.to_string(bytes)
- |> result.replace_error(path <> ": not valid UTF-8"),
- )
- let doc = load_ssp.parse(text)
- Ok([
- Language(
- source_value(doc.source, "language"),
- doc.source_filename,
- sha256_hex(bytes),
- ),
- ])
- }
- load.Pod -> {
- use archive <- result.try(load_pod.archive_bytes(path))
- use entries <- result.try(zip.entries(archive))
- use found <- result.try(
- zip.find_suffix(entries, ".ssp")
- |> list.try_map(fn(e) {
- use bytes <- result.try(zip.read(archive, e))
- use text <- result.try(
- bit_array.to_string(bytes)
- |> result.replace_error(e.name <> ": not valid UTF-8"),
- )
- let doc = load_ssp.parse(text)
- let lang = result.unwrap(load_pod.lang_of(e.name), "")
- Ok(#(
- Language(lang, doc.source_filename, sha256_hex(bytes)),
- string.split(source_value(doc.source, "languages"), " "),
- ))
- }),
- )
- let order = case found {
- [#(_, o), ..] -> list.filter(o, fn(l) { l != "" })
- [] -> []
- }
- let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list
- // the manifest's order, and anything it did not name after it
- let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) })
- let rest =
- list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) })
- |> list.map(fn(f) { f.0 })
- Ok(list.append(named, rest))
- }
- load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown))
- }
-}
-
-/// Only a `.ocda.db` carries files.
-pub fn is_carrier(path: String) -> Bool {
- load.source_of(path) == load.OcdaDb
-}
-
-pub fn carried(path: String) -> Result(List(Carried), String) {
- case is_carrier(path) {
- False -> Ok([])
- True -> {
- use rows <- result.try(load_db.carried_of_file(path))
- Ok(
- list.filter_map(rows, fn(r) {
- case r {
- [role, compression, n, bytes, stored] ->
- Ok(Carried(role, compression, n, bytes, stored))
- _ -> Error(Nil)
- }
- }),
- )
- }
- }
-}
-
-fn source_value(source: dict.Dict(String, String), key: String) -> String {
- result.unwrap(dict.get(source, key), "")
-}
-
-@external(erlang, "ocda_ffi", "sha256_hex")
-fn sha256_hex(bytes: BitArray) -> String
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-ocda.org b/org/gleam-ocda.org
index a7cac9a..2cb2e20 100644
--- a/org/gleam-ocda.org
+++ b/org/gleam-ocda.org
@@ -6,9 +6,9 @@
#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty
#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
#+PROPERTY: header-args+ :results silent :cache no
#+PROPERTY: header-args+ :mkdirp yes
#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
@@ -16,159 +16,7 @@
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
* gleam :gleam:
-** README.md
-*** tangle
-
-#+HEADER: :tangle ../gleam/README.md
-#+HEADER: :noweb yes
-#+BEGIN_SRC md
-# ocda-lib, in Gleam
-
-The abstraction's model and both its loaders, as a Gleam library, with its
-reader and the two programs that put it to the question.
-
-| | |
-|------------------|-----------------------------------------------------------------------------------------|
-| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
-| `ocda/load_ssp` | text -> model |
-| `ocda/load_db` | sqlite -> model |
-| `ocda/load` | what a path is, and loading it |
-| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
-| `ocda/dump` | the model written out canonically, as an oracle |
-| `read/main` | `ocda-read`, the reader |
-| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
-
-## Use
-
-```gleam
-import ocda/load
-import ocda/model
-
-case load.load(path, "en", False) {
- Error(why) -> io.println_error(why)
- Ok(doc) -> {
- // and from here a writer neither knows nor cares which of the two
- // files it was handed
- list.each(model.section(doc, "body"), fn(obj) { ... })
- }
-}
-```
-
-## The reader
-
-```sh
-nix build
-./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
- <file>.ssp|<file>.ocda.db|<file>.zip [lang]
-```
-
-`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
-had, over the library, so each reads every form this library reads; the
-others are the D reader's. **Its output is the D `ocda-read`'s, byte for
-byte**, in every mode both have. In `src/read/`, because Gleam compiles
-nothing outside `src/`; a program depending on the library compiles it too,
-and does not use it.
-
-A `.sisupod` it refuses, saying why: the zstd frame, below.
-
-## The two programs
-
-```sh
-nix build
-./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
-./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
-```
-
-`agree` asks whether the two loaders give the same model. `dump` writes
-the whole model out canonically, and is the stronger check of the two,
-because it is **byte for byte what the D library's `ocda-dump` writes**.
-So the four questions can all be asked with `diff`:
-
-| | |
-|-----------------------------|-----------------------------|
-| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
-| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
-| `.ssp`: D == Gleam | 36 of 36 |
-| `.ocda.db`: D == Gleam | 36 of 36 |
-
-## Deliberately a mirror of the D library
-
-Field for field, name for name, including where that is not the most
-Gleamish choice:
-
-- `text` is one `String` with the lines joined by newline, not a
- `List(String)`. So an object whose only text line is blank is
- indistinguishable from an object with no text, *in both languages, in the
- same place*. A difference that existed in one and not the other would be
- worse than a loss both share, because it would show up in every dump and
- mean nothing.
-- the properties are typed fields rather than a list of pairs. A writer
- wants `obj.parent_ocn`, not a lookup that might miss.
-
-The point is that the two agree. Making them agree is much cheaper when
-they are shaped alike, and the shape is the specification's, not either
-language's.
-
-## What the agreement found
-
-Four disagreements, none of which a comparison of section counts or of a
-handful of fields would have shown. All four are recorded in the D
-library's history; the short version:
-
-1. `.identifier` and `.anchor` are omitted by the text form when they can
- be worked out (the identifier is the ocn, the anchor is the segment).
- Both loaders now derive them. Over one document that is 1,314
- identifiers.
-2. `make` and `doc_has` are one metadata table in the database,
- distinguished by a key prefix. Both loaders unprefix them.
-3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
- as `\/` and every heading with a slash in it came back wrong.
-4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
- which is one language. It is a fact about the file, so neither loader
- puts it in the document.
-
-## Build
-
-```sh
-nix build # sandboxed, no network
-nix develop -c gleam test
-```
-
-Shipped as an Erlang shipment rather than an escript, because sqlite on the
-BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
-The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
-one and the other must change with it.
-
-## Format
-
-Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
-`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
-two serialisations. At format **2.x**.
-#+END_SRC
-
-** gleam.toml
-*** tangle
-
-#+HEADER: :tangle ../gleam/gleam.toml
-#+HEADER: :noweb yes
-#+BEGIN_SRC toml
-name = "ocda_lib"
-version = "0.1.0"
-description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
-licences = ["AGPL-3.0-or-later"]
-
-[dependencies]
-gleam_stdlib = ">= 1.0.0 and < 2.0.0"
-argv = ">= 1.1.0 and < 2.0.0"
-simplifile = ">= 2.7.0 and < 3.0.0"
-sqlight = ">= 1.0.0 and < 2.0.0"
-gleam_json = ">= 3.0.0 and < 4.0.0"
-
-[dev-dependencies]
-gleeunit = ">= 1.0.0 and < 2.0.0"
-#+END_SRC
-
-** gleam ocda_lib
+** ocda lib
*** tangle
#+HEADER: :tangle ../gleam/src/ocda_lib.gleam
@@ -178,7 +26,7 @@ gleeunit = ">= 1.0.0 and < 2.0.0"
<<ocda_lib>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: ocda_lib
#+HEADER: :no-expand
@@ -329,31 +177,7 @@ fn fail(message: String) -> Nil {
fn halt(code: Int) -> Nil
#+END_SRC
-** erlang
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda_ffi.erl
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-%%<<doc_header_including_copyright_and_license>>
-<<erlang_ffi>>
-#+END_SRC
-
-*** erlang source
-
-#+NAME: erlang_ffi
-#+HEADER: :no-expand
-#+BEGIN_SRC erlang
-%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's
-%% crypto has and the Gleam standard library does not. Upper case hex, as
-%% the database stores it and the D library writes it.
--module(ocda_ffi).
--export([sha256_hex/1]).
-
-sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
-#+END_SRC
-
-** model gleam
+** model
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/model.gleam
@@ -363,6 +187,8 @@ sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
<<model>>
#+END_SRC
+*** source
+
#+NAME: model
#+HEADER: :no-expand
#+BEGIN_SRC gleam
@@ -650,7 +476,7 @@ pub fn is_preamble_block(name: String) -> Bool {
}
#+END_SRC
-** inventory gleam
+** inventory
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/inventory.gleam
@@ -660,6 +486,8 @@ pub fn is_preamble_block(name: String) -> Bool {
<<inventory>>
#+END_SRC
+*** source
+
#+NAME: inventory
#+HEADER: :no-expand
#+BEGIN_SRC gleam
@@ -803,7 +631,7 @@ fn source_value(source: dict.Dict(String, String), key: String) -> String {
fn sha256_hex(bytes: BitArray) -> String
#+END_SRC
-** load gleam
+** load
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/load.gleam
@@ -813,7 +641,7 @@ fn sha256_hex(bytes: BitArray) -> String
<<load>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: load
#+HEADER: :no-expand
@@ -909,7 +737,7 @@ pub fn load(
<<load_pod>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: load_pod
#+HEADER: :no-expand
@@ -1064,7 +892,7 @@ pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
<<load_ssp>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: load_ssp
#+HEADER: :no-expand
@@ -1567,7 +1395,7 @@ fn numbers(value: String) -> List(Int) {
<<load_db>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: load_db
#+HEADER: :no-expand
@@ -2243,7 +2071,7 @@ pub fn carried_of_file(path: String) -> Result(List(List(String)), String) {
}
#+END_SRC
-** zip gleam
+** zip
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/zip.gleam
@@ -2253,7 +2081,7 @@ pub fn carried_of_file(path: String) -> Result(List(List(String)), String) {
<<zip>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: zip
#+HEADER: :no-expand
@@ -2430,7 +2258,7 @@ fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
}
#+END_SRC
-** imapges gleam
+** imapges
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/images.gleam
@@ -2440,7 +2268,7 @@ fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
<<images>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: images
#+HEADER: :no-expand
@@ -2582,7 +2410,7 @@ pub fn byte_size(b: BitArray) -> Int {
}
#+END_SRC
-** dump gleam
+** dump
*** tangle
#+HEADER: :tangle ../gleam/src/ocda/dump.gleam
@@ -2592,7 +2420,7 @@ pub fn byte_size(b: BitArray) -> Int {
<<dump>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: dump
#+HEADER: :no-expand
@@ -2848,6 +2676,31 @@ fn ints(xs: List(Int)) -> String {
}
#+END_SRC
+* erlang
+** erlang
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda_ffi.erl
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+%%<<doc_header_including_copyright_and_license>>
+<<erlang_ffi>>
+#+END_SRC
+
+*** erlang source
+
+#+NAME: erlang_ffi
+#+HEADER: :no-expand
+#+BEGIN_SRC erlang
+%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's
+%% crypto has and the Gleam standard library does not. Upper case hex, as
+%% the database stores it and the D library writes it.
+-module(ocda_ffi).
+-export([sha256_hex/1]).
+
+sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
+#+END_SRC
+
* org includes
** project version
diff --git a/org/gleam-ocda_lib.org b/org/gleam-ocda_lib.org
deleted file mode 100644
index e19715b..0000000
--- a/org/gleam-ocda_lib.org
+++ /dev/null
@@ -1,381 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** README.md
-*** tangle
-
-#+HEADER: :tangle ../gleam/README.md
-#+HEADER: :noweb yes
-#+BEGIN_SRC md
-# ocda-lib, in Gleam
-
-The abstraction's model and both its loaders, as a Gleam library, with its
-reader and the two programs that put it to the question.
-
-| | |
-|------------------|-----------------------------------------------------------------------------------------|
-| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
-| `ocda/load_ssp` | text -> model |
-| `ocda/load_db` | sqlite -> model |
-| `ocda/load` | what a path is, and loading it |
-| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
-| `ocda/dump` | the model written out canonically, as an oracle |
-| `read/main` | `ocda-read`, the reader |
-| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
-
-## Use
-
-```gleam
-import ocda/load
-import ocda/model
-
-case load.load(path, "en", False) {
- Error(why) -> io.println_error(why)
- Ok(doc) -> {
- // and from here a writer neither knows nor cares which of the two
- // files it was handed
- list.each(model.section(doc, "body"), fn(obj) { ... })
- }
-}
-```
-
-## The reader
-
-```sh
-nix build
-./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
- <file>.ssp|<file>.ocda.db|<file>.zip [lang]
-```
-
-`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
-had, over the library, so each reads every form this library reads; the
-others are the D reader's. **Its output is the D `ocda-read`'s, byte for
-byte**, in every mode both have. In `src/read/`, because Gleam compiles
-nothing outside `src/`; a program depending on the library compiles it too,
-and does not use it.
-
-A `.sisupod` it refuses, saying why: the zstd frame, below.
-
-## The two programs
-
-```sh
-nix build
-./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
-./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
-```
-
-`agree` asks whether the two loaders give the same model. `dump` writes
-the whole model out canonically, and is the stronger check of the two,
-because it is **byte for byte what the D library's `ocda-dump` writes**.
-So the four questions can all be asked with `diff`:
-
-| | |
-|-----------------------------|-----------------------------|
-| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
-| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
-| `.ssp`: D == Gleam | 36 of 36 |
-| `.ocda.db`: D == Gleam | 36 of 36 |
-
-## Deliberately a mirror of the D library
-
-Field for field, name for name, including where that is not the most
-Gleamish choice:
-
-- `text` is one `String` with the lines joined by newline, not a
- `List(String)`. So an object whose only text line is blank is
- indistinguishable from an object with no text, *in both languages, in the
- same place*. A difference that existed in one and not the other would be
- worse than a loss both share, because it would show up in every dump and
- mean nothing.
-- the properties are typed fields rather than a list of pairs. A writer
- wants `obj.parent_ocn`, not a lookup that might miss.
-
-The point is that the two agree. Making them agree is much cheaper when
-they are shaped alike, and the shape is the specification's, not either
-language's.
-
-## What the agreement found
-
-Four disagreements, none of which a comparison of section counts or of a
-handful of fields would have shown. All four are recorded in the D
-library's history; the short version:
-
-1. `.identifier` and `.anchor` are omitted by the text form when they can
- be worked out (the identifier is the ocn, the anchor is the segment).
- Both loaders now derive them. Over one document that is 1,314
- identifiers.
-2. `make` and `doc_has` are one metadata table in the database,
- distinguished by a key prefix. Both loaders unprefix them.
-3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
- as `\/` and every heading with a slash in it came back wrong.
-4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
- which is one language. It is a fact about the file, so neither loader
- puts it in the document.
-
-## Build
-
-```sh
-nix build # sandboxed, no network
-nix develop -c gleam test
-```
-
-Shipped as an Erlang shipment rather than an escript, because sqlite on the
-BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
-The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
-one and the other must change with it.
-
-## Format
-
-Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
-`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
-two serialisations. At format **2.x**.
-#+END_SRC
-
-** gleam.toml
-*** tangle
-
-#+HEADER: :tangle ../gleam/gleam.toml
-#+HEADER: :noweb yes
-#+BEGIN_SRC toml
-name = "ocda_lib"
-version = "0.1.0"
-description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
-licences = ["AGPL-3.0-or-later"]
-
-[dependencies]
-gleam_stdlib = ">= 1.0.0 and < 2.0.0"
-argv = ">= 1.1.0 and < 2.0.0"
-simplifile = ">= 2.7.0 and < 3.0.0"
-sqlight = ">= 1.0.0 and < 2.0.0"
-gleam_json = ">= 3.0.0 and < 4.0.0"
-
-[dev-dependencies]
-gleeunit = ">= 1.0.0 and < 2.0.0"
-#+END_SRC
-
-** gleam ocda_lib
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda_lib.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<ocda_lib>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: ocda_lib
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// ocda-lib, in Gleam: the abstraction as a library, put to the question.
-////
-//// A library has no binary, so this is the programs over it: the reader,
-//// and the two that test its claims.
-////
-//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
-//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
-//// ocda-lib read [options] <file> [lang] (also installed as ocda-read)
-////
-//// `dump` writes the whole loaded model out canonically, byte for byte as
-//// the D library's `ocda-dump` writes it, so that the two implementations
-//// can be diffed. `agree` asks the same question the D library's
-//// `ocda-agree` asks, within this language: does loading a document's
-//// `.ssp` give what loading its `.ocda.db` gives?
-
-import argv
-import gleam/int
-import gleam/io
-import gleam/list
-import gleam/string
-import ocda/dump
-import ocda/load
-import ocda/model.{type Document}
-import read/main as reader
-
-pub fn main() {
- case argv.load().arguments {
- ["dump", "--unknown", path] -> dump_file(path, "", True)
- ["dump", "--unknown", path, lang] -> dump_file(path, lang, True)
- ["dump", path] -> dump_file(path, "", False)
- ["dump", path, lang] -> dump_file(path, lang, False)
- ["agree", ssp, db] -> agree(ssp, db, "")
- ["agree", ssp, db, lang] -> agree(ssp, db, lang)
- ["read", ..rest] ->
- case reader.run(rest) {
- 0 -> Nil
- code -> halt(code)
- }
- _ -> usage()
- }
-}
-
-fn usage() -> Nil {
- io.println_error(
- "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
- ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
- ocda-lib read [--help] [options] <file> [lang]",
- )
- halt(2)
-}
-
-fn dump_file(path: String, lang: String, unknown: Bool) -> Nil {
- case load.load(path, lang, False) {
- Error(message) -> fail(message)
- Ok(doc) ->
- case unknown {
- True -> io.print(dump.unknown(doc))
- False -> io.print(dump.document(doc))
- }
- }
-}
-
-/// The library's whole purpose is that a writer cannot tell which
-/// serialisation it was handed. That is only true if the two loaders give
-/// the same model, so this asks, and says where they part company rather
-/// than only that they did.
-fn agree(ssp_path: String, db_path: String, lang: String) -> Nil {
- case load.load(ssp_path, "", False), load.load(db_path, lang, False) {
- Error(message), _ -> fail(message)
- _, Error(message) -> fail(message)
- Ok(from_ssp), Ok(from_db) -> {
- let differences = compare(from_ssp, from_db)
- case differences {
- [] -> io.println("agree: the two loaders give the same model")
- _ -> {
- list.each(list.take(differences, 12), io.println)
- case list.length(differences) > 12 {
- True -> io.println(" ... stopping after 12")
- False -> Nil
- }
- io.println(int.to_string(list.length(differences)) <> " difference(s)")
- halt(1)
- }
- }
- }
- }
-}
-
-/// Compared through the dump rather than field by field, which is both
-/// shorter and stricter: the dump carries every field, so nothing can
-/// differ without showing here.
-fn compare(from_ssp: Document, from_db: Document) -> List(String) {
- let a = dump.document(from_ssp)
- let b = dump.document(from_db)
- case a == b {
- True -> []
- False -> diff_lines(a, b)
- }
-}
-
-fn diff_lines(a: String, b: String) -> List(String) {
- let la = split_lines(a)
- let lb = split_lines(b)
- case list.length(la) == list.length(lb) {
- False -> [
- " differ: line count",
- " .ssp: " <> int.to_string(list.length(la)),
- " .ocda.db: " <> int.to_string(list.length(lb)),
- ..pairwise(la, lb)
- ]
- True -> pairwise(la, lb)
- }
-}
-
-fn pairwise(la: List(String), lb: List(String)) -> List(String) {
- list.zip(la, lb)
- |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) })
- |> list.filter(fn(t) { t.1 != t.2 })
- |> list.map(fn(t) {
- " differ: line "
- <> int.to_string(t.0 + 1)
- <> "\n .ssp: "
- <> t.1
- <> "\n .ocda.db: "
- <> t.2
- })
-}
-
-fn split_lines(text: String) -> List(String) {
- case text {
- "" -> []
- _ -> string.split(text, on: "\n")
- }
-}
-
-fn fail(message: String) -> Nil {
- io.println_error("error: " <> message)
- halt(2)
-}
-
-/// Gleam has no standard exit, and the exit status is what a sweep script
-/// reads.
-@external(erlang, "erlang", "halt")
-fn halt(code: Int) -> Nil
-#+END_SRC
-
-** erlang
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/ocda_ffi.erl
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-%%<<doc_header_including_copyright_and_license>>
-<<erlang_ffi>>
-#+END_SRC
-
-*** erlang source
-
-#+NAME: erlang_ffi
-#+HEADER: :no-expand
-#+BEGIN_SRC erlang
-%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's
-%% crypto has and the Gleam standard library does not. Upper case hex, as
-%% the database stores it and the D library writes it.
--module(ocda_ffi).
--export([sha256_hex/1]).
-
-sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-read-main.org b/org/gleam-read.org
index f968fd1..05fa2d5 100644
--- a/org/gleam-read-main.org
+++ b/org/gleam-read.org
@@ -6,9 +6,9 @@
#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty
#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
#+PROPERTY: header-args+ :results silent :cache no
#+PROPERTY: header-args+ :mkdirp yes
#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
@@ -16,7 +16,7 @@
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
* gleam :gleam:
-** gleam
+** main
*** tangle
#+HEADER: :tangle ../gleam/src/read/main.gleam
@@ -26,7 +26,7 @@
<<read>>
#+END_SRC
-*** gleam source
+*** source
#+NAME: read
#+HEADER: :no-expand