summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-10-04 12:12:51 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-10-04 12:12:51 -0400
commit828c686ed361771b684806e5b37c5dee52514063 (patch)
treec0e28a351ae7cd30aec0c7e7cd2a6b92053558ff
parenttrack orgify (diff)
track orgify
-rw-r--r--org/config.org468
-rw-r--r--org/d-ocda.org1854
-rw-r--r--org/d-read.org1732
-rw-r--r--org/d-test.org178
-rw-r--r--org/d.org3895
-rw-r--r--org/gleam-read.org520
-rw-r--r--org/gleam-test.org209
-rw-r--r--org/gleam.org (renamed from org/gleam-ocda.org)914
8 files changed, 4807 insertions, 4963 deletions
diff --git a/org/config.org b/org/config.org
index 2c1294f..8420c3b 100644
--- a/org/config.org
+++ b/org/config.org
@@ -280,474 +280,6 @@ erl_crash.dump
!**/*.yaml
#+END_SRC
-* ./dlang
-** nix flake.nix :flake:
-
-#+HEADER: :tangle ../dlang/flake.nix
-#+HEADER: :noweb yes
-#+BEGIN_SRC nix
-{
- description = "ocda-lib: the object-centric document abstraction as a D library";
- inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
- outputs = { self, nixpkgs }:
- let
- system = "x86_64-linux"; # adjust for aarch64-linux / darwin
- pkgs = import nixpkgs { inherit system; };
- /* A library, so the library itself is not a build product: a program
- that uses it compiles its modules alongside its own, as the writer's
- flake does, or takes it as a dub dependency. What is built here are
- programs over it: the reader, and the two that put the library to
- the question - the agreement check, which loads a document's .ssp
- and its .ocda.db and compares the two models, and the dump, which
- writes a loaded model out canonically so that an implementation in
- another language can be diffed against this one.
- .
- `mains` is what is compiled besides the library: the checks are one
- file each in test/, the reader is src/read/.
- */
- mkTool = { tool, binary, what, mains, compiler, compilerBin }:
- pkgs.stdenv.mkDerivation {
- pname = "ocda-lib-${tool}";
- version = "0.1.0";
- src = self;
- nativeBuildInputs = [ compiler ];
- buildInputs = [ pkgs.sqlite pkgs.zstd ];
- buildPhase = ''
- runHook preBuild
- export HOME=$TMPDIR
- ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd
- runHook postBuild
- '';
- installPhase = ''
- runHook preInstall
- mkdir -p $out/bin
- install -m755 ${tool} $out/bin/${binary}
- runHook postInstall
- '';
- meta = {
- description = what;
- mainProgram = binary;
- };
- };
- ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
- ocda-read = mkTool (ldc // {
- tool = "read"; binary = "ocda-read"; mains = "src/read/*.d";
- what = "reads the abstraction in any of its forms and says what is in it";
- });
- ocda-agree = mkTool (ldc // {
- tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d";
- what = "checks that the .ssp and .ocda.db loaders give one model";
- });
- ocda-dump = mkTool (ldc // {
- tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d";
- what = "writes a loaded abstraction out canonically, as an oracle";
- });
- in {
- packages.${system} = {
- /* All three in one bin/, so that one build leaves every program in
- one result link. Built separately into the same link, each build
- replaced the one before, and a script looking for ocda-dump beside
- ocda-agree found nothing.
- */
- default = pkgs.symlinkJoin {
- name = "ocda-lib-programs-0.1.0";
- paths = [ ocda-read ocda-agree ocda-dump ];
- };
- inherit ocda-read ocda-agree ocda-dump;
- };
-
- devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
- };
- };
-}
-#+END_SRC
-
-** README.md
-
-#+HEADER: :tangle ../dlang/README.md
-#+HEADER: :noweb yes
-#+BEGIN_SRC md
-# ocda-lib, in D
-
-The abstraction's model and both its loaders, as a dub library.
-
-## Modules
-
-| | |
-|---|---|
-| `ocda.model` | the document, the object, the eight sections. Imports nothing |
-| `ocda.load_ssp` | text -> model |
-| `ocda.load_db` | sqlite -> model |
-| `ocda.load` | what a path is, and loading it |
-| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries |
-| `ocda.dump` | the model written out canonically, as an oracle |
-| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon |
-| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) |
-
-`ocda.model` importing nothing is the property that matters: a writer
-depends on the model and on neither loader.
-
-## Use
-
-```json
-"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } }
-```
-
-```d
-import ocda.load;
-auto doc = load("document.ocda.db", "de"); // or "document.ssp"
-foreach (obj; doc.body) { ... }
-```
-
-Link `sqlite3`; the library declares the dozen functions it needs rather
-than taking a binding from the registry, so there is nothing to fetch.
-
-## The reader
-
-```sh
-nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl|
- --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate]
- <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]
-```
-
-The library's reader, one program where `sisudoc-ssp-reader` and
-`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp`
-reader's, moved onto the library's model, so every mode reads every form
-and **says the same thing of a document's `.ssp`, `.ocda.db` and
-`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries
-files; `--validate` checks a `.ssp` as text and refuses anything else.
-
-It lives in `src/read/` beside the library, and dub keeps it out of the
-library: the configuration `library` excludes `src/read/*`, the
-configuration `ocda-read` builds it. A program using the library by dub, or
-by naming `src/ocda/*.d` as the writer's flake does, never compiles it.
-
-## The two checks
-
-```sh
-nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang]
-nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang]
-```
-
-`agree` loads both and compares ten fields of each object, naming the field
-where they part company. `agree` found the blank-first-text-line defect and
-was not enough for anything subtler.
-
-**`ocda-dump` is the stronger check**: the whole loaded model, every field,
-written out in a form fully determined by the model, so that two
-implementations can be compared with `diff`. It is an oracle, not an output
-format; anything reading it to reconstruct a document has misunderstood it.
-It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is
-the point of having it.
-
-It found four disagreements `agree` could not see. They are in the
-repository README and in the history.
-
-## Build
-
-```sh
-nix build # all three programs, in one result/bin
-nix build .#ocda-read # the reader
-nix build .#ocda-agree # the agreement check
-nix build .#ocda-dump # the canonical dump
-nix develop # ldc, dub, dtools, sqlite, zstd
-dub build # the library
-dub build --config=ocda-read # the reader
-```
-#+END_SRC
-
-** dub.json
-
-#+HEADER: :tangle ../dlang/dub.json
-#+HEADER: :noweb yes
-#+BEGIN_SRC json
-{
- "authors": [ "Ralph Amissah" ],
- "copyright": "Copyright (C) 2026 Ralph Amissah",
- "name": "ocda-lib",
- "version": "0.1.0",
- "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations",
- "homepage": "https://sisudoc.org",
- "license": "AGPL-3.0+",
- "sourcePaths": [ "./src" ],
- "importPaths": [ "./src" ],
- "libs": [ "sqlite3", "zstd" ],
- "configurations": [
- {
- "name": "library",
- "targetType": "library",
- "excludedSourceFiles": [ "src/read/*" ]
- },
- {
- "name": "ocda-read",
- "targetType": "executable",
- "targetName": "ocda-read",
- "mainSourceFile": "src/read/main.d"
- }
- ]
-}
-#+END_SRC
-
-* ./gleam
-** nix flake.nix :flake:
-
-#+HEADER: :tangle ../gleam/flake.nix
-#+HEADER: :noweb yes
-#+BEGIN_SRC nix
-{
- description = "ocda-lib: the object-centric document abstraction as a Gleam library";
- inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
- outputs = { self, nixpkgs }:
- let
- system = "x86_64-linux"; # adjust for aarch64-linux / darwin
- pkgs = import nixpkgs { inherit system; };
-
- name = "ocda_lib";
-
- /* rebar3 with the port compiler already in it.
- esqlite builds a NIF, and its rebar3 configuration asks for the
- `pc` plugin, which rebar3 would otherwise fetch from Hex at build
- time: impossible in a sandbox, and the build fails with
- "missing_package pc". nixpkgs carries the plugin, so the answer is
- a rebar3 that has it before it starts.
- */
- rebar3-pc = pkgs.beamPackages.rebar3WithPlugins {
- plugins = [ pkgs.beamPackages.pc ];
- };
-
- /* The Hex dependencies, as manifest.toml pins them.
- .
- There is no Gleam builder in nixpkgs and `gleam build` fetches from
- Hex, so a sandboxed build cannot reach them. A fixed output
- derivation may, because its result is pinned by hash.
- .
- When a dependency changes, manifest.toml changes and this hash must
- change with it: build, take the hash nix reports, put it here.
- */
- deps = pkgs.stdenv.mkDerivation {
- pname = "${name}-deps";
- version = "0.1.0";
- src = ./.;
- nativeBuildInputs = [ pkgs.gleam pkgs.cacert ];
- buildPhase = ''
- export HOME=$TMPDIR
- gleam deps download
- '';
- installPhase = ''
- mkdir -p $out
- cp -r build/packages/* $out/
- '';
- dontFixup = true;
- outputHashMode = "recursive";
- outputHashAlgo = "sha256";
- outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c=";
- };
- in {
- packages.${system} = {
- default = self.packages.${system}.${name};
- /* Shipped as an Erlang shipment rather than as an escript, because
- sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into
- a shared object, and a NIF cannot be loaded from inside an
- escript archive, the runtime needing a real file on disk. The
- shipment keeps priv/ on disk, so it loads.
- */
- ${name} = pkgs.stdenv.mkDerivation {
- pname = "ocda-lib";
- version = "0.1.0";
- src = ./.;
- nativeBuildInputs = [
- pkgs.gleam
- pkgs.erlang
- rebar3-pc
- pkgs.makeWrapper
- ];
- buildPhase = ''
- runHook preBuild
- export HOME=$TMPDIR
- # the dependencies, in place of the fetch gleam would do
- mkdir -p build/packages
- cp -r ${deps}/* build/packages/
- chmod -R u+w build/packages
- gleam export erlang-shipment
- runHook postBuild
- '';
- installPhase = ''
- runHook preInstall
- mkdir -p $out/lib $out/bin
- cp -r build/erlang-shipment $out/lib/${name}
- makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \
- --add-flags run \
- --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
- # the reader under its own name, as the D flake installs it
- makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \
- --add-flags "run read" \
- --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
- runHook postInstall
- '';
- meta = {
- description = "the abstraction as a Gleam library, with its reader, dump and agreement check";
- mainProgram = "ocda-lib";
- };
- };
- };
- devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ];
- };
- };
-}
-#+END_SRC
-
-** README.md
-
-#+HEADER: :tangle ../gleam/README.md
-#+HEADER: :noweb yes
-#+BEGIN_SRC md
-# ocda-lib, in Gleam
-
-The abstraction's model and both its loaders, as a Gleam library, with its
-reader and the two programs that put it to the question.
-
-| | |
-|------------------|-----------------------------------------------------------------------------------------|
-| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
-| `ocda/load_ssp` | text -> model |
-| `ocda/load_db` | sqlite -> model |
-| `ocda/load` | what a path is, and loading it |
-| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
-| `ocda/dump` | the model written out canonically, as an oracle |
-| `read/main` | `ocda-read`, the reader |
-| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
-
-## Use
-
-```gleam
-import ocda/load
-import ocda/model
-
-case load.load(path, "en", False) {
- Error(why) -> io.println_error(why)
- Ok(doc) -> {
- // and from here a writer neither knows nor cares which of the two
- // files it was handed
- list.each(model.section(doc, "body"), fn(obj) { ... })
- }
-}
-```
-
-## The reader
-
-```sh
-nix build
-./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
- <file>.ssp|<file>.ocda.db|<file>.zip [lang]
-```
-
-`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
-had, over the library, so each reads every form this library reads; the
-others are the D reader's. **Its output is the D `ocda-read`'s, byte for
-byte**, in every mode both have. In `src/read/`, because Gleam compiles
-nothing outside `src/`; a program depending on the library compiles it too,
-and does not use it.
-
-A `.sisupod` it refuses, saying why: the zstd frame, below.
-
-## The two programs
-
-```sh
-nix build
-./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
-./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
-```
-
-`agree` asks whether the two loaders give the same model. `dump` writes
-the whole model out canonically, and is the stronger check of the two,
-because it is **byte for byte what the D library's `ocda-dump` writes**.
-So the four questions can all be asked with `diff`:
-
-| | |
-|-----------------------------|-----------------------------|
-| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
-| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
-| `.ssp`: D == Gleam | 36 of 36 |
-| `.ocda.db`: D == Gleam | 36 of 36 |
-
-## Deliberately a mirror of the D library
-
-Field for field, name for name, including where that is not the most
-Gleamish choice:
-
-- `text` is one `String` with the lines joined by newline, not a
- `List(String)`. So an object whose only text line is blank is
- indistinguishable from an object with no text, *in both languages, in the
- same place*. A difference that existed in one and not the other would be
- worse than a loss both share, because it would show up in every dump and
- mean nothing.
-- the properties are typed fields rather than a list of pairs. A writer
- wants `obj.parent_ocn`, not a lookup that might miss.
-
-The point is that the two agree. Making them agree is much cheaper when
-they are shaped alike, and the shape is the specification's, not either
-language's.
-
-## What the agreement found
-
-Four disagreements, none of which a comparison of section counts or of a
-handful of fields would have shown. All four are recorded in the D
-library's history; the short version:
-
-1. `.identifier` and `.anchor` are omitted by the text form when they can
- be worked out (the identifier is the ocn, the anchor is the segment).
- Both loaders now derive them. Over one document that is 1,314
- identifiers.
-2. `make` and `doc_has` are one metadata table in the database,
- distinguished by a key prefix. Both loaders unprefix them.
-3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
- as `\/` and every heading with a slash in it came back wrong.
-4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
- which is one language. It is a fact about the file, so neither loader
- puts it in the document.
-
-## Build
-
-```sh
-nix build # sandboxed, no network
-nix develop -c gleam test
-```
-
-Shipped as an Erlang shipment rather than an escript, because sqlite on the
-BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
-The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
-one and the other must change with it.
-
-## Format
-
-Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
-`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
-two serialisations. At format **2.x**.
-#+END_SRC
-
-** gleam.toml
-
-#+HEADER: :tangle ../gleam/gleam.toml
-#+HEADER: :noweb yes
-#+BEGIN_SRC toml
-name = "ocda_lib"
-version = "0.1.0"
-description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
-licences = ["AGPL-3.0-or-later"]
-
-[dependencies]
-gleam_stdlib = ">= 1.0.0 and < 2.0.0"
-argv = ">= 1.1.0 and < 2.0.0"
-simplifile = ">= 2.7.0 and < 3.0.0"
-sqlight = ">= 1.0.0 and < 2.0.0"
-gleam_json = ">= 3.0.0 and < 4.0.0"
-
-[dev-dependencies]
-gleeunit = ">= 1.0.0 and < 2.0.0"
-#+END_SRC
-
* versions GET
** dlang overlays
*** ldc
diff --git a/org/d-ocda.org b/org/d-ocda.org
deleted file mode 100644
index c4510af..0000000
--- a/org/d-ocda.org
+++ /dev/null
@@ -1,1854 +0,0 @@
--*- mode: org -*-
-#+TITLE: dlang load
-#+DESCRIPTION: dlang lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* dlang
-** model
-
-#+HEADER: :tangle "../dlang/src/ocda/model.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.model;
-
-struct OcdaImage {
- string name; // image filename as referenced by the object
- string sha256; // hex-encoded (uppercase) digest of the file
- ulong bytes; // file size
- int width; // file's own pixel width (not display width)
- int height; // file's own pixel height
- bool missing; // file absent or unreadable, no sha256/bytes
- string[string] fields; // any further/unrecognised key:value fields
-}
-
-struct OcdaObject {
- int ocn;
- string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb
- string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb
- string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings)
- string identifier; // heading: from the declaration line; otherwise .identifier
- string part; // frontmatter, body, backmatter
- string is_of_type; // para, block, comment: how the text behaves
- string own_section; // .section, when it differs from the block it sits in
- string segment_html_is;// the html segment this object sits in
- string segment_epub; // the epub segment this heading opens
- string segment_epub_is;// the epub segment this object sits in
- string table_aligns; // l c r per column
- string[] stow_link; // links extracted from the object
- string[] anchor_tags; // named targets the markup placed on this object
- int parent_ocn;
- int last_descendant_ocn;
- int ocn_range_first;// a container (a poem): the first and last
- int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1)
- int[] children; // direct child heading OCNs
- int[] ancestors; // markedup_ancestors[8]
- bool dummy_heading;
- bool ocn_off;
- int indent_base;
- int indent_hang;
- bool bullet;
- string lang;
- bool has_links;
- bool has_notes_reg;
- bool has_notes_star;
- bool has_images;
- string segment;
- string segment_prev;
- string segment_next;
- string anchor;
- string heading_lev_anchor; // a heading's own named target
- int table_cols;
- string table_widths;
- bool table_header;
- string code_syntax;
- bool code_linenumbers;
- string heading_ancestors_text;
- string sha256;
- OcdaImage[] images; // repeatable .image records, in order
- string text; // content (lines joined with newline)
-
- /+ ↓ whether this object is rendered at all.
- is_of_type "comment" is markup commentary: carried by the
- abstraction so that nothing is lost, and not put into output. An
- output that renders it leaks the author notes into the document,
- which is what spine itself does not do.
- +/
- bool renders() const {
- return is_of_type != "comment";
- }
-
- /+ ↓ the section this object belongs to, which is the block it sits in
- unless it says otherwise +/
- string sectionOf() const {
- return (own_section.length > 0) ? own_section : section;
- }
-}
-
-/+ ↓ what the reader met and did not understand.
- A reader that silently drops what it does not know cannot say whether
- it understood the document, and cannot be used to check a
- specification against an implementation: silence and understanding
- look alike. So the parser counts what it ignored, and
- --report-unknown prints it.
-+/
-struct OcdaUnknown {
- size_t[string] property; // .name -> times seen
- size_t[string] block; // @name -> times seen
- size_t[string] line; // first char -> times seen, lines of no known shape
-
- bool any() const {
- return property.length > 0 || block.length > 0 || line.length > 0;
- }
-}
-
-struct OcdaDocument {
- string format_version; // as declared on line 1, "" when absent
- string source_filename; // from the "% Source:" line
- string[string] source; // @source: language, languages, digest
- OcdaUnknown unknown; // what was met and not understood
- string[string] metadata; // key -> value
- string[string] make; // key -> value
- string[string] doc_has; // key -> value
- OcdaObject[] head;
- OcdaObject[] toc;
- OcdaObject[] body;
- OcdaObject[] endnotes;
- OcdaObject[] glossary;
- OcdaObject[] bibliography;
- OcdaObject[] bookindex;
- OcdaObject[] blurb;
- OcdaObject[] tail;
-
- /+ ↓ the major part of the declared version, "" when none was declared +/
- string formatVersionMajor() const {
- import std.string : indexOf;
- if (format_version.length == 0) { return ""; }
- auto dot = format_version.indexOf('.');
- return (dot > 0) ? format_version[0 .. dot] : format_version;
- }
-
- string meta(string key) {
- if (auto p = key in metadata) return *p;
- return "";
- }
-
- OcdaObject[]* sectionByName(string name) {
- switch (name) {
- case "head": return &head;
- case "toc": return &toc;
- case "body": return &body;
- case "endnotes": return &endnotes;
- case "glossary": return &glossary;
- case "bibliography": return &bibliography;
- case "bookindex": return &bookindex;
- case "blurb": return &blurb;
- case "tail": return &tail;
- default: return null;
- }
- }
-}
-
-/+ ↓ the format major version this reader knows.
- The .ssp and the .ocda.db carry one version between them, declared in
- band: the .ssp on its first line, the database in its schema.version
- row. The minor part is raised when a property is added, and a reader of
- the newer format still reads the older one, so only the major part is
- checked here. Raise this when the reader has been brought to a new
- major, not before: the point of the gate is to refuse what has not been
- looked at.
-+/
-enum ssp_format_major_read = "2";
-#+END_SRC
-
-** load
-** what to load
-*** inventory
-
-#+HEADER: :tangle "../dlang/src/ocda/inventory.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.inventory;
-
-/+ ↓ what a source holds, beyond the one document the model is loaded from.
- .
- The model is one language of one document. Two questions a reader asks
- are about the file instead: which languages it holds, and, for a
- database, what else it carries. They are here rather than in a reader
- because the answer depends on the input form, which is the library's
- to know, and so that a reader asking them does not learn which form it
- was handed.
- .
- languages() answers for every form, and answers the same: the
- languages in the order the manifest gave them (which every .ssp
- records, in @source languages), each with the markup file it was made
- from and the sha256 of its .ssp. The database stores that digest; for a
- .ssp and a pod it is taken from the bytes, which are the same bytes.
- .
- carried() is the database's: a .ocda.db holds the markup, images,
- configuration and catalogues it was built from, a .ssp holds none of
- them, and a pod's are its archive entries rather than anything the
- abstraction says. So it answers for a database and says so otherwise.
-+/
-
-import std.algorithm : endsWith, countUntil;
-import std.array : array, split;
-import std.conv : to;
-import std.digest : toHexString, LetterCase;
-import std.digest.sha : sha256Of;
-import std.file : read;
-import std.string : strip;
-import ocda.model;
-import ocda.load;
-import ocda.load_ssp;
-import ocda.load_db;
-import ocda.load_pod;
-import ocda.sqlite;
-import ocda.zip;
-
-struct OcdaLanguage {
- string lang;
- string source_filename; // the markup file the abstraction was made from
- string ssp_digest; // sha256 of that language's .ssp, upper case hex
-}
-
-struct OcdaCarried {
- string role; // source, image, conf, manifest, tools
- string compression; // how it is stored: none, zstd
- string count;
- string bytes; // the files' own size, whatever the storage
- string stored; // what they take in the database
-}
-
-OcdaLanguage[] languages(string path) {
- final switch (sourceOf(path)) {
- case Source.ocda_db:
- auto db = Db.open(path);
- scope(exit) { db.close(); }
- OcdaLanguage[] out_;
- foreach (l; dbLanguages(db)) {
- out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest);
- }
- return out_;
- case Source.ssp:
- auto bytes = cast(ubyte[]) read(path);
- auto doc = parseSsp(cast(string) bytes.idup);
- return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ];
- case Source.pod:
- auto archive = archiveBytes(path);
- OcdaLanguage[string] found;
- string[] order;
- foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) {
- auto bytes = zipRead(archive, e);
- auto doc = parseSsp(cast(string) bytes.idup);
- string lang = langOfSspName(e.name);
- found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes));
- if (order.length == 0) {
- order = sourceValue(doc, "languages").split;
- }
- }
- /+ ↓ the manifest's order, and anything it did not name after it +/
- OcdaLanguage[] out_;
- foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } }
- foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } }
- return out_;
- case Source.unknown:
- throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown));
- }
-}
-
-/+ ↓ what a database carries, by role and storage; empty for any other form,
- which isCarrier() says in advance +/
-bool isCarrier(string path) {
- return sourceOf(path) == Source.ocda_db;
-}
-
-OcdaCarried[] carried(string path) {
- if (!isCarrier(path)) { return []; }
- auto db = Db.open(path);
- scope(exit) { db.close(); }
- if (!db.has("files")) { return []; }
- OcdaCarried[] out_;
- foreach (r; db.rows(
- "SELECT role, compression, count(*), sum(bytes), sum(length(data))
- FROM files GROUP BY role, compression ORDER BY role")) {
- out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]);
- }
- return out_;
-}
-
-private string sourceValue(ref OcdaDocument doc, string key) {
- if (auto p = key in doc.source) { return *p; }
- return "";
-}
-
-private string hexOf(const(ubyte)[] bytes) {
- return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup;
-}
-#+END_SRC
-
-*** load
-**** load
-
-#+HEADER: :tangle "../dlang/src/ocda/load.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load;
-
-/+ ↓ what a path is, and loading it.
- .
- One place that decides, so that every reader and every writer accepts
- the same inputs and a new input type is added once. This mirrors
- spine's own ocda/abstraction/load.d, which names the sources it can
- enter the pipeline from and tells them apart by suffix.
- .
- Deliberately *not* here: .sst and .ssm markup. Reading markup means
- implementing the parser, which is spine's work and a far larger thing
- than a reader or a sample writer needs. This library reads
- abstractions.
-+/
-
-import std.algorithm : endsWith;
-import ocda.model;
-import ocda.load_ssp;
-import ocda.load_db;
-import ocda.load_pod;
-
-enum Source {
- ssp, // .ssp, the abstraction as text
- ocda_db, // .ocda.db, the abstraction as sqlite
- pod, // .sisupod or .zip, an archive carrying a .ssp
- unknown,
-}
-
-Source sourceOf(string path) {
- if (path.endsWith(".ocda.db")) { return Source.ocda_db; }
- if (path.endsWith(".ssp")) { return Source.ssp; }
- /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote
- before, and every pod already published carries that name, so both
- are read. What is inside decides how it is opened, not the name.
- +/
- if (path.endsWith(".sisupod")) { return Source.pod; }
- if (path.endsWith(".zip")) { return Source.pod; }
- if (path.endsWith(".db")) { return Source.ocda_db; }
- return Source.unknown;
-}
-
-string sourceName(Source s) {
- final switch (s) {
- case Source.ssp: return ".ssp (the abstraction as text)";
- case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)";
- case Source.pod: return "a pod archive carrying a .ssp";
- case Source.unknown: return "not an abstraction this library reads";
- }
-}
-
-class OcdaLoadError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ load whatever the path is into the model.
- lang names which language to take from a .ocda.db, which holds
- every language of its document, and from a pod, which may carry more
- than one .ssp. A .ssp is one language by construction and ignores it.
-+/
-OcdaDocument load(string path, string lang = "", bool any_version = false) {
- final switch (sourceOf(path)) {
- case Source.ssp: return parseSspFile(path);
- case Source.ocda_db: return loadDbFile(path, lang, any_version);
- case Source.pod: return loadPodFile(path, lang);
- case Source.unknown:
- throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)
- ~ " (expected .ssp, .ocda.db, .sisupod or .zip)");
- }
-}
-#+END_SRC
-
-**** load db
-
-#+HEADER: :tangle "../dlang/src/ocda/load_db.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_db;
-
-/+ ↓ build the model from a .ocda.db.
- .
- The point of this module is that it returns *the same value*
- ocda.load_ssp returns from the same document's .ssp. A writer is then
- a function from the model to bytes, and cannot tell which
- serialisation it was given, exactly as spine's own writers cannot.
- .
- Written from the sisudoc-abstraction-format specification,
- ocda-db-format-specification.md, whose section 4 gives the column by
- column correspondence this module implements.
-+/
-
-import std.array : join, split;
-import std.json : JSONType, parseJSON;
-import std.conv : to;
-import std.algorithm : startsWith;
-import std.string : strip;
-import ocda.model;
-import ocda.sqlite;
-
-/+ ↓ the schema major this loader knows (specification section 2) +/
-enum schema_major_read = "2";
-
-class OcdaDbError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ every language the database holds, in manifest order +/
-struct DbLanguage {
- int doc_id;
- string lang;
- string source_filename;
- string ssp_digest;
-}
-
-DbLanguage[] dbLanguages(ref Db db) {
- DbLanguage[] out_;
- foreach (r; db.rows(
- "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
- FROM documents ORDER BY id")) {
- out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]);
- }
- return out_;
-}
-
-/+ ↓ load one language of a database as the model.
- lang == "" takes the first language the manifest gave.
-+/
-OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) {
- auto db = Db.open(path);
- scope(exit) { db.close(); }
-
- if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) {
- throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;"
- ~ " it does not look like a .ocda.db");
- }
-
- /+ ↓ specification sections 2 and 3.2: the rows describing the file rather
- than a document in it are the only ones with doc_id null +/
- string schema_version = db.one(
- "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'");
- if (schema_version.length > 0 && !any_version) {
- auto major = schema_version.split(".")[0];
- if (major != schema_major_read) {
- throw new OcdaDbError(path ~ " is schema v" ~ schema_version
- ~ ", this loader knows v" ~ schema_major_read ~ ".x");
- }
- }
-
- auto languages = dbLanguages(db);
- if (languages.length == 0) {
- throw new OcdaDbError(path ~ ": no documents rows");
- }
- DbLanguage chosen = languages[0];
- if (lang.length > 0) {
- bool found = false;
- foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } }
- if (!found) {
- string have;
- foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; }
- throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have);
- }
- }
-
- OcdaDocument doc;
- doc.format_version = schema_version;
- doc.source_filename = chosen.source_filename;
-
- /+ ↓ @source of the .ssp corresponds to the documents row and the
- source.digest metadata (specification section 5.1 of the .ssp, and
- section 7 here) +/
- doc.source["language"] = chosen.lang;
- string languages_joined;
- foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; }
- doc.source["languages"] = languages_joined;
- auto digest = db.one("SELECT value FROM metadata WHERE doc_id = "
- ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'");
- if (digest.length > 0) { doc.source["digest"] = digest; }
-
- /+ ↓ @meta: every metadata row of this language that is not a source.*
- property. source.* describes the artefact rather than the document
- and lives in @source in the text form.
- +/
- foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
- ~ chosen.doc_id.to!string ~ " ORDER BY key")) {
- string key = r[0];
- if (key.startsWith("source.")) { continue; }
- /+ ↓ translation.ocn_aligned says whether the languages of this
- document share one numbering. A .ocda.db can know it because it
- holds every language; a .ssp is one language and cannot. So it is
- a fact about the *file* rather than about the document, and putting
- it in the document's metadata would let a writer tell which
- serialisation it was handed, which is the one thing this library
- exists to prevent. Left out until the model has a place for
- collection-wide facts.
- +/
- if (key.startsWith("translation.")) { continue; }
- /+ ↓ the database has one metadata table where the text form has three
- blocks, and tells them apart by a key prefix. Unprefixing them
- here is what makes the two loaders give one model: a writer asks
- doc.make["footer"] whichever serialisation it was handed, and
- never doc.metadata["make.footer"].
- +/
- if (key.startsWith("make.")) {
- doc.make[key["make.".length .. $]] = r[1].strip;
- } else if (key.startsWith("doc_has.")) {
- doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
- } else {
- doc.metadata[key] = r[1].strip;
- }
- }
-
- /+ ↓ 2.1 added the container range as two columns. A 2.0 database has
- neither, and reads as having no range, which is what 2.0 meant. +/
- bool has_range = false;
- foreach (c; db.rows("PRAGMA table_info(objects)")) {
- if (c[1] == "ocn_range_first") { has_range = true; }
- }
- string range_cols = has_range
- ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
- : "0, 0";
-
- /+ ↓ the objects, in document order within each section +/
- foreach (r; db.rows(
- "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''),
- coalesce(is_of_section,''), coalesce(is_of_type,''),
- coalesce(heading_level,-1), coalesce(identifier,''),
- coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
- coalesce(children,''), coalesce(ancestors,''),
- coalesce(dummy_heading,0), coalesce(object_number_off,0),
- coalesce(indent_base,0), coalesce(indent_hang,0),
- coalesce(bullet,0), coalesce(lang,''),
- coalesce(has_links,0), coalesce(has_notes_reg,0),
- coalesce(has_notes_star,0), coalesce(has_images,0),
- coalesce(segment,''), coalesce(segment_prev,''),
- coalesce(segment_next,''), coalesce(anchor,''),
- coalesce(segment_html_is,''),
- coalesce(table_cols,0), coalesce(table_widths,''),
- coalesce(table_aligns,''), coalesce(table_header,0),
- coalesce(code_linenumbers,0),
- coalesce(heading_ancestors_text,''), coalesce(sha256,''),
- coalesce(text,''), id, coalesce(heading_lev_anchor,''),
- coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ "
- FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
- ~ " ORDER BY CASE section
- WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
- WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5
- WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7
- WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) {
- OcdaObject obj;
- obj.section = r[0];
- obj.ocn = r[2].to!int;
- obj.is_a = r[3];
- obj.part = r[4];
- obj.own_section = (r[5] == r[0]) ? "" : r[5];
- obj.is_of_type = r[6];
- /+ ↓ the database holds the level as a number, the text form as the
- letter or digit the markup wrote. Model section 4 is the mapping. +/
- obj.heading_level = levelName(r[7].to!int);
- obj.identifier = r[8];
- obj.parent_ocn = r[9].to!int;
- obj.last_descendant_ocn = r[10].to!int;
- obj.children = jsonInts(r[11]);
- obj.ancestors = jsonInts(r[12]);
- obj.dummy_heading = (r[13] != "0");
- obj.ocn_off = (r[14] != "0");
- obj.indent_base = r[15].to!int;
- obj.indent_hang = r[16].to!int;
- obj.bullet = (r[17] != "0");
- obj.lang = r[18];
- obj.has_links = (r[19] != "0");
- obj.has_notes_reg = (r[20] != "0");
- obj.has_notes_star = (r[21] != "0");
- obj.has_images = (r[22] != "0");
- obj.segment = r[23];
- obj.segment_prev = r[24];
- obj.segment_next = r[25];
- obj.anchor = r[26];
- obj.segment_html_is = r[27];
- obj.heading_lev_anchor = r[37];
- obj.segment_epub = r[38];
- obj.segment_epub_is = r[39];
- obj.ocn_range_first = r[40].to!int;
- obj.ocn_range_last = r[41].to!int;
- obj.table_cols = r[28].to!int;
- obj.table_widths = jsonWords(r[29]);
- obj.table_aligns = jsonWords(r[30]);
- obj.table_header = (r[31] != "0");
- obj.code_linenumbers = (r[32] != "0");
- obj.heading_ancestors_text = jsonJoined(r[33]);
- obj.sha256 = r[34];
- obj.text = r[35];
-
- /+ ↓ the four repeatable properties of the text form are child tables
- here (specification section 4) +/
- string object_id = r[36];
- foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0),
- coalesce(width,0), coalesce(height,0),
- coalesce(missing,0)
- FROM object_images WHERE object_id = " ~ object_id
- ~ " ORDER BY seq")) {
- OcdaImage img;
- img.name = i[0];
- img.sha256 = i[1];
- img.bytes = i[2].to!ulong;
- img.width = i[3].to!int;
- img.height = i[4].to!int;
- img.missing = (i[5] != "0");
- obj.images ~= img;
- }
- foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = "
- ~ object_id ~ " ORDER BY seq")) {
- obj.stow_link ~= l[0];
- }
- foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = "
- ~ object_id ~ " ORDER BY seq")) {
- obj.anchor_tags ~= a[0];
- }
-
- auto section = doc.sectionByName(r[0]);
- if (section !is null) { *section ~= obj; }
- }
- countUnread(db, chosen.doc_id, doc);
- return doc;
-}
-
-/+ ↓ what the database holds and the model does not.
- .
- The text loader meets every property line and counts those it does not
- keep, so a reader can say what was read and not understood. This
- loader asks for columns by name, and would never see the ones it did
- not ask for: it would report that it had understood everything, which
- is the silence the count exists to prevent. So it counts them, on the
- conditions under which the .ssp writes the same property (specification
- section 6.3), and the report is the same from either form: checked,
- 36 of 36.
-+/
-private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) {
- string id = doc_id.to!string;
- void count(string property, string where) {
- auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id
- ~ " AND " ~ where).to!size_t;
- if (n > 0) { doc.unknown.property[property] = n; }
- }
- count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''");
- count("dom_status", "coalesce(dom_status,'') != ''");
- count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''");
- count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL");
- count("parent_lev", "coalesce(parent_lev,0) != 0");
- count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''");
- count("attrib", "coalesce(attrib,'') != ''");
- if (db.has("object_subtoc")) {
- auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o"
- ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t;
- if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; }
- }
-}
-
-/+ ↓ model section 4: the eight levels and their indices +/
-private string levelName(int index) {
- switch (index) {
- case 0: return "A";
- case 1: return "B";
- case 2: return "C";
- case 3: return "D";
- case 4: return "1";
- case 5: return "2";
- case 6: return "3";
- case 7: return "4";
- default: return "";
- }
-}
-
-/+ ↓ the JSON columns, read as JSON.
- .
- The fixed width arrays are JSON in the database and space separated
- (or "|" separated) in the text form. This was first written as a
- character scan that threw away brackets and quotes, which is wrong the
- moment a value holds an escape: Phobos' JSONValue.toString escapes
- "/" as "\/", so every heading containing a slash came back with a
- backslash in it. 36 of 36 documents differed on
- heading_ancestors_text and nothing else had shown it.
- .
- So: parse. The values are small, there are few of them, and a real
- parser is shorter than the scan it replaces.
-+/
-private string[] jsonStrings(string v) {
- if (v.length == 0) { return []; }
- string[] out_;
- try {
- auto j = parseJSON(v);
- if (j.type != JSONType.array) { return []; }
- foreach (e; j.array) {
- switch (e.type) {
- case JSONType.string: out_ ~= e.str; break;
- case JSONType.integer: out_ ~= e.integer.to!string; break;
- case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
- case JSONType.float_: out_ ~= e.floating.to!string; break;
- case JSONType.true_: out_ ~= "true"; break;
- case JSONType.false_: out_ ~= "false"; break;
- default: out_ ~= ""; break;
- }
- }
- } catch (Exception) {
- return [];
- }
- return out_;
-}
-
-private int[] jsonInts(string v) {
- int[] out_;
- foreach (s; jsonStrings(v)) {
- if (s.length == 0) { continue; }
- try { out_ ~= s.to!int; } catch (Exception) {}
- }
- return out_;
-}
-
-/+ ↓ space separated in the text form: table_widths, table_aligns +/
-private string jsonWords(string v) {
- return jsonStrings(v).join(" ");
-}
-
-/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
- form and a JSON array here. The empty slots are significant: they say
- which levels the heading sits under, so the separators carry as much
- as the text does.
-+/
-private string jsonJoined(string v) {
- auto parts = jsonStrings(v);
- if (parts.length == 0) { return ""; }
- return parts.join("|");
-}
-#+END_SRC
-
-**** load pod
-
-#+HEADER: :tangle "../dlang/src/ocda/load_pod.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_pod;
-
-/+ ↓ the abstraction out of a pod archive.
- .
- A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same
- archive unwrapped, which is what spine published before it wrapped them.
- Either way, a pod built with --pod2 carries the abstraction inside it,
- at media/abstraction/<uid>.<lang>.ssp, and that is what is read.
- .
- So this is a third input form and *not* a third model: the .ssp inside
- is parsed by ocda.load_ssp and comes back as the same value. A writer
- over the library gains .sisupod without being touched, which is the
- whole reason the dispatch is in one place.
- .
- A pod built without --pod2 carries markup and no abstraction. That is
- refused, and the refusal says which flag makes a pod this reader can
- read, because "no abstraction in it" is a fact about the pod rather
- than about the reader.
-+/
-
-import std.algorithm : endsWith, sort;
-import std.array : array;
-import std.file : read;
-import std.string : indexOf;
-import ocda.model;
-import ocda.load_ssp;
-import ocda.zip;
-import ocda.zstd;
-
-/+ ↓ a limit on what a frame may expand to.
- A reader of an artefact it did not make declines to be told how much
- memory to allocate. 512 MiB is far above the largest pod in the
- reference collection (4 MiB) and far below anything that matters.
-+/
-enum size_t max_pod_size = 512 * 1024 * 1024;
-
-class OcdaPodError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ which languages of the abstraction the pod carries, in sorted order +/
-string[] podLanguages(string path) {
- auto bytes = archiveBytes(path);
- string[] out_;
- foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) {
- string lang = langOfSspName(e.name);
- if (lang.length > 0) { out_ ~= lang; }
- }
- return out_.sort.array;
-}
-
-/+ ↓ load one language of the abstraction a pod carries.
- lang == "" takes the only one there is, and where there is a choice it
- says so rather than choosing, as the .ocda.db loader does.
-+/
-OcdaDocument loadPodFile(string path, string lang = "") {
- auto bytes = archiveBytes(path);
- auto entries = zipEntries(bytes);
- auto ssps = zipFindSuffix(entries, ".ssp");
- if (ssps.length == 0) {
- throw new OcdaPodError(path ~ ": carries no .ssp."
- ~ " A pod carries the abstraction only when spine was asked for it:"
- ~ " spine --pod2 puts it in, spine --source does not.");
- }
- ZipEntry chosen = ssps[0];
- if (lang.length > 0) {
- bool found = false;
- foreach (e; ssps) {
- if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; }
- }
- if (!found) {
- string have;
- foreach (i, e; ssps) {
- have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
- }
- throw new OcdaPodError(path ~ ": no language '" ~ lang
- ~ "'; it holds: " ~ have);
- }
- } else if (ssps.length > 1) {
- string have;
- foreach (i, e; ssps) {
- have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
- }
- throw new OcdaPodError(path ~ ": holds more than one language ("
- ~ have ~ "); name the one wanted");
- }
- auto ssp = zipRead(bytes, chosen);
- return parseSsp(cast(string) ssp.idup);
-}
-
-/+ ↓ the archive, unwrapped if it was wrapped.
- The bytes decide, not the name: a .sisupod is a zstd frame and a .zip
- is not, and a file misnamed either way still reads.
-+/
-package ubyte[] archiveBytes(string path) {
- auto bytes = cast(ubyte[]) read(path);
- return zstdIsFrame(bytes)
- ? zstdDecompress(bytes, max_pod_size)
- : bytes;
-}
-
-/+ ↓ <uid>.<lang>.ssp -> <lang> +/
-package string langOfSspName(string name) {
- if (!name.endsWith(".ssp")) { return ""; }
- string stem = name[0 .. $ - 4];
- auto dot = stem.lastIndexOf('.');
- return (dot > 0) ? stem[dot + 1 .. $] : "";
-}
-
-private ptrdiff_t lastIndexOf(string s, char c) {
- for (ptrdiff_t i = s.length - 1; i >= 0; i--) {
- if (s[i] == c) { return i; }
- }
- return -1;
-}
-#+END_SRC
-
-**** load ssp
-
-#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.load_ssp;
-
-import std.algorithm : splitter, startsWith;
-import std.array : array, split;
-import std.conv : to, ConvException;
-import std.file : readText;
-import std.stdio : writeln;
-import std.string : strip, stripLeft, indexOf;
-import ocda.model;
-
-OcdaDocument parseSspFile(string filename) {
- string content = readText(filename);
- return parseSsp(content);
-}
-
-OcdaDocument parseSsp(string content) {
- OcdaDocument doc;
- OcdaObject current_obj;
- bool in_object = false;
- /+ ↓ whether a text line has been seen for the object being read.
- Not "is the text non-empty": an object whose first text line is blank
- (a verse opening on an empty line, which the sample set has seven of)
- would then lose it, the next line overwriting rather than appending.
- +/
- bool seen_text = false;
- string current_section = "";
- string current_block = ""; // meta, make, doc_has, or section name
-
- foreach (line; content.splitter("\n")) {
- /+ ↓ comment. The first line of a .ssp declares the format version, and
- it is the one comment that carries meaning: a reader checks the
- major part and refuses what it does not know rather than loading
- an unfamiliar artefact half populated in silence.
- +/
- if (line.length > 0 && line[0] == '%') {
- enum _banner = "% SiSU Document Abstraction v";
- enum _source = "% Source: ";
- if (doc.format_version.length == 0 && line.startsWith(_banner)) {
- doc.format_version = line[_banner.length .. $].strip;
- } else if (doc.source_filename.length == 0 && line.startsWith(_source)) {
- doc.source_filename = line[_source.length .. $].strip;
- }
- continue;
- }
-
- /+ ↓ blank line - finalize current object +/
- if (line.length == 0 || line.strip.length == 0) {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- current_obj = OcdaObject.init;
- in_object = false;
- seen_text = false;
- }
- continue;
- }
-
- /+ ↓ section/block open +/
- if (line[0] == '@') {
- auto brace_pos = line.indexOf('{');
- if (brace_pos > 0) {
- string name = line[1 .. brace_pos].strip;
- current_block = name;
- if (!isPreambleBlock(name)) {
- current_section = name;
- /+ ↓ a block that is neither preamble nor a section this reader
- knows: counted, so that --report-unknown can say so +/
- if (doc.sectionByName(name) is null) {
- doc.unknown.block[name]++;
- }
- }
- }
- continue;
- }
-
- /+ ↓ section/block close +/
- if (line[0] == '}') {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- current_obj = OcdaObject.init;
- in_object = false;
- seen_text = false;
- }
- current_block = "";
- continue;
- }
-
- /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/
- if (isPreambleBlock(current_block)) {
- string stripped = line.stripLeft;
- auto colon_pos = stripped.indexOf(':');
- if (colon_pos > 0) {
- string key = stripped[0 .. colon_pos].strip;
- string value = (colon_pos + 1 < stripped.length)
- ? stripped[colon_pos + 1 .. $].strip : "";
- if (current_block == "meta") { doc.metadata[key] = value; }
- else if (current_block == "make") { doc.make[key] = value; }
- else if (current_block == "source") { doc.source[key] = value; }
- else { doc.doc_has[key] = value; }
- }
- continue;
- }
-
- /+ ↓ object declaration: [N] type ... +/
- if (line[0] == '[') {
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- }
- current_obj = OcdaObject.init;
- current_obj.section = current_section;
- in_object = true;
- seen_text = false;
- parseObjectDeclaration(line, current_obj);
- continue;
- }
-
- /+ ↓ property: .name: value +/
- if (line[0] == '.') {
- if (in_object) {
- parseProperty(line, current_obj, doc.unknown);
- }
- continue;
- }
-
- /+ ↓ text content: | content +/
- if (line.length >= 1 && line[0] == '|') {
- if (in_object) {
- string text_content = (line.length > 2) ? line[2 .. $] : "";
- if (seen_text) {
- current_obj.text ~= "\n" ~ text_content;
- } else {
- current_obj.text = text_content;
- seen_text = true;
- }
- }
- continue;
- }
-
- /+ ↓ no known shape. Counted by its first character so that a reader
- can be told the file held something this one did not expect
- +/
- doc.unknown.line[line[0 .. 1].idup]++;
- }
-
- /+ ↓ finalize last object +/
- if (in_object) {
- commitObject(doc, current_section, current_obj);
- }
-
- return doc;
-}
-
-private void parseObjectDeclaration(string line, ref OcdaObject obj) {
- /+ parse: [N] type :level identifier +/
- auto bracket_close = line.indexOf(']');
- if (bracket_close < 0) return;
- try {
- obj.ocn = line[1 .. bracket_close].strip.to!int;
- } catch (ConvException) {
- obj.ocn = 0;
- }
- string rest = (bracket_close + 1 < line.length)
- ? line[bracket_close + 1 .. $].strip : "";
- if (rest.length == 0) return;
-
- auto parts = rest.split(" ");
- if (parts.length == 0) return;
- obj.is_a = parts[0];
-
- if (obj.is_a == "heading" && parts.length >= 2) {
- /+ parse :A, :B, :1, etc. +/
- string lev = parts[1];
- if (lev.length >= 2 && lev[0] == ':') {
- obj.heading_level = lev[1 .. $];
- }
- /+ optional identifier after level +/
- if (parts.length >= 3) {
- obj.identifier = parts[2];
- }
- }
-}
-
-/+ ↓ the blocks that are document preamble rather than sections of objects +/
-private bool isPreambleBlock(string name) {
- return name == "source" || name == "meta" || name == "make" || name == "doc_has";
-}
-
-private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) {
- /+ parse: .name: value +/
- auto colon_pos = line.indexOf(':');
- if (colon_pos < 1) return;
- string name = line[1 .. colon_pos].strip;
- string value = (colon_pos + 1 < line.length)
- ? line[colon_pos + 1 .. $].strip : "";
-
- switch (name) {
- case "part": obj.part = value; break;
- case "is_of_type": obj.is_of_type = value; break;
- case "section": obj.own_section = value; break;
- case "identifier": obj.identifier = value; break;
- case "segment_html_is": obj.segment_html_is = value; break;
- /+ ↓ the epub segments at heading levels 1 to 4 where the html
- segments at level 4 alone, so the two are different files and
- the abstraction records both +/
- case "segment_epub": obj.segment_epub = value; break;
- case "segment_epub_is": obj.segment_epub_is = value; break;
- case "table_aligns": obj.table_aligns = value; break;
- case "stow_link": obj.stow_link ~= value; break;
- case "anchor_tag": obj.anchor_tags ~= value; break;
- case "parent": obj.parent_ocn = toIntSafe(value); break;
- case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break;
- case "ocn_range": {
- auto r = parseIntArray(value);
- if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; }
- break;
- }
- case "children": obj.children = parseIntArray(value); break;
- case "ancestors": obj.ancestors = parseIntArray(value); break;
- case "dummy": obj.dummy_heading = (value == "true"); break;
- case "ocn_off": obj.ocn_off = (value == "true"); break;
- case "indent":
- auto parts = value.split(" ");
- if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]);
- if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]);
- break;
- case "bullet": obj.bullet = (value == "true"); break;
- case "lang": obj.lang = value; break;
- case "has":
- foreach (flag; value.split(" ")) {
- switch (flag) {
- case "links": obj.has_links = true; break;
- case "notes_reg": obj.has_notes_reg = true; break;
- case "notes_star": obj.has_notes_star = true; break;
- case "images": obj.has_images = true; break;
- default: break;
- }
- }
- break;
- case "segment": obj.segment = value; break;
- case "segment_prev": obj.segment_prev = value; break;
- case "segment_next": obj.segment_next = value; break;
- case "anchor": obj.anchor = value; break;
- case "heading_lev_anchor": obj.heading_lev_anchor = value; break;
- case "table_cols": obj.table_cols = toIntSafe(value); break;
- case "table_widths": obj.table_widths = value; break;
- case "table_header": obj.table_header = (value == "true"); break;
- case "code_syntax": obj.code_syntax = value; break;
- case "code_linenumbers": obj.code_linenumbers = (value == "true"); break;
- case "heading_ancestors_text": obj.heading_ancestors_text = value; break;
- case "sha256": obj.sha256 = value; break;
- case "image": obj.images ~= parseImageRecord(value); break;
- /+ ↓ a property this reader has no use for, or has never heard of.
- Ignored, which is the conforming behaviour within a major
- version, but counted rather than passed over in silence
- +/
- default: unknown.property[name]++; break;
- }
-}
-
-private OcdaImage parseImageRecord(string value) {
- /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...]
- filename first, remaining fields order-independent; unrecognised
- fields are kept rather than discarded +/
- OcdaImage img;
- bool first = true;
- foreach (field; value.split(" ")) {
- if (field.length == 0) continue;
- if (first) {
- img.name = field;
- first = false;
- continue;
- }
- auto colon_pos = field.indexOf(':');
- if (colon_pos < 1) continue;
- string key = field[0 .. colon_pos];
- string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : "";
- switch (key) {
- case "sha256": img.sha256 = val; break;
- case "bytes": img.bytes = toUlongSafe(val); break;
- case "px":
- auto dims = val.split("x");
- if (dims.length == 2) {
- img.width = toIntSafe(dims[0]);
- img.height = toIntSafe(dims[1]);
- }
- break;
- case "missing": img.missing = (val == "true"); break;
- default: img.fields[key] = val; break;
- }
- }
- return img;
-}
-
-/+ ↓ two values the text form leaves out because they can be worked out.
- .
- The .ssp emits a property only when it carries information, so
- .identifier is absent when it is simply the ocn, and .anchor is absent
- when it is the segment name. The .ocda.db, being columns rather than
- lines, stores both unconditionally. So a loader that takes the text
- form literally gives a *different model* from the one the database
- gives, and the difference is not small: in the reference collection
- the identifier is omitted on all but a handful of objects.
- .
- The model is the thing both serialisations describe, so the loader
- fills them in. Found by dumping both models and diffing them, which no
- amount of comparing section counts would have shown.
-+/
-private void deriveOmitted(ref OcdaObject obj) {
- import std.conv : to;
- if (obj.identifier.length == 0 && obj.ocn != 0) {
- obj.identifier = obj.ocn.to!string;
- }
- if (obj.anchor.length == 0 && obj.segment.length > 0) {
- obj.anchor = obj.segment;
- }
-}
-
-private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) {
- deriveOmitted(obj);
- auto sec = doc.sectionByName(section);
- if (sec !is null) {
- ,*sec ~= obj;
- }
-}
-
-private int toIntSafe(string s) {
- try { return s.strip.to!int; }
- catch (ConvException) { return 0; }
-}
-
-private ulong toUlongSafe(string s) {
- try { return s.strip.to!ulong; }
- catch (ConvException) { return 0; }
-}
-
-private int[] parseIntArray(string s) {
- int[] result;
- foreach (part; s.split(" ")) {
- auto v = toIntSafe(part);
- result ~= v;
- }
- return result;
-}
-#+END_SRC
-
-*** images
-
-#+HEADER: :tangle "../dlang/src/ocda/images.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.images;
-
-/+ ↓ an image's bytes, from whichever source the document came from.
- .
- The model *names* images, with digest, byte size and pixel size; it does
- not hold them. That is right - an abstraction of a document is not a
- bundle of its assets - but a writer of any format that shows images
- needs the bytes, so something has to know where they are. This is that
- something, and it is in the library rather than in a writer because the
- answer depends on the *input form* and not on the output.
- .
- Three answers, and one refusal:
- .
- .ocda.db the files table, role 'image', by name
- .sisupod the archive entry whose path ends in /image/<name>
- .zip the same
- .ssp beside it: a pod puts the abstraction in
- media/abstraction/ and its images in media/image/, so the
- path is documented rather than guessed
- .
- A .ssp that is not in a pod has nowhere to look, and that is reported as
- absent rather than as an error: a reader that cannot find an image
- should say so and carry on, because the document is still readable.
-+/
-
-import std.algorithm : endsWith;
-import std.array : array;
-import std.file : exists, isFile, read;
-import std.path : buildPath, dirName;
-import ocda.load;
-import ocda.sqlite;
-import ocda.zip;
-import ocda.zstd;
-
-/+ ↓ the bytes, or an empty slice when the source has no such image +/
-ubyte[] imageBytes(string source, string name) {
- final switch (sourceOf(source)) {
- case Source.ocda_db: return fromDb(source, name);
- case Source.pod: return fromArchive(source, name);
- case Source.ssp: return fromBeside(source, name);
- case Source.unknown: return [];
- }
-}
-
-/+ ↓ can this source give images at all?
- A writer asks before it decides whether to emit them, because an <img>
- pointing at a file it did not include is worse than no image.
-+/
-bool canGiveImages(string source) {
- final switch (sourceOf(source)) {
- case Source.ocda_db: return true;
- case Source.pod: return true;
- case Source.ssp: return imageDirBeside(source).length > 0;
- case Source.unknown: return false;
- }
-}
-
-private ubyte[] fromDb(string path, string name) {
- auto db = Db.open(path);
- scope(exit) { db.close(); }
- if (!db.has("files")) { return []; }
- /+ ↓ the name is from the abstraction, not from a caller, and the quote
- doubling is what keeps it a value rather than syntax +/
- return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '"
- ~ sqlQuote(name) ~ "'");
-}
-
-private ubyte[] fromArchive(string path, string name) {
- auto bytes = cast(ubyte[]) read(path);
- if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); }
- auto entries = zipEntries(bytes);
- foreach (e; entries) {
- if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); }
- }
- return [];
-}
-
-private ubyte[] fromBeside(string path, string name) {
- string dir = imageDirBeside(path);
- if (dir.length == 0) { return []; }
- string file = buildPath(dir, name);
- return file.exists && file.isFile ? cast(ubyte[]) read(file) : [];
-}
-
-/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/
-private string imageDirBeside(string path) {
- string dir = path.dirName;
- if (!dir.endsWith("abstraction")) { return ""; }
- string image_dir = buildPath(dir.dirName, "image");
- return image_dir.exists ? image_dir : "";
-}
-
-/+ ↓ a single quote doubled is how sql says a single quote +/
-private string sqlQuote(string s) {
- string out_;
- foreach (c; s) {
- out_ ~= (c == '\'') ? "''" : [c];
- }
- return out_;
-}
-#+END_SRC
-
-** sqlite
-
-#+HEADER: :tangle "../dlang/src/ocda/sqlite.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.sqlite;
-
-/+ ↓ the sqlite3 C API, declared rather than depended upon.
- .
- A binding from the dub registry would be a dependency to fetch, and
- the point of this reader is that it builds in a sandbox with no
- network. The dozen functions a reader needs are declared here and the
- library is linked directly, which costs a page of extern(C) and saves
- a dependency.
-+/
-
-import std.conv : to;
-import std.string : fromStringz, toStringz;
-
-extern (C) {
- struct sqlite3;
- struct sqlite3_stmt;
-
- int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs);
- int sqlite3_close(sqlite3* db);
- int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte,
- sqlite3_stmt** stmt, const(char)** tail);
- int sqlite3_step(sqlite3_stmt* stmt);
- int sqlite3_finalize(sqlite3_stmt* stmt);
- int sqlite3_column_count(sqlite3_stmt* stmt);
- const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col);
- const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col);
- int sqlite3_column_int(sqlite3_stmt* stmt, int col);
- int sqlite3_column_type(sqlite3_stmt* stmt, int col);
- const(char)* sqlite3_errmsg(sqlite3* db);
- const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col);
- int sqlite3_column_bytes(sqlite3_stmt* stmt, int col);
-}
-
-enum SQLITE_OK = 0;
-enum SQLITE_ROW = 100;
-enum SQLITE_DONE = 101;
-enum SQLITE_NULL = 5;
-enum SQLITE_OPEN_READONLY = 0x00000001;
-
-class SqliteError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ a database, opened read only.
- Read only is deliberate: a reader has no business writing, and a
- request that opens a database read-write can create an empty one where
- a path is mistyped.
-+/
-struct Db {
- private sqlite3* _handle;
-
- static Db open(string path) {
- sqlite3* handle;
- auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null);
- if (rc != SQLITE_OK) {
- string msg = (handle is null)
- ? "cannot open " ~ path
- : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup;
- if (handle !is null) { sqlite3_close(handle); }
- throw new SqliteError(msg);
- }
- Db db;
- db._handle = handle;
- return db;
- }
-
- void close() {
- if (_handle !is null) { sqlite3_close(_handle); _handle = null; }
- }
-
- /+ ↓ every row of a query, each row as its columns in order +/
- string[][] rows(string sql) {
- sqlite3_stmt* stmt;
- auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null);
- if (rc != SQLITE_OK) {
- throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql);
- }
- scope(exit) { sqlite3_finalize(stmt); }
- string[][] out_rows;
- int cols = sqlite3_column_count(stmt);
- while (sqlite3_step(stmt) == SQLITE_ROW) {
- string[] row;
- foreach (c; 0 .. cols) {
- row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL)
- ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup;
- }
- out_rows ~= row;
- }
- return out_rows;
- }
-
- /+ ↓ the first column of the first row, "" when there is none +/
- string one(string sql) {
- auto r = rows(sql);
- return (r.length > 0 && r[0].length > 0) ? r[0][0] : "";
- }
-
- /+ ↓ the first column of the first row as bytes, for a blob.
- Separate from one(), because a blob is not text: an image read as text
- stops at the first zero byte, and a png's ninth byte is zero.
- +/
- ubyte[] blob(string sql) {
- sqlite3_stmt* stmt;
- if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) {
- throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup
- ~ " [" ~ sql ~ "]");
- }
- scope(exit) { sqlite3_finalize(stmt); }
- if (sqlite3_step(stmt) != SQLITE_ROW) { return []; }
- int n = sqlite3_column_bytes(stmt, 0);
- if (n <= 0) { return []; }
- auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0);
- if (p is null) { return []; }
- return p[0 .. n].dup;
- }
-
- /+ ↓ does a table or view of this name exist +/
- bool has(string name) {
- return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0";
- }
-}
-#+END_SRC
-
-** compression
-*** dlang read zip
-
-#+HEADER: :tangle "../dlang/src/ocda/zip.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.zip;
-
-/+ ↓ enough of the zip format to find one file in a pod and read it.
- .
- Not a zip library. A pod archive holds a handful of entries, spine
- writes them **stored** (uncompressed) so that the outer zstd frame does
- the compressing once over the whole thing, and what is wanted from it
- here is one named entry. So: read the central directory, find the
- entry, return its bytes.
- .
- Deflate is *refused rather than mishandled*. Phobos can inflate, but a
- library that says it read an entry it in fact skipped is worse than one
- that says it cannot, and spine's own archives do not need it.
-+/
-
-import std.algorithm : endsWith;
-import std.conv : to;
-
-class OcdaZipError : Exception {
- this(string msg) { super(msg); }
-}
-
-struct ZipEntry {
- string name;
- uint method; // 0 stored, 8 deflate
- uint compressed;
- uint uncompressed;
- uint local_offset;
-}
-
-/+ ↓ the entries, from the central directory at the end of the file +/
-ZipEntry[] zipEntries(const(ubyte)[] bytes) {
- enum uint eocd_sig = 0x06054b50;
- enum uint cd_sig = 0x02014b50;
- if (bytes.length < 22) {
- throw new OcdaZipError("too short to be a zip archive");
- }
- /+ ↓ the end of central directory record is last, and may be followed by a
- comment of up to 65535 bytes, so it is searched for backwards +/
- ptrdiff_t eocd = -1;
- ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0;
- for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) {
- if (u32(bytes, i) == eocd_sig) { eocd = i; break; }
- }
- if (eocd < 0) {
- throw new OcdaZipError("no end of central directory record;"
- ~ " it does not look like a zip archive");
- }
- uint count = u16(bytes, eocd + 10);
- uint offset = u32(bytes, eocd + 16);
- ZipEntry[] out_;
- size_t p = offset;
- foreach (_; 0 .. count) {
- if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) {
- throw new OcdaZipError("central directory entry "
- ~ out_.length.to!string ~ " is malformed");
- }
- ZipEntry e;
- e.method = u16(bytes, p + 10);
- e.compressed = u32(bytes, p + 20);
- e.uncompressed = u32(bytes, p + 24);
- uint name_len = u16(bytes, p + 28);
- uint extra_len = u16(bytes, p + 30);
- uint cmnt_len = u16(bytes, p + 32);
- e.local_offset = u32(bytes, p + 42);
- e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup;
- out_ ~= e;
- p += 46 + name_len + extra_len + cmnt_len;
- }
- return out_;
-}
-
-/+ ↓ one entry's bytes, from its local header +/
-ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) {
- enum uint local_sig = 0x04034b50;
- size_t p = e.local_offset;
- if (p + 30 > bytes.length || u32(bytes, p) != local_sig) {
- throw new OcdaZipError(e.name ~ ": local header is malformed");
- }
- uint name_len = u16(bytes, p + 26);
- uint extra_len = u16(bytes, p + 28);
- size_t start = p + 30 + name_len + extra_len;
- size_t end = start + e.compressed;
- if (end > bytes.length) {
- throw new OcdaZipError(e.name ~ ": runs past the end of the archive");
- }
- if (e.method != 0) {
- throw new OcdaZipError(e.name ~ ": compression method "
- ~ e.method.to!string ~ "; this reader takes only stored entries,"
- ~ " which is what spine writes");
- }
- return bytes[start .. end].dup;
-}
-
-/+ ↓ the first entry whose name ends this way, or none.
- Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid.
-+/
-ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) {
- ZipEntry[] out_;
- foreach (e; entries) {
- if (e.name.endsWith(suffix)) { out_ ~= e; }
- }
- return out_;
-}
-
-private uint u16(const(ubyte)[] b, size_t i) {
- return cast(uint) b[i] | (cast(uint) b[i + 1] << 8);
-}
-
-private uint u32(const(ubyte)[] b, size_t i) {
- return cast(uint) b[i]
- | (cast(uint) b[i + 1] << 8)
- | (cast(uint) b[i + 2] << 16)
- | (cast(uint) b[i + 3] << 24);
-}
-#+END_SRC
-
-*** dlang read zstd
-
-#+HEADER: :tangle "../dlang/src/ocda/zstd.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.zstd;
-
-/+ ↓ libzstd, declared rather than bound.
- .
- A .sisupod is one zstd frame wrapping a zip archive, so reading one
- needs exactly two things from the library: is this a frame, and what is
- in it. Both are declared here and the C library is linked, which is the
- same arrangement ocda.sqlite uses: no vendored tree, no generated
- header, and the prototypes say what is being called.
-+/
-
-extern (C) @nogc nothrow private {
- size_t ZSTD_decompress(void* dst, size_t dstCapacity,
- const(void)* src, size_t compressedSize);
- ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
- uint ZSTD_isError(size_t code);
- const(char)* ZSTD_getErrorName(size_t code);
-}
-
-/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/
-private enum ulong contentSizeUnknown = ulong.max;
-private enum ulong contentSizeError = ulong.max - 1;
-
-/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/
-private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD];
-
-class OcdaZstdError : Exception {
- this(string msg) { super(msg); }
-}
-
-/+ ↓ does this begin with a zstd frame?
- Asked of the bytes rather than of the filename, because what is inside
- decides how a thing is opened. A .sisupod is a frame; a .zip published
- before spine wrapped them is not, and both are read.
-+/
-bool zstdIsFrame(const(ubyte)[] bytes) {
- if (bytes.length < zstd_magic.length) { return false; }
- foreach (i, b; zstd_magic) {
- if (bytes[i] != b) { return false; }
- }
- return true;
-}
-
-/+ ↓ one frame in, its contents out.
- max_size is a refusal rather than a buffer size: a frame declares how
- much it expands to, and a frame that claims more than the caller is
- willing to hold is not decompressed at all. A reader of an artefact it
- did not make wants that limit.
-+/
-ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) {
- if (!zstdIsFrame(bytes)) {
- throw new OcdaZstdError("not a zstd frame");
- }
- ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length);
- if (declared == contentSizeError) {
- throw new OcdaZstdError("zstd frame: could not read its content size");
- }
- if (declared == contentSizeUnknown) {
- throw new OcdaZstdError("zstd frame declares no content size;"
- ~ " this reader takes only frames that do");
- }
- if (declared > max_size) {
- import std.conv : to;
- throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string
- ~ " bytes, over the " ~ max_size.to!string ~ " byte limit");
- }
- auto out_ = new ubyte[](cast(size_t) declared);
- size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length);
- if (ZSTD_isError(got)) {
- import std.string : fromStringz;
- throw new OcdaZstdError("zstd: "
- ~ ZSTD_getErrorName(got).fromStringz.idup);
- }
- if (got != declared) {
- throw new OcdaZstdError("zstd frame expanded to a different size"
- ~ " than it declared");
- }
- return out_;
-}
-#+END_SRC
-
-** dump
-
-#+HEADER: :tangle "../dlang/src/ocda/dump.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module ocda.dump;
-
-/+ ↓ the model, written out canonically.
- .
- Not an output format: an *oracle*. Two libraries in two languages
- implement the same model from the same specification, and the cheap
- way to ask whether they agree is to have each write the whole loaded
- document as text and diff the two. That is only useful if the text is
- fully determined by the model, so:
- .
- - every line is "<key> <value>", the value running to end of line,
- and only one free-form value per line;
- - a field at its default is *omitted*, so the output says what the
- document has rather than what the struct declares;
- - dictionaries are emitted in sorted key order, because neither
- language's hash map has an order worth relying on;
- - sections come in the fixed order of the format, not
- alphabetically.
- .
- Anything that reads this to reconstruct a document has misunderstood
- it. Load the .ssp or the .ocda.db.
-+/
-
-import std.algorithm : sort, splitter;
-import std.array : appender, array, join;
-import std.conv : to;
-import ocda.model;
-
-/+ ↓ the nine sections, in format order +/
-static immutable string[] dump_sections = ["head", "toc", "body", "endnotes",
- "glossary", "bibliography", "bookindex", "blurb", "tail"];
-
-string dumpDocument(ref OcdaDocument doc) {
- auto out_ = appender!string;
-
- void kv(string key, string value) {
- if (value.length == 0) { return; }
- out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n";
- }
- void kn(string key, long value) {
- if (value == 0) { return; }
- kv(key, value.to!string);
- }
- void kb(string key, bool value) {
- if (!value) { return; }
- kv(key, "1");
- }
- void dict(string prefix, string[string] d) {
- foreach (key; d.keys.dup.sort) {
- out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
- out_ ~= d[key]; out_ ~= "\n";
- }
- }
-
- kv("version", doc.format_version);
- kv("source_filename", doc.source_filename);
- dict("source", doc.source);
- dict("meta", doc.metadata);
- dict("make", doc.make);
- dict("doc_has", doc.doc_has);
-
- foreach (name; dump_sections) {
- auto sec = doc.sectionByName(name);
- if (sec is null || (*sec).length == 0) { continue; }
- kv("section", name ~ " " ~ (*sec).length.to!string);
- }
-
- foreach (name; dump_sections) {
- auto sec = doc.sectionByName(name);
- if (sec is null) { continue; }
- foreach (i, obj; *sec) {
- out_ ~= "object "; out_ ~= name; out_ ~= " ";
- out_ ~= i.to!string; out_ ~= "\n";
- kn("ocn", obj.ocn);
- kv("is_a", obj.is_a);
- kv("own_section", obj.own_section);
- kv("level", obj.heading_level);
- kv("identifier", obj.identifier);
- kv("part", obj.part);
- kv("is_of_type", obj.is_of_type);
- kv("segment_html_is", obj.segment_html_is);
- kv("segment_epub", obj.segment_epub);
- kv("segment_epub_is", obj.segment_epub_is);
- kn("parent", obj.parent_ocn);
- kn("last_descendant", obj.last_descendant_ocn);
- kn("ocn_range_first", obj.ocn_range_first);
- kn("ocn_range_last", obj.ocn_range_last);
- kv("children", ints(obj.children));
- kv("ancestors", ints(obj.ancestors));
- kb("dummy", obj.dummy_heading);
- kb("ocn_off", obj.ocn_off);
- kn("indent_base", obj.indent_base);
- kn("indent_hang", obj.indent_hang);
- kb("bullet", obj.bullet);
- kv("lang", obj.lang);
- kb("has_links", obj.has_links);
- kb("has_notes_reg", obj.has_notes_reg);
- kb("has_notes_star", obj.has_notes_star);
- kb("has_images", obj.has_images);
- kv("segment", obj.segment);
- kv("segment_prev", obj.segment_prev);
- kv("segment_next", obj.segment_next);
- kv("anchor", obj.anchor);
- kv("heading_lev_anchor", obj.heading_lev_anchor);
- kn("table_cols", obj.table_cols);
- kv("table_widths", obj.table_widths);
- kv("table_aligns", obj.table_aligns);
- kb("table_header", obj.table_header);
- kv("code_syntax", obj.code_syntax);
- kb("code_linenumbers", obj.code_linenumbers);
- kv("heading_ancestors_text", obj.heading_ancestors_text);
- kv("sha256", obj.sha256);
- foreach (url; obj.stow_link) { kv("stow_link", url); }
- foreach (a; obj.anchor_tags) { kv("anchor_tag", a); }
- foreach (img; obj.images) {
- /+ ↓ the name last, because a filename may hold a space where
- none of the other fields can +/
- kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " "
- ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " "
- ~ (img.missing ? "1" : "0") ~ " " ~ img.name);
- foreach (key; img.fields.keys.dup.sort) {
- kv("image_field", key ~ " " ~ img.fields[key]);
- }
- }
- /+ ↓ text, when there is any. An object whose only text line is
- blank holds "" here, indistinguishable from no text at all,
- because the model joins the lines into one string. Both
- libraries lose the same distinction in the same place, which is
- what makes them comparable; it is recorded rather than hidden.
- +/
- if (obj.text.length > 0) {
- auto lines = obj.text.splitter("\n").array;
- kv("text", lines.length.to!string);
- foreach (line; lines) {
- out_ ~= "| "; out_ ~= line; out_ ~= "\n";
- }
- }
- }
- }
- return out_.data;
-}
-
-/+ ↓ what the reader met and did not understand, sorted. Separate from the
- document dump: it describes the reader, not the document, and the
- .ocda.db loader has nothing to report where the .ssp loader does.
-+/
-string dumpUnknown(ref OcdaDocument doc) {
- auto out_ = appender!string;
- void counts(string prefix, size_t[string] d) {
- foreach (key; d.keys.dup.sort) {
- out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
- out_ ~= d[key].to!string; out_ ~= "\n";
- }
- }
- counts("property", doc.unknown.property);
- counts("block", doc.unknown.block);
- counts("line", doc.unknown.line);
- return out_.data;
-}
-
-private string ints(int[] xs) {
- if (xs.length == 0) { return ""; }
- string[] parts;
- foreach (x; xs) { parts ~= x.to!string; }
- return parts.join(",");
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/d-read.org b/org/d-read.org
deleted file mode 100644
index 9652e21..0000000
--- a/org/d-read.org
+++ /dev/null
@@ -1,1732 +0,0 @@
--*- mode: org -*-
-#+TITLE: dlang load
-#+DESCRIPTION: dlang lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* dlang :dlang:
-** main
-
-#+HEADER: :tangle "../dlang/src/read/main.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-/++
- ocda-read - reads the object-centric document abstraction, in any of the
- forms it is written in, and says what is in it.
-
- ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]
-
- One reader where there were two: sisudoc-ssp-reader read a .ssp and
- sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is
- built on the library, so every mode reads every form, and a mode given the
- .ssp, the .ocda.db and the .sisupod of a document says the same thing of
- each: it is told what the document is, not which file it came from.
-
- Two things are about the file rather than the document, and say so:
- --files, what a .ocda.db carries besides the abstraction, and --validate,
- which checks a .ssp as text. Everything else is the model's.
-
- The output modules are the .ssp reader's, moved onto the library's model.
-+/
-module read.main;
-
-import std.algorithm : canFind, map;
-import std.array : array, join;
-import std.getopt;
-import std.stdio : writeln, writefln, stderr;
-import ocda.model;
-import ocda.load;
-import ocda.inventory;
-import read.tree;
-import read.html;
-import read.llm_text;
-import read.llm_jsonl;
-import read.images;
-import read.verify;
-import read.report_unknown;
-import read.validate;
-
-int main(string[] args) {
- bool do_tree = false;
- bool do_html = false;
- bool do_llm_text = false;
- bool do_llm_jsonl = false;
- bool do_images = false;
- bool do_verify = false;
- bool do_report_unknown = false;
- bool do_validate = false;
- bool do_languages = false;
- bool do_files = false;
- bool any_version = false;
- string image_dir = "";
- string output_file = "";
- string lang = "";
-
- GetoptResult helpInfo;
- try {
- helpInfo = getopt(args,
- "tree", "the headings, as a tree", &do_tree,
- "outline", "the same as --tree", &do_tree,
- "html", "the document as one html page", &do_html,
- "llm-text", "plain text with heading breadcrumbs", &do_llm_text,
- "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl,
- "images", "the image records: digest, size, pixels", &do_images,
- "verify", "check the image records against --image-dir", &do_verify,
- "image-dir", "the images to verify against", &image_dir,
- "languages", "the languages the file holds", &do_languages,
- "files", "what a .ocda.db carries besides the abstraction", &do_files,
- "report-unknown", "what was read and not understood", &do_report_unknown,
- "validate", "check a .ssp against its specification, as text", &do_validate,
- "lang", "=<code> the language to read (default: the first)", &lang,
- "output", "write the mode's output to a file", &output_file,
- "any-version", "read a format major this reader does not know", &any_version,
- );
- } catch (GetOptException e) {
- stderr.writeln("ocda-read: ", e.msg, "; --help lists the options");
- return 2;
- }
-
- if (helpInfo.helpWanted || args.length < 2) {
- defaultGetoptPrinter(
- "ocda-read - reads the document abstraction, in any of its forms\n\n"
- ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n",
- helpInfo.options,
- );
- return 0;
- }
-
- string input_file = args[1];
- if (lang.length == 0 && args.length > 2) { lang = args[2]; }
- bool lang_named = (lang.length > 0);
-
- /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not
- at all: there is no text form of a database to check, and a pod's
- .ssp is checked by taking it out +/
- if (do_validate && sourceOf(input_file) != Source.ssp) {
- stderr.writeln("--validate checks a .ssp as text, against the .ssp",
- " specification; ", input_file, " is ", sourceName(sourceOf(input_file)));
- return 2;
- }
-
- /+ ↓ which language. A .ocda.db and a pod may hold several; where none is
- named, every form reads the first in the manifest's order and says
- so, rather than the database taking one and the pod refusing +/
- OcdaLanguage[] held;
- OcdaDocument doc;
- try {
- held = languages(input_file);
- if (lang.length == 0 && held.length > 0) { lang = held[0].lang; }
- if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) {
- stderr.writeln("error: no language '", lang, "' in ", input_file,
- "; it holds: ", held.map!(l => l.lang).array.join(" "));
- return 2;
- }
- doc = load(input_file, lang, any_version);
- } catch (Exception e) {
- stderr.writeln("Error reading ", input_file, ": ", e.msg);
- return 2;
- }
-
- /+ ↓ the version gate: the major part, and refuse what is not known +/
- if (doc.format_version.length == 0) {
- stderr.writeln("warning: ", input_file, " declares no format version");
- } else if (doc.formatVersionMajor != ssp_format_major_read) {
- if (any_version) {
- stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
- ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
- } else {
- stderr.writeln("error: ", input_file, " is format v", doc.format_version,
- ", this reader knows v", ssp_format_major_read, ".x");
- stderr.writeln(" a major version differs: a property may have",
- " changed meaning or gone away.");
- stderr.writeln(" --any-version reads it regardless.");
- return 3;
- }
- }
-
- writeln("Read: ", input_file);
- if (doc.format_version.length > 0) {
- writeln(" Format: v", doc.format_version);
- }
- if (auto _lang = "language" in doc.source) {
- writeln(" Source: ", *_lang,
- ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
- ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
- }
- if (held.length > 1 && !lang_named) {
- writeln(" (reading ", lang, "; --lang=<code> for another)");
- }
- writeln(" Title: ", doc.meta("title.main"));
- writeln(" Author: ", doc.meta("creator.author"));
- writeln(" Sections: head=", doc.head.length,
- " toc=", doc.toc.length,
- " body=", doc.body.length,
- " endnotes=", doc.endnotes.length,
- " glossary=", doc.glossary.length,
- " bibliography=", doc.bibliography.length,
- " bookindex=", doc.bookindex.length,
- " blurb=", doc.blurb.length,
- " tail=", doc.tail.length);
- auto image_records = imageRecordCount(doc);
- if (image_records > 0) {
- writeln(" Images: ", image_records, " record",
- (image_records == 1) ? "" : "s");
- }
-
- if (do_languages) { printLanguages(held); }
- if (do_files) { printFiles(input_file); }
- if (do_tree) { outputTree(doc, output_file); }
- if (do_html) { outputHtmlScroll(doc, output_file); }
- if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); }
- if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); }
- if (do_images) { outputImages(doc, output_file); }
-
- int _verify_failures = 0;
- if (do_verify) {
- if (image_dir.length == 0) {
- stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
- return 2;
- }
- _verify_failures = verifyImages(doc, image_dir, output_file);
- }
-
- if (do_report_unknown) { reportUnknown(doc); }
-
- int _validation_findings = 0;
- if (do_validate) {
- auto _found = validateSspFile(input_file, ssp_format_major_read);
- reportValidation(input_file, _found);
- _validation_findings = cast(int) _found.length;
- }
-
- if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
- && !do_verify && !do_report_unknown && !do_validate
- && !do_languages && !do_files) {
- writeln("\n(this is the summary: --help lists the modes, each adding its output)");
- }
-
- return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
-}
-
-/+ ↓ the languages, each with the markup it was made from and its .ssp's
- digest: the same from every form, the database storing the digest and
- the others taking it from the bytes +/
-void printLanguages(OcdaLanguage[] held) {
- writeln();
- writeln(" lang source ssp digest");
- foreach (l; held) {
- writefln(" %-7s %-31s %s", l.lang, l.source_filename,
- (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest);
- }
-}
-
-/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries
- nothing, and a pod's files are its archive entries rather than anything
- the abstraction records, so for those it says so +/
-void printFiles(string path) {
- writeln();
- if (!isCarrier(path)) {
- writeln(" carried files: only a .ocda.db carries files; this is ",
- sourceName(sourceOf(path)));
- return;
- }
- auto rows = carried(path);
- if (rows.length == 0) { writeln(" carries no files"); return; }
- writeln(" carried files:");
- writeln(" role stored n original in the file");
- foreach (r; rows) {
- writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored);
- }
-}
-#+END_SRC
-
-** check
-*** read validate =.ssp=
-
-#+HEADER: :tangle "../dlang/src/read/validate.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.validate;
-
-/+ ↓ check a .ssp against the format specification.
- .
- Written from specs/sisudoc-abstraction-format/ssp-format-specification.md
- and from nothing else: the point of it is to be what the specification
- says, executed, so that a file can be held against the document rather
- than against an implementation. Each check names the section it comes
- from, so a disagreement can be taken back to the text.
-+/
-
-import std.algorithm : canFind, startsWith;
-import std.array : array, split;
-import std.conv : to;
-import std.file : readText;
-import std.stdio : writeln, writefln;
-import std.string : indexOf, strip;
-
-struct SspFinding {
- size_t line; // 1-based
- string section; // the specification section the rule is from
- string detail;
-}
-
-/+ ↓ specification section 6.3: the properties, in the order they are
- emitted. A reader may rely on the order, so a validator can check it +/
-private immutable string[] property_order = [
- "identifier", "part", "section", "parent", "last_descendant", "ocn_range",
- "children",
- "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed",
- "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type",
- "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent",
- "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns",
- "table_header", "code_linenumbers", "stow_link", "segment", "anchor",
- "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub",
- "segment_html_is", "segment_epub_is", "segment_lv4_is",
- "heading_ancestors_text", "lev4_subtoc", "anchor_tag",
-];
-
-/+ ↓ section 6.1: the sixteen kinds +/
-private immutable string[] kinds = [
- "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography",
- "comment", "verse", "glossary", "quote", "poem", "block", "table",
- "blurb", "group",
-];
-
-/+ ↓ section 4: the nine sections, in the order they appear +/
-private immutable string[] section_order = [
- "head", "toc", "body", "endnotes", "glossary", "bibliography",
- "bookindex", "blurb", "tail",
-];
-
-/+ ↓ section 4: the four preamble blocks, in the order they appear +/
-private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"];
-
-/+ ↓ section 6.1: the heading levels +/
-private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"];
-
-private int propertyRank(string name) {
- foreach (i, p; property_order) { if (p == name) { return cast(int) i; } }
- return -1;
-}
-
-SspFinding[] validateSspFile(string filename, string major_known) {
- SspFinding[] found;
- auto lines = readText(filename).split("\n");
-
- bool seen_banner = false;
- bool seen_source_line = false;
- string block = ""; // the block currently open, "" when none
- int preamble_at = -1; // how far through the preamble order
- int section_at = -1; // how far through the section order
- bool in_object = false;
- int prev_rank = -1;
- size_t doc_has_keys = 0;
- bool in_text = false; // text lines come after properties
-
- foreach (i, line; lines) {
- size_t no = i + 1;
-
- /+ ↓ section 2: the version, on the first line +/
- if (no == 1) {
- enum banner = "% SiSU Document Abstraction v";
- if (!line.startsWith(banner)) {
- found ~= SspFinding(no, "2", "first line is not the version banner");
- } else {
- seen_banner = true;
- string v = line[banner.length .. $].strip;
- auto dot = v.indexOf('.');
- string major = (dot > 0) ? v[0 .. dot] : v;
- if (major != major_known) {
- found ~= SspFinding(no, "2.1",
- "format major v" ~ major ~ ", this validator knows v" ~ major_known);
- }
- }
- continue;
- }
- if (no == 2) {
- if (!line.startsWith("% Source: ")) {
- found ~= SspFinding(no, "4", "second line is not \"% Source: \"");
- } else { seen_source_line = true; }
- continue;
- }
-
- if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; }
-
- /+ ↓ section 3: a comment is the file header only +/
- if (line[0] == '%') {
- found ~= SspFinding(no, "3", "a comment outside the two header lines");
- continue;
- }
-
- /+ ↓ block open +/
- if (line[0] == '@') {
- auto brace = line.indexOf(" {");
- if (brace < 1 || line[$ - 1] != '{') {
- found ~= SspFinding(no, "3", "block open is not \"@name {\"");
- continue;
- }
- string name = line[1 .. brace];
- if (preamble_order.canFind(name)) {
- int at = 0;
- foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } }
- if (at <= preamble_at) {
- found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order");
- }
- preamble_at = at;
- if (section_at >= 0) {
- found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section");
- }
- } else if (section_order.canFind(name)) {
- int at = 0;
- foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } }
- if (at <= section_at) {
- found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order");
- }
- section_at = at;
- } else {
- found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section");
- }
- block = name;
- doc_has_keys = 0;
- continue;
- }
-
- if (line[0] == '}') {
- if (line.length != 1) {
- found ~= SspFinding(no, "3", "block close carries more than \"}\"");
- }
- /+ ↓ section 5.4: ten keys, always +/
- if (block == "doc_has" && doc_has_keys != 10) {
- found ~= SspFinding(no, "5.4",
- "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten");
- }
- block = "";
- in_object = false;
- prev_rank = -1;
- continue;
- }
-
- /+ ↓ section 5: inside a preamble block, " key: value" +/
- if (preamble_order.canFind(block)) {
- if (!line.startsWith(" ") || line.indexOf(": ") < 3) {
- found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block);
- } else if (block == "doc_has") {
- doc_has_keys++;
- }
- continue;
- }
-
- /+ ↓ section 6.1: the object declaration line +/
- if (line[0] == '[') {
- auto close = line.indexOf("] ");
- if (close < 2) {
- found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\"");
- continue;
- }
- string ocn = line[1 .. close];
- foreach (c; ocn) {
- if (c < '0' || c > '9') {
- found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number");
- break;
- }
- }
- auto rest = line[close + 2 .. $].split(" ");
- if (rest.length == 0 || !kinds.canFind(rest[0])) {
- found ~= SspFinding(no, "6.1",
- "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds");
- } else if (rest[0] == "heading") {
- if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') {
- found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\"");
- } else if (!levels.canFind(rest[1][1 .. $])) {
- found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight");
- }
- }
- in_object = true;
- in_text = false;
- prev_rank = -1;
- continue;
- }
-
- /+ ↓ section 6.3: properties, known and in order +/
- if (line[0] == '.') {
- if (!in_object) {
- found ~= SspFinding(no, "6", "a property outside an object record");
- continue;
- }
- if (in_text) {
- found ~= SspFinding(no, "6", "a property after the object text began");
- }
- auto colon = line.indexOf(": ");
- if (colon < 2) {
- found ~= SspFinding(no, "3", "property is not \".name: value\"");
- continue;
- }
- string name = line[1 .. colon];
- int rank = propertyRank(name);
- if (rank < 0) {
- found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties");
- } else {
- /+ ↓ the four repeatable properties may recur at the same rank +/
- if (rank < prev_rank) {
- found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order");
- }
- prev_rank = rank;
- }
- continue;
- }
-
- /+ ↓ section 6.2: text lines +/
- if (line[0] == '|') {
- if (!in_object) {
- found ~= SspFinding(no, "6.2", "a text line outside an object record");
- }
- if (line.length < 2 || line[1] != ' ') {
- found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text");
- }
- in_text = true;
- continue;
- }
-
- found ~= SspFinding(no, "3", "line of no shape the specification defines");
- }
-
- if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); }
- if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); }
- return found;
-}
-
-void reportValidation(string filename, SspFinding[] found) {
- if (found.length == 0) {
- writeln("\nvalid: ", filename,
- " conforms to the .ssp specification as this validator reads it");
- return;
- }
- writefln("\n%d finding%s against the specification:",
- found.length, (found.length == 1) ? "" : "s");
- foreach (f; found) {
- if (f.line > 0) {
- writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail);
- } else {
- writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail);
- }
- }
-}
-#+END_SRC
-
-*** read verify images
-
-#+HEADER: :tangle "../dlang/src/read/verify.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.verify;
-
-import std.algorithm : sort;
-import std.array : array, join;
-import std.conv : to;
-import std.digest : toHexString;
-import std.digest.sha : SHA256, digest;
-import std.file : dirEntries, exists, isDir, SpanMode, getSize;
-import std.path : baseName, buildPath, extension;
-import std.stdio : writeln, File;
-import std.string : toLower;
-import ocda.model;
-import read.images : gatherImageRecords;
-
-/+ ↓ verification: recompute what the .image records claim, against the
- images as they stand in a directory (a pod's media/image).
-
- ssp-reader --verify --image-dir=<pod>/media/image doc.ssp
-
- Reports one line per distinct file:
-
- OK won_benkler_2_1.png
- MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D...
- ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory)
- PRESENT ffa.png (recorded missing, but the file is there)
- EXTRA unused.png (in directory, no record in this .ssp)
-
- Returns the number of failures. EXTRA is reported but is not a failure:
- a pod carries the images of every language version of a document, while
- one .ssp covers one language.
-+/
-int verifyImages(OcdaDocument doc, string image_dir, string output_file) {
- string[] lines;
- int failures = 0;
- int ok = 0;
- int mismatch = 0;
- int absent = 0;
- int extra = 0;
-
- if (!(image_dir.exists) || !(image_dir.isDir)) {
- writeln("ERROR: image directory not found: ", image_dir);
- return 1;
- }
- auto gathered = gatherImageRecords(doc);
- bool[string] recorded;
-
- foreach (e; gathered.entries) {
- recorded[e.name] = true;
- string pth = buildPath(image_dir, e.name);
- if (!(pth.exists)) {
- if (e.missing) {
- /+ recorded missing, and missing here too: consistent +/
- ok += 1;
- lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)";
- } else {
- absent += 1;
- failures += 1;
- lines ~= "ABSENT " ~ e.name
- ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)";
- }
- continue;
- }
- if (e.missing) {
- mismatch += 1;
- failures += 1;
- lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)";
- continue;
- }
- auto file_ = readFileStat(pth);
- string[] differs;
- if (file_.sha256 != e.sha256) {
- differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256;
- }
- if (file_.bytes != e.bytes) {
- differs ~= "bytes recorded " ~ e.bytes.to!string
- ~ " file " ~ file_.bytes.to!string;
- }
- /+ px is written only where the dimensions could be read, and read here
- only for the formats below, so compare only when both are known +/
- if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0
- && (file_.width != e.width || file_.height != e.height)) {
- differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string
- ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string;
- }
- if (differs.length > 0) {
- mismatch += 1;
- failures += 1;
- lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; ");
- } else {
- ok += 1;
- lines ~= "OK " ~ e.name;
- }
- }
-
- /+ ↓ files in the directory that no record names +/
- string[] in_dir;
- foreach (entry; dirEntries(image_dir, SpanMode.shallow)) {
- if (!(entry.isFile)) continue;
- string nm = entry.name.baseName;
- switch (nm.extension.toLower) {
- case ".png": case ".jpg": case ".jpeg": case ".gif":
- in_dir ~= nm;
- break;
- default: break;
- }
- }
- in_dir.sort;
- foreach (nm; in_dir) {
- if (nm !in recorded) {
- extra += 1;
- lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)";
- }
- }
-
- string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file"
- ~ ((gathered.entries.length == 1) ? "" : "s")
- ~ " in " ~ gathered.records.to!string ~ " record"
- ~ ((gathered.records == 1) ? "" : "s")
- ~ ", against " ~ image_dir;
- lines = [head, ""] ~ lines;
- lines ~= "";
- lines ~= "verify: " ~ ok.to!string ~ " ok, "
- ~ mismatch.to!string ~ " mismatch, "
- ~ absent.to!string ~ " absent, "
- ~ extra.to!string ~ " extra"
- ~ ((failures == 0) ? " PASS" : " FAIL");
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
- return failures;
-}
-
-private struct FileStat_ {
- string sha256;
- ulong bytes;
- int width;
- int height;
-}
-
-private FileStat_ readFileStat(string pth) {
- FileStat_ ret;
- auto f = File(pth, "rb");
- ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024));
- ret.sha256 = sha.toHexString.to!string;
- ret.bytes = pth.getSize;
- auto dim = readImageDimensions(pth);
- ret.width = dim[0];
- ret.height = dim[1];
- return ret;
-}
-
-/+ ↓ pixel dimensions from the file header, for the formats spine's markup
- admits (png, gif, jpg). Header read only, no decoding, and no dependency
- beyond phobos. Returns [0, 0] where the header cannot be read
-+/
-private int[2] readImageDimensions(string pth) {
- int[2] ret = [0, 0];
- ubyte[] head;
- try {
- auto f = File(pth, "rb");
- head = new ubyte[](64 * 1024);
- head = f.rawRead(head);
- } catch (Exception ex) {
- return ret;
- }
- if (head.length >= 24
- && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
- /+ png: IHDR width & height, big endian, at 16 and 20 +/
- ret[0] = beInt(head[16 .. 20]);
- ret[1] = beInt(head[20 .. 24]);
- return ret;
- }
- if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) {
- /+ gif: logical screen width & height, little endian, at 6 and 8 +/
- ret[0] = head[6] | (head[7] << 8);
- ret[1] = head[8] | (head[9] << 8);
- return ret;
- }
- if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) {
- /+ jpeg: walk the segments to a start of frame, height then width,
- big endian, at offsets 5 and 7 of the segment
- +/
- size_t i = 2;
- while (i + 9 < head.length) {
- if (head[i] != 0xff) { i += 1; continue; }
- ubyte marker = head[i + 1];
- if (marker == 0xd8 || marker == 0x01
- || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }
- if (marker == 0xff) { i += 1; continue; }
- size_t seg_len = (head[i + 2] << 8) | head[i + 3];
- bool is_sof = ((marker >= 0xc0 && marker <= 0xc3)
- || (marker >= 0xc5 && marker <= 0xc7)
- || (marker >= 0xc9 && marker <= 0xcb)
- || (marker >= 0xcd && marker <= 0xcf));
- if (is_sof) {
- ret[1] = (head[i + 5] << 8) | head[i + 6];
- ret[0] = (head[i + 7] << 8) | head[i + 8];
- return ret;
- }
- if (seg_len < 2) break;
- i += 2 + seg_len;
- }
- }
- return ret;
-}
-
-private int beInt(const(ubyte)[] b) {
- return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]);
-}
-#+END_SRC
-
-*** read report unknown
-
-#+HEADER: :tangle "../dlang/src/read/report_unknown.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.report_unknown;
-
-import std.algorithm : sort;
-import std.array : array;
-import std.stdio : writeln, writefln;
-import ocda.model;
-
-/+ ↓ say what was read and not understood.
- This is what makes the reader usable as a check on the format
- specification: a reader that ignores what it does not know is
- conforming, but one that cannot say what it ignored cannot tell you
- whether the specification it was written from was complete. Silence
- and understanding look the same from outside.
-+/
-void reportUnknown(ref OcdaDocument doc) {
- if (!doc.unknown.any) {
- writeln("\nread and understood: every property, block and line was known");
- return;
- }
- writeln("\nread and NOT understood:");
- if (doc.unknown.property.length > 0) {
- writeln(" properties ignored (", doc.unknown.property.length, " distinct):");
- foreach (name; doc.unknown.property.keys.sort) {
- writefln(" .%-26s %8d", name, doc.unknown.property[name]);
- }
- }
- if (doc.unknown.block.length > 0) {
- writeln(" blocks ignored (", doc.unknown.block.length, " distinct):");
- foreach (name; doc.unknown.block.keys.sort) {
- writefln(" @%-26s %8d", name, doc.unknown.block[name]);
- }
- }
- if (doc.unknown.line.length > 0) {
- writeln(" lines of no known shape, by first character:");
- foreach (c; doc.unknown.line.keys.sort) {
- writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]);
- }
- }
- writeln(" (ignoring these is conforming within a major version;",
- " the count is so that it is not silent)");
-}
-#+END_SRC
-
-** misc
-*** read tree
-
-#+HEADER: :tangle "../dlang/src/read/tree.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.tree;
-
-import std.conv : to;
-import std.stdio : writeln, writef, writefln, File;
-import std.array : replicate;
-import ocda.model;
-
-void outputTree(OcdaDocument doc, string output_file) {
- string[] lines;
-
- lines ~= "Document: " ~ doc.meta("title.main");
- if (doc.meta("title.subtitle").length > 0)
- lines ~= " " ~ doc.meta("title.subtitle");
- lines ~= "Author: " ~ doc.meta("creator.author");
- lines ~= "";
-
- /+ ↓ build OCN -> object lookup for body headings +/
- OcdaObject[int] headings;
- foreach (obj; doc.body) {
- if (obj.is_a == "heading" && obj.ocn > 0) {
- headings[obj.ocn] = obj;
- }
- }
-
- /+ ↓ find root heading (from head section, or first body heading) +/
- if (doc.head.length > 0 && doc.head[0].is_a == "heading") {
- auto root = doc.head[0];
- lines ~= formatHeading(root, 0);
- walkChildren(root, headings, lines, 1);
- }
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
-}
-
-private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) {
- foreach (child_ocn; parent.children) {
- if (auto p = child_ocn in headings) {
- lines ~= formatHeading(*p, depth);
- walkChildren(*p, headings, lines, depth + 1);
- }
- }
-}
-
-private string formatHeading(OcdaObject obj, int depth) {
- string indent = replicate(" ", depth);
- string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?";
- string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
- /+ ↓ strip internal markup from heading text for clean display +/
- string text = stripInternalMarkup(obj.text);
- return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text;
-}
-
-private string stripInternalMarkup(string text) {
- /+ ↓ simple removal of common internal markers for display +/
- import std.regex : replaceAll, regex;
- import std.array : replace;
- string result = text;
- result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
- .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
- .replace("┨", "").replace("┣", "")
- .replace("┙", " ").replace("┘", "\n");
- return result;
-}
-#+END_SRC
-
-*** read html
-
-#+HEADER: :tangle "../dlang/src/read/html.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.html;
-
-import std.array : replace, split, join, appender;
-import std.conv : to;
-import std.stdio : writeln, File;
-import std.string : strip, indexOf, startsWith;
-import ocda.model;
-
-void outputHtmlScroll(OcdaDocument doc, string output_file) {
- auto buf = appender!(string[]);
-
- buf ~= htmlHead(doc);
-
- /+ ↓ title +/
- if (doc.head.length > 0) {
- auto title_obj = doc.head[0];
- buf ~= `<div class="title">`;
- buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`;
- if (doc.meta("title.subtitle").length > 0)
- buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`;
- buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`;
- buf ~= `</div>`;
- }
-
- /+ ↓ table of contents +/
- if (doc.toc.length > 0) {
- buf ~= `<div class="toc">`;
- buf ~= `<h2>Table of Contents</h2>`;
- foreach (obj; doc.toc) {
- if (obj.is_a == "toc") {
- string text = convertInlineMarkup(obj.text);
- int indent = obj.indent_base;
- string cls = (indent > 4) ? "toc-sub" : "toc-entry";
- buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`;
- }
- }
- buf ~= `</div>`;
- buf ~= `<hr/>`;
- }
-
- /+ ↓ body +/
- buf ~= `<div class="body">`;
- foreach (obj; doc.body) {
- buf ~= renderObject(obj);
- }
- buf ~= `</div>`;
-
- /+ ↓ endnotes +/
- bool has_endnotes = false;
- foreach (obj; doc.endnotes) {
- if (obj.is_a == "endnote") { has_endnotes = true; break; }
- }
- if (has_endnotes) {
- buf ~= `<div class="endnotes">`;
- buf ~= `<hr/>`;
- buf ~= `<h2>Endnotes</h2>`;
- foreach (obj; doc.endnotes) {
- if (obj.is_a == "endnote") {
- string text = convertInlineMarkup(obj.text);
- string parent_ref = (obj.parent_ocn > 0)
- ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>`
- : "";
- buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`;
- }
- }
- buf ~= `</div>`;
- }
-
- /+ ↓ glossary +/
- if (doc.glossary.length > 0) {
- buf ~= `<div class="glossary">`;
- buf ~= `<hr/>`;
- buf ~= `<h2>Glossary</h2>`;
- foreach (obj; doc.glossary) {
- string text = convertInlineMarkup(obj.text);
- buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`;
- }
- buf ~= `</div>`;
- }
-
- /+ ↓ bookindex +/
- bool has_bookindex = false;
- foreach (obj; doc.bookindex) {
- if (obj.is_a == "bookindex") { has_bookindex = true; break; }
- }
- if (has_bookindex) {
- buf ~= `<div class="bookindex">`;
- buf ~= `<hr/>`;
- buf ~= `<h2>Book Index</h2>`;
- foreach (obj; doc.bookindex) {
- if (obj.is_a == "bookindex") {
- string text = convertInlineMarkup(obj.text);
- buf ~= `<p class="index-entry">` ~ text ~ `</p>`;
- }
- }
- buf ~= `</div>`;
- }
-
- buf ~= htmlTail();
-
- /+ ↓ write output +/
- string[] output = buf.data;
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; output) f.writeln(line);
- } else {
- foreach (line; output) writeln(line);
- }
-}
-
-private string[] renderObject(OcdaObject obj) {
- /+ ↓ markup commentary is carried by the abstraction and is not output.
- Rendering it would put the author notes into the document, which
- spine itself does not do
- +/
- if (!obj.renders) { return []; }
- string[] lines;
- string text = convertInlineMarkup(obj.text);
- string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off)
- ? ` id="o` ~ obj.ocn.to!string ~ `"`
- : "";
- string ocn_display = (obj.ocn > 0 && !obj.ocn_off)
- ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>`
- : "";
-
- switch (obj.is_a) {
- case "heading":
- string tag = headingTag(obj.heading_level);
- string seg_anchor = (obj.segment.length > 0)
- ? ` id="` ~ htmlEscape(obj.segment) ~ `"`
- : ocn_anchor;
- lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`;
- break;
- case "para":
- string indent_cls = "";
- if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`;
- else if (obj.bullet) indent_cls = ` class="bullet"`;
- lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`;
- break;
- case "code":
- lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`;
- break;
- case "table":
- lines ~= renderTable(obj, ocn_anchor);
- break;
- case "group":
- lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`;
- break;
- case "block":
- lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`;
- break;
- case "verse":
- lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`;
- break;
- case "quote":
- lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`;
- break;
- default:
- if (text.length > 0)
- lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`;
- break;
- }
- return lines;
-}
-
-private string headingTag(string level) {
- switch (level) {
- case "A": return "h1";
- case "B": return "h2";
- case "C": return "h3";
- case "D": return "h4";
- case "1": return "h3";
- case "2": return "h4";
- case "3": return "h5";
- case "4": return "h6";
- default: return "h4";
- }
-}
-
-private string renderTable(OcdaObject obj, string ocn_anchor) {
- string[] lines;
- lines ~= `<table` ~ ocn_anchor ~ `>`;
- /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/
- auto rows = obj.text.split("\n");
- bool first_row = obj.table_header;
- foreach (row; rows) {
- if (row.strip.length == 0) continue;
- string tag = first_row ? "th" : "td";
- lines ~= `<tr>`;
- foreach (cell; row.split("┊")) {
- lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`;
- }
- lines ~= `</tr>`;
- first_row = false;
- }
- lines ~= `</table>`;
- return lines.join("\n");
-}
-
-/+ ↓ convert spine internal Unicode markup to HTML +/
-string convertInlineMarkup(string text) {
- if (text.length == 0) return "";
- string result = htmlEscape(text);
-
- /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc.
- Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X
- The ┨┣ form is used within heading content +/
-
- /+ ↓ bold +/
- result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>");
- result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>");
-
- /+ ↓ emphasis +/
- result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>");
- result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>");
-
- /+ ↓ italic +/
- result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>");
- result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>");
-
- /+ ↓ underscore +/
- result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>");
- result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>");
-
- /+ ↓ superscript +/
- result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>");
- result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>");
-
- /+ ↓ subscript +/
- result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>");
- result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>");
-
- /+ ↓ monospace +/
- result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>");
-
- /+ ↓ links: ┥link text┝┤url├ +/
- result = replaceLinks(result);
-
- /+ ↓ footnote markers: 【N text】 +/
- result = replaceFootnotes(result);
-
- /+ ↓ line breaks +/
- result = result.replace("┘", "<br/>");
- result = result.replace("┙", "<br/>");
-
- /+ ↓ non-breaking spaces +/
- result = result.replace("░", "&nbsp;");
-
- /+ ↓ bullet markers +/
- result = result.replace("● ", "&#8226; ");
-
- /+ ↓ page break markers +/
- result = result.replace("┼", "").replace("┿", "").replace("╂", "");
-
- return result;
-}
-
-private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) {
- /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/
- string result = text;
- while (true) {
- auto open_pos = result.indexOf(open_mark);
- if (open_pos < 0) break;
- auto after_open = open_pos + open_mark.length;
- /+ ↓ look for closing ⑆X +/
- string close_mark = "⑆" ~ close_suffix;
- auto close_pos = indexOf(result[after_open .. $], close_mark);
- if (close_pos < 0) break;
- close_pos += after_open;
- string content = result[after_open .. close_pos];
- result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
- ~ result[close_pos + close_mark.length .. $];
- }
- return result;
-}
-
-private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) {
- /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/
- string result = text;
- string full_open = open_mark ~ bracket_open;
- while (true) {
- auto open_pos = result.indexOf(full_open);
- if (open_pos < 0) break;
- auto after_open = open_pos + full_open.length;
- auto close_pos = indexOf(result[after_open .. $], bracket_close);
- if (close_pos < 0) break;
- close_pos += after_open;
- string content = result[after_open .. close_pos];
- result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
- ~ result[close_pos + bracket_close.length .. $];
- }
- return result;
-}
-
-private string replaceLinks(string text) {
- /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/
- string result = text;
- while (true) {
- auto link_open = result.indexOf("┥");
- if (link_open < 0) break;
- auto link_mid = indexOf(result[link_open .. $], "┝┤");
- if (link_mid < 0) break;
- link_mid += link_open;
- auto link_close = indexOf(result[link_mid .. $], "├");
- if (link_close < 0) break;
- link_close += link_mid;
- string link_text = result[link_open + "┥".length .. link_mid];
- string url = result[link_mid + "┝┤".length .. link_close];
- /+ ↓ handle internal links (¤ prefix) +/
- if (url.length > 0 && url[0] == '¤') {
- url = url[("¤").length .. $];
- }
- /+ ↓ handle OCN-only links (#N) +/
- if (url.length > 0 && url[0] == '#') {
- url = "#o" ~ url[1 .. $];
- }
- result = result[0 .. link_open]
- ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>`
- ~ result[link_close + "├".length .. $];
- }
- return result;
-}
-
-private string replaceFootnotes(string text) {
- /+ ↓ replace 【N content】 with superscript note number +/
- string result = text;
- while (true) {
- auto fn_open = result.indexOf("【");
- if (fn_open < 0) break;
- auto fn_close = indexOf(result[fn_open .. $], "】");
- if (fn_close < 0) break;
- fn_close += fn_open;
- string fn_content = result[fn_open + "【".length .. fn_close];
- /+ ↓ extract note number (first token before space) +/
- auto space_pos = fn_content.indexOf(' ');
- string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content;
- result = result[0 .. fn_open]
- ~ `<sup class="fn">` ~ note_num ~ `</sup>`
- ~ result[fn_close + "】".length .. $];
- }
- return result;
-}
-
-string htmlEscape(string text) {
- return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
- .replace("\"", "&quot;");
-}
-
-private string[] htmlHead(OcdaDocument doc) {
- string[] lines;
- lines ~= `<!DOCTYPE html>`;
- lines ~= `<html>`;
- lines ~= `<head>`;
- lines ~= `<meta charset="utf-8"/>`;
- lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`;
- lines ~= `<style>`;
- lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`;
- lines ~= ` .title { text-align: center; margin-bottom: 2em; }`;
- lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`;
- lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`;
- lines ~= ` .toc { margin-bottom: 2em; }`;
- lines ~= ` .toc-entry { margin: 0.2em 0; }`;
- lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`;
- lines ~= ` .indent-1 { margin-left: 2em; }`;
- lines ~= ` .indent-2 { margin-left: 4em; }`;
- lines ~= ` .indent-3 { margin-left: 6em; }`;
- lines ~= ` .bullet { margin-left: 2em; }`;
- lines ~= ` .bullet:before { content: "\2022 "; }`;
- lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`;
- lines ~= ` .group { margin: 1em 2em; }`;
- lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`;
- lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`;
- lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`;
- lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`;
- lines ~= ` th { background: #f0f0f0; }`;
- lines ~= ` .endnote { font-size: 0.9em; }`;
- lines ~= ` .fn { color: #666; }`;
- lines ~= ` .index-entry { font-size: 0.9em; }`;
- lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`;
- lines ~= `</style>`;
- lines ~= `</head>`;
- lines ~= `<body>`;
- return lines;
-}
-
-private string[] htmlTail() {
- return [`</body>`, `</html>`];
-}
-#+END_SRC
-
-*** read images
-
-#+HEADER: :tangle "../dlang/src/read/images.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.images;
-
-import std.array : join;
-import std.conv : to;
-import std.stdio : writeln, File;
-import ocda.model;
-
-/+ ↓ the .image records of a document, gathered per distinct file, with the
- objects referencing each file, in order of first appearance +/
-struct ImageEntry {
- string name;
- string sha256;
- ulong bytes;
- int width;
- int height;
- bool missing;
- string[] refs;
- string[] other_sha256; // same filename, differing digest
-}
-
-struct ImageRecords {
- int records; // .image lines, an image referenced twice counts twice
- ImageEntry[] entries; // distinct files
-}
-
-ImageRecords gatherImageRecords(OcdaDocument doc) {
- ImageRecords ret;
- size_t[string] seen;
- foreach (section; sectionNames) {
- auto objs = doc.sectionByName(section);
- if (objs is null) continue;
- foreach (obj; *objs) {
- foreach (img; obj.images) {
- ret.records += 1;
- string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]";
- if (auto at = img.name in seen) {
- if (ret.entries[*at].sha256 != img.sha256
- && img.sha256.length > 0
- && ret.entries[*at].sha256.length > 0) {
- ret.entries[*at].other_sha256 ~= img.sha256;
- }
- ret.entries[*at].refs ~= _ref;
- } else {
- ImageEntry e_;
- e_.name = img.name;
- e_.sha256 = img.sha256;
- e_.bytes = img.bytes;
- e_.width = img.width;
- e_.height = img.height;
- e_.missing = img.missing;
- e_.refs = [_ref];
- seen[img.name] = ret.entries.length;
- ret.entries ~= e_;
- }
- }
- }
- }
- return ret;
-}
-
-/+ ↓ image manifest, line oriented and greppable:
-
- <sha256> <bytes> <px> <filename> refs: body[97] body[412]
- MISSING <filename> refs: body[840]
-
- The point of the record in the .ssp is verification (is the image
- shipped in this pod the image the abstraction was built from), so this
- output is the shape a verifier wants: one line per distinct file.
-+/
-void outputImages(OcdaDocument doc, string output_file) {
- auto gathered = gatherImageRecords(doc);
- string[] lines;
- int missing = 0;
- foreach (e; gathered.entries) {
- if (e.missing) missing += 1;
- }
- lines ~= "Images: " ~ gathered.records.to!string ~ " record"
- ~ ((gathered.records == 1) ? "" : "s")
- ~ ", " ~ gathered.entries.length.to!string ~ " distinct file"
- ~ ((gathered.entries.length == 1) ? "" : "s")
- ~ ", " ~ missing.to!string ~ " missing";
- lines ~= "";
- foreach (e; gathered.entries) {
- string px = (e.width > 0 && e.height > 0)
- ? e.width.to!string ~ "x" ~ e.height.to!string : "-";
- lines ~= (e.missing)
- ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ")
- : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px
- ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ");
- /+ ↓ one filename, more than one digest: the .ssp is inconsistent,
- or two distinct files share a name +/
- foreach (other; e.other_sha256) {
- lines ~= " WARNING differing digest for same filename: " ~ other;
- }
- }
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
-}
-
-/+ ↓ count of .image records in a document, for the parse summary +/
-int imageRecordCount(OcdaDocument doc) {
- return gatherImageRecords(doc).records;
-}
-
-package immutable string[] sectionNames = [
- "head", "toc", "body", "endnotes",
- "glossary", "bibliography", "bookindex", "blurb",
-];
-#+END_SRC
-
-** llm
-*** read llm text
-
-#+HEADER: :tangle "../dlang/src/read/llm_text.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.llm_text;
-
-import std.array : replace, split, join;
-import std.conv : to;
-import std.stdio : writeln, File;
-import std.string : strip, indexOf, stripRight;
-import ocda.model;
-
-/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption
- format:
- ## Document Title > Chapter > Section > [OCN]
-
- Clean paragraph text with inline markup stripped.
-+/
-void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) {
- string[] lines;
- string doc_title = doc.meta("title.main");
-
- /+ ↓ document header +/
- lines ~= "# " ~ doc_title;
- if (doc.meta("title.subtitle").length > 0)
- lines ~= "# " ~ doc.meta("title.subtitle");
- lines ~= "# " ~ doc.meta("creator.author");
- lines ~= "";
-
- /+ ↓ build heading text lookup from ancestors +/
- string[int] heading_text;
- foreach (obj; doc.head) {
- if (obj.is_a == "heading" && obj.ocn > 0)
- heading_text[obj.ocn] = stripMarkup(obj.text);
- }
- foreach (obj; doc.body) {
- if (obj.is_a == "heading")
- heading_text[obj.ocn] = stripMarkup(obj.text);
- }
-
- /+ ↓ body objects +/
- string current_breadcrumb = "";
- foreach (obj; doc.body) {
- if (obj.is_a == "heading") {
- current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj);
- lines ~= "";
- lines ~= "## " ~ current_breadcrumb;
- lines ~= "";
- continue;
- }
-
- /+ ↓ markup commentary is not content +/
- if (!obj.renders) { continue; }
-
- string text = stripMarkup(obj.text);
- if (text.strip.length == 0) continue;
-
- string ocn_ref = (obj.ocn > 0 && !obj.ocn_off)
- ? " [" ~ obj.ocn.to!string ~ "]"
- : "";
-
- switch (obj.is_a) {
- case "para":
- lines ~= text ~ ocn_ref;
- lines ~= "";
- break;
- case "code":
- lines ~= "```";
- lines ~= obj.text; /+ preserve raw text for code +/
- lines ~= "```";
- lines ~= "";
- break;
- case "table":
- lines ~= text ~ ocn_ref;
- lines ~= "";
- break;
- case "verse":
- lines ~= text ~ ocn_ref;
- lines ~= "";
- break;
- case "quote":
- lines ~= "> " ~ text ~ ocn_ref;
- lines ~= "";
- break;
- case "group":
- case "block":
- lines ~= text ~ ocn_ref;
- lines ~= "";
- break;
- default:
- if (text.length > 0) {
- lines ~= text ~ ocn_ref;
- lines ~= "";
- }
- break;
- }
- }
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
-}
-
-private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) {
- /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/
- string[] parts;
- if (ancestors_text.length > 0) {
- foreach (part; ancestors_text.split("|")) {
- string clean = stripMarkup(part).strip;
- if (clean.length > 0 && clean != doc_title) {
- /+ ↓ truncate very long heading text for breadcrumb +/
- if (clean.length > 60)
- clean = clean[0 .. 57] ~ "...";
- parts ~= clean;
- }
- }
- }
- /+ ↓ add the current heading itself +/
- string heading_text = stripMarkup(obj.text).strip;
- if (heading_text.length > 60)
- heading_text = heading_text[0 .. 57] ~ "...";
-
- string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
-
- if (parts.length > 0)
- return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref;
- else
- return doc_title ~ " > " ~ heading_text ~ ocn_ref;
-}
-
-/+ ↓ strip all internal Unicode markup to produce clean plain text +/
-string stripMarkup(string text) {
- if (text.length == 0) return "";
- string r = text;
-
- /+ ↓ links: ┥link text┝┤url├ -> link text +/
- while (true) {
- auto lo = r.indexOf("┥");
- if (lo < 0) break;
- auto lm = indexOf(r[lo .. $], "┝┤");
- if (lm < 0) break;
- lm += lo;
- auto lc = indexOf(r[lm .. $], "├");
- if (lc < 0) break;
- lc += lm;
- string link_text = r[lo + "┥".length .. lm];
- r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $];
- }
-
- /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/
- while (true) {
- auto fo = r.indexOf("【");
- if (fo < 0) break;
- auto fc = indexOf(r[fo .. $], "】");
- if (fc < 0) break;
- fc += fo;
- r = r[0 .. fo] ~ r[fc + "】".length .. $];
- }
-
- /+ ↓ font face markers +/
- r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
- .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
- .replace("⑆■", "")
- .replace("┨", "").replace("┣*", "").replace("┣!", "")
- .replace("┣/", "").replace("┣_", "").replace("┣^", "")
- .replace("┣,", "").replace("┣", "");
-
- /+ ↓ line breaks and special chars +/
- r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n");
- r = r.replace("░", " ");
- r = r.replace("● ", "- ");
- r = r.replace("┼", "").replace("┿", "").replace("╂", "");
-
- /+ ↓ table cell separators +/
- r = r.replace("┊", " | ");
-
- /+ ↓ clean up multiple spaces +/
- while (r.indexOf(" ") >= 0)
- r = r.replace(" ", " ");
-
- return r.strip;
-}
-#+END_SRC
-
-*** read llm jsonl
-
-#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.llm_jsonl;
-
-import std.array : split, join, replace;
-import std.conv : to;
-import std.stdio : writeln, File;
-import std.string : strip, indexOf, stripRight;
-import ocda.model;
-import read.llm_text : stripMarkup;
-
-/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
- one JSON object per line, each containing:
- doc, ocn, path (breadcrumb), type, text (clean),
- images (only where the object embeds one, so that a chunk whose
- text reads around a figure carries the figure's identity),
- index_terms (from bookindex entries associated with this object)
-+/
-void outputLlmJsonl(OcdaDocument doc, string output_file) {
- string[] lines;
- string doc_id = doc.meta("title.main");
- string doc_author = doc.meta("creator.author");
-
- /+ ↓ build bookindex -> OCN mapping for index_terms
- bookindex objects in the .ssp have parent_ocn linking to body objects
- but the index terms are in the text of bookindex objects, not
- directly attached to body objects. We use a simpler approach:
- headings carry heading_ancestors_text which gives context. +/
-
- /+ ↓ track current heading path +/
- string current_path = "";
-
- foreach (obj; doc.body) {
- if (obj.is_a == "heading") {
- current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
- /+ ↓ emit heading as a chunk too +/
- string heading_text = stripMarkup(obj.text);
- if (heading_text.strip.length > 0) {
- lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
- }
- continue;
- }
-
- /+ ↓ markup commentary is not content, and least of all is it a chunk
- to embed: it would be indexed as though the author had written it
- into the document
- +/
- if (!obj.renders) { continue; }
-
- string text = stripMarkup(obj.text);
- if (text.strip.length == 0) continue;
-
- lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
- }
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
-}
-
-private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
- return `{"doc":` ~ jsonStr(doc_id)
- ~ `,"author":` ~ jsonStr(doc_author)
- ~ `,"ocn":` ~ ocn.to!string
- ~ `,"path":` ~ jsonStr(path)
- ~ `,"type":` ~ jsonStr(type)
- ~ `,"text":` ~ jsonStr(text)
- ~ jsonImages(images)
- ~ `}`;
-}
-
-private string jsonImages(OcdaImage[] images) {
- /+ ↓ omitted entirely where the object embeds no image +/
- if (images.length == 0) return "";
- string[] items;
- foreach (img; images) {
- string item = `{"name":` ~ jsonStr(img.name);
- if (img.missing) {
- item ~= `,"missing":true`;
- } else {
- item ~= `,"sha256":` ~ jsonStr(img.sha256)
- ~ `,"bytes":` ~ img.bytes.to!string;
- if (img.width > 0 && img.height > 0) {
- item ~= `,"width":` ~ img.width.to!string
- ~ `,"height":` ~ img.height.to!string;
- }
- }
- items ~= item ~ `}`;
- }
- return `,"images":[` ~ items.join(",") ~ `]`;
-}
-
-private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
- string[] parts;
- if (ancestors_text.length > 0) {
- foreach (part; ancestors_text.split("|")) {
- string clean = stripMarkup(part).strip;
- if (clean.length > 0 && clean != doc_title) {
- parts ~= clean;
- }
- }
- }
- string heading_text = stripMarkup(obj.text).strip;
- if (parts.length > 0)
- return parts.join(" > ") ~ " > " ~ heading_text;
- else
- return heading_text;
-}
-
-private string jsonStr(string s) {
- /+ ↓ JSON string escaping +/
- string r = s;
- r = r.replace(`\`, `\\`);
- r = r.replace(`"`, `\"`);
- r = r.replace("\n", `\n`);
- r = r.replace("\r", `\r`);
- r = r.replace("\t", `\t`);
- return `"` ~ r ~ `"`;
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/d-test.org b/org/d-test.org
deleted file mode 100644
index d159819..0000000
--- a/org/d-test.org
+++ /dev/null
@@ -1,178 +0,0 @@
--*- mode: org -*-
-#+TITLE: dlang load
-#+DESCRIPTION: dlang lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* dlang :dlang:
-** tests
-*** test agree
-
-#+HEADER: :tangle "../dlang/test/agree.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-/++
- Do the two loaders agree?
-
- The library's whole purpose is that a writer cannot tell which
- serialisation it was handed. That is only true if loading a document's
- .ssp and loading its .ocda.db give the same model, so this asks.
-
- Usage: agree <doc>.ssp <doc>.ocda.db [lang]
-+/
-import std.conv : to;
-import std.stdio;
-import ocda.model;
-import ocda.load;
-
-int main(string[] args) {
- if (args.length < 3) {
- stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]");
- return 2;
- }
- string lang = (args.length > 3) ? args[3] : "";
- OcdaDocument a, b;
- try {
- a = load(args[1]);
- b = load(args[2], lang);
- } catch (Exception e) {
- stderr.writeln("error: ", e.msg);
- return 2;
- }
-
- int differences = 0;
- void differ(string what, string from_ssp, string from_db) {
- writeln(" differ: ", what);
- writeln(" .ssp: ", from_ssp);
- writeln(" .ocda.db: ", from_db);
- differences++;
- }
-
- if (a.format_version != b.format_version) {
- differ("format_version", a.format_version, b.format_version);
- }
- if (a.source_filename != b.source_filename) {
- differ("source_filename", a.source_filename, b.source_filename);
- }
-
- /+ ↓ the sections, object by object. Position within a section is the
- identity to compare on: an ocn is not unique and 0 is shared +/
- static immutable sections = ["head", "toc", "body", "endnotes", "glossary",
- "bibliography", "bookindex", "blurb", "tail"];
- foreach (name; sections) {
- auto sa = a.sectionByName(name);
- auto sb = b.sectionByName(name);
- if (sa is null || sb is null) { continue; }
- if ((*sa).length != (*sb).length) {
- differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string);
- continue;
- }
- foreach (i, oa; *sa) {
- auto ob = (*sb)[i];
- string at = name ~ "[" ~ i.to!string ~ "]";
- if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); }
- if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); }
- if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); }
- if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); }
- if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); }
- if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); }
- if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); }
- if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); }
- if (oa.ancestors != ob.ancestors) {
- differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string);
- }
- if (oa.images.length != ob.images.length) {
- differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string);
- }
- if (differences > 12) { writeln(" ... stopping after 12"); return 1; }
- }
- }
-
- if (differences == 0) {
- writeln("agree: the two loaders give the same model");
- return 0;
- }
- writeln(differences, " difference(s)");
- return 1;
-}
-#+END_SRC
-
-*** test dump
-
-#+HEADER: :tangle "../dlang/test/dump.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-/++
- Write a loaded abstraction out canonically.
-
- The oracle for everything that claims to implement this model: load a
- .ssp or a .ocda.db and print the model, deterministically, so that two
- implementations can be compared with diff.
-
- Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
-+/
-import std.stdio;
-import ocda.model;
-import ocda.load;
-import ocda.dump;
-
-int main(string[] args) {
- bool unknown = false;
- string[] rest;
- foreach (a; args[1 .. $]) {
- if (a == "--unknown") { unknown = true; } else { rest ~= a; }
- }
- if (rest.length < 1) {
- stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]");
- return 2;
- }
- string lang = (rest.length > 1) ? rest[1] : "";
- OcdaDocument doc;
- try {
- doc = load(rest[0], lang);
- } catch (Exception e) {
- stderr.writeln("error: ", e.msg);
- return 2;
- }
- write(unknown ? dumpUnknown(doc) : dumpDocument(doc));
- return 0;
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/d.org b/org/d.org
new file mode 100644
index 0000000..1eade53
--- /dev/null
+++ b/org/d.org
@@ -0,0 +1,3895 @@
+-*- mode: org -*-
+#+TITLE: dlang load
+#+DESCRIPTION: dlang lib
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* ocda
+** model
+
+#+HEADER: :tangle "../dlang/src/ocda/model.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.model;
+
+struct OcdaImage {
+ string name; // image filename as referenced by the object
+ string sha256; // hex-encoded (uppercase) digest of the file
+ ulong bytes; // file size
+ int width; // file's own pixel width (not display width)
+ int height; // file's own pixel height
+ bool missing; // file absent or unreadable, no sha256/bytes
+ string[string] fields; // any further/unrecognised key:value fields
+}
+
+struct OcdaObject {
+ int ocn;
+ string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb
+ string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb
+ string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings)
+ string identifier; // heading: from the declaration line; otherwise .identifier
+ string part; // frontmatter, body, backmatter
+ string is_of_type; // para, block, comment: how the text behaves
+ string own_section; // .section, when it differs from the block it sits in
+ string segment_html_is;// the html segment this object sits in
+ string segment_epub; // the epub segment this heading opens
+ string segment_epub_is;// the epub segment this object sits in
+ string table_aligns; // l c r per column
+ string[] stow_link; // links extracted from the object
+ string[] anchor_tags; // named targets the markup placed on this object
+ int parent_ocn;
+ int last_descendant_ocn;
+ int ocn_range_first;// a container (a poem): the first and last
+ int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1)
+ int[] children; // direct child heading OCNs
+ int[] ancestors; // markedup_ancestors[8]
+ bool dummy_heading;
+ bool ocn_off;
+ int indent_base;
+ int indent_hang;
+ bool bullet;
+ string lang;
+ bool has_links;
+ bool has_notes_reg;
+ bool has_notes_star;
+ bool has_images;
+ string segment;
+ string segment_prev;
+ string segment_next;
+ string anchor;
+ string heading_lev_anchor; // a heading's own named target
+ int table_cols;
+ string table_widths;
+ bool table_header;
+ string code_syntax;
+ bool code_linenumbers;
+ string heading_ancestors_text;
+ string sha256;
+ OcdaImage[] images; // repeatable .image records, in order
+ string text; // content (lines joined with newline)
+
+ /+ ↓ whether this object is rendered at all.
+ is_of_type "comment" is markup commentary: carried by the
+ abstraction so that nothing is lost, and not put into output. An
+ output that renders it leaks the author notes into the document,
+ which is what spine itself does not do.
+ +/
+ bool renders() const {
+ return is_of_type != "comment";
+ }
+
+ /+ ↓ the section this object belongs to, which is the block it sits in
+ unless it says otherwise +/
+ string sectionOf() const {
+ return (own_section.length > 0) ? own_section : section;
+ }
+}
+
+/+ ↓ what the reader met and did not understand.
+ A reader that silently drops what it does not know cannot say whether
+ it understood the document, and cannot be used to check a
+ specification against an implementation: silence and understanding
+ look alike. So the parser counts what it ignored, and
+ --report-unknown prints it.
++/
+struct OcdaUnknown {
+ size_t[string] property; // .name -> times seen
+ size_t[string] block; // @name -> times seen
+ size_t[string] line; // first char -> times seen, lines of no known shape
+
+ bool any() const {
+ return property.length > 0 || block.length > 0 || line.length > 0;
+ }
+}
+
+struct OcdaDocument {
+ string format_version; // as declared on line 1, "" when absent
+ string source_filename; // from the "% Source:" line
+ string[string] source; // @source: language, languages, digest
+ OcdaUnknown unknown; // what was met and not understood
+ string[string] metadata; // key -> value
+ string[string] make; // key -> value
+ string[string] doc_has; // key -> value
+ OcdaObject[] head;
+ OcdaObject[] toc;
+ OcdaObject[] body;
+ OcdaObject[] endnotes;
+ OcdaObject[] glossary;
+ OcdaObject[] bibliography;
+ OcdaObject[] bookindex;
+ OcdaObject[] blurb;
+ OcdaObject[] tail;
+
+ /+ ↓ the major part of the declared version, "" when none was declared +/
+ string formatVersionMajor() const {
+ import std.string : indexOf;
+ if (format_version.length == 0) { return ""; }
+ auto dot = format_version.indexOf('.');
+ return (dot > 0) ? format_version[0 .. dot] : format_version;
+ }
+
+ string meta(string key) {
+ if (auto p = key in metadata) return *p;
+ return "";
+ }
+
+ OcdaObject[]* sectionByName(string name) {
+ switch (name) {
+ case "head": return &head;
+ case "toc": return &toc;
+ case "body": return &body;
+ case "endnotes": return &endnotes;
+ case "glossary": return &glossary;
+ case "bibliography": return &bibliography;
+ case "bookindex": return &bookindex;
+ case "blurb": return &blurb;
+ case "tail": return &tail;
+ default: return null;
+ }
+ }
+}
+
+/+ ↓ the format major version this reader knows.
+ The .ssp and the .ocda.db carry one version between them, declared in
+ band: the .ssp on its first line, the database in its schema.version
+ row. The minor part is raised when a property is added, and a reader of
+ the newer format still reads the older one, so only the major part is
+ checked here. Raise this when the reader has been brought to a new
+ major, not before: the point of the gate is to refuse what has not been
+ looked at.
++/
+enum ssp_format_major_read = "2";
+#+END_SRC
+
+** load
+** what to load
+*** inventory
+
+#+HEADER: :tangle "../dlang/src/ocda/inventory.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.inventory;
+
+/+ ↓ what a source holds, beyond the one document the model is loaded from.
+ .
+ The model is one language of one document. Two questions a reader asks
+ are about the file instead: which languages it holds, and, for a
+ database, what else it carries. They are here rather than in a reader
+ because the answer depends on the input form, which is the library's
+ to know, and so that a reader asking them does not learn which form it
+ was handed.
+ .
+ languages() answers for every form, and answers the same: the
+ languages in the order the manifest gave them (which every .ssp
+ records, in @source languages), each with the markup file it was made
+ from and the sha256 of its .ssp. The database stores that digest; for a
+ .ssp and a pod it is taken from the bytes, which are the same bytes.
+ .
+ carried() is the database's: a .ocda.db holds the markup, images,
+ configuration and catalogues it was built from, a .ssp holds none of
+ them, and a pod's are its archive entries rather than anything the
+ abstraction says. So it answers for a database and says so otherwise.
++/
+
+import std.algorithm : endsWith, countUntil;
+import std.array : array, split;
+import std.conv : to;
+import std.digest : toHexString, LetterCase;
+import std.digest.sha : sha256Of;
+import std.file : read;
+import std.string : strip;
+import ocda.model;
+import ocda.load;
+import ocda.load_ssp;
+import ocda.load_db;
+import ocda.load_pod;
+import ocda.sqlite;
+import ocda.zip;
+
+struct OcdaLanguage {
+ string lang;
+ string source_filename; // the markup file the abstraction was made from
+ string ssp_digest; // sha256 of that language's .ssp, upper case hex
+}
+
+struct OcdaCarried {
+ string role; // source, image, conf, manifest, tools
+ string compression; // how it is stored: none, zstd
+ string count;
+ string bytes; // the files' own size, whatever the storage
+ string stored; // what they take in the database
+}
+
+OcdaLanguage[] languages(string path) {
+ final switch (sourceOf(path)) {
+ case Source.ocda_db:
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ OcdaLanguage[] out_;
+ foreach (l; dbLanguages(db)) {
+ out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest);
+ }
+ return out_;
+ case Source.ssp:
+ auto bytes = cast(ubyte[]) read(path);
+ auto doc = parseSsp(cast(string) bytes.idup);
+ return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ];
+ case Source.pod:
+ auto archive = archiveBytes(path);
+ OcdaLanguage[string] found;
+ string[] order;
+ foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) {
+ auto bytes = zipRead(archive, e);
+ auto doc = parseSsp(cast(string) bytes.idup);
+ string lang = langOfSspName(e.name);
+ found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes));
+ if (order.length == 0) {
+ order = sourceValue(doc, "languages").split;
+ }
+ }
+ /+ ↓ the manifest's order, and anything it did not name after it +/
+ OcdaLanguage[] out_;
+ foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } }
+ foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } }
+ return out_;
+ case Source.unknown:
+ throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown));
+ }
+}
+
+/+ ↓ what a database carries, by role and storage; empty for any other form,
+ which isCarrier() says in advance +/
+bool isCarrier(string path) {
+ return sourceOf(path) == Source.ocda_db;
+}
+
+OcdaCarried[] carried(string path) {
+ if (!isCarrier(path)) { return []; }
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ if (!db.has("files")) { return []; }
+ OcdaCarried[] out_;
+ foreach (r; db.rows(
+ "SELECT role, compression, count(*), sum(bytes), sum(length(data))
+ FROM files GROUP BY role, compression ORDER BY role")) {
+ out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]);
+ }
+ return out_;
+}
+
+private string sourceValue(ref OcdaDocument doc, string key) {
+ if (auto p = key in doc.source) { return *p; }
+ return "";
+}
+
+private string hexOf(const(ubyte)[] bytes) {
+ return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup;
+}
+#+END_SRC
+
+*** load
+**** load
+
+#+HEADER: :tangle "../dlang/src/ocda/load.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load;
+
+/+ ↓ what a path is, and loading it.
+ .
+ One place that decides, so that every reader and every writer accepts
+ the same inputs and a new input type is added once. This mirrors
+ spine's own ocda/abstraction/load.d, which names the sources it can
+ enter the pipeline from and tells them apart by suffix.
+ .
+ Deliberately *not* here: .sst and .ssm markup. Reading markup means
+ implementing the parser, which is spine's work and a far larger thing
+ than a reader or a sample writer needs. This library reads
+ abstractions.
++/
+
+import std.algorithm : endsWith;
+import ocda.model;
+import ocda.load_ssp;
+import ocda.load_db;
+import ocda.load_pod;
+
+enum Source {
+ ssp, // .ssp, the abstraction as text
+ ocda_db, // .ocda.db, the abstraction as sqlite
+ pod, // .sisupod or .zip, an archive carrying a .ssp
+ unknown,
+}
+
+Source sourceOf(string path) {
+ if (path.endsWith(".ocda.db")) { return Source.ocda_db; }
+ if (path.endsWith(".ssp")) { return Source.ssp; }
+ /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote
+ before, and every pod already published carries that name, so both
+ are read. What is inside decides how it is opened, not the name.
+ +/
+ if (path.endsWith(".sisupod")) { return Source.pod; }
+ if (path.endsWith(".zip")) { return Source.pod; }
+ if (path.endsWith(".db")) { return Source.ocda_db; }
+ return Source.unknown;
+}
+
+string sourceName(Source s) {
+ final switch (s) {
+ case Source.ssp: return ".ssp (the abstraction as text)";
+ case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)";
+ case Source.pod: return "a pod archive carrying a .ssp";
+ case Source.unknown: return "not an abstraction this library reads";
+ }
+}
+
+class OcdaLoadError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ load whatever the path is into the model.
+ lang names which language to take from a .ocda.db, which holds
+ every language of its document, and from a pod, which may carry more
+ than one .ssp. A .ssp is one language by construction and ignores it.
++/
+OcdaDocument load(string path, string lang = "", bool any_version = false) {
+ final switch (sourceOf(path)) {
+ case Source.ssp: return parseSspFile(path);
+ case Source.ocda_db: return loadDbFile(path, lang, any_version);
+ case Source.pod: return loadPodFile(path, lang);
+ case Source.unknown:
+ throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)
+ ~ " (expected .ssp, .ocda.db, .sisupod or .zip)");
+ }
+}
+#+END_SRC
+
+**** load db
+
+#+HEADER: :tangle "../dlang/src/ocda/load_db.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_db;
+
+/+ ↓ build the model from a .ocda.db.
+ .
+ The point of this module is that it returns *the same value*
+ ocda.load_ssp returns from the same document's .ssp. A writer is then
+ a function from the model to bytes, and cannot tell which
+ serialisation it was given, exactly as spine's own writers cannot.
+ .
+ Written from the sisudoc-abstraction-format specification,
+ ocda-db-format-specification.md, whose section 4 gives the column by
+ column correspondence this module implements.
++/
+
+import std.array : join, split;
+import std.json : JSONType, parseJSON;
+import std.conv : to;
+import std.algorithm : startsWith;
+import std.string : strip;
+import ocda.model;
+import ocda.sqlite;
+
+/+ ↓ the schema major this loader knows (specification section 2) +/
+enum schema_major_read = "2";
+
+class OcdaDbError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ every language the database holds, in manifest order +/
+struct DbLanguage {
+ int doc_id;
+ string lang;
+ string source_filename;
+ string ssp_digest;
+}
+
+DbLanguage[] dbLanguages(ref Db db) {
+ DbLanguage[] out_;
+ foreach (r; db.rows(
+ "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
+ FROM documents ORDER BY id")) {
+ out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]);
+ }
+ return out_;
+}
+
+/+ ↓ load one language of a database as the model.
+ lang == "" takes the first language the manifest gave.
++/
+OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+
+ if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) {
+ throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;"
+ ~ " it does not look like a .ocda.db");
+ }
+
+ /+ ↓ specification sections 2 and 3.2: the rows describing the file rather
+ than a document in it are the only ones with doc_id null +/
+ string schema_version = db.one(
+ "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'");
+ if (schema_version.length > 0 && !any_version) {
+ auto major = schema_version.split(".")[0];
+ if (major != schema_major_read) {
+ throw new OcdaDbError(path ~ " is schema v" ~ schema_version
+ ~ ", this loader knows v" ~ schema_major_read ~ ".x");
+ }
+ }
+
+ auto languages = dbLanguages(db);
+ if (languages.length == 0) {
+ throw new OcdaDbError(path ~ ": no documents rows");
+ }
+ DbLanguage chosen = languages[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } }
+ if (!found) {
+ string have;
+ foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; }
+ throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have);
+ }
+ }
+
+ OcdaDocument doc;
+ doc.format_version = schema_version;
+ doc.source_filename = chosen.source_filename;
+
+ /+ ↓ @source of the .ssp corresponds to the documents row and the
+ source.digest metadata (specification section 5.1 of the .ssp, and
+ section 7 here) +/
+ doc.source["language"] = chosen.lang;
+ string languages_joined;
+ foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; }
+ doc.source["languages"] = languages_joined;
+ auto digest = db.one("SELECT value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'");
+ if (digest.length > 0) { doc.source["digest"] = digest; }
+
+ /+ ↓ @meta: every metadata row of this language that is not a source.*
+ property. source.* describes the artefact rather than the document
+ and lives in @source in the text form.
+ +/
+ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " ORDER BY key")) {
+ string key = r[0];
+ if (key.startsWith("source.")) { continue; }
+ /+ ↓ translation.ocn_aligned says whether the languages of this
+ document share one numbering. A .ocda.db can know it because it
+ holds every language; a .ssp is one language and cannot. So it is
+ a fact about the *file* rather than about the document, and putting
+ it in the document's metadata would let a writer tell which
+ serialisation it was handed, which is the one thing this library
+ exists to prevent. Left out until the model has a place for
+ collection-wide facts.
+ +/
+ if (key.startsWith("translation.")) { continue; }
+ /+ ↓ the database has one metadata table where the text form has three
+ blocks, and tells them apart by a key prefix. Unprefixing them
+ here is what makes the two loaders give one model: a writer asks
+ doc.make["footer"] whichever serialisation it was handed, and
+ never doc.metadata["make.footer"].
+ +/
+ if (key.startsWith("make.")) {
+ doc.make[key["make.".length .. $]] = r[1].strip;
+ } else if (key.startsWith("doc_has.")) {
+ doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
+ } else {
+ doc.metadata[key] = r[1].strip;
+ }
+ }
+
+ /+ ↓ 2.1 added the container range as two columns. A 2.0 database has
+ neither, and reads as having no range, which is what 2.0 meant. +/
+ bool has_range = false;
+ foreach (c; db.rows("PRAGMA table_info(objects)")) {
+ if (c[1] == "ocn_range_first") { has_range = true; }
+ }
+ string range_cols = has_range
+ ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
+ : "0, 0";
+
+ /+ ↓ the objects, in document order within each section +/
+ foreach (r; db.rows(
+ "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''),
+ coalesce(is_of_section,''), coalesce(is_of_type,''),
+ coalesce(heading_level,-1), coalesce(identifier,''),
+ coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
+ coalesce(children,''), coalesce(ancestors,''),
+ coalesce(dummy_heading,0), coalesce(object_number_off,0),
+ coalesce(indent_base,0), coalesce(indent_hang,0),
+ coalesce(bullet,0), coalesce(lang,''),
+ coalesce(has_links,0), coalesce(has_notes_reg,0),
+ coalesce(has_notes_star,0), coalesce(has_images,0),
+ coalesce(segment,''), coalesce(segment_prev,''),
+ coalesce(segment_next,''), coalesce(anchor,''),
+ coalesce(segment_html_is,''),
+ coalesce(table_cols,0), coalesce(table_widths,''),
+ coalesce(table_aligns,''), coalesce(table_header,0),
+ coalesce(code_linenumbers,0),
+ coalesce(heading_ancestors_text,''), coalesce(sha256,''),
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ "
+ FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
+ ~ " ORDER BY CASE section
+ WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
+ WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5
+ WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7
+ WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) {
+ OcdaObject obj;
+ obj.section = r[0];
+ obj.ocn = r[2].to!int;
+ obj.is_a = r[3];
+ obj.part = r[4];
+ obj.own_section = (r[5] == r[0]) ? "" : r[5];
+ obj.is_of_type = r[6];
+ /+ ↓ the database holds the level as a number, the text form as the
+ letter or digit the markup wrote. Model section 4 is the mapping. +/
+ obj.heading_level = levelName(r[7].to!int);
+ obj.identifier = r[8];
+ obj.parent_ocn = r[9].to!int;
+ obj.last_descendant_ocn = r[10].to!int;
+ obj.children = jsonInts(r[11]);
+ obj.ancestors = jsonInts(r[12]);
+ obj.dummy_heading = (r[13] != "0");
+ obj.ocn_off = (r[14] != "0");
+ obj.indent_base = r[15].to!int;
+ obj.indent_hang = r[16].to!int;
+ obj.bullet = (r[17] != "0");
+ obj.lang = r[18];
+ obj.has_links = (r[19] != "0");
+ obj.has_notes_reg = (r[20] != "0");
+ obj.has_notes_star = (r[21] != "0");
+ obj.has_images = (r[22] != "0");
+ obj.segment = r[23];
+ obj.segment_prev = r[24];
+ obj.segment_next = r[25];
+ obj.anchor = r[26];
+ obj.segment_html_is = r[27];
+ obj.heading_lev_anchor = r[37];
+ obj.segment_epub = r[38];
+ obj.segment_epub_is = r[39];
+ obj.ocn_range_first = r[40].to!int;
+ obj.ocn_range_last = r[41].to!int;
+ obj.table_cols = r[28].to!int;
+ obj.table_widths = jsonWords(r[29]);
+ obj.table_aligns = jsonWords(r[30]);
+ obj.table_header = (r[31] != "0");
+ obj.code_linenumbers = (r[32] != "0");
+ obj.heading_ancestors_text = jsonJoined(r[33]);
+ obj.sha256 = r[34];
+ obj.text = r[35];
+
+ /+ ↓ the four repeatable properties of the text form are child tables
+ here (specification section 4) +/
+ string object_id = r[36];
+ foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0),
+ coalesce(width,0), coalesce(height,0),
+ coalesce(missing,0)
+ FROM object_images WHERE object_id = " ~ object_id
+ ~ " ORDER BY seq")) {
+ OcdaImage img;
+ img.name = i[0];
+ img.sha256 = i[1];
+ img.bytes = i[2].to!ulong;
+ img.width = i[3].to!int;
+ img.height = i[4].to!int;
+ img.missing = (i[5] != "0");
+ obj.images ~= img;
+ }
+ foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.stow_link ~= l[0];
+ }
+ foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.anchor_tags ~= a[0];
+ }
+
+ auto section = doc.sectionByName(r[0]);
+ if (section !is null) { *section ~= obj; }
+ }
+ countUnread(db, chosen.doc_id, doc);
+ return doc;
+}
+
+/+ ↓ what the database holds and the model does not.
+ .
+ The text loader meets every property line and counts those it does not
+ keep, so a reader can say what was read and not understood. This
+ loader asks for columns by name, and would never see the ones it did
+ not ask for: it would report that it had understood everything, which
+ is the silence the count exists to prevent. So it counts them, on the
+ conditions under which the .ssp writes the same property (specification
+ section 6.3), and the report is the same from either form: checked,
+ 36 of 36.
++/
+private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) {
+ string id = doc_id.to!string;
+ void count(string property, string where) {
+ auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id
+ ~ " AND " ~ where).to!size_t;
+ if (n > 0) { doc.unknown.property[property] = n; }
+ }
+ count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''");
+ count("dom_status", "coalesce(dom_status,'') != ''");
+ count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''");
+ count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL");
+ count("parent_lev", "coalesce(parent_lev,0) != 0");
+ count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''");
+ count("attrib", "coalesce(attrib,'') != ''");
+ if (db.has("object_subtoc")) {
+ auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o"
+ ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t;
+ if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; }
+ }
+}
+
+/+ ↓ model section 4: the eight levels and their indices +/
+private string levelName(int index) {
+ switch (index) {
+ case 0: return "A";
+ case 1: return "B";
+ case 2: return "C";
+ case 3: return "D";
+ case 4: return "1";
+ case 5: return "2";
+ case 6: return "3";
+ case 7: return "4";
+ default: return "";
+ }
+}
+
+/+ ↓ the JSON columns, read as JSON.
+ .
+ The fixed width arrays are JSON in the database and space separated
+ (or "|" separated) in the text form. This was first written as a
+ character scan that threw away brackets and quotes, which is wrong the
+ moment a value holds an escape: Phobos' JSONValue.toString escapes
+ "/" as "\/", so every heading containing a slash came back with a
+ backslash in it. 36 of 36 documents differed on
+ heading_ancestors_text and nothing else had shown it.
+ .
+ So: parse. The values are small, there are few of them, and a real
+ parser is shorter than the scan it replaces.
++/
+private string[] jsonStrings(string v) {
+ if (v.length == 0) { return []; }
+ string[] out_;
+ try {
+ auto j = parseJSON(v);
+ if (j.type != JSONType.array) { return []; }
+ foreach (e; j.array) {
+ switch (e.type) {
+ case JSONType.string: out_ ~= e.str; break;
+ case JSONType.integer: out_ ~= e.integer.to!string; break;
+ case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
+ case JSONType.float_: out_ ~= e.floating.to!string; break;
+ case JSONType.true_: out_ ~= "true"; break;
+ case JSONType.false_: out_ ~= "false"; break;
+ default: out_ ~= ""; break;
+ }
+ }
+ } catch (Exception) {
+ return [];
+ }
+ return out_;
+}
+
+private int[] jsonInts(string v) {
+ int[] out_;
+ foreach (s; jsonStrings(v)) {
+ if (s.length == 0) { continue; }
+ try { out_ ~= s.to!int; } catch (Exception) {}
+ }
+ return out_;
+}
+
+/+ ↓ space separated in the text form: table_widths, table_aligns +/
+private string jsonWords(string v) {
+ return jsonStrings(v).join(" ");
+}
+
+/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
+ form and a JSON array here. The empty slots are significant: they say
+ which levels the heading sits under, so the separators carry as much
+ as the text does.
++/
+private string jsonJoined(string v) {
+ auto parts = jsonStrings(v);
+ if (parts.length == 0) { return ""; }
+ return parts.join("|");
+}
+#+END_SRC
+
+**** load pod
+
+#+HEADER: :tangle "../dlang/src/ocda/load_pod.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_pod;
+
+/+ ↓ the abstraction out of a pod archive.
+ .
+ A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same
+ archive unwrapped, which is what spine published before it wrapped them.
+ Either way, a pod built with --pod2 carries the abstraction inside it,
+ at media/abstraction/<uid>.<lang>.ssp, and that is what is read.
+ .
+ So this is a third input form and *not* a third model: the .ssp inside
+ is parsed by ocda.load_ssp and comes back as the same value. A writer
+ over the library gains .sisupod without being touched, which is the
+ whole reason the dispatch is in one place.
+ .
+ A pod built without --pod2 carries markup and no abstraction. That is
+ refused, and the refusal says which flag makes a pod this reader can
+ read, because "no abstraction in it" is a fact about the pod rather
+ than about the reader.
++/
+
+import std.algorithm : endsWith, sort;
+import std.array : array;
+import std.file : read;
+import std.string : indexOf;
+import ocda.model;
+import ocda.load_ssp;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ a limit on what a frame may expand to.
+ A reader of an artefact it did not make declines to be told how much
+ memory to allocate. 512 MiB is far above the largest pod in the
+ reference collection (4 MiB) and far below anything that matters.
++/
+enum size_t max_pod_size = 512 * 1024 * 1024;
+
+class OcdaPodError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ which languages of the abstraction the pod carries, in sorted order +/
+string[] podLanguages(string path) {
+ auto bytes = archiveBytes(path);
+ string[] out_;
+ foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) {
+ string lang = langOfSspName(e.name);
+ if (lang.length > 0) { out_ ~= lang; }
+ }
+ return out_.sort.array;
+}
+
+/+ ↓ load one language of the abstraction a pod carries.
+ lang == "" takes the only one there is, and where there is a choice it
+ says so rather than choosing, as the .ocda.db loader does.
++/
+OcdaDocument loadPodFile(string path, string lang = "") {
+ auto bytes = archiveBytes(path);
+ auto entries = zipEntries(bytes);
+ auto ssps = zipFindSuffix(entries, ".ssp");
+ if (ssps.length == 0) {
+ throw new OcdaPodError(path ~ ": carries no .ssp."
+ ~ " A pod carries the abstraction only when spine was asked for it:"
+ ~ " spine --pod2 puts it in, spine --source does not.");
+ }
+ ZipEntry chosen = ssps[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (e; ssps) {
+ if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; }
+ }
+ if (!found) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": no language '" ~ lang
+ ~ "'; it holds: " ~ have);
+ }
+ } else if (ssps.length > 1) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": holds more than one language ("
+ ~ have ~ "); name the one wanted");
+ }
+ auto ssp = zipRead(bytes, chosen);
+ return parseSsp(cast(string) ssp.idup);
+}
+
+/+ ↓ the archive, unwrapped if it was wrapped.
+ The bytes decide, not the name: a .sisupod is a zstd frame and a .zip
+ is not, and a file misnamed either way still reads.
++/
+package ubyte[] archiveBytes(string path) {
+ auto bytes = cast(ubyte[]) read(path);
+ return zstdIsFrame(bytes)
+ ? zstdDecompress(bytes, max_pod_size)
+ : bytes;
+}
+
+/+ ↓ <uid>.<lang>.ssp -> <lang> +/
+package string langOfSspName(string name) {
+ if (!name.endsWith(".ssp")) { return ""; }
+ string stem = name[0 .. $ - 4];
+ auto dot = stem.lastIndexOf('.');
+ return (dot > 0) ? stem[dot + 1 .. $] : "";
+}
+
+private ptrdiff_t lastIndexOf(string s, char c) {
+ for (ptrdiff_t i = s.length - 1; i >= 0; i--) {
+ if (s[i] == c) { return i; }
+ }
+ return -1;
+}
+#+END_SRC
+
+**** load ssp
+
+#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_ssp;
+
+import std.algorithm : splitter, startsWith;
+import std.array : array, split;
+import std.conv : to, ConvException;
+import std.file : readText;
+import std.stdio : writeln;
+import std.string : strip, stripLeft, indexOf;
+import ocda.model;
+
+OcdaDocument parseSspFile(string filename) {
+ string content = readText(filename);
+ return parseSsp(content);
+}
+
+OcdaDocument parseSsp(string content) {
+ OcdaDocument doc;
+ OcdaObject current_obj;
+ bool in_object = false;
+ /+ ↓ whether a text line has been seen for the object being read.
+ Not "is the text non-empty": an object whose first text line is blank
+ (a verse opening on an empty line, which the sample set has seven of)
+ would then lose it, the next line overwriting rather than appending.
+ +/
+ bool seen_text = false;
+ string current_section = "";
+ string current_block = ""; // meta, make, doc_has, or section name
+
+ foreach (line; content.splitter("\n")) {
+ /+ ↓ comment. The first line of a .ssp declares the format version, and
+ it is the one comment that carries meaning: a reader checks the
+ major part and refuses what it does not know rather than loading
+ an unfamiliar artefact half populated in silence.
+ +/
+ if (line.length > 0 && line[0] == '%') {
+ enum _banner = "% SiSU Document Abstraction v";
+ enum _source = "% Source: ";
+ if (doc.format_version.length == 0 && line.startsWith(_banner)) {
+ doc.format_version = line[_banner.length .. $].strip;
+ } else if (doc.source_filename.length == 0 && line.startsWith(_source)) {
+ doc.source_filename = line[_source.length .. $].strip;
+ }
+ continue;
+ }
+
+ /+ ↓ blank line - finalize current object +/
+ if (line.length == 0 || line.strip.length == 0) {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = OcdaObject.init;
+ in_object = false;
+ seen_text = false;
+ }
+ continue;
+ }
+
+ /+ ↓ section/block open +/
+ if (line[0] == '@') {
+ auto brace_pos = line.indexOf('{');
+ if (brace_pos > 0) {
+ string name = line[1 .. brace_pos].strip;
+ current_block = name;
+ if (!isPreambleBlock(name)) {
+ current_section = name;
+ /+ ↓ a block that is neither preamble nor a section this reader
+ knows: counted, so that --report-unknown can say so +/
+ if (doc.sectionByName(name) is null) {
+ doc.unknown.block[name]++;
+ }
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ section/block close +/
+ if (line[0] == '}') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = OcdaObject.init;
+ in_object = false;
+ seen_text = false;
+ }
+ current_block = "";
+ continue;
+ }
+
+ /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/
+ if (isPreambleBlock(current_block)) {
+ string stripped = line.stripLeft;
+ auto colon_pos = stripped.indexOf(':');
+ if (colon_pos > 0) {
+ string key = stripped[0 .. colon_pos].strip;
+ string value = (colon_pos + 1 < stripped.length)
+ ? stripped[colon_pos + 1 .. $].strip : "";
+ if (current_block == "meta") { doc.metadata[key] = value; }
+ else if (current_block == "make") { doc.make[key] = value; }
+ else if (current_block == "source") { doc.source[key] = value; }
+ else { doc.doc_has[key] = value; }
+ }
+ continue;
+ }
+
+ /+ ↓ object declaration: [N] type ... +/
+ if (line[0] == '[') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+ current_obj = OcdaObject.init;
+ current_obj.section = current_section;
+ in_object = true;
+ seen_text = false;
+ parseObjectDeclaration(line, current_obj);
+ continue;
+ }
+
+ /+ ↓ property: .name: value +/
+ if (line[0] == '.') {
+ if (in_object) {
+ parseProperty(line, current_obj, doc.unknown);
+ }
+ continue;
+ }
+
+ /+ ↓ text content: | content +/
+ if (line.length >= 1 && line[0] == '|') {
+ if (in_object) {
+ string text_content = (line.length > 2) ? line[2 .. $] : "";
+ if (seen_text) {
+ current_obj.text ~= "\n" ~ text_content;
+ } else {
+ current_obj.text = text_content;
+ seen_text = true;
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ no known shape. Counted by its first character so that a reader
+ can be told the file held something this one did not expect
+ +/
+ doc.unknown.line[line[0 .. 1].idup]++;
+ }
+
+ /+ ↓ finalize last object +/
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+
+ return doc;
+}
+
+private void parseObjectDeclaration(string line, ref OcdaObject obj) {
+ /+ parse: [N] type :level identifier +/
+ auto bracket_close = line.indexOf(']');
+ if (bracket_close < 0) return;
+ try {
+ obj.ocn = line[1 .. bracket_close].strip.to!int;
+ } catch (ConvException) {
+ obj.ocn = 0;
+ }
+ string rest = (bracket_close + 1 < line.length)
+ ? line[bracket_close + 1 .. $].strip : "";
+ if (rest.length == 0) return;
+
+ auto parts = rest.split(" ");
+ if (parts.length == 0) return;
+ obj.is_a = parts[0];
+
+ if (obj.is_a == "heading" && parts.length >= 2) {
+ /+ parse :A, :B, :1, etc. +/
+ string lev = parts[1];
+ if (lev.length >= 2 && lev[0] == ':') {
+ obj.heading_level = lev[1 .. $];
+ }
+ /+ optional identifier after level +/
+ if (parts.length >= 3) {
+ obj.identifier = parts[2];
+ }
+ }
+}
+
+/+ ↓ the blocks that are document preamble rather than sections of objects +/
+private bool isPreambleBlock(string name) {
+ return name == "source" || name == "meta" || name == "make" || name == "doc_has";
+}
+
+private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) {
+ /+ parse: .name: value +/
+ auto colon_pos = line.indexOf(':');
+ if (colon_pos < 1) return;
+ string name = line[1 .. colon_pos].strip;
+ string value = (colon_pos + 1 < line.length)
+ ? line[colon_pos + 1 .. $].strip : "";
+
+ switch (name) {
+ case "part": obj.part = value; break;
+ case "is_of_type": obj.is_of_type = value; break;
+ case "section": obj.own_section = value; break;
+ case "identifier": obj.identifier = value; break;
+ case "segment_html_is": obj.segment_html_is = value; break;
+ /+ ↓ the epub segments at heading levels 1 to 4 where the html
+ segments at level 4 alone, so the two are different files and
+ the abstraction records both +/
+ case "segment_epub": obj.segment_epub = value; break;
+ case "segment_epub_is": obj.segment_epub_is = value; break;
+ case "table_aligns": obj.table_aligns = value; break;
+ case "stow_link": obj.stow_link ~= value; break;
+ case "anchor_tag": obj.anchor_tags ~= value; break;
+ case "parent": obj.parent_ocn = toIntSafe(value); break;
+ case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break;
+ case "ocn_range": {
+ auto r = parseIntArray(value);
+ if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; }
+ break;
+ }
+ case "children": obj.children = parseIntArray(value); break;
+ case "ancestors": obj.ancestors = parseIntArray(value); break;
+ case "dummy": obj.dummy_heading = (value == "true"); break;
+ case "ocn_off": obj.ocn_off = (value == "true"); break;
+ case "indent":
+ auto parts = value.split(" ");
+ if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]);
+ if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]);
+ break;
+ case "bullet": obj.bullet = (value == "true"); break;
+ case "lang": obj.lang = value; break;
+ case "has":
+ foreach (flag; value.split(" ")) {
+ switch (flag) {
+ case "links": obj.has_links = true; break;
+ case "notes_reg": obj.has_notes_reg = true; break;
+ case "notes_star": obj.has_notes_star = true; break;
+ case "images": obj.has_images = true; break;
+ default: break;
+ }
+ }
+ break;
+ case "segment": obj.segment = value; break;
+ case "segment_prev": obj.segment_prev = value; break;
+ case "segment_next": obj.segment_next = value; break;
+ case "anchor": obj.anchor = value; break;
+ case "heading_lev_anchor": obj.heading_lev_anchor = value; break;
+ case "table_cols": obj.table_cols = toIntSafe(value); break;
+ case "table_widths": obj.table_widths = value; break;
+ case "table_header": obj.table_header = (value == "true"); break;
+ case "code_syntax": obj.code_syntax = value; break;
+ case "code_linenumbers": obj.code_linenumbers = (value == "true"); break;
+ case "heading_ancestors_text": obj.heading_ancestors_text = value; break;
+ case "sha256": obj.sha256 = value; break;
+ case "image": obj.images ~= parseImageRecord(value); break;
+ /+ ↓ a property this reader has no use for, or has never heard of.
+ Ignored, which is the conforming behaviour within a major
+ version, but counted rather than passed over in silence
+ +/
+ default: unknown.property[name]++; break;
+ }
+}
+
+private OcdaImage parseImageRecord(string value) {
+ /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...]
+ filename first, remaining fields order-independent; unrecognised
+ fields are kept rather than discarded +/
+ OcdaImage img;
+ bool first = true;
+ foreach (field; value.split(" ")) {
+ if (field.length == 0) continue;
+ if (first) {
+ img.name = field;
+ first = false;
+ continue;
+ }
+ auto colon_pos = field.indexOf(':');
+ if (colon_pos < 1) continue;
+ string key = field[0 .. colon_pos];
+ string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : "";
+ switch (key) {
+ case "sha256": img.sha256 = val; break;
+ case "bytes": img.bytes = toUlongSafe(val); break;
+ case "px":
+ auto dims = val.split("x");
+ if (dims.length == 2) {
+ img.width = toIntSafe(dims[0]);
+ img.height = toIntSafe(dims[1]);
+ }
+ break;
+ case "missing": img.missing = (val == "true"); break;
+ default: img.fields[key] = val; break;
+ }
+ }
+ return img;
+}
+
+/+ ↓ two values the text form leaves out because they can be worked out.
+ .
+ The .ssp emits a property only when it carries information, so
+ .identifier is absent when it is simply the ocn, and .anchor is absent
+ when it is the segment name. The .ocda.db, being columns rather than
+ lines, stores both unconditionally. So a loader that takes the text
+ form literally gives a *different model* from the one the database
+ gives, and the difference is not small: in the reference collection
+ the identifier is omitted on all but a handful of objects.
+ .
+ The model is the thing both serialisations describe, so the loader
+ fills them in. Found by dumping both models and diffing them, which no
+ amount of comparing section counts would have shown.
++/
+private void deriveOmitted(ref OcdaObject obj) {
+ import std.conv : to;
+ if (obj.identifier.length == 0 && obj.ocn != 0) {
+ obj.identifier = obj.ocn.to!string;
+ }
+ if (obj.anchor.length == 0 && obj.segment.length > 0) {
+ obj.anchor = obj.segment;
+ }
+}
+
+private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) {
+ deriveOmitted(obj);
+ auto sec = doc.sectionByName(section);
+ if (sec !is null) {
+ ,*sec ~= obj;
+ }
+}
+
+private int toIntSafe(string s) {
+ try { return s.strip.to!int; }
+ catch (ConvException) { return 0; }
+}
+
+private ulong toUlongSafe(string s) {
+ try { return s.strip.to!ulong; }
+ catch (ConvException) { return 0; }
+}
+
+private int[] parseIntArray(string s) {
+ int[] result;
+ foreach (part; s.split(" ")) {
+ auto v = toIntSafe(part);
+ result ~= v;
+ }
+ return result;
+}
+#+END_SRC
+
+*** images
+
+#+HEADER: :tangle "../dlang/src/ocda/images.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.images;
+
+/+ ↓ an image's bytes, from whichever source the document came from.
+ .
+ The model *names* images, with digest, byte size and pixel size; it does
+ not hold them. That is right - an abstraction of a document is not a
+ bundle of its assets - but a writer of any format that shows images
+ needs the bytes, so something has to know where they are. This is that
+ something, and it is in the library rather than in a writer because the
+ answer depends on the *input form* and not on the output.
+ .
+ Three answers, and one refusal:
+ .
+ .ocda.db the files table, role 'image', by name
+ .sisupod the archive entry whose path ends in /image/<name>
+ .zip the same
+ .ssp beside it: a pod puts the abstraction in
+ media/abstraction/ and its images in media/image/, so the
+ path is documented rather than guessed
+ .
+ A .ssp that is not in a pod has nowhere to look, and that is reported as
+ absent rather than as an error: a reader that cannot find an image
+ should say so and carry on, because the document is still readable.
++/
+
+import std.algorithm : endsWith;
+import std.array : array;
+import std.file : exists, isFile, read;
+import std.path : buildPath, dirName;
+import ocda.load;
+import ocda.sqlite;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ the bytes, or an empty slice when the source has no such image +/
+ubyte[] imageBytes(string source, string name) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return fromDb(source, name);
+ case Source.pod: return fromArchive(source, name);
+ case Source.ssp: return fromBeside(source, name);
+ case Source.unknown: return [];
+ }
+}
+
+/+ ↓ can this source give images at all?
+ A writer asks before it decides whether to emit them, because an <img>
+ pointing at a file it did not include is worse than no image.
++/
+bool canGiveImages(string source) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return true;
+ case Source.pod: return true;
+ case Source.ssp: return imageDirBeside(source).length > 0;
+ case Source.unknown: return false;
+ }
+}
+
+private ubyte[] fromDb(string path, string name) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ if (!db.has("files")) { return []; }
+ /+ ↓ the name is from the abstraction, not from a caller, and the quote
+ doubling is what keeps it a value rather than syntax +/
+ return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '"
+ ~ sqlQuote(name) ~ "'");
+}
+
+private ubyte[] fromArchive(string path, string name) {
+ auto bytes = cast(ubyte[]) read(path);
+ if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); }
+ auto entries = zipEntries(bytes);
+ foreach (e; entries) {
+ if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); }
+ }
+ return [];
+}
+
+private ubyte[] fromBeside(string path, string name) {
+ string dir = imageDirBeside(path);
+ if (dir.length == 0) { return []; }
+ string file = buildPath(dir, name);
+ return file.exists && file.isFile ? cast(ubyte[]) read(file) : [];
+}
+
+/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/
+private string imageDirBeside(string path) {
+ string dir = path.dirName;
+ if (!dir.endsWith("abstraction")) { return ""; }
+ string image_dir = buildPath(dir.dirName, "image");
+ return image_dir.exists ? image_dir : "";
+}
+
+/+ ↓ a single quote doubled is how sql says a single quote +/
+private string sqlQuote(string s) {
+ string out_;
+ foreach (c; s) {
+ out_ ~= (c == '\'') ? "''" : [c];
+ }
+ return out_;
+}
+#+END_SRC
+
+** sqlite
+
+#+HEADER: :tangle "../dlang/src/ocda/sqlite.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.sqlite;
+
+/+ ↓ the sqlite3 C API, declared rather than depended upon.
+ .
+ A binding from the dub registry would be a dependency to fetch, and
+ the point of this reader is that it builds in a sandbox with no
+ network. The dozen functions a reader needs are declared here and the
+ library is linked directly, which costs a page of extern(C) and saves
+ a dependency.
++/
+
+import std.conv : to;
+import std.string : fromStringz, toStringz;
+
+extern (C) {
+ struct sqlite3;
+ struct sqlite3_stmt;
+
+ int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs);
+ int sqlite3_close(sqlite3* db);
+ int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte,
+ sqlite3_stmt** stmt, const(char)** tail);
+ int sqlite3_step(sqlite3_stmt* stmt);
+ int sqlite3_finalize(sqlite3_stmt* stmt);
+ int sqlite3_column_count(sqlite3_stmt* stmt);
+ const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col);
+ const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_int(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_type(sqlite3_stmt* stmt, int col);
+ const(char)* sqlite3_errmsg(sqlite3* db);
+ const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_bytes(sqlite3_stmt* stmt, int col);
+}
+
+enum SQLITE_OK = 0;
+enum SQLITE_ROW = 100;
+enum SQLITE_DONE = 101;
+enum SQLITE_NULL = 5;
+enum SQLITE_OPEN_READONLY = 0x00000001;
+
+class SqliteError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ a database, opened read only.
+ Read only is deliberate: a reader has no business writing, and a
+ request that opens a database read-write can create an empty one where
+ a path is mistyped.
++/
+struct Db {
+ private sqlite3* _handle;
+
+ static Db open(string path) {
+ sqlite3* handle;
+ auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null);
+ if (rc != SQLITE_OK) {
+ string msg = (handle is null)
+ ? "cannot open " ~ path
+ : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup;
+ if (handle !is null) { sqlite3_close(handle); }
+ throw new SqliteError(msg);
+ }
+ Db db;
+ db._handle = handle;
+ return db;
+ }
+
+ void close() {
+ if (_handle !is null) { sqlite3_close(_handle); _handle = null; }
+ }
+
+ /+ ↓ every row of a query, each row as its columns in order +/
+ string[][] rows(string sql) {
+ sqlite3_stmt* stmt;
+ auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null);
+ if (rc != SQLITE_OK) {
+ throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql);
+ }
+ scope(exit) { sqlite3_finalize(stmt); }
+ string[][] out_rows;
+ int cols = sqlite3_column_count(stmt);
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ string[] row;
+ foreach (c; 0 .. cols) {
+ row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL)
+ ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup;
+ }
+ out_rows ~= row;
+ }
+ return out_rows;
+ }
+
+ /+ ↓ the first column of the first row, "" when there is none +/
+ string one(string sql) {
+ auto r = rows(sql);
+ return (r.length > 0 && r[0].length > 0) ? r[0][0] : "";
+ }
+
+ /+ ↓ the first column of the first row as bytes, for a blob.
+ Separate from one(), because a blob is not text: an image read as text
+ stops at the first zero byte, and a png's ninth byte is zero.
+ +/
+ ubyte[] blob(string sql) {
+ sqlite3_stmt* stmt;
+ if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) {
+ throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup
+ ~ " [" ~ sql ~ "]");
+ }
+ scope(exit) { sqlite3_finalize(stmt); }
+ if (sqlite3_step(stmt) != SQLITE_ROW) { return []; }
+ int n = sqlite3_column_bytes(stmt, 0);
+ if (n <= 0) { return []; }
+ auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0);
+ if (p is null) { return []; }
+ return p[0 .. n].dup;
+ }
+
+ /+ ↓ does a table or view of this name exist +/
+ bool has(string name) {
+ return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0";
+ }
+}
+#+END_SRC
+
+** compression
+*** dlang read zip
+
+#+HEADER: :tangle "../dlang/src/ocda/zip.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.zip;
+
+/+ ↓ enough of the zip format to find one file in a pod and read it.
+ .
+ Not a zip library. A pod archive holds a handful of entries, spine
+ writes them **stored** (uncompressed) so that the outer zstd frame does
+ the compressing once over the whole thing, and what is wanted from it
+ here is one named entry. So: read the central directory, find the
+ entry, return its bytes.
+ .
+ Deflate is *refused rather than mishandled*. Phobos can inflate, but a
+ library that says it read an entry it in fact skipped is worse than one
+ that says it cannot, and spine's own archives do not need it.
++/
+
+import std.algorithm : endsWith;
+import std.conv : to;
+
+class OcdaZipError : Exception {
+ this(string msg) { super(msg); }
+}
+
+struct ZipEntry {
+ string name;
+ uint method; // 0 stored, 8 deflate
+ uint compressed;
+ uint uncompressed;
+ uint local_offset;
+}
+
+/+ ↓ the entries, from the central directory at the end of the file +/
+ZipEntry[] zipEntries(const(ubyte)[] bytes) {
+ enum uint eocd_sig = 0x06054b50;
+ enum uint cd_sig = 0x02014b50;
+ if (bytes.length < 22) {
+ throw new OcdaZipError("too short to be a zip archive");
+ }
+ /+ ↓ the end of central directory record is last, and may be followed by a
+ comment of up to 65535 bytes, so it is searched for backwards +/
+ ptrdiff_t eocd = -1;
+ ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0;
+ for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) {
+ if (u32(bytes, i) == eocd_sig) { eocd = i; break; }
+ }
+ if (eocd < 0) {
+ throw new OcdaZipError("no end of central directory record;"
+ ~ " it does not look like a zip archive");
+ }
+ uint count = u16(bytes, eocd + 10);
+ uint offset = u32(bytes, eocd + 16);
+ ZipEntry[] out_;
+ size_t p = offset;
+ foreach (_; 0 .. count) {
+ if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) {
+ throw new OcdaZipError("central directory entry "
+ ~ out_.length.to!string ~ " is malformed");
+ }
+ ZipEntry e;
+ e.method = u16(bytes, p + 10);
+ e.compressed = u32(bytes, p + 20);
+ e.uncompressed = u32(bytes, p + 24);
+ uint name_len = u16(bytes, p + 28);
+ uint extra_len = u16(bytes, p + 30);
+ uint cmnt_len = u16(bytes, p + 32);
+ e.local_offset = u32(bytes, p + 42);
+ e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup;
+ out_ ~= e;
+ p += 46 + name_len + extra_len + cmnt_len;
+ }
+ return out_;
+}
+
+/+ ↓ one entry's bytes, from its local header +/
+ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) {
+ enum uint local_sig = 0x04034b50;
+ size_t p = e.local_offset;
+ if (p + 30 > bytes.length || u32(bytes, p) != local_sig) {
+ throw new OcdaZipError(e.name ~ ": local header is malformed");
+ }
+ uint name_len = u16(bytes, p + 26);
+ uint extra_len = u16(bytes, p + 28);
+ size_t start = p + 30 + name_len + extra_len;
+ size_t end = start + e.compressed;
+ if (end > bytes.length) {
+ throw new OcdaZipError(e.name ~ ": runs past the end of the archive");
+ }
+ if (e.method != 0) {
+ throw new OcdaZipError(e.name ~ ": compression method "
+ ~ e.method.to!string ~ "; this reader takes only stored entries,"
+ ~ " which is what spine writes");
+ }
+ return bytes[start .. end].dup;
+}
+
+/+ ↓ the first entry whose name ends this way, or none.
+ Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid.
++/
+ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) {
+ ZipEntry[] out_;
+ foreach (e; entries) {
+ if (e.name.endsWith(suffix)) { out_ ~= e; }
+ }
+ return out_;
+}
+
+private uint u16(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i] | (cast(uint) b[i + 1] << 8);
+}
+
+private uint u32(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i]
+ | (cast(uint) b[i + 1] << 8)
+ | (cast(uint) b[i + 2] << 16)
+ | (cast(uint) b[i + 3] << 24);
+}
+#+END_SRC
+
+*** dlang read zstd
+
+#+HEADER: :tangle "../dlang/src/ocda/zstd.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.zstd;
+
+/+ ↓ libzstd, declared rather than bound.
+ .
+ A .sisupod is one zstd frame wrapping a zip archive, so reading one
+ needs exactly two things from the library: is this a frame, and what is
+ in it. Both are declared here and the C library is linked, which is the
+ same arrangement ocda.sqlite uses: no vendored tree, no generated
+ header, and the prototypes say what is being called.
++/
+
+extern (C) @nogc nothrow private {
+ size_t ZSTD_decompress(void* dst, size_t dstCapacity,
+ const(void)* src, size_t compressedSize);
+ ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
+ uint ZSTD_isError(size_t code);
+ const(char)* ZSTD_getErrorName(size_t code);
+}
+
+/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/
+private enum ulong contentSizeUnknown = ulong.max;
+private enum ulong contentSizeError = ulong.max - 1;
+
+/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/
+private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD];
+
+class OcdaZstdError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ does this begin with a zstd frame?
+ Asked of the bytes rather than of the filename, because what is inside
+ decides how a thing is opened. A .sisupod is a frame; a .zip published
+ before spine wrapped them is not, and both are read.
++/
+bool zstdIsFrame(const(ubyte)[] bytes) {
+ if (bytes.length < zstd_magic.length) { return false; }
+ foreach (i, b; zstd_magic) {
+ if (bytes[i] != b) { return false; }
+ }
+ return true;
+}
+
+/+ ↓ one frame in, its contents out.
+ max_size is a refusal rather than a buffer size: a frame declares how
+ much it expands to, and a frame that claims more than the caller is
+ willing to hold is not decompressed at all. A reader of an artefact it
+ did not make wants that limit.
++/
+ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) {
+ if (!zstdIsFrame(bytes)) {
+ throw new OcdaZstdError("not a zstd frame");
+ }
+ ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length);
+ if (declared == contentSizeError) {
+ throw new OcdaZstdError("zstd frame: could not read its content size");
+ }
+ if (declared == contentSizeUnknown) {
+ throw new OcdaZstdError("zstd frame declares no content size;"
+ ~ " this reader takes only frames that do");
+ }
+ if (declared > max_size) {
+ import std.conv : to;
+ throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string
+ ~ " bytes, over the " ~ max_size.to!string ~ " byte limit");
+ }
+ auto out_ = new ubyte[](cast(size_t) declared);
+ size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length);
+ if (ZSTD_isError(got)) {
+ import std.string : fromStringz;
+ throw new OcdaZstdError("zstd: "
+ ~ ZSTD_getErrorName(got).fromStringz.idup);
+ }
+ if (got != declared) {
+ throw new OcdaZstdError("zstd frame expanded to a different size"
+ ~ " than it declared");
+ }
+ return out_;
+}
+#+END_SRC
+
+** dump
+
+#+HEADER: :tangle "../dlang/src/ocda/dump.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.dump;
+
+/+ ↓ the model, written out canonically.
+ .
+ Not an output format: an *oracle*. Two libraries in two languages
+ implement the same model from the same specification, and the cheap
+ way to ask whether they agree is to have each write the whole loaded
+ document as text and diff the two. That is only useful if the text is
+ fully determined by the model, so:
+ .
+ - every line is "<key> <value>", the value running to end of line,
+ and only one free-form value per line;
+ - a field at its default is *omitted*, so the output says what the
+ document has rather than what the struct declares;
+ - dictionaries are emitted in sorted key order, because neither
+ language's hash map has an order worth relying on;
+ - sections come in the fixed order of the format, not
+ alphabetically.
+ .
+ Anything that reads this to reconstruct a document has misunderstood
+ it. Load the .ssp or the .ocda.db.
++/
+
+import std.algorithm : sort, splitter;
+import std.array : appender, array, join;
+import std.conv : to;
+import ocda.model;
+
+/+ ↓ the nine sections, in format order +/
+static immutable string[] dump_sections = ["head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb", "tail"];
+
+string dumpDocument(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+
+ void kv(string key, string value) {
+ if (value.length == 0) { return; }
+ out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n";
+ }
+ void kn(string key, long value) {
+ if (value == 0) { return; }
+ kv(key, value.to!string);
+ }
+ void kb(string key, bool value) {
+ if (!value) { return; }
+ kv(key, "1");
+ }
+ void dict(string prefix, string[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key]; out_ ~= "\n";
+ }
+ }
+
+ kv("version", doc.format_version);
+ kv("source_filename", doc.source_filename);
+ dict("source", doc.source);
+ dict("meta", doc.metadata);
+ dict("make", doc.make);
+ dict("doc_has", doc.doc_has);
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null || (*sec).length == 0) { continue; }
+ kv("section", name ~ " " ~ (*sec).length.to!string);
+ }
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null) { continue; }
+ foreach (i, obj; *sec) {
+ out_ ~= "object "; out_ ~= name; out_ ~= " ";
+ out_ ~= i.to!string; out_ ~= "\n";
+ kn("ocn", obj.ocn);
+ kv("is_a", obj.is_a);
+ kv("own_section", obj.own_section);
+ kv("level", obj.heading_level);
+ kv("identifier", obj.identifier);
+ kv("part", obj.part);
+ kv("is_of_type", obj.is_of_type);
+ kv("segment_html_is", obj.segment_html_is);
+ kv("segment_epub", obj.segment_epub);
+ kv("segment_epub_is", obj.segment_epub_is);
+ kn("parent", obj.parent_ocn);
+ kn("last_descendant", obj.last_descendant_ocn);
+ kn("ocn_range_first", obj.ocn_range_first);
+ kn("ocn_range_last", obj.ocn_range_last);
+ kv("children", ints(obj.children));
+ kv("ancestors", ints(obj.ancestors));
+ kb("dummy", obj.dummy_heading);
+ kb("ocn_off", obj.ocn_off);
+ kn("indent_base", obj.indent_base);
+ kn("indent_hang", obj.indent_hang);
+ kb("bullet", obj.bullet);
+ kv("lang", obj.lang);
+ kb("has_links", obj.has_links);
+ kb("has_notes_reg", obj.has_notes_reg);
+ kb("has_notes_star", obj.has_notes_star);
+ kb("has_images", obj.has_images);
+ kv("segment", obj.segment);
+ kv("segment_prev", obj.segment_prev);
+ kv("segment_next", obj.segment_next);
+ kv("anchor", obj.anchor);
+ kv("heading_lev_anchor", obj.heading_lev_anchor);
+ kn("table_cols", obj.table_cols);
+ kv("table_widths", obj.table_widths);
+ kv("table_aligns", obj.table_aligns);
+ kb("table_header", obj.table_header);
+ kv("code_syntax", obj.code_syntax);
+ kb("code_linenumbers", obj.code_linenumbers);
+ kv("heading_ancestors_text", obj.heading_ancestors_text);
+ kv("sha256", obj.sha256);
+ foreach (url; obj.stow_link) { kv("stow_link", url); }
+ foreach (a; obj.anchor_tags) { kv("anchor_tag", a); }
+ foreach (img; obj.images) {
+ /+ ↓ the name last, because a filename may hold a space where
+ none of the other fields can +/
+ kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " "
+ ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " "
+ ~ (img.missing ? "1" : "0") ~ " " ~ img.name);
+ foreach (key; img.fields.keys.dup.sort) {
+ kv("image_field", key ~ " " ~ img.fields[key]);
+ }
+ }
+ /+ ↓ text, when there is any. An object whose only text line is
+ blank holds "" here, indistinguishable from no text at all,
+ because the model joins the lines into one string. Both
+ libraries lose the same distinction in the same place, which is
+ what makes them comparable; it is recorded rather than hidden.
+ +/
+ if (obj.text.length > 0) {
+ auto lines = obj.text.splitter("\n").array;
+ kv("text", lines.length.to!string);
+ foreach (line; lines) {
+ out_ ~= "| "; out_ ~= line; out_ ~= "\n";
+ }
+ }
+ }
+ }
+ return out_.data;
+}
+
+/+ ↓ what the reader met and did not understand, sorted. Separate from the
+ document dump: it describes the reader, not the document, and the
+ .ocda.db loader has nothing to report where the .ssp loader does.
++/
+string dumpUnknown(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+ void counts(string prefix, size_t[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key].to!string; out_ ~= "\n";
+ }
+ }
+ counts("property", doc.unknown.property);
+ counts("block", doc.unknown.block);
+ counts("line", doc.unknown.line);
+ return out_.data;
+}
+
+private string ints(int[] xs) {
+ if (xs.length == 0) { return ""; }
+ string[] parts;
+ foreach (x; xs) { parts ~= x.to!string; }
+ return parts.join(",");
+}
+#+END_SRC
+
+* read
+** main
+
+#+HEADER: :tangle "../dlang/src/read/main.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ ocda-read - reads the object-centric document abstraction, in any of the
+ forms it is written in, and says what is in it.
+
+ ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]
+
+ One reader where there were two: sisudoc-ssp-reader read a .ssp and
+ sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is
+ built on the library, so every mode reads every form, and a mode given the
+ .ssp, the .ocda.db and the .sisupod of a document says the same thing of
+ each: it is told what the document is, not which file it came from.
+
+ Two things are about the file rather than the document, and say so:
+ --files, what a .ocda.db carries besides the abstraction, and --validate,
+ which checks a .ssp as text. Everything else is the model's.
+
+ The output modules are the .ssp reader's, moved onto the library's model.
++/
+module read.main;
+
+import std.algorithm : canFind, map;
+import std.array : array, join;
+import std.getopt;
+import std.stdio : writeln, writefln, stderr;
+import ocda.model;
+import ocda.load;
+import ocda.inventory;
+import read.tree;
+import read.html;
+import read.llm_text;
+import read.llm_jsonl;
+import read.images;
+import read.verify;
+import read.report_unknown;
+import read.validate;
+
+int main(string[] args) {
+ bool do_tree = false;
+ bool do_html = false;
+ bool do_llm_text = false;
+ bool do_llm_jsonl = false;
+ bool do_images = false;
+ bool do_verify = false;
+ bool do_report_unknown = false;
+ bool do_validate = false;
+ bool do_languages = false;
+ bool do_files = false;
+ bool any_version = false;
+ string image_dir = "";
+ string output_file = "";
+ string lang = "";
+
+ GetoptResult helpInfo;
+ try {
+ helpInfo = getopt(args,
+ "tree", "the headings, as a tree", &do_tree,
+ "outline", "the same as --tree", &do_tree,
+ "html", "the document as one html page", &do_html,
+ "llm-text", "plain text with heading breadcrumbs", &do_llm_text,
+ "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl,
+ "images", "the image records: digest, size, pixels", &do_images,
+ "verify", "check the image records against --image-dir", &do_verify,
+ "image-dir", "the images to verify against", &image_dir,
+ "languages", "the languages the file holds", &do_languages,
+ "files", "what a .ocda.db carries besides the abstraction", &do_files,
+ "report-unknown", "what was read and not understood", &do_report_unknown,
+ "validate", "check a .ssp against its specification, as text", &do_validate,
+ "lang", "=<code> the language to read (default: the first)", &lang,
+ "output", "write the mode's output to a file", &output_file,
+ "any-version", "read a format major this reader does not know", &any_version,
+ );
+ } catch (GetOptException e) {
+ stderr.writeln("ocda-read: ", e.msg, "; --help lists the options");
+ return 2;
+ }
+
+ if (helpInfo.helpWanted || args.length < 2) {
+ defaultGetoptPrinter(
+ "ocda-read - reads the document abstraction, in any of its forms\n\n"
+ ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n",
+ helpInfo.options,
+ );
+ return 0;
+ }
+
+ string input_file = args[1];
+ if (lang.length == 0 && args.length > 2) { lang = args[2]; }
+ bool lang_named = (lang.length > 0);
+
+ /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not
+ at all: there is no text form of a database to check, and a pod's
+ .ssp is checked by taking it out +/
+ if (do_validate && sourceOf(input_file) != Source.ssp) {
+ stderr.writeln("--validate checks a .ssp as text, against the .ssp",
+ " specification; ", input_file, " is ", sourceName(sourceOf(input_file)));
+ return 2;
+ }
+
+ /+ ↓ which language. A .ocda.db and a pod may hold several; where none is
+ named, every form reads the first in the manifest's order and says
+ so, rather than the database taking one and the pod refusing +/
+ OcdaLanguage[] held;
+ OcdaDocument doc;
+ try {
+ held = languages(input_file);
+ if (lang.length == 0 && held.length > 0) { lang = held[0].lang; }
+ if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) {
+ stderr.writeln("error: no language '", lang, "' in ", input_file,
+ "; it holds: ", held.map!(l => l.lang).array.join(" "));
+ return 2;
+ }
+ doc = load(input_file, lang, any_version);
+ } catch (Exception e) {
+ stderr.writeln("Error reading ", input_file, ": ", e.msg);
+ return 2;
+ }
+
+ /+ ↓ the version gate: the major part, and refuse what is not known +/
+ if (doc.format_version.length == 0) {
+ stderr.writeln("warning: ", input_file, " declares no format version");
+ } else if (doc.formatVersionMajor != ssp_format_major_read) {
+ if (any_version) {
+ stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
+ } else {
+ stderr.writeln("error: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x");
+ stderr.writeln(" a major version differs: a property may have",
+ " changed meaning or gone away.");
+ stderr.writeln(" --any-version reads it regardless.");
+ return 3;
+ }
+ }
+
+ writeln("Read: ", input_file);
+ if (doc.format_version.length > 0) {
+ writeln(" Format: v", doc.format_version);
+ }
+ if (auto _lang = "language" in doc.source) {
+ writeln(" Source: ", *_lang,
+ ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
+ ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
+ }
+ if (held.length > 1 && !lang_named) {
+ writeln(" (reading ", lang, "; --lang=<code> for another)");
+ }
+ writeln(" Title: ", doc.meta("title.main"));
+ writeln(" Author: ", doc.meta("creator.author"));
+ writeln(" Sections: head=", doc.head.length,
+ " toc=", doc.toc.length,
+ " body=", doc.body.length,
+ " endnotes=", doc.endnotes.length,
+ " glossary=", doc.glossary.length,
+ " bibliography=", doc.bibliography.length,
+ " bookindex=", doc.bookindex.length,
+ " blurb=", doc.blurb.length,
+ " tail=", doc.tail.length);
+ auto image_records = imageRecordCount(doc);
+ if (image_records > 0) {
+ writeln(" Images: ", image_records, " record",
+ (image_records == 1) ? "" : "s");
+ }
+
+ if (do_languages) { printLanguages(held); }
+ if (do_files) { printFiles(input_file); }
+ if (do_tree) { outputTree(doc, output_file); }
+ if (do_html) { outputHtmlScroll(doc, output_file); }
+ if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); }
+ if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); }
+ if (do_images) { outputImages(doc, output_file); }
+
+ int _verify_failures = 0;
+ if (do_verify) {
+ if (image_dir.length == 0) {
+ stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
+ return 2;
+ }
+ _verify_failures = verifyImages(doc, image_dir, output_file);
+ }
+
+ if (do_report_unknown) { reportUnknown(doc); }
+
+ int _validation_findings = 0;
+ if (do_validate) {
+ auto _found = validateSspFile(input_file, ssp_format_major_read);
+ reportValidation(input_file, _found);
+ _validation_findings = cast(int) _found.length;
+ }
+
+ if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
+ && !do_verify && !do_report_unknown && !do_validate
+ && !do_languages && !do_files) {
+ writeln("\n(this is the summary: --help lists the modes, each adding its output)");
+ }
+
+ return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
+}
+
+/+ ↓ the languages, each with the markup it was made from and its .ssp's
+ digest: the same from every form, the database storing the digest and
+ the others taking it from the bytes +/
+void printLanguages(OcdaLanguage[] held) {
+ writeln();
+ writeln(" lang source ssp digest");
+ foreach (l; held) {
+ writefln(" %-7s %-31s %s", l.lang, l.source_filename,
+ (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest);
+ }
+}
+
+/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries
+ nothing, and a pod's files are its archive entries rather than anything
+ the abstraction records, so for those it says so +/
+void printFiles(string path) {
+ writeln();
+ if (!isCarrier(path)) {
+ writeln(" carried files: only a .ocda.db carries files; this is ",
+ sourceName(sourceOf(path)));
+ return;
+ }
+ auto rows = carried(path);
+ if (rows.length == 0) { writeln(" carries no files"); return; }
+ writeln(" carried files:");
+ writeln(" role stored n original in the file");
+ foreach (r; rows) {
+ writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored);
+ }
+}
+#+END_SRC
+
+** check
+*** read validate =.ssp=
+
+#+HEADER: :tangle "../dlang/src/read/validate.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.validate;
+
+/+ ↓ check a .ssp against the format specification.
+ .
+ Written from specs/sisudoc-abstraction-format/ssp-format-specification.md
+ and from nothing else: the point of it is to be what the specification
+ says, executed, so that a file can be held against the document rather
+ than against an implementation. Each check names the section it comes
+ from, so a disagreement can be taken back to the text.
++/
+
+import std.algorithm : canFind, startsWith;
+import std.array : array, split;
+import std.conv : to;
+import std.file : readText;
+import std.stdio : writeln, writefln;
+import std.string : indexOf, strip;
+
+struct SspFinding {
+ size_t line; // 1-based
+ string section; // the specification section the rule is from
+ string detail;
+}
+
+/+ ↓ specification section 6.3: the properties, in the order they are
+ emitted. A reader may rely on the order, so a validator can check it +/
+private immutable string[] property_order = [
+ "identifier", "part", "section", "parent", "last_descendant", "ocn_range",
+ "children",
+ "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed",
+ "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type",
+ "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent",
+ "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns",
+ "table_header", "code_linenumbers", "stow_link", "segment", "anchor",
+ "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub",
+ "segment_html_is", "segment_epub_is", "segment_lv4_is",
+ "heading_ancestors_text", "lev4_subtoc", "anchor_tag",
+];
+
+/+ ↓ section 6.1: the sixteen kinds +/
+private immutable string[] kinds = [
+ "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography",
+ "comment", "verse", "glossary", "quote", "poem", "block", "table",
+ "blurb", "group",
+];
+
+/+ ↓ section 4: the nine sections, in the order they appear +/
+private immutable string[] section_order = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography",
+ "bookindex", "blurb", "tail",
+];
+
+/+ ↓ section 4: the four preamble blocks, in the order they appear +/
+private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"];
+
+/+ ↓ section 6.1: the heading levels +/
+private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"];
+
+private int propertyRank(string name) {
+ foreach (i, p; property_order) { if (p == name) { return cast(int) i; } }
+ return -1;
+}
+
+SspFinding[] validateSspFile(string filename, string major_known) {
+ SspFinding[] found;
+ auto lines = readText(filename).split("\n");
+
+ bool seen_banner = false;
+ bool seen_source_line = false;
+ string block = ""; // the block currently open, "" when none
+ int preamble_at = -1; // how far through the preamble order
+ int section_at = -1; // how far through the section order
+ bool in_object = false;
+ int prev_rank = -1;
+ size_t doc_has_keys = 0;
+ bool in_text = false; // text lines come after properties
+
+ foreach (i, line; lines) {
+ size_t no = i + 1;
+
+ /+ ↓ section 2: the version, on the first line +/
+ if (no == 1) {
+ enum banner = "% SiSU Document Abstraction v";
+ if (!line.startsWith(banner)) {
+ found ~= SspFinding(no, "2", "first line is not the version banner");
+ } else {
+ seen_banner = true;
+ string v = line[banner.length .. $].strip;
+ auto dot = v.indexOf('.');
+ string major = (dot > 0) ? v[0 .. dot] : v;
+ if (major != major_known) {
+ found ~= SspFinding(no, "2.1",
+ "format major v" ~ major ~ ", this validator knows v" ~ major_known);
+ }
+ }
+ continue;
+ }
+ if (no == 2) {
+ if (!line.startsWith("% Source: ")) {
+ found ~= SspFinding(no, "4", "second line is not \"% Source: \"");
+ } else { seen_source_line = true; }
+ continue;
+ }
+
+ if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; }
+
+ /+ ↓ section 3: a comment is the file header only +/
+ if (line[0] == '%') {
+ found ~= SspFinding(no, "3", "a comment outside the two header lines");
+ continue;
+ }
+
+ /+ ↓ block open +/
+ if (line[0] == '@') {
+ auto brace = line.indexOf(" {");
+ if (brace < 1 || line[$ - 1] != '{') {
+ found ~= SspFinding(no, "3", "block open is not \"@name {\"");
+ continue;
+ }
+ string name = line[1 .. brace];
+ if (preamble_order.canFind(name)) {
+ int at = 0;
+ foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } }
+ if (at <= preamble_at) {
+ found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order");
+ }
+ preamble_at = at;
+ if (section_at >= 0) {
+ found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section");
+ }
+ } else if (section_order.canFind(name)) {
+ int at = 0;
+ foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } }
+ if (at <= section_at) {
+ found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order");
+ }
+ section_at = at;
+ } else {
+ found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section");
+ }
+ block = name;
+ doc_has_keys = 0;
+ continue;
+ }
+
+ if (line[0] == '}') {
+ if (line.length != 1) {
+ found ~= SspFinding(no, "3", "block close carries more than \"}\"");
+ }
+ /+ ↓ section 5.4: ten keys, always +/
+ if (block == "doc_has" && doc_has_keys != 10) {
+ found ~= SspFinding(no, "5.4",
+ "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten");
+ }
+ block = "";
+ in_object = false;
+ prev_rank = -1;
+ continue;
+ }
+
+ /+ ↓ section 5: inside a preamble block, " key: value" +/
+ if (preamble_order.canFind(block)) {
+ if (!line.startsWith(" ") || line.indexOf(": ") < 3) {
+ found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block);
+ } else if (block == "doc_has") {
+ doc_has_keys++;
+ }
+ continue;
+ }
+
+ /+ ↓ section 6.1: the object declaration line +/
+ if (line[0] == '[') {
+ auto close = line.indexOf("] ");
+ if (close < 2) {
+ found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\"");
+ continue;
+ }
+ string ocn = line[1 .. close];
+ foreach (c; ocn) {
+ if (c < '0' || c > '9') {
+ found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number");
+ break;
+ }
+ }
+ auto rest = line[close + 2 .. $].split(" ");
+ if (rest.length == 0 || !kinds.canFind(rest[0])) {
+ found ~= SspFinding(no, "6.1",
+ "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds");
+ } else if (rest[0] == "heading") {
+ if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') {
+ found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\"");
+ } else if (!levels.canFind(rest[1][1 .. $])) {
+ found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight");
+ }
+ }
+ in_object = true;
+ in_text = false;
+ prev_rank = -1;
+ continue;
+ }
+
+ /+ ↓ section 6.3: properties, known and in order +/
+ if (line[0] == '.') {
+ if (!in_object) {
+ found ~= SspFinding(no, "6", "a property outside an object record");
+ continue;
+ }
+ if (in_text) {
+ found ~= SspFinding(no, "6", "a property after the object text began");
+ }
+ auto colon = line.indexOf(": ");
+ if (colon < 2) {
+ found ~= SspFinding(no, "3", "property is not \".name: value\"");
+ continue;
+ }
+ string name = line[1 .. colon];
+ int rank = propertyRank(name);
+ if (rank < 0) {
+ found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties");
+ } else {
+ /+ ↓ the four repeatable properties may recur at the same rank +/
+ if (rank < prev_rank) {
+ found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order");
+ }
+ prev_rank = rank;
+ }
+ continue;
+ }
+
+ /+ ↓ section 6.2: text lines +/
+ if (line[0] == '|') {
+ if (!in_object) {
+ found ~= SspFinding(no, "6.2", "a text line outside an object record");
+ }
+ if (line.length < 2 || line[1] != ' ') {
+ found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text");
+ }
+ in_text = true;
+ continue;
+ }
+
+ found ~= SspFinding(no, "3", "line of no shape the specification defines");
+ }
+
+ if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); }
+ if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); }
+ return found;
+}
+
+void reportValidation(string filename, SspFinding[] found) {
+ if (found.length == 0) {
+ writeln("\nvalid: ", filename,
+ " conforms to the .ssp specification as this validator reads it");
+ return;
+ }
+ writefln("\n%d finding%s against the specification:",
+ found.length, (found.length == 1) ? "" : "s");
+ foreach (f; found) {
+ if (f.line > 0) {
+ writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail);
+ } else {
+ writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail);
+ }
+ }
+}
+#+END_SRC
+
+*** read verify images
+
+#+HEADER: :tangle "../dlang/src/read/verify.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.verify;
+
+import std.algorithm : sort;
+import std.array : array, join;
+import std.conv : to;
+import std.digest : toHexString;
+import std.digest.sha : SHA256, digest;
+import std.file : dirEntries, exists, isDir, SpanMode, getSize;
+import std.path : baseName, buildPath, extension;
+import std.stdio : writeln, File;
+import std.string : toLower;
+import ocda.model;
+import read.images : gatherImageRecords;
+
+/+ ↓ verification: recompute what the .image records claim, against the
+ images as they stand in a directory (a pod's media/image).
+
+ ssp-reader --verify --image-dir=<pod>/media/image doc.ssp
+
+ Reports one line per distinct file:
+
+ OK won_benkler_2_1.png
+ MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D...
+ ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory)
+ PRESENT ffa.png (recorded missing, but the file is there)
+ EXTRA unused.png (in directory, no record in this .ssp)
+
+ Returns the number of failures. EXTRA is reported but is not a failure:
+ a pod carries the images of every language version of a document, while
+ one .ssp covers one language.
++/
+int verifyImages(OcdaDocument doc, string image_dir, string output_file) {
+ string[] lines;
+ int failures = 0;
+ int ok = 0;
+ int mismatch = 0;
+ int absent = 0;
+ int extra = 0;
+
+ if (!(image_dir.exists) || !(image_dir.isDir)) {
+ writeln("ERROR: image directory not found: ", image_dir);
+ return 1;
+ }
+ auto gathered = gatherImageRecords(doc);
+ bool[string] recorded;
+
+ foreach (e; gathered.entries) {
+ recorded[e.name] = true;
+ string pth = buildPath(image_dir, e.name);
+ if (!(pth.exists)) {
+ if (e.missing) {
+ /+ recorded missing, and missing here too: consistent +/
+ ok += 1;
+ lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)";
+ } else {
+ absent += 1;
+ failures += 1;
+ lines ~= "ABSENT " ~ e.name
+ ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)";
+ }
+ continue;
+ }
+ if (e.missing) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)";
+ continue;
+ }
+ auto file_ = readFileStat(pth);
+ string[] differs;
+ if (file_.sha256 != e.sha256) {
+ differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256;
+ }
+ if (file_.bytes != e.bytes) {
+ differs ~= "bytes recorded " ~ e.bytes.to!string
+ ~ " file " ~ file_.bytes.to!string;
+ }
+ /+ px is written only where the dimensions could be read, and read here
+ only for the formats below, so compare only when both are known +/
+ if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0
+ && (file_.width != e.width || file_.height != e.height)) {
+ differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string
+ ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string;
+ }
+ if (differs.length > 0) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; ");
+ } else {
+ ok += 1;
+ lines ~= "OK " ~ e.name;
+ }
+ }
+
+ /+ ↓ files in the directory that no record names +/
+ string[] in_dir;
+ foreach (entry; dirEntries(image_dir, SpanMode.shallow)) {
+ if (!(entry.isFile)) continue;
+ string nm = entry.name.baseName;
+ switch (nm.extension.toLower) {
+ case ".png": case ".jpg": case ".jpeg": case ".gif":
+ in_dir ~= nm;
+ break;
+ default: break;
+ }
+ }
+ in_dir.sort;
+ foreach (nm; in_dir) {
+ if (nm !in recorded) {
+ extra += 1;
+ lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)";
+ }
+ }
+
+ string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ " in " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", against " ~ image_dir;
+ lines = [head, ""] ~ lines;
+ lines ~= "";
+ lines ~= "verify: " ~ ok.to!string ~ " ok, "
+ ~ mismatch.to!string ~ " mismatch, "
+ ~ absent.to!string ~ " absent, "
+ ~ extra.to!string ~ " extra"
+ ~ ((failures == 0) ? " PASS" : " FAIL");
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+ return failures;
+}
+
+private struct FileStat_ {
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+}
+
+private FileStat_ readFileStat(string pth) {
+ FileStat_ ret;
+ auto f = File(pth, "rb");
+ ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024));
+ ret.sha256 = sha.toHexString.to!string;
+ ret.bytes = pth.getSize;
+ auto dim = readImageDimensions(pth);
+ ret.width = dim[0];
+ ret.height = dim[1];
+ return ret;
+}
+
+/+ ↓ pixel dimensions from the file header, for the formats spine's markup
+ admits (png, gif, jpg). Header read only, no decoding, and no dependency
+ beyond phobos. Returns [0, 0] where the header cannot be read
++/
+private int[2] readImageDimensions(string pth) {
+ int[2] ret = [0, 0];
+ ubyte[] head;
+ try {
+ auto f = File(pth, "rb");
+ head = new ubyte[](64 * 1024);
+ head = f.rawRead(head);
+ } catch (Exception ex) {
+ return ret;
+ }
+ if (head.length >= 24
+ && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
+ /+ png: IHDR width & height, big endian, at 16 and 20 +/
+ ret[0] = beInt(head[16 .. 20]);
+ ret[1] = beInt(head[20 .. 24]);
+ return ret;
+ }
+ if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) {
+ /+ gif: logical screen width & height, little endian, at 6 and 8 +/
+ ret[0] = head[6] | (head[7] << 8);
+ ret[1] = head[8] | (head[9] << 8);
+ return ret;
+ }
+ if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) {
+ /+ jpeg: walk the segments to a start of frame, height then width,
+ big endian, at offsets 5 and 7 of the segment
+ +/
+ size_t i = 2;
+ while (i + 9 < head.length) {
+ if (head[i] != 0xff) { i += 1; continue; }
+ ubyte marker = head[i + 1];
+ if (marker == 0xd8 || marker == 0x01
+ || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }
+ if (marker == 0xff) { i += 1; continue; }
+ size_t seg_len = (head[i + 2] << 8) | head[i + 3];
+ bool is_sof = ((marker >= 0xc0 && marker <= 0xc3)
+ || (marker >= 0xc5 && marker <= 0xc7)
+ || (marker >= 0xc9 && marker <= 0xcb)
+ || (marker >= 0xcd && marker <= 0xcf));
+ if (is_sof) {
+ ret[1] = (head[i + 5] << 8) | head[i + 6];
+ ret[0] = (head[i + 7] << 8) | head[i + 8];
+ return ret;
+ }
+ if (seg_len < 2) break;
+ i += 2 + seg_len;
+ }
+ }
+ return ret;
+}
+
+private int beInt(const(ubyte)[] b) {
+ return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]);
+}
+#+END_SRC
+
+*** read report unknown
+
+#+HEADER: :tangle "../dlang/src/read/report_unknown.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.report_unknown;
+
+import std.algorithm : sort;
+import std.array : array;
+import std.stdio : writeln, writefln;
+import ocda.model;
+
+/+ ↓ say what was read and not understood.
+ This is what makes the reader usable as a check on the format
+ specification: a reader that ignores what it does not know is
+ conforming, but one that cannot say what it ignored cannot tell you
+ whether the specification it was written from was complete. Silence
+ and understanding look the same from outside.
++/
+void reportUnknown(ref OcdaDocument doc) {
+ if (!doc.unknown.any) {
+ writeln("\nread and understood: every property, block and line was known");
+ return;
+ }
+ writeln("\nread and NOT understood:");
+ if (doc.unknown.property.length > 0) {
+ writeln(" properties ignored (", doc.unknown.property.length, " distinct):");
+ foreach (name; doc.unknown.property.keys.sort) {
+ writefln(" .%-26s %8d", name, doc.unknown.property[name]);
+ }
+ }
+ if (doc.unknown.block.length > 0) {
+ writeln(" blocks ignored (", doc.unknown.block.length, " distinct):");
+ foreach (name; doc.unknown.block.keys.sort) {
+ writefln(" @%-26s %8d", name, doc.unknown.block[name]);
+ }
+ }
+ if (doc.unknown.line.length > 0) {
+ writeln(" lines of no known shape, by first character:");
+ foreach (c; doc.unknown.line.keys.sort) {
+ writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]);
+ }
+ }
+ writeln(" (ignoring these is conforming within a major version;",
+ " the count is so that it is not silent)");
+}
+#+END_SRC
+
+** misc
+*** read tree
+
+#+HEADER: :tangle "../dlang/src/read/tree.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.tree;
+
+import std.conv : to;
+import std.stdio : writeln, writef, writefln, File;
+import std.array : replicate;
+import ocda.model;
+
+void outputTree(OcdaDocument doc, string output_file) {
+ string[] lines;
+
+ lines ~= "Document: " ~ doc.meta("title.main");
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= " " ~ doc.meta("title.subtitle");
+ lines ~= "Author: " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build OCN -> object lookup for body headings +/
+ OcdaObject[int] headings;
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading" && obj.ocn > 0) {
+ headings[obj.ocn] = obj;
+ }
+ }
+
+ /+ ↓ find root heading (from head section, or first body heading) +/
+ if (doc.head.length > 0 && doc.head[0].is_a == "heading") {
+ auto root = doc.head[0];
+ lines ~= formatHeading(root, 0);
+ walkChildren(root, headings, lines, 1);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) {
+ foreach (child_ocn; parent.children) {
+ if (auto p = child_ocn in headings) {
+ lines ~= formatHeading(*p, depth);
+ walkChildren(*p, headings, lines, depth + 1);
+ }
+ }
+}
+
+private string formatHeading(OcdaObject obj, int depth) {
+ string indent = replicate(" ", depth);
+ string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?";
+ string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+ /+ ↓ strip internal markup from heading text for clean display +/
+ string text = stripInternalMarkup(obj.text);
+ return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text;
+}
+
+private string stripInternalMarkup(string text) {
+ /+ ↓ simple removal of common internal markers for display +/
+ import std.regex : replaceAll, regex;
+ import std.array : replace;
+ string result = text;
+ result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("┨", "").replace("┣", "")
+ .replace("┙", " ").replace("┘", "\n");
+ return result;
+}
+#+END_SRC
+
+*** read html
+
+#+HEADER: :tangle "../dlang/src/read/html.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.html;
+
+import std.array : replace, split, join, appender;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, startsWith;
+import ocda.model;
+
+void outputHtmlScroll(OcdaDocument doc, string output_file) {
+ auto buf = appender!(string[]);
+
+ buf ~= htmlHead(doc);
+
+ /+ ↓ title +/
+ if (doc.head.length > 0) {
+ auto title_obj = doc.head[0];
+ buf ~= `<div class="title">`;
+ buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`;
+ if (doc.meta("title.subtitle").length > 0)
+ buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`;
+ buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`;
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ table of contents +/
+ if (doc.toc.length > 0) {
+ buf ~= `<div class="toc">`;
+ buf ~= `<h2>Table of Contents</h2>`;
+ foreach (obj; doc.toc) {
+ if (obj.is_a == "toc") {
+ string text = convertInlineMarkup(obj.text);
+ int indent = obj.indent_base;
+ string cls = (indent > 4) ? "toc-sub" : "toc-entry";
+ buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ buf ~= `<hr/>`;
+ }
+
+ /+ ↓ body +/
+ buf ~= `<div class="body">`;
+ foreach (obj; doc.body) {
+ buf ~= renderObject(obj);
+ }
+ buf ~= `</div>`;
+
+ /+ ↓ endnotes +/
+ bool has_endnotes = false;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") { has_endnotes = true; break; }
+ }
+ if (has_endnotes) {
+ buf ~= `<div class="endnotes">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Endnotes</h2>`;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") {
+ string text = convertInlineMarkup(obj.text);
+ string parent_ref = (obj.parent_ocn > 0)
+ ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>`
+ : "";
+ buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ glossary +/
+ if (doc.glossary.length > 0) {
+ buf ~= `<div class="glossary">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Glossary</h2>`;
+ foreach (obj; doc.glossary) {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`;
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ bookindex +/
+ bool has_bookindex = false;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") { has_bookindex = true; break; }
+ }
+ if (has_bookindex) {
+ buf ~= `<div class="bookindex">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Book Index</h2>`;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="index-entry">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ buf ~= htmlTail();
+
+ /+ ↓ write output +/
+ string[] output = buf.data;
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; output) f.writeln(line);
+ } else {
+ foreach (line; output) writeln(line);
+ }
+}
+
+private string[] renderObject(OcdaObject obj) {
+ /+ ↓ markup commentary is carried by the abstraction and is not output.
+ Rendering it would put the author notes into the document, which
+ spine itself does not do
+ +/
+ if (!obj.renders) { return []; }
+ string[] lines;
+ string text = convertInlineMarkup(obj.text);
+ string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off)
+ ? ` id="o` ~ obj.ocn.to!string ~ `"`
+ : "";
+ string ocn_display = (obj.ocn > 0 && !obj.ocn_off)
+ ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>`
+ : "";
+
+ switch (obj.is_a) {
+ case "heading":
+ string tag = headingTag(obj.heading_level);
+ string seg_anchor = (obj.segment.length > 0)
+ ? ` id="` ~ htmlEscape(obj.segment) ~ `"`
+ : ocn_anchor;
+ lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`;
+ break;
+ case "para":
+ string indent_cls = "";
+ if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`;
+ else if (obj.bullet) indent_cls = ` class="bullet"`;
+ lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ case "code":
+ lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`;
+ break;
+ case "table":
+ lines ~= renderTable(obj, ocn_anchor);
+ break;
+ case "group":
+ lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`;
+ break;
+ case "block":
+ lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`;
+ break;
+ case "verse":
+ lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`;
+ break;
+ case "quote":
+ lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`;
+ break;
+ default:
+ if (text.length > 0)
+ lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ }
+ return lines;
+}
+
+private string headingTag(string level) {
+ switch (level) {
+ case "A": return "h1";
+ case "B": return "h2";
+ case "C": return "h3";
+ case "D": return "h4";
+ case "1": return "h3";
+ case "2": return "h4";
+ case "3": return "h5";
+ case "4": return "h6";
+ default: return "h4";
+ }
+}
+
+private string renderTable(OcdaObject obj, string ocn_anchor) {
+ string[] lines;
+ lines ~= `<table` ~ ocn_anchor ~ `>`;
+ /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/
+ auto rows = obj.text.split("\n");
+ bool first_row = obj.table_header;
+ foreach (row; rows) {
+ if (row.strip.length == 0) continue;
+ string tag = first_row ? "th" : "td";
+ lines ~= `<tr>`;
+ foreach (cell; row.split("┊")) {
+ lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`;
+ }
+ lines ~= `</tr>`;
+ first_row = false;
+ }
+ lines ~= `</table>`;
+ return lines.join("\n");
+}
+
+/+ ↓ convert spine internal Unicode markup to HTML +/
+string convertInlineMarkup(string text) {
+ if (text.length == 0) return "";
+ string result = htmlEscape(text);
+
+ /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc.
+ Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X
+ The ┨┣ form is used within heading content +/
+
+ /+ ↓ bold +/
+ result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>");
+ result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>");
+
+ /+ ↓ emphasis +/
+ result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>");
+ result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>");
+
+ /+ ↓ italic +/
+ result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>");
+ result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>");
+
+ /+ ↓ underscore +/
+ result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>");
+ result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>");
+
+ /+ ↓ superscript +/
+ result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>");
+ result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>");
+
+ /+ ↓ subscript +/
+ result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>");
+ result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>");
+
+ /+ ↓ monospace +/
+ result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>");
+
+ /+ ↓ links: ┥link text┝┤url├ +/
+ result = replaceLinks(result);
+
+ /+ ↓ footnote markers: 【N text】 +/
+ result = replaceFootnotes(result);
+
+ /+ ↓ line breaks +/
+ result = result.replace("┘", "<br/>");
+ result = result.replace("┙", "<br/>");
+
+ /+ ↓ non-breaking spaces +/
+ result = result.replace("░", "&nbsp;");
+
+ /+ ↓ bullet markers +/
+ result = result.replace("● ", "&#8226; ");
+
+ /+ ↓ page break markers +/
+ result = result.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ return result;
+}
+
+private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) {
+ /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/
+ string result = text;
+ while (true) {
+ auto open_pos = result.indexOf(open_mark);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + open_mark.length;
+ /+ ↓ look for closing ⑆X +/
+ string close_mark = "⑆" ~ close_suffix;
+ auto close_pos = indexOf(result[after_open .. $], close_mark);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + close_mark.length .. $];
+ }
+ return result;
+}
+
+private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) {
+ /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/
+ string result = text;
+ string full_open = open_mark ~ bracket_open;
+ while (true) {
+ auto open_pos = result.indexOf(full_open);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + full_open.length;
+ auto close_pos = indexOf(result[after_open .. $], bracket_close);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + bracket_close.length .. $];
+ }
+ return result;
+}
+
+private string replaceLinks(string text) {
+ /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/
+ string result = text;
+ while (true) {
+ auto link_open = result.indexOf("┥");
+ if (link_open < 0) break;
+ auto link_mid = indexOf(result[link_open .. $], "┝┤");
+ if (link_mid < 0) break;
+ link_mid += link_open;
+ auto link_close = indexOf(result[link_mid .. $], "├");
+ if (link_close < 0) break;
+ link_close += link_mid;
+ string link_text = result[link_open + "┥".length .. link_mid];
+ string url = result[link_mid + "┝┤".length .. link_close];
+ /+ ↓ handle internal links (¤ prefix) +/
+ if (url.length > 0 && url[0] == '¤') {
+ url = url[("¤").length .. $];
+ }
+ /+ ↓ handle OCN-only links (#N) +/
+ if (url.length > 0 && url[0] == '#') {
+ url = "#o" ~ url[1 .. $];
+ }
+ result = result[0 .. link_open]
+ ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>`
+ ~ result[link_close + "├".length .. $];
+ }
+ return result;
+}
+
+private string replaceFootnotes(string text) {
+ /+ ↓ replace 【N content】 with superscript note number +/
+ string result = text;
+ while (true) {
+ auto fn_open = result.indexOf("【");
+ if (fn_open < 0) break;
+ auto fn_close = indexOf(result[fn_open .. $], "】");
+ if (fn_close < 0) break;
+ fn_close += fn_open;
+ string fn_content = result[fn_open + "【".length .. fn_close];
+ /+ ↓ extract note number (first token before space) +/
+ auto space_pos = fn_content.indexOf(' ');
+ string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content;
+ result = result[0 .. fn_open]
+ ~ `<sup class="fn">` ~ note_num ~ `</sup>`
+ ~ result[fn_close + "】".length .. $];
+ }
+ return result;
+}
+
+string htmlEscape(string text) {
+ return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+ .replace("\"", "&quot;");
+}
+
+private string[] htmlHead(OcdaDocument doc) {
+ string[] lines;
+ lines ~= `<!DOCTYPE html>`;
+ lines ~= `<html>`;
+ lines ~= `<head>`;
+ lines ~= `<meta charset="utf-8"/>`;
+ lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`;
+ lines ~= `<style>`;
+ lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`;
+ lines ~= ` .title { text-align: center; margin-bottom: 2em; }`;
+ lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`;
+ lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`;
+ lines ~= ` .toc { margin-bottom: 2em; }`;
+ lines ~= ` .toc-entry { margin: 0.2em 0; }`;
+ lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`;
+ lines ~= ` .indent-1 { margin-left: 2em; }`;
+ lines ~= ` .indent-2 { margin-left: 4em; }`;
+ lines ~= ` .indent-3 { margin-left: 6em; }`;
+ lines ~= ` .bullet { margin-left: 2em; }`;
+ lines ~= ` .bullet:before { content: "\2022 "; }`;
+ lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`;
+ lines ~= ` .group { margin: 1em 2em; }`;
+ lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`;
+ lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`;
+ lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`;
+ lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`;
+ lines ~= ` th { background: #f0f0f0; }`;
+ lines ~= ` .endnote { font-size: 0.9em; }`;
+ lines ~= ` .fn { color: #666; }`;
+ lines ~= ` .index-entry { font-size: 0.9em; }`;
+ lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`;
+ lines ~= `</style>`;
+ lines ~= `</head>`;
+ lines ~= `<body>`;
+ return lines;
+}
+
+private string[] htmlTail() {
+ return [`</body>`, `</html>`];
+}
+#+END_SRC
+
+*** read images
+
+#+HEADER: :tangle "../dlang/src/read/images.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.images;
+
+import std.array : join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import ocda.model;
+
+/+ ↓ the .image records of a document, gathered per distinct file, with the
+ objects referencing each file, in order of first appearance +/
+struct ImageEntry {
+ string name;
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+ bool missing;
+ string[] refs;
+ string[] other_sha256; // same filename, differing digest
+}
+
+struct ImageRecords {
+ int records; // .image lines, an image referenced twice counts twice
+ ImageEntry[] entries; // distinct files
+}
+
+ImageRecords gatherImageRecords(OcdaDocument doc) {
+ ImageRecords ret;
+ size_t[string] seen;
+ foreach (section; sectionNames) {
+ auto objs = doc.sectionByName(section);
+ if (objs is null) continue;
+ foreach (obj; *objs) {
+ foreach (img; obj.images) {
+ ret.records += 1;
+ string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]";
+ if (auto at = img.name in seen) {
+ if (ret.entries[*at].sha256 != img.sha256
+ && img.sha256.length > 0
+ && ret.entries[*at].sha256.length > 0) {
+ ret.entries[*at].other_sha256 ~= img.sha256;
+ }
+ ret.entries[*at].refs ~= _ref;
+ } else {
+ ImageEntry e_;
+ e_.name = img.name;
+ e_.sha256 = img.sha256;
+ e_.bytes = img.bytes;
+ e_.width = img.width;
+ e_.height = img.height;
+ e_.missing = img.missing;
+ e_.refs = [_ref];
+ seen[img.name] = ret.entries.length;
+ ret.entries ~= e_;
+ }
+ }
+ }
+ }
+ return ret;
+}
+
+/+ ↓ image manifest, line oriented and greppable:
+
+ <sha256> <bytes> <px> <filename> refs: body[97] body[412]
+ MISSING <filename> refs: body[840]
+
+ The point of the record in the .ssp is verification (is the image
+ shipped in this pod the image the abstraction was built from), so this
+ output is the shape a verifier wants: one line per distinct file.
++/
+void outputImages(OcdaDocument doc, string output_file) {
+ auto gathered = gatherImageRecords(doc);
+ string[] lines;
+ int missing = 0;
+ foreach (e; gathered.entries) {
+ if (e.missing) missing += 1;
+ }
+ lines ~= "Images: " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ ", " ~ missing.to!string ~ " missing";
+ lines ~= "";
+ foreach (e; gathered.entries) {
+ string px = (e.width > 0 && e.height > 0)
+ ? e.width.to!string ~ "x" ~ e.height.to!string : "-";
+ lines ~= (e.missing)
+ ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ")
+ : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px
+ ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ");
+ /+ ↓ one filename, more than one digest: the .ssp is inconsistent,
+ or two distinct files share a name +/
+ foreach (other; e.other_sha256) {
+ lines ~= " WARNING differing digest for same filename: " ~ other;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+/+ ↓ count of .image records in a document, for the parse summary +/
+int imageRecordCount(OcdaDocument doc) {
+ return gatherImageRecords(doc).records;
+}
+
+package immutable string[] sectionNames = [
+ "head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb",
+];
+#+END_SRC
+
+** llm
+*** read llm text
+
+#+HEADER: :tangle "../dlang/src/read/llm_text.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.llm_text;
+
+import std.array : replace, split, join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ocda.model;
+
+/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption
+ format:
+ ## Document Title > Chapter > Section > [OCN]
+
+ Clean paragraph text with inline markup stripped.
++/
+void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) {
+ string[] lines;
+ string doc_title = doc.meta("title.main");
+
+ /+ ↓ document header +/
+ lines ~= "# " ~ doc_title;
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= "# " ~ doc.meta("title.subtitle");
+ lines ~= "# " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build heading text lookup from ancestors +/
+ string[int] heading_text;
+ foreach (obj; doc.head) {
+ if (obj.is_a == "heading" && obj.ocn > 0)
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading")
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+
+ /+ ↓ body objects +/
+ string current_breadcrumb = "";
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj);
+ lines ~= "";
+ lines ~= "## " ~ current_breadcrumb;
+ lines ~= "";
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ string ocn_ref = (obj.ocn > 0 && !obj.ocn_off)
+ ? " [" ~ obj.ocn.to!string ~ "]"
+ : "";
+
+ switch (obj.is_a) {
+ case "para":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "code":
+ lines ~= "```";
+ lines ~= obj.text; /+ preserve raw text for code +/
+ lines ~= "```";
+ lines ~= "";
+ break;
+ case "table":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "verse":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "quote":
+ lines ~= "> " ~ text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "group":
+ case "block":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ default:
+ if (text.length > 0) {
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ }
+ break;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) {
+ /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ /+ ↓ truncate very long heading text for breadcrumb +/
+ if (clean.length > 60)
+ clean = clean[0 .. 57] ~ "...";
+ parts ~= clean;
+ }
+ }
+ }
+ /+ ↓ add the current heading itself +/
+ string heading_text = stripMarkup(obj.text).strip;
+ if (heading_text.length > 60)
+ heading_text = heading_text[0 .. 57] ~ "...";
+
+ string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+
+ if (parts.length > 0)
+ return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref;
+ else
+ return doc_title ~ " > " ~ heading_text ~ ocn_ref;
+}
+
+/+ ↓ strip all internal Unicode markup to produce clean plain text +/
+string stripMarkup(string text) {
+ if (text.length == 0) return "";
+ string r = text;
+
+ /+ ↓ links: ┥link text┝┤url├ -> link text +/
+ while (true) {
+ auto lo = r.indexOf("┥");
+ if (lo < 0) break;
+ auto lm = indexOf(r[lo .. $], "┝┤");
+ if (lm < 0) break;
+ lm += lo;
+ auto lc = indexOf(r[lm .. $], "├");
+ if (lc < 0) break;
+ lc += lm;
+ string link_text = r[lo + "┥".length .. lm];
+ r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $];
+ }
+
+ /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/
+ while (true) {
+ auto fo = r.indexOf("【");
+ if (fo < 0) break;
+ auto fc = indexOf(r[fo .. $], "】");
+ if (fc < 0) break;
+ fc += fo;
+ r = r[0 .. fo] ~ r[fc + "】".length .. $];
+ }
+
+ /+ ↓ font face markers +/
+ r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("⑆■", "")
+ .replace("┨", "").replace("┣*", "").replace("┣!", "")
+ .replace("┣/", "").replace("┣_", "").replace("┣^", "")
+ .replace("┣,", "").replace("┣", "");
+
+ /+ ↓ line breaks and special chars +/
+ r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n");
+ r = r.replace("░", " ");
+ r = r.replace("● ", "- ");
+ r = r.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ /+ ↓ table cell separators +/
+ r = r.replace("┊", " | ");
+
+ /+ ↓ clean up multiple spaces +/
+ while (r.indexOf(" ") >= 0)
+ r = r.replace(" ", " ");
+
+ return r.strip;
+}
+#+END_SRC
+
+*** read llm jsonl
+
+#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.llm_jsonl;
+
+import std.array : split, join, replace;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ocda.model;
+import read.llm_text : stripMarkup;
+
+/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
+ one JSON object per line, each containing:
+ doc, ocn, path (breadcrumb), type, text (clean),
+ images (only where the object embeds one, so that a chunk whose
+ text reads around a figure carries the figure's identity),
+ index_terms (from bookindex entries associated with this object)
++/
+void outputLlmJsonl(OcdaDocument doc, string output_file) {
+ string[] lines;
+ string doc_id = doc.meta("title.main");
+ string doc_author = doc.meta("creator.author");
+
+ /+ ↓ build bookindex -> OCN mapping for index_terms
+ bookindex objects in the .ssp have parent_ocn linking to body objects
+ but the index terms are in the text of bookindex objects, not
+ directly attached to body objects. We use a simpler approach:
+ headings carry heading_ancestors_text which gives context. +/
+
+ /+ ↓ track current heading path +/
+ string current_path = "";
+
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
+ /+ ↓ emit heading as a chunk too +/
+ string heading_text = stripMarkup(obj.text);
+ if (heading_text.strip.length > 0) {
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
+ }
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content, and least of all is it a chunk
+ to embed: it would be indexed as though the author had written it
+ into the document
+ +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
+ return `{"doc":` ~ jsonStr(doc_id)
+ ~ `,"author":` ~ jsonStr(doc_author)
+ ~ `,"ocn":` ~ ocn.to!string
+ ~ `,"path":` ~ jsonStr(path)
+ ~ `,"type":` ~ jsonStr(type)
+ ~ `,"text":` ~ jsonStr(text)
+ ~ jsonImages(images)
+ ~ `}`;
+}
+
+private string jsonImages(OcdaImage[] images) {
+ /+ ↓ omitted entirely where the object embeds no image +/
+ if (images.length == 0) return "";
+ string[] items;
+ foreach (img; images) {
+ string item = `{"name":` ~ jsonStr(img.name);
+ if (img.missing) {
+ item ~= `,"missing":true`;
+ } else {
+ item ~= `,"sha256":` ~ jsonStr(img.sha256)
+ ~ `,"bytes":` ~ img.bytes.to!string;
+ if (img.width > 0 && img.height > 0) {
+ item ~= `,"width":` ~ img.width.to!string
+ ~ `,"height":` ~ img.height.to!string;
+ }
+ }
+ items ~= item ~ `}`;
+ }
+ return `,"images":[` ~ items.join(",") ~ `]`;
+}
+
+private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ parts ~= clean;
+ }
+ }
+ }
+ string heading_text = stripMarkup(obj.text).strip;
+ if (parts.length > 0)
+ return parts.join(" > ") ~ " > " ~ heading_text;
+ else
+ return heading_text;
+}
+
+private string jsonStr(string s) {
+ /+ ↓ JSON string escaping +/
+ string r = s;
+ r = r.replace(`\`, `\\`);
+ r = r.replace(`"`, `\"`);
+ r = r.replace("\n", `\n`);
+ r = r.replace("\r", `\r`);
+ r = r.replace("\t", `\t`);
+ return `"` ~ r ~ `"`;
+}
+#+END_SRC
+
+* test
+** tests
+*** test agree
+
+#+HEADER: :tangle "../dlang/test/agree.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ Do the two loaders agree?
+
+ The library's whole purpose is that a writer cannot tell which
+ serialisation it was handed. That is only true if loading a document's
+ .ssp and loading its .ocda.db give the same model, so this asks.
+
+ Usage: agree <doc>.ssp <doc>.ocda.db [lang]
++/
+import std.conv : to;
+import std.stdio;
+import ocda.model;
+import ocda.load;
+
+int main(string[] args) {
+ if (args.length < 3) {
+ stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]");
+ return 2;
+ }
+ string lang = (args.length > 3) ? args[3] : "";
+ OcdaDocument a, b;
+ try {
+ a = load(args[1]);
+ b = load(args[2], lang);
+ } catch (Exception e) {
+ stderr.writeln("error: ", e.msg);
+ return 2;
+ }
+
+ int differences = 0;
+ void differ(string what, string from_ssp, string from_db) {
+ writeln(" differ: ", what);
+ writeln(" .ssp: ", from_ssp);
+ writeln(" .ocda.db: ", from_db);
+ differences++;
+ }
+
+ if (a.format_version != b.format_version) {
+ differ("format_version", a.format_version, b.format_version);
+ }
+ if (a.source_filename != b.source_filename) {
+ differ("source_filename", a.source_filename, b.source_filename);
+ }
+
+ /+ ↓ the sections, object by object. Position within a section is the
+ identity to compare on: an ocn is not unique and 0 is shared +/
+ static immutable sections = ["head", "toc", "body", "endnotes", "glossary",
+ "bibliography", "bookindex", "blurb", "tail"];
+ foreach (name; sections) {
+ auto sa = a.sectionByName(name);
+ auto sb = b.sectionByName(name);
+ if (sa is null || sb is null) { continue; }
+ if ((*sa).length != (*sb).length) {
+ differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string);
+ continue;
+ }
+ foreach (i, oa; *sa) {
+ auto ob = (*sb)[i];
+ string at = name ~ "[" ~ i.to!string ~ "]";
+ if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); }
+ if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); }
+ if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); }
+ if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); }
+ if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); }
+ if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); }
+ if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); }
+ if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); }
+ if (oa.ancestors != ob.ancestors) {
+ differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string);
+ }
+ if (oa.images.length != ob.images.length) {
+ differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string);
+ }
+ if (differences > 12) { writeln(" ... stopping after 12"); return 1; }
+ }
+ }
+
+ if (differences == 0) {
+ writeln("agree: the two loaders give the same model");
+ return 0;
+ }
+ writeln(differences, " difference(s)");
+ return 1;
+}
+#+END_SRC
+
+*** test dump
+
+#+HEADER: :tangle "../dlang/test/dump.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+/++
+ Write a loaded abstraction out canonically.
+
+ The oracle for everything that claims to implement this model: load a
+ .ssp or a .ocda.db and print the model, deterministically, so that two
+ implementations can be compared with diff.
+
+ Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
++/
+import std.stdio;
+import ocda.model;
+import ocda.load;
+import ocda.dump;
+
+int main(string[] args) {
+ bool unknown = false;
+ string[] rest;
+ foreach (a; args[1 .. $]) {
+ if (a == "--unknown") { unknown = true; } else { rest ~= a; }
+ }
+ if (rest.length < 1) {
+ stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]");
+ return 2;
+ }
+ string lang = (rest.length > 1) ? rest[1] : "";
+ OcdaDocument doc;
+ try {
+ doc = load(rest[0], lang);
+ } catch (Exception e) {
+ stderr.writeln("error: ", e.msg);
+ return 2;
+ }
+ write(unknown ? dumpUnknown(doc) : dumpDocument(doc));
+ return 0;
+}
+#+END_SRC
+
+* config
+** nix flake.nix :flake:
+
+#+HEADER: :tangle ../dlang/flake.nix
+#+HEADER: :noweb yes
+#+BEGIN_SRC nix
+{
+ description = "ocda-lib: the object-centric document abstraction as a D library";
+ inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+ outputs = { self, nixpkgs }:
+ let
+ system = "x86_64-linux"; # adjust for aarch64-linux / darwin
+ pkgs = import nixpkgs { inherit system; };
+ /* A library, so the library itself is not a build product: a program
+ that uses it compiles its modules alongside its own, as the writer's
+ flake does, or takes it as a dub dependency. What is built here are
+ programs over it: the reader, and the two that put the library to
+ the question - the agreement check, which loads a document's .ssp
+ and its .ocda.db and compares the two models, and the dump, which
+ writes a loaded model out canonically so that an implementation in
+ another language can be diffed against this one.
+ .
+ `mains` is what is compiled besides the library: the checks are one
+ file each in test/, the reader is src/read/.
+ */
+ mkTool = { tool, binary, what, mains, compiler, compilerBin }:
+ pkgs.stdenv.mkDerivation {
+ pname = "ocda-lib-${tool}";
+ version = "0.1.0";
+ src = self;
+ nativeBuildInputs = [ compiler ];
+ buildInputs = [ pkgs.sqlite pkgs.zstd ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/bin
+ install -m755 ${tool} $out/bin/${binary}
+ runHook postInstall
+ '';
+ meta = {
+ description = what;
+ mainProgram = binary;
+ };
+ };
+ ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
+ ocda-read = mkTool (ldc // {
+ tool = "read"; binary = "ocda-read"; mains = "src/read/*.d";
+ what = "reads the abstraction in any of its forms and says what is in it";
+ });
+ ocda-agree = mkTool (ldc // {
+ tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d";
+ what = "checks that the .ssp and .ocda.db loaders give one model";
+ });
+ ocda-dump = mkTool (ldc // {
+ tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d";
+ what = "writes a loaded abstraction out canonically, as an oracle";
+ });
+ in {
+ packages.${system} = {
+ /* All three in one bin/, so that one build leaves every program in
+ one result link. Built separately into the same link, each build
+ replaced the one before, and a script looking for ocda-dump beside
+ ocda-agree found nothing.
+ */
+ default = pkgs.symlinkJoin {
+ name = "ocda-lib-programs-0.1.0";
+ paths = [ ocda-read ocda-agree ocda-dump ];
+ };
+ inherit ocda-read ocda-agree ocda-dump;
+ };
+
+ devShells.${system}.default = pkgs.mkShell {
+ packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
+ };
+ };
+}
+#+END_SRC
+
+** README.md
+
+#+HEADER: :tangle ../dlang/README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# ocda-lib, in D
+
+The abstraction's model and both its loaders, as a dub library.
+
+## Modules
+
+| | |
+|---|---|
+| `ocda.model` | the document, the object, the eight sections. Imports nothing |
+| `ocda.load_ssp` | text -> model |
+| `ocda.load_db` | sqlite -> model |
+| `ocda.load` | what a path is, and loading it |
+| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries |
+| `ocda.dump` | the model written out canonically, as an oracle |
+| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon |
+| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) |
+
+`ocda.model` importing nothing is the property that matters: a writer
+depends on the model and on neither loader.
+
+## Use
+
+```json
+"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } }
+```
+
+```d
+import ocda.load;
+auto doc = load("document.ocda.db", "de"); // or "document.ssp"
+foreach (obj; doc.body) { ... }
+```
+
+Link `sqlite3`; the library declares the dozen functions it needs rather
+than taking a binding from the registry, so there is nothing to fetch.
+
+## The reader
+
+```sh
+nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl|
+ --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate]
+ <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]
+```
+
+The library's reader, one program where `sisudoc-ssp-reader` and
+`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp`
+reader's, moved onto the library's model, so every mode reads every form
+and **says the same thing of a document's `.ssp`, `.ocda.db` and
+`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries
+files; `--validate` checks a `.ssp` as text and refuses anything else.
+
+It lives in `src/read/` beside the library, and dub keeps it out of the
+library: the configuration `library` excludes `src/read/*`, the
+configuration `ocda-read` builds it. A program using the library by dub, or
+by naming `src/ocda/*.d` as the writer's flake does, never compiles it.
+
+## The two checks
+
+```sh
+nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang]
+nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang]
+```
+
+`agree` loads both and compares ten fields of each object, naming the field
+where they part company. `agree` found the blank-first-text-line defect and
+was not enough for anything subtler.
+
+**`ocda-dump` is the stronger check**: the whole loaded model, every field,
+written out in a form fully determined by the model, so that two
+implementations can be compared with `diff`. It is an oracle, not an output
+format; anything reading it to reconstruct a document has misunderstood it.
+It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is
+the point of having it.
+
+It found four disagreements `agree` could not see. They are in the
+repository README and in the history.
+
+## Build
+
+```sh
+nix build # all three programs, in one result/bin
+nix build .#ocda-read # the reader
+nix build .#ocda-agree # the agreement check
+nix build .#ocda-dump # the canonical dump
+nix develop # ldc, dub, dtools, sqlite, zstd
+dub build # the library
+dub build --config=ocda-read # the reader
+```
+#+END_SRC
+
+** dub.json
+
+#+HEADER: :tangle ../dlang/dub.json
+#+HEADER: :noweb yes
+#+BEGIN_SRC json
+{
+ "authors": [ "Ralph Amissah" ],
+ "copyright": "Copyright (C) 2026 Ralph Amissah",
+ "name": "ocda-lib",
+ "version": "0.1.0",
+ "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations",
+ "homepage": "https://sisudoc.org",
+ "license": "AGPL-3.0+",
+ "sourcePaths": [ "./src" ],
+ "importPaths": [ "./src" ],
+ "libs": [ "sqlite3", "zstd" ],
+ "configurations": [
+ {
+ "name": "library",
+ "targetType": "library",
+ "excludedSourceFiles": [ "src/read/*" ]
+ },
+ {
+ "name": "ocda-read",
+ "targetType": "executable",
+ "targetName": "ocda-read",
+ "mainSourceFile": "src/read/main.d"
+ }
+ ]
+}
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* nixpkgs source
+
+#+NAME: nixpkgs_unstable
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>>
+#+END_SRC
+
+* __END__
diff --git a/org/gleam-read.org b/org/gleam-read.org
deleted file mode 100644
index 05fa2d5..0000000
--- a/org/gleam-read.org
+++ /dev/null
@@ -1,520 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** main
-*** tangle
-
-#+HEADER: :tangle ../gleam/src/read/main.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<read>>
-#+END_SRC
-
-*** source
-
-#+NAME: read
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-//// ocda-read, in Gleam: reads the abstraction, in any of the forms the
-//// library reads, and says what is in it.
-////
-//// ocda-read [--tree|--outline|--languages|--files|--report-unknown]
-//// <file>.ssp | <file>.ocda.db | <file>.zip [lang]
-////
-//// The modes the Gleam readers had (the summary, the outline, the unknown
-//// report, and the database's languages and files), over the library, so
-//// that each reads every form. Their output is the D ocda-read's, byte for
-//// byte: two readers in unrelated languages, neither written from the other
-//// line by line, agreeing is what says each has read the model and not its
-//// own idea of it. A `.sisupod` is the one form the Gleam library cannot
-//// open (its zstd frame; see `ocda/load_pod`), and it says so.
-
-import gleam/dict
-import gleam/int
-import gleam/io
-import gleam/list
-import gleam/result
-import gleam/string
-import ocda/inventory.{type Language}
-import ocda/load
-import ocda/model.{type Document, type Object}
-
-/// The format major this reader knows.
-const major_read = "2"
-
-type Options {
- Options(
- tree: Bool,
- languages: Bool,
- files: Bool,
- report_unknown: Bool,
- any_version: Bool,
- lang: String,
- positional: List(String),
- )
-}
-
-/// Run with the arguments after the program name; returns the exit status.
-pub fn run(args: List(String)) -> Int {
- case parse(args, Options(False, False, False, False, False, "", [])) {
- Error(message) -> {
- io.println_error("ocda-read: " <> message <> "; --help lists the options")
- 2
- }
- Ok(Options(positional: [], ..)) -> {
- help()
- 0
- }
- Ok(options) -> read(options)
- }
-}
-
-fn parse(args: List(String), o: Options) -> Result(Options, String) {
- case args {
- [] -> Ok(Options(..o, positional: list.reverse(o.positional)))
- ["--help", ..] | ["-h", ..] -> Ok(Options(..o, positional: []))
- ["--tree", ..rest] | ["--outline", ..rest] ->
- parse(rest, Options(..o, tree: True))
- ["--languages", ..rest] -> parse(rest, Options(..o, languages: True))
- ["--files", ..rest] -> parse(rest, Options(..o, files: True))
- ["--report-unknown", ..rest] ->
- parse(rest, Options(..o, report_unknown: True))
- ["--any-version", ..rest] -> parse(rest, Options(..o, any_version: True))
- ["--lang", code, ..rest] -> parse(rest, Options(..o, lang: code))
- [flag, ..rest] ->
- case string.starts_with(flag, "--lang=") {
- True -> parse(rest, Options(..o, lang: string.drop_start(flag, 7)))
- False ->
- case string.starts_with(flag, "-") {
- True ->
- case
- list.contains(
- [
- "--html", "--llm-text", "--llm-jsonl", "--images", "--verify",
- "--validate",
- ],
- flag,
- )
- {
- True ->
- Error(flag <> " is a mode of the D ocda-read, not of this one")
- False -> Error("Unrecognized option " <> flag)
- }
- False ->
- parse(rest, Options(..o, positional: [flag, ..o.positional]))
- }
- }
- }
-}
-
-fn help() -> Nil {
- io.println("ocda-read - reads the document abstraction, in any of its forms")
- io.println("")
- io.println(
- "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.zip [lang]",
- )
- io.println("")
- io.println(" --tree the headings, as a tree")
- io.println(" --outline the same as --tree")
- io.println(" --languages the languages the file holds")
- io.println(" --files what a .ocda.db carries besides the abstraction")
- io.println(" --report-unknown what was read and not understood")
- io.println(" --lang=<code> the language to read (default: the first)")
- io.println(" --any-version read a format major this reader does not know")
-}
-
-fn read(o: Options) -> Int {
- let assert [path, ..more] = o.positional
- let lang = case o.lang, more {
- "", [given, ..] -> given
- l, _ -> l
- }
- let lang_named = lang != ""
- let loaded = {
- use held <- result.try(inventory.languages(path))
- let lang = case lang, held {
- "", [first, ..] -> first.lang
- l, _ -> l
- }
- use _ <- result.try(case
- held != [] && !list.any(held, fn(l) { l.lang == lang })
- {
- True ->
- Error(
- "no language '"
- <> lang
- <> "' in "
- <> path
- <> "; it holds: "
- <> string.join(list.map(held, fn(l) { l.lang }), " "),
- )
- False -> Ok(Nil)
- })
- use doc <- result.try(load.load(path, lang, o.any_version))
- Ok(#(held, lang, doc))
- }
- case loaded {
- Error(message) -> {
- io.println_error("Error reading " <> path <> ": " <> message)
- 2
- }
- Ok(#(held, lang, doc)) ->
- case gate(path, doc, o.any_version) {
- False -> 3
- True -> {
- summary(path, doc, held, lang, lang_named)
- case o.languages {
- True -> print_languages(held)
- False -> Nil
- }
- case o.files {
- True -> print_files(path)
- False -> Nil
- }
- case o.tree {
- True -> tree(doc)
- False -> Nil
- }
- case o.report_unknown {
- True -> report_unknown(doc)
- False -> Nil
- }
- case o.tree || o.languages || o.files || o.report_unknown {
- True -> Nil
- False ->
- io.println(
- "\n(this is the summary: --help lists the modes, each adding its output)",
- )
- }
- 0
- }
- }
- }
-}
-
-/// The major part, and refuse what is not known.
-fn gate(path: String, doc: Document, any_version: Bool) -> Bool {
- case doc.format_version, model.version_major(doc) == major_read, any_version {
- "", _, _ -> {
- io.println_error("warning: " <> path <> " declares no format version")
- True
- }
- _, True, _ -> True
- v, False, True -> {
- io.println_error(
- "warning: "
- <> path
- <> " is format v"
- <> v
- <> ", this reader knows v"
- <> major_read
- <> ".x; reading anyway",
- )
- True
- }
- v, False, False -> {
- io.println_error(
- "error: "
- <> path
- <> " is format v"
- <> v
- <> ", this reader knows v"
- <> major_read
- <> ".x",
- )
- io.println_error(
- " a major version differs: a property may have changed meaning or gone away.",
- )
- io.println_error(" --any-version reads it regardless.")
- False
- }
- }
-}
-
-const sections = [
- "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
- "blurb", "tail",
-]
-
-/// The sections whose images are counted: every one but the tail, as the D
-/// reader counts them.
-const image_sections = [
- "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
- "blurb",
-]
-
-fn summary(
- path: String,
- doc: Document,
- held: List(Language),
- lang: String,
- lang_named: Bool,
-) -> Nil {
- io.println("Read: " <> path)
- case doc.format_version {
- "" -> Nil
- v -> io.println(" Format: v" <> v)
- }
- case dict.get(doc.source, "language") {
- Error(_) -> Nil
- Ok(language) -> {
- let of = case dict.get(doc.source, "languages") {
- Ok(ls) -> " of " <> ls
- Error(_) -> ""
- }
- let digest = case dict.get(doc.source, "digest") {
- Ok(d) -> ", markup digest " <> string.slice(d, 0, 16) <> "..."
- Error(_) -> ""
- }
- io.println(" Source: " <> language <> of <> digest)
- }
- }
- case list.length(held) > 1 && !lang_named {
- True -> io.println(" (reading " <> lang <> "; --lang=<code> for another)")
- False -> Nil
- }
- io.println(" Title: " <> model.meta(doc, "title.main"))
- io.println(" Author: " <> model.meta(doc, "creator.author"))
- io.println(
- " Sections: "
- <> string.join(
- list.map(sections, fn(s) {
- s <> "=" <> int.to_string(list.length(model.section(doc, s)))
- }),
- " ",
- ),
- )
- let records =
- list.fold(image_sections, 0, fn(n, s) {
- list.fold(model.section(doc, s), n, fn(n, o) {
- n + list.length(o.images)
- })
- })
- case records {
- 0 -> Nil
- 1 -> io.println(" Images: 1 record")
- n -> io.println(" Images: " <> int.to_string(n) <> " records")
- }
-}
-
-fn print_languages(held: List(Language)) -> Nil {
- io.println("")
- io.println(" lang source ssp digest")
- list.each(held, fn(l) {
- let digest = case string.length(l.ssp_digest) > 16 {
- True -> string.slice(l.ssp_digest, 0, 16) <> "..."
- False -> l.ssp_digest
- }
- io.println(
- " "
- <> string.pad_end(l.lang, 7, " ")
- <> " "
- <> string.pad_end(l.source_filename, 31, " ")
- <> " "
- <> digest,
- )
- })
-}
-
-fn print_files(path: String) -> Nil {
- io.println("")
- case inventory.is_carrier(path) {
- False ->
- io.println(
- " carried files: only a .ocda.db carries files; this is "
- <> load.source_name(load.source_of(path)),
- )
- True ->
- case inventory.carried(path) {
- Error(message) -> io.println_error("error: " <> message)
- Ok([]) -> io.println(" carries no files")
- Ok(rows) -> {
- io.println(" carried files:")
- io.println(" role stored n original in the file")
- list.each(rows, fn(r) {
- io.println(
- " "
- <> string.pad_end(r.role, 9, " ")
- <> " "
- <> string.pad_end(r.compression, 7, " ")
- <> " "
- <> string.pad_end(r.count, 5, " ")
- <> " "
- <> string.pad_end(r.bytes, 13, " ")
- <> " "
- <> r.stored,
- )
- })
- }
- }
- }
-}
-
-/// The headings, as a tree: from the document's root heading, down the
-/// children each heading lists, as the D reader walks it.
-fn tree(doc: Document) -> Nil {
- io.println("Document: " <> model.meta(doc, "title.main"))
- case model.meta(doc, "title.subtitle") {
- "" -> Nil
- sub -> io.println(" " <> sub)
- }
- io.println("Author: " <> model.meta(doc, "creator.author"))
- io.println("")
- let headings =
- model.section(doc, "body")
- |> list.filter(fn(o) { o.is_a == "heading" && o.ocn > 0 })
- |> list.map(fn(o) { #(o.ocn, o) })
- |> dict.from_list
- case model.section(doc, "head") {
- [root, ..] if root.is_a == "heading" -> {
- io.println(heading_line(root, 0))
- walk(root, headings, 1)
- }
- _ -> Nil
- }
-}
-
-fn walk(parent: Object, headings: dict.Dict(Int, Object), depth: Int) -> Nil {
- list.each(parent.children, fn(ocn) {
- case dict.get(headings, ocn) {
- Ok(child) -> {
- io.println(heading_line(child, depth))
- walk(child, headings, depth + 1)
- }
- Error(_) -> Nil
- }
- })
-}
-
-fn heading_line(o: Object, depth: Int) -> String {
- let level = case o.heading_level {
- "" -> "?"
- l -> l
- }
- let ocn = case o.ocn > 0 {
- True -> " [" <> int.to_string(o.ocn) <> "]"
- False -> ""
- }
- string.repeat(" ", depth) <> level <> "~" <> ocn <> " " <> strip_markup(o.text)
-}
-
-/// The internal markers a heading's text may carry, removed for display, as
-/// the D reader removes them.
-fn strip_markup(text: String) -> String {
- text
- |> string.replace("⑆*", "")
- |> string.replace("⑆!", "")
- |> string.replace("⑆/", "")
- |> string.replace("⑆_", "")
- |> string.replace("⑆^", "")
- |> string.replace("⑆,", "")
- |> string.replace("┨", "")
- |> string.replace("┣", "")
- |> string.replace("┙", " ")
- |> string.replace("┘", "\n")
-}
-
-/// Say what was read and not understood.
-fn report_unknown(doc: Document) -> Nil {
- let u = doc.unknown
- case model.any_unknown(u) {
- False ->
- io.println(
- "\nread and understood: every property, block and line was known",
- )
- True -> {
- io.println("\nread and NOT understood:")
- section_of_unknown(u.property, " properties ignored (", fn(name, n) {
- " ."
- <> string.pad_end(name, 26, " ")
- <> " "
- <> string.pad_start(int.to_string(n), 8, " ")
- })
- section_of_unknown(u.block, " blocks ignored (", fn(name, n) {
- " @"
- <> string.pad_end(name, 26, " ")
- <> " "
- <> string.pad_start(int.to_string(n), 8, " ")
- })
- case dict.size(u.line) {
- 0 -> Nil
- _ -> {
- io.println(" lines of no known shape, by first character:")
- dict.to_list(u.line)
- |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
- |> list.each(fn(e) {
- io.println(
- " "
- <> string.pad_end("\"" <> e.0 <> "\"", 27, " ")
- <> " "
- <> string.pad_start(int.to_string(e.1), 8, " "),
- )
- })
- }
- }
- io.println(
- " (ignoring these is conforming within a major version; the count is so that it is not silent)",
- )
- }
- }
-}
-
-fn section_of_unknown(
- counts: dict.Dict(String, Int),
- heading: String,
- line: fn(String, Int) -> String,
-) -> Nil {
- case dict.size(counts) {
- 0 -> Nil
- n -> {
- io.println(heading <> int.to_string(n) <> " distinct):")
- dict.to_list(counts)
- |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
- |> list.each(fn(e) { io.println(line(e.0, e.1)) })
- }
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-test.org b/org/gleam-test.org
deleted file mode 100644
index 402c6e0..0000000
--- a/org/gleam-test.org
+++ /dev/null
@@ -1,209 +0,0 @@
--*- mode: org -*-
-#+TITLE: gleam lib
-#+DESCRIPTION: gleam lib
-#+FILETAGS: :spine:build:tools:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-* gleam :gleam:
-** gleam
-*** tangle
-
-#+HEADER: :tangle ../gleam/test/ocda_lib_test.gleam
-#+HEADER: :noweb yes
-#+BEGIN_SRC org
-////<<doc_header_including_copyright_and_license>>
-<<test>>
-#+END_SRC
-
-*** gleam source
-
-#+NAME: test
-#+HEADER: :no-expand
-#+BEGIN_SRC gleam
-import gleam/dict
-import gleam/list
-import simplifile
-import gleam/string
-import gleeunit
-import gleeunit/should
-import ocda/dump
-import ocda/load
-import ocda/load_ssp
-import ocda/model
-
-pub fn main() {
- gleeunit.main()
-}
-
-const minimal = "% SiSU Document Abstraction v2.0
-% Source: example.sst
-
-@source {
- language: en
- languages: en
- digest: ABC
-}
-
-@meta {
- title.main: An Example
-}
-
-@body {
-[1] heading :A the_title
-.part: frontmatter
-.sha256: DEAD
-| An Example
-
-[2] para
-.part: body
-.parent: 1
-| First line
-|
-| Third line, after a blank one
-
-}
-"
-
-pub fn version_test() {
- let doc = load_ssp.parse(minimal)
- doc.format_version |> should.equal("2.0")
- model.version_major(doc) |> should.equal("2")
- doc.source_filename |> should.equal("example.sst")
-}
-
-pub fn preamble_test() {
- let doc = load_ssp.parse(minimal)
- model.meta(doc, "title.main") |> should.equal("An Example")
-}
-
-pub fn objects_test() {
- let doc = load_ssp.parse(minimal)
- let body = model.section(doc, "body")
- body |> list.length |> should.equal(2)
-}
-
-/// The blank line in the middle of an object's text is kept, and so is the
-/// count: three lines, not two.
-pub fn blank_text_line_test() {
- let doc = load_ssp.parse(minimal)
- case model.section(doc, "body") {
- [_, second] ->
- second.text
- |> should.equal("First line\n\nThird line, after a blank one")
- _ -> should.fail()
- }
-}
-
-/// A heading's identifier comes from its declaration line; an object whose
-/// identifier is simply its ocn has it derived, because the text form does
-/// not write what it can work out.
-pub fn derived_identifier_test() {
- let doc = load_ssp.parse(minimal)
- case model.section(doc, "body") {
- [first, second] -> {
- first.identifier |> should.equal("the_title")
- second.identifier |> should.equal("2")
- }
- _ -> should.fail()
- }
-}
-
-pub fn source_of_test() {
- load.source_of("a/b.ssp") |> should.equal(load.Ssp)
- load.source_of("a/b.ocda.db") |> should.equal(load.OcdaDb)
- load.source_of("a/b.sst") |> should.equal(load.Unknown)
-}
-
-/// The dump omits a field at its default, so a minimal document gives a
-/// short dump and every line in it is a fact.
-pub fn dump_test() {
- let text = dump.document(load_ssp.parse(minimal))
- text |> string.contains(contain: "version 2.0") |> should.be_true
- text |> string.contains(contain: "meta title.main An Example") |> should.be_true
- text |> string.contains(contain: "object body 0") |> should.be_true
- text |> string.contains(contain: "level A") |> should.be_true
- // a false boolean is not written at all
- text |> string.contains(contain: "bullet") |> should.be_false
-}
-
-
-/// A property is read, which the fixture above had to be corrected to show:
-/// a property line is `.name: value` at the start of the line, and an
-/// indented one is a line of no known shape. The fixture indented them, so
-/// every property in it was being counted as malformed and the tests around
-/// it passed for the wrong reason.
-pub fn property_test() {
- let doc = load_ssp.parse(minimal)
- case model.section(doc, "body") {
- [first, second] -> {
- first.part |> should.equal("frontmatter")
- first.sha256 |> should.equal("DEAD")
- second.parent_ocn |> should.equal(1)
- }
- _ -> should.fail()
- }
- // and nothing in the fixture is malformed any more
- dict_size(doc.unknown.line) |> should.equal(0)
-}
-
-@external(erlang, "maps", "size")
-fn dict_size(d: dict.Dict(a, b)) -> Int
-
-/// What a path is, including the two archive forms.
-pub fn pod_source_test() {
- load.source_of("a/b.sisupod") |> should.equal(load.Pod)
- load.source_of("a/b.zip") |> should.equal(load.Pod)
-}
-
-/// A zstd-wrapped archive is refused, and the refusal says why rather than
-/// claiming the file is malformed. This is the one place the D and Gleam
-/// libraries differ in what they accept, and it is a packaging limit: the
-/// only zstd on the BEAM is a NIF that fetches zstd in its own build.
-pub fn zstd_refusal_test() {
- let path = "test/fixture-zstd-frame.sisupod"
- let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>)
- let assert Error(message) = load.load(path, "", False)
- message |> string.contains(contain: "zstd") |> should.be_true
- message |> string.contains(contain: "zstd -d") |> should.be_true
- let _ = simplifile.delete(path)
- Nil
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__
diff --git a/org/gleam-ocda.org b/org/gleam.org
index c10554e..56d8f0d 100644
--- a/org/gleam-ocda.org
+++ b/org/gleam.org
@@ -15,7 +15,7 @@
- magic single double-quote → " ← FIX changes hilighting behavior (occuring
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-* gleam :gleam:
+* ocda
** ocda lib
*** tangle
@@ -177,7 +177,7 @@ fn fail(message: String) -> Nil {
fn halt(code: Int) -> Nil
#+END_SRC
-* gleam :gleam:
+* ocda
** model
*** tangle
@@ -2703,6 +2703,908 @@ fn ints(xs: List(Int)) -> String {
sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
#+END_SRC
+* read
+** main
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/read/main.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<read>>
+#+END_SRC
+
+*** source
+
+#+NAME: read
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// ocda-read, in Gleam: reads the abstraction, in any of the forms the
+//// library reads, and says what is in it.
+////
+//// ocda-read [--tree|--outline|--languages|--files|--report-unknown]
+//// <file>.ssp | <file>.ocda.db | <file>.zip [lang]
+////
+//// The modes the Gleam readers had (the summary, the outline, the unknown
+//// report, and the database's languages and files), over the library, so
+//// that each reads every form. Their output is the D ocda-read's, byte for
+//// byte: two readers in unrelated languages, neither written from the other
+//// line by line, agreeing is what says each has read the model and not its
+//// own idea of it. A `.sisupod` is the one form the Gleam library cannot
+//// open (its zstd frame; see `ocda/load_pod`), and it says so.
+
+import gleam/dict
+import gleam/int
+import gleam/io
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/inventory.{type Language}
+import ocda/load
+import ocda/model.{type Document, type Object}
+
+/// The format major this reader knows.
+const major_read = "2"
+
+type Options {
+ Options(
+ tree: Bool,
+ languages: Bool,
+ files: Bool,
+ report_unknown: Bool,
+ any_version: Bool,
+ lang: String,
+ positional: List(String),
+ )
+}
+
+/// Run with the arguments after the program name; returns the exit status.
+pub fn run(args: List(String)) -> Int {
+ case parse(args, Options(False, False, False, False, False, "", [])) {
+ Error(message) -> {
+ io.println_error("ocda-read: " <> message <> "; --help lists the options")
+ 2
+ }
+ Ok(Options(positional: [], ..)) -> {
+ help()
+ 0
+ }
+ Ok(options) -> read(options)
+ }
+}
+
+fn parse(args: List(String), o: Options) -> Result(Options, String) {
+ case args {
+ [] -> Ok(Options(..o, positional: list.reverse(o.positional)))
+ ["--help", ..] | ["-h", ..] -> Ok(Options(..o, positional: []))
+ ["--tree", ..rest] | ["--outline", ..rest] ->
+ parse(rest, Options(..o, tree: True))
+ ["--languages", ..rest] -> parse(rest, Options(..o, languages: True))
+ ["--files", ..rest] -> parse(rest, Options(..o, files: True))
+ ["--report-unknown", ..rest] ->
+ parse(rest, Options(..o, report_unknown: True))
+ ["--any-version", ..rest] -> parse(rest, Options(..o, any_version: True))
+ ["--lang", code, ..rest] -> parse(rest, Options(..o, lang: code))
+ [flag, ..rest] ->
+ case string.starts_with(flag, "--lang=") {
+ True -> parse(rest, Options(..o, lang: string.drop_start(flag, 7)))
+ False ->
+ case string.starts_with(flag, "-") {
+ True ->
+ case
+ list.contains(
+ [
+ "--html", "--llm-text", "--llm-jsonl", "--images", "--verify",
+ "--validate",
+ ],
+ flag,
+ )
+ {
+ True ->
+ Error(flag <> " is a mode of the D ocda-read, not of this one")
+ False -> Error("Unrecognized option " <> flag)
+ }
+ False ->
+ parse(rest, Options(..o, positional: [flag, ..o.positional]))
+ }
+ }
+ }
+}
+
+fn help() -> Nil {
+ io.println("ocda-read - reads the document abstraction, in any of its forms")
+ io.println("")
+ io.println(
+ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.zip [lang]",
+ )
+ io.println("")
+ io.println(" --tree the headings, as a tree")
+ io.println(" --outline the same as --tree")
+ io.println(" --languages the languages the file holds")
+ io.println(" --files what a .ocda.db carries besides the abstraction")
+ io.println(" --report-unknown what was read and not understood")
+ io.println(" --lang=<code> the language to read (default: the first)")
+ io.println(" --any-version read a format major this reader does not know")
+}
+
+fn read(o: Options) -> Int {
+ let assert [path, ..more] = o.positional
+ let lang = case o.lang, more {
+ "", [given, ..] -> given
+ l, _ -> l
+ }
+ let lang_named = lang != ""
+ let loaded = {
+ use held <- result.try(inventory.languages(path))
+ let lang = case lang, held {
+ "", [first, ..] -> first.lang
+ l, _ -> l
+ }
+ use _ <- result.try(case
+ held != [] && !list.any(held, fn(l) { l.lang == lang })
+ {
+ True ->
+ Error(
+ "no language '"
+ <> lang
+ <> "' in "
+ <> path
+ <> "; it holds: "
+ <> string.join(list.map(held, fn(l) { l.lang }), " "),
+ )
+ False -> Ok(Nil)
+ })
+ use doc <- result.try(load.load(path, lang, o.any_version))
+ Ok(#(held, lang, doc))
+ }
+ case loaded {
+ Error(message) -> {
+ io.println_error("Error reading " <> path <> ": " <> message)
+ 2
+ }
+ Ok(#(held, lang, doc)) ->
+ case gate(path, doc, o.any_version) {
+ False -> 3
+ True -> {
+ summary(path, doc, held, lang, lang_named)
+ case o.languages {
+ True -> print_languages(held)
+ False -> Nil
+ }
+ case o.files {
+ True -> print_files(path)
+ False -> Nil
+ }
+ case o.tree {
+ True -> tree(doc)
+ False -> Nil
+ }
+ case o.report_unknown {
+ True -> report_unknown(doc)
+ False -> Nil
+ }
+ case o.tree || o.languages || o.files || o.report_unknown {
+ True -> Nil
+ False ->
+ io.println(
+ "\n(this is the summary: --help lists the modes, each adding its output)",
+ )
+ }
+ 0
+ }
+ }
+ }
+}
+
+/// The major part, and refuse what is not known.
+fn gate(path: String, doc: Document, any_version: Bool) -> Bool {
+ case doc.format_version, model.version_major(doc) == major_read, any_version {
+ "", _, _ -> {
+ io.println_error("warning: " <> path <> " declares no format version")
+ True
+ }
+ _, True, _ -> True
+ v, False, True -> {
+ io.println_error(
+ "warning: "
+ <> path
+ <> " is format v"
+ <> v
+ <> ", this reader knows v"
+ <> major_read
+ <> ".x; reading anyway",
+ )
+ True
+ }
+ v, False, False -> {
+ io.println_error(
+ "error: "
+ <> path
+ <> " is format v"
+ <> v
+ <> ", this reader knows v"
+ <> major_read
+ <> ".x",
+ )
+ io.println_error(
+ " a major version differs: a property may have changed meaning or gone away.",
+ )
+ io.println_error(" --any-version reads it regardless.")
+ False
+ }
+ }
+}
+
+const sections = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
+ "blurb", "tail",
+]
+
+/// The sections whose images are counted: every one but the tail, as the D
+/// reader counts them.
+const image_sections = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
+ "blurb",
+]
+
+fn summary(
+ path: String,
+ doc: Document,
+ held: List(Language),
+ lang: String,
+ lang_named: Bool,
+) -> Nil {
+ io.println("Read: " <> path)
+ case doc.format_version {
+ "" -> Nil
+ v -> io.println(" Format: v" <> v)
+ }
+ case dict.get(doc.source, "language") {
+ Error(_) -> Nil
+ Ok(language) -> {
+ let of = case dict.get(doc.source, "languages") {
+ Ok(ls) -> " of " <> ls
+ Error(_) -> ""
+ }
+ let digest = case dict.get(doc.source, "digest") {
+ Ok(d) -> ", markup digest " <> string.slice(d, 0, 16) <> "..."
+ Error(_) -> ""
+ }
+ io.println(" Source: " <> language <> of <> digest)
+ }
+ }
+ case list.length(held) > 1 && !lang_named {
+ True -> io.println(" (reading " <> lang <> "; --lang=<code> for another)")
+ False -> Nil
+ }
+ io.println(" Title: " <> model.meta(doc, "title.main"))
+ io.println(" Author: " <> model.meta(doc, "creator.author"))
+ io.println(
+ " Sections: "
+ <> string.join(
+ list.map(sections, fn(s) {
+ s <> "=" <> int.to_string(list.length(model.section(doc, s)))
+ }),
+ " ",
+ ),
+ )
+ let records =
+ list.fold(image_sections, 0, fn(n, s) {
+ list.fold(model.section(doc, s), n, fn(n, o) {
+ n + list.length(o.images)
+ })
+ })
+ case records {
+ 0 -> Nil
+ 1 -> io.println(" Images: 1 record")
+ n -> io.println(" Images: " <> int.to_string(n) <> " records")
+ }
+}
+
+fn print_languages(held: List(Language)) -> Nil {
+ io.println("")
+ io.println(" lang source ssp digest")
+ list.each(held, fn(l) {
+ let digest = case string.length(l.ssp_digest) > 16 {
+ True -> string.slice(l.ssp_digest, 0, 16) <> "..."
+ False -> l.ssp_digest
+ }
+ io.println(
+ " "
+ <> string.pad_end(l.lang, 7, " ")
+ <> " "
+ <> string.pad_end(l.source_filename, 31, " ")
+ <> " "
+ <> digest,
+ )
+ })
+}
+
+fn print_files(path: String) -> Nil {
+ io.println("")
+ case inventory.is_carrier(path) {
+ False ->
+ io.println(
+ " carried files: only a .ocda.db carries files; this is "
+ <> load.source_name(load.source_of(path)),
+ )
+ True ->
+ case inventory.carried(path) {
+ Error(message) -> io.println_error("error: " <> message)
+ Ok([]) -> io.println(" carries no files")
+ Ok(rows) -> {
+ io.println(" carried files:")
+ io.println(" role stored n original in the file")
+ list.each(rows, fn(r) {
+ io.println(
+ " "
+ <> string.pad_end(r.role, 9, " ")
+ <> " "
+ <> string.pad_end(r.compression, 7, " ")
+ <> " "
+ <> string.pad_end(r.count, 5, " ")
+ <> " "
+ <> string.pad_end(r.bytes, 13, " ")
+ <> " "
+ <> r.stored,
+ )
+ })
+ }
+ }
+ }
+}
+
+/// The headings, as a tree: from the document's root heading, down the
+/// children each heading lists, as the D reader walks it.
+fn tree(doc: Document) -> Nil {
+ io.println("Document: " <> model.meta(doc, "title.main"))
+ case model.meta(doc, "title.subtitle") {
+ "" -> Nil
+ sub -> io.println(" " <> sub)
+ }
+ io.println("Author: " <> model.meta(doc, "creator.author"))
+ io.println("")
+ let headings =
+ model.section(doc, "body")
+ |> list.filter(fn(o) { o.is_a == "heading" && o.ocn > 0 })
+ |> list.map(fn(o) { #(o.ocn, o) })
+ |> dict.from_list
+ case model.section(doc, "head") {
+ [root, ..] if root.is_a == "heading" -> {
+ io.println(heading_line(root, 0))
+ walk(root, headings, 1)
+ }
+ _ -> Nil
+ }
+}
+
+fn walk(parent: Object, headings: dict.Dict(Int, Object), depth: Int) -> Nil {
+ list.each(parent.children, fn(ocn) {
+ case dict.get(headings, ocn) {
+ Ok(child) -> {
+ io.println(heading_line(child, depth))
+ walk(child, headings, depth + 1)
+ }
+ Error(_) -> Nil
+ }
+ })
+}
+
+fn heading_line(o: Object, depth: Int) -> String {
+ let level = case o.heading_level {
+ "" -> "?"
+ l -> l
+ }
+ let ocn = case o.ocn > 0 {
+ True -> " [" <> int.to_string(o.ocn) <> "]"
+ False -> ""
+ }
+ string.repeat(" ", depth) <> level <> "~" <> ocn <> " " <> strip_markup(o.text)
+}
+
+/// The internal markers a heading's text may carry, removed for display, as
+/// the D reader removes them.
+fn strip_markup(text: String) -> String {
+ text
+ |> string.replace("⑆*", "")
+ |> string.replace("⑆!", "")
+ |> string.replace("⑆/", "")
+ |> string.replace("⑆_", "")
+ |> string.replace("⑆^", "")
+ |> string.replace("⑆,", "")
+ |> string.replace("┨", "")
+ |> string.replace("┣", "")
+ |> string.replace("┙", " ")
+ |> string.replace("┘", "\n")
+}
+
+/// Say what was read and not understood.
+fn report_unknown(doc: Document) -> Nil {
+ let u = doc.unknown
+ case model.any_unknown(u) {
+ False ->
+ io.println(
+ "\nread and understood: every property, block and line was known",
+ )
+ True -> {
+ io.println("\nread and NOT understood:")
+ section_of_unknown(u.property, " properties ignored (", fn(name, n) {
+ " ."
+ <> string.pad_end(name, 26, " ")
+ <> " "
+ <> string.pad_start(int.to_string(n), 8, " ")
+ })
+ section_of_unknown(u.block, " blocks ignored (", fn(name, n) {
+ " @"
+ <> string.pad_end(name, 26, " ")
+ <> " "
+ <> string.pad_start(int.to_string(n), 8, " ")
+ })
+ case dict.size(u.line) {
+ 0 -> Nil
+ _ -> {
+ io.println(" lines of no known shape, by first character:")
+ dict.to_list(u.line)
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.each(fn(e) {
+ io.println(
+ " "
+ <> string.pad_end("\"" <> e.0 <> "\"", 27, " ")
+ <> " "
+ <> string.pad_start(int.to_string(e.1), 8, " "),
+ )
+ })
+ }
+ }
+ io.println(
+ " (ignoring these is conforming within a major version; the count is so that it is not silent)",
+ )
+ }
+ }
+}
+
+fn section_of_unknown(
+ counts: dict.Dict(String, Int),
+ heading: String,
+ line: fn(String, Int) -> String,
+) -> Nil {
+ case dict.size(counts) {
+ 0 -> Nil
+ n -> {
+ io.println(heading <> int.to_string(n) <> " distinct):")
+ dict.to_list(counts)
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.each(fn(e) { io.println(line(e.0, e.1)) })
+ }
+ }
+}
+#+END_SRC
+
+* test
+** gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/test/ocda_lib_test.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<test>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: test
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+import gleam/dict
+import gleam/list
+import simplifile
+import gleam/string
+import gleeunit
+import gleeunit/should
+import ocda/dump
+import ocda/load
+import ocda/load_ssp
+import ocda/model
+
+pub fn main() {
+ gleeunit.main()
+}
+
+const minimal = "% SiSU Document Abstraction v2.0
+% Source: example.sst
+
+@source {
+ language: en
+ languages: en
+ digest: ABC
+}
+
+@meta {
+ title.main: An Example
+}
+
+@body {
+[1] heading :A the_title
+.part: frontmatter
+.sha256: DEAD
+| An Example
+
+[2] para
+.part: body
+.parent: 1
+| First line
+|
+| Third line, after a blank one
+
+}
+"
+
+pub fn version_test() {
+ let doc = load_ssp.parse(minimal)
+ doc.format_version |> should.equal("2.0")
+ model.version_major(doc) |> should.equal("2")
+ doc.source_filename |> should.equal("example.sst")
+}
+
+pub fn preamble_test() {
+ let doc = load_ssp.parse(minimal)
+ model.meta(doc, "title.main") |> should.equal("An Example")
+}
+
+pub fn objects_test() {
+ let doc = load_ssp.parse(minimal)
+ let body = model.section(doc, "body")
+ body |> list.length |> should.equal(2)
+}
+
+/// The blank line in the middle of an object's text is kept, and so is the
+/// count: three lines, not two.
+pub fn blank_text_line_test() {
+ let doc = load_ssp.parse(minimal)
+ case model.section(doc, "body") {
+ [_, second] ->
+ second.text
+ |> should.equal("First line\n\nThird line, after a blank one")
+ _ -> should.fail()
+ }
+}
+
+/// A heading's identifier comes from its declaration line; an object whose
+/// identifier is simply its ocn has it derived, because the text form does
+/// not write what it can work out.
+pub fn derived_identifier_test() {
+ let doc = load_ssp.parse(minimal)
+ case model.section(doc, "body") {
+ [first, second] -> {
+ first.identifier |> should.equal("the_title")
+ second.identifier |> should.equal("2")
+ }
+ _ -> should.fail()
+ }
+}
+
+pub fn source_of_test() {
+ load.source_of("a/b.ssp") |> should.equal(load.Ssp)
+ load.source_of("a/b.ocda.db") |> should.equal(load.OcdaDb)
+ load.source_of("a/b.sst") |> should.equal(load.Unknown)
+}
+
+/// The dump omits a field at its default, so a minimal document gives a
+/// short dump and every line in it is a fact.
+pub fn dump_test() {
+ let text = dump.document(load_ssp.parse(minimal))
+ text |> string.contains(contain: "version 2.0") |> should.be_true
+ text |> string.contains(contain: "meta title.main An Example") |> should.be_true
+ text |> string.contains(contain: "object body 0") |> should.be_true
+ text |> string.contains(contain: "level A") |> should.be_true
+ // a false boolean is not written at all
+ text |> string.contains(contain: "bullet") |> should.be_false
+}
+
+
+/// A property is read, which the fixture above had to be corrected to show:
+/// a property line is `.name: value` at the start of the line, and an
+/// indented one is a line of no known shape. The fixture indented them, so
+/// every property in it was being counted as malformed and the tests around
+/// it passed for the wrong reason.
+pub fn property_test() {
+ let doc = load_ssp.parse(minimal)
+ case model.section(doc, "body") {
+ [first, second] -> {
+ first.part |> should.equal("frontmatter")
+ first.sha256 |> should.equal("DEAD")
+ second.parent_ocn |> should.equal(1)
+ }
+ _ -> should.fail()
+ }
+ // and nothing in the fixture is malformed any more
+ dict_size(doc.unknown.line) |> should.equal(0)
+}
+
+@external(erlang, "maps", "size")
+fn dict_size(d: dict.Dict(a, b)) -> Int
+
+/// What a path is, including the two archive forms.
+pub fn pod_source_test() {
+ load.source_of("a/b.sisupod") |> should.equal(load.Pod)
+ load.source_of("a/b.zip") |> should.equal(load.Pod)
+}
+
+/// A zstd-wrapped archive is refused, and the refusal says why rather than
+/// claiming the file is malformed. This is the one place the D and Gleam
+/// libraries differ in what they accept, and it is a packaging limit: the
+/// only zstd on the BEAM is a NIF that fetches zstd in its own build.
+pub fn zstd_refusal_test() {
+ let path = "test/fixture-zstd-frame.sisupod"
+ let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>)
+ let assert Error(message) = load.load(path, "", False)
+ message |> string.contains(contain: "zstd") |> should.be_true
+ message |> string.contains(contain: "zstd -d") |> should.be_true
+ let _ = simplifile.delete(path)
+ Nil
+}
+#+END_SRC
+
+* config
+** nix flake.nix :flake:
+
+#+HEADER: :tangle ../gleam/flake.nix
+#+HEADER: :noweb yes
+#+BEGIN_SRC nix
+{
+ description = "ocda-lib: the object-centric document abstraction as a Gleam library";
+ inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+ outputs = { self, nixpkgs }:
+ let
+ system = "x86_64-linux"; # adjust for aarch64-linux / darwin
+ pkgs = import nixpkgs { inherit system; };
+
+ name = "ocda_lib";
+
+ /* rebar3 with the port compiler already in it.
+ esqlite builds a NIF, and its rebar3 configuration asks for the
+ `pc` plugin, which rebar3 would otherwise fetch from Hex at build
+ time: impossible in a sandbox, and the build fails with
+ "missing_package pc". nixpkgs carries the plugin, so the answer is
+ a rebar3 that has it before it starts.
+ */
+ rebar3-pc = pkgs.beamPackages.rebar3WithPlugins {
+ plugins = [ pkgs.beamPackages.pc ];
+ };
+
+ /* The Hex dependencies, as manifest.toml pins them.
+ .
+ There is no Gleam builder in nixpkgs and `gleam build` fetches from
+ Hex, so a sandboxed build cannot reach them. A fixed output
+ derivation may, because its result is pinned by hash.
+ .
+ When a dependency changes, manifest.toml changes and this hash must
+ change with it: build, take the hash nix reports, put it here.
+ */
+ deps = pkgs.stdenv.mkDerivation {
+ pname = "${name}-deps";
+ version = "0.1.0";
+ src = ./.;
+ nativeBuildInputs = [ pkgs.gleam pkgs.cacert ];
+ buildPhase = ''
+ export HOME=$TMPDIR
+ gleam deps download
+ '';
+ installPhase = ''
+ mkdir -p $out
+ cp -r build/packages/* $out/
+ '';
+ dontFixup = true;
+ outputHashMode = "recursive";
+ outputHashAlgo = "sha256";
+ outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c=";
+ };
+ in {
+ packages.${system} = {
+ default = self.packages.${system}.${name};
+ /* Shipped as an Erlang shipment rather than as an escript, because
+ sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into
+ a shared object, and a NIF cannot be loaded from inside an
+ escript archive, the runtime needing a real file on disk. The
+ shipment keeps priv/ on disk, so it loads.
+ */
+ ${name} = pkgs.stdenv.mkDerivation {
+ pname = "ocda-lib";
+ version = "0.1.0";
+ src = ./.;
+ nativeBuildInputs = [
+ pkgs.gleam
+ pkgs.erlang
+ rebar3-pc
+ pkgs.makeWrapper
+ ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ # the dependencies, in place of the fetch gleam would do
+ mkdir -p build/packages
+ cp -r ${deps}/* build/packages/
+ chmod -R u+w build/packages
+ gleam export erlang-shipment
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/lib $out/bin
+ cp -r build/erlang-shipment $out/lib/${name}
+ makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \
+ --add-flags run \
+ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
+ # the reader under its own name, as the D flake installs it
+ makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \
+ --add-flags "run read" \
+ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]}
+ runHook postInstall
+ '';
+ meta = {
+ description = "the abstraction as a Gleam library, with its reader, dump and agreement check";
+ mainProgram = "ocda-lib";
+ };
+ };
+ };
+ devShells.${system}.default = pkgs.mkShell {
+ packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ];
+ };
+ };
+}
+#+END_SRC
+
+** README.md
+
+#+HEADER: :tangle ../gleam/README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# ocda-lib, in Gleam
+
+The abstraction's model and both its loaders, as a Gleam library, with its
+reader and the two programs that put it to the question.
+
+| | |
+|------------------|-----------------------------------------------------------------------------------------|
+| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
+| `ocda/load_ssp` | text -> model |
+| `ocda/load_db` | sqlite -> model |
+| `ocda/load` | what a path is, and loading it |
+| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
+| `ocda/dump` | the model written out canonically, as an oracle |
+| `read/main` | `ocda-read`, the reader |
+| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
+
+## Use
+
+```gleam
+import ocda/load
+import ocda/model
+
+case load.load(path, "en", False) {
+ Error(why) -> io.println_error(why)
+ Ok(doc) -> {
+ // and from here a writer neither knows nor cares which of the two
+ // files it was handed
+ list.each(model.section(doc, "body"), fn(obj) { ... })
+ }
+}
+```
+
+## The reader
+
+```sh
+nix build
+./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
+ <file>.ssp|<file>.ocda.db|<file>.zip [lang]
+```
+
+`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
+had, over the library, so each reads every form this library reads; the
+others are the D reader's. **Its output is the D `ocda-read`'s, byte for
+byte**, in every mode both have. In `src/read/`, because Gleam compiles
+nothing outside `src/`; a program depending on the library compiles it too,
+and does not use it.
+
+A `.sisupod` it refuses, saying why: the zstd frame, below.
+
+## The two programs
+
+```sh
+nix build
+./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+```
+
+`agree` asks whether the two loaders give the same model. `dump` writes
+the whole model out canonically, and is the stronger check of the two,
+because it is **byte for byte what the D library's `ocda-dump` writes**.
+So the four questions can all be asked with `diff`:
+
+| | |
+|-----------------------------|-----------------------------|
+| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
+| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
+| `.ssp`: D == Gleam | 36 of 36 |
+| `.ocda.db`: D == Gleam | 36 of 36 |
+
+## Deliberately a mirror of the D library
+
+Field for field, name for name, including where that is not the most
+Gleamish choice:
+
+- `text` is one `String` with the lines joined by newline, not a
+ `List(String)`. So an object whose only text line is blank is
+ indistinguishable from an object with no text, *in both languages, in the
+ same place*. A difference that existed in one and not the other would be
+ worse than a loss both share, because it would show up in every dump and
+ mean nothing.
+- the properties are typed fields rather than a list of pairs. A writer
+ wants `obj.parent_ocn`, not a lookup that might miss.
+
+The point is that the two agree. Making them agree is much cheaper when
+they are shaped alike, and the shape is the specification's, not either
+language's.
+
+## What the agreement found
+
+Four disagreements, none of which a comparison of section counts or of a
+handful of fields would have shown. All four are recorded in the D
+library's history; the short version:
+
+1. `.identifier` and `.anchor` are omitted by the text form when they can
+ be worked out (the identifier is the ocn, the anchor is the segment).
+ Both loaders now derive them. Over one document that is 1,314
+ identifiers.
+2. `make` and `doc_has` are one metadata table in the database,
+ distinguished by a key prefix. Both loaders unprefix them.
+3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
+ as `\/` and every heading with a slash in it came back wrong.
+4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
+ which is one language. It is a fact about the file, so neither loader
+ puts it in the document.
+
+## Build
+
+```sh
+nix build # sandboxed, no network
+nix develop -c gleam test
+```
+
+Shipped as an Erlang shipment rather than an escript, because sqlite on the
+BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
+The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
+one and the other must change with it.
+
+## Format
+
+Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
+`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
+two serialisations. At format **2.x**.
+#+END_SRC
+
+** gleam.toml
+
+#+HEADER: :tangle ../gleam/gleam.toml
+#+HEADER: :noweb yes
+#+BEGIN_SRC toml
+name = "ocda_lib"
+version = "0.1.0"
+description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
+licences = ["AGPL-3.0-or-later"]
+
+[dependencies]
+gleam_stdlib = ">= 1.0.0 and < 2.0.0"
+argv = ">= 1.1.0 and < 2.0.0"
+simplifile = ">= 2.7.0 and < 3.0.0"
+sqlight = ">= 1.0.0 and < 2.0.0"
+gleam_json = ">= 3.0.0 and < 4.0.0"
+
+[dev-dependencies]
+gleeunit = ">= 1.0.0 and < 2.0.0"
+#+END_SRC
+
* org includes
** project version
@@ -2728,4 +3630,12 @@ sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
#+END_SRC
+* nixpkgs source
+
+#+NAME: nixpkgs_unstable
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>>
+#+END_SRC
+
* __END__