diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-04 12:12:51 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-04 12:12:51 -0400 |
| commit | 828c686ed361771b684806e5b37c5dee52514063 (patch) | |
| tree | c0e28a351ae7cd30aec0c7e7cd2a6b92053558ff | |
| parent | track orgify (diff) | |
track orgify
| -rw-r--r-- | org/config.org | 468 | ||||
| -rw-r--r-- | org/d-ocda.org | 1854 | ||||
| -rw-r--r-- | org/d-read.org | 1732 | ||||
| -rw-r--r-- | org/d-test.org | 178 | ||||
| -rw-r--r-- | org/d.org | 3895 | ||||
| -rw-r--r-- | org/gleam-read.org | 520 | ||||
| -rw-r--r-- | org/gleam-test.org | 209 | ||||
| -rw-r--r-- | org/gleam.org (renamed from org/gleam-ocda.org) | 914 |
8 files changed, 4807 insertions, 4963 deletions
diff --git a/org/config.org b/org/config.org index 2c1294f..8420c3b 100644 --- a/org/config.org +++ b/org/config.org @@ -280,474 +280,6 @@ erl_crash.dump !**/*.yaml #+END_SRC -* ./dlang -** nix flake.nix :flake: - -#+HEADER: :tangle ../dlang/flake.nix -#+HEADER: :noweb yes -#+BEGIN_SRC nix -{ - description = "ocda-lib: the object-centric document abstraction as a D library"; - inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; - outputs = { self, nixpkgs }: - let - system = "x86_64-linux"; # adjust for aarch64-linux / darwin - pkgs = import nixpkgs { inherit system; }; - /* A library, so the library itself is not a build product: a program - that uses it compiles its modules alongside its own, as the writer's - flake does, or takes it as a dub dependency. What is built here are - programs over it: the reader, and the two that put the library to - the question - the agreement check, which loads a document's .ssp - and its .ocda.db and compares the two models, and the dump, which - writes a loaded model out canonically so that an implementation in - another language can be diffed against this one. - . - `mains` is what is compiled besides the library: the checks are one - file each in test/, the reader is src/read/. - */ - mkTool = { tool, binary, what, mains, compiler, compilerBin }: - pkgs.stdenv.mkDerivation { - pname = "ocda-lib-${tool}"; - version = "0.1.0"; - src = self; - nativeBuildInputs = [ compiler ]; - buildInputs = [ pkgs.sqlite pkgs.zstd ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/bin - install -m755 ${tool} $out/bin/${binary} - runHook postInstall - ''; - meta = { - description = what; - mainProgram = binary; - }; - }; - ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; - ocda-read = mkTool (ldc // { - tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; - what = "reads the abstraction in any of its forms and says what is in it"; - }); - ocda-agree = mkTool (ldc // { - tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; - what = "checks that the .ssp and .ocda.db loaders give one model"; - }); - ocda-dump = mkTool (ldc // { - tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; - what = "writes a loaded abstraction out canonically, as an oracle"; - }); - in { - packages.${system} = { - /* All three in one bin/, so that one build leaves every program in - one result link. Built separately into the same link, each build - replaced the one before, and a script looking for ocda-dump beside - ocda-agree found nothing. - */ - default = pkgs.symlinkJoin { - name = "ocda-lib-programs-0.1.0"; - paths = [ ocda-read ocda-agree ocda-dump ]; - }; - inherit ocda-read ocda-agree ocda-dump; - }; - - devShells.${system}.default = pkgs.mkShell { - packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; - }; - }; -} -#+END_SRC - -** README.md - -#+HEADER: :tangle ../dlang/README.md -#+HEADER: :noweb yes -#+BEGIN_SRC md -# ocda-lib, in D - -The abstraction's model and both its loaders, as a dub library. - -## Modules - -| | | -|---|---| -| `ocda.model` | the document, the object, the eight sections. Imports nothing | -| `ocda.load_ssp` | text -> model | -| `ocda.load_db` | sqlite -> model | -| `ocda.load` | what a path is, and loading it | -| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries | -| `ocda.dump` | the model written out canonically, as an oracle | -| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon | -| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) | - -`ocda.model` importing nothing is the property that matters: a writer -depends on the model and on neither loader. - -## Use - -```json -"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } } -``` - -```d -import ocda.load; -auto doc = load("document.ocda.db", "de"); // or "document.ssp" -foreach (obj; doc.body) { ... } -``` - -Link `sqlite3`; the library declares the dozen functions it needs rather -than taking a binding from the registry, so there is nothing to fetch. - -## The reader - -```sh -nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl| - --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate] - <file>.ssp|<file>.ocda.db|<file>.sisupod [lang] -``` - -The library's reader, one program where `sisudoc-ssp-reader` and -`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp` -reader's, moved onto the library's model, so every mode reads every form -and **says the same thing of a document's `.ssp`, `.ocda.db` and -`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries -files; `--validate` checks a `.ssp` as text and refuses anything else. - -It lives in `src/read/` beside the library, and dub keeps it out of the -library: the configuration `library` excludes `src/read/*`, the -configuration `ocda-read` builds it. A program using the library by dub, or -by naming `src/ocda/*.d` as the writer's flake does, never compiles it. - -## The two checks - -```sh -nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang] -nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang] -``` - -`agree` loads both and compares ten fields of each object, naming the field -where they part company. `agree` found the blank-first-text-line defect and -was not enough for anything subtler. - -**`ocda-dump` is the stronger check**: the whole loaded model, every field, -written out in a form fully determined by the model, so that two -implementations can be compared with `diff`. It is an oracle, not an output -format; anything reading it to reconstruct a document has misunderstood it. -It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is -the point of having it. - -It found four disagreements `agree` could not see. They are in the -repository README and in the history. - -## Build - -```sh -nix build # all three programs, in one result/bin -nix build .#ocda-read # the reader -nix build .#ocda-agree # the agreement check -nix build .#ocda-dump # the canonical dump -nix develop # ldc, dub, dtools, sqlite, zstd -dub build # the library -dub build --config=ocda-read # the reader -``` -#+END_SRC - -** dub.json - -#+HEADER: :tangle ../dlang/dub.json -#+HEADER: :noweb yes -#+BEGIN_SRC json -{ - "authors": [ "Ralph Amissah" ], - "copyright": "Copyright (C) 2026 Ralph Amissah", - "name": "ocda-lib", - "version": "0.1.0", - "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations", - "homepage": "https://sisudoc.org", - "license": "AGPL-3.0+", - "sourcePaths": [ "./src" ], - "importPaths": [ "./src" ], - "libs": [ "sqlite3", "zstd" ], - "configurations": [ - { - "name": "library", - "targetType": "library", - "excludedSourceFiles": [ "src/read/*" ] - }, - { - "name": "ocda-read", - "targetType": "executable", - "targetName": "ocda-read", - "mainSourceFile": "src/read/main.d" - } - ] -} -#+END_SRC - -* ./gleam -** nix flake.nix :flake: - -#+HEADER: :tangle ../gleam/flake.nix -#+HEADER: :noweb yes -#+BEGIN_SRC nix -{ - description = "ocda-lib: the object-centric document abstraction as a Gleam library"; - inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; - outputs = { self, nixpkgs }: - let - system = "x86_64-linux"; # adjust for aarch64-linux / darwin - pkgs = import nixpkgs { inherit system; }; - - name = "ocda_lib"; - - /* rebar3 with the port compiler already in it. - esqlite builds a NIF, and its rebar3 configuration asks for the - `pc` plugin, which rebar3 would otherwise fetch from Hex at build - time: impossible in a sandbox, and the build fails with - "missing_package pc". nixpkgs carries the plugin, so the answer is - a rebar3 that has it before it starts. - */ - rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { - plugins = [ pkgs.beamPackages.pc ]; - }; - - /* The Hex dependencies, as manifest.toml pins them. - . - There is no Gleam builder in nixpkgs and `gleam build` fetches from - Hex, so a sandboxed build cannot reach them. A fixed output - derivation may, because its result is pinned by hash. - . - When a dependency changes, manifest.toml changes and this hash must - change with it: build, take the hash nix reports, put it here. - */ - deps = pkgs.stdenv.mkDerivation { - pname = "${name}-deps"; - version = "0.1.0"; - src = ./.; - nativeBuildInputs = [ pkgs.gleam pkgs.cacert ]; - buildPhase = '' - export HOME=$TMPDIR - gleam deps download - ''; - installPhase = '' - mkdir -p $out - cp -r build/packages/* $out/ - ''; - dontFixup = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c="; - }; - in { - packages.${system} = { - default = self.packages.${system}.${name}; - /* Shipped as an Erlang shipment rather than as an escript, because - sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into - a shared object, and a NIF cannot be loaded from inside an - escript archive, the runtime needing a real file on disk. The - shipment keeps priv/ on disk, so it loads. - */ - ${name} = pkgs.stdenv.mkDerivation { - pname = "ocda-lib"; - version = "0.1.0"; - src = ./.; - nativeBuildInputs = [ - pkgs.gleam - pkgs.erlang - rebar3-pc - pkgs.makeWrapper - ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - # the dependencies, in place of the fetch gleam would do - mkdir -p build/packages - cp -r ${deps}/* build/packages/ - chmod -R u+w build/packages - gleam export erlang-shipment - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/lib $out/bin - cp -r build/erlang-shipment $out/lib/${name} - makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \ - --add-flags run \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} - # the reader under its own name, as the D flake installs it - makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \ - --add-flags "run read" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} - runHook postInstall - ''; - meta = { - description = "the abstraction as a Gleam library, with its reader, dump and agreement check"; - mainProgram = "ocda-lib"; - }; - }; - }; - devShells.${system}.default = pkgs.mkShell { - packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ]; - }; - }; -} -#+END_SRC - -** README.md - -#+HEADER: :tangle ../gleam/README.md -#+HEADER: :noweb yes -#+BEGIN_SRC md -# ocda-lib, in Gleam - -The abstraction's model and both its loaders, as a Gleam library, with its -reader and the two programs that put it to the question. - -| | | -|------------------|-----------------------------------------------------------------------------------------| -| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | -| `ocda/load_ssp` | text -> model | -| `ocda/load_db` | sqlite -> model | -| `ocda/load` | what a path is, and loading it | -| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | -| `ocda/dump` | the model written out canonically, as an oracle | -| `read/main` | `ocda-read`, the reader | -| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | - -## Use - -```gleam -import ocda/load -import ocda/model - -case load.load(path, "en", False) { - Error(why) -> io.println_error(why) - Ok(doc) -> { - // and from here a writer neither knows nor cares which of the two - // files it was handed - list.each(model.section(doc, "body"), fn(obj) { ... }) - } -} -``` - -## The reader - -```sh -nix build -./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] - <file>.ssp|<file>.ocda.db|<file>.zip [lang] -``` - -`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers -had, over the library, so each reads every form this library reads; the -others are the D reader's. **Its output is the D `ocda-read`'s, byte for -byte**, in every mode both have. In `src/read/`, because Gleam compiles -nothing outside `src/`; a program depending on the library compiles it too, -and does not use it. - -A `.sisupod` it refuses, saying why: the zstd frame, below. - -## The two programs - -```sh -nix build -./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] -./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] -``` - -`agree` asks whether the two loaders give the same model. `dump` writes -the whole model out canonically, and is the stronger check of the two, -because it is **byte for byte what the D library's `ocda-dump` writes**. -So the four questions can all be asked with `diff`: - -| | | -|-----------------------------|-----------------------------| -| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | -| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | -| `.ssp`: D == Gleam | 36 of 36 | -| `.ocda.db`: D == Gleam | 36 of 36 | - -## Deliberately a mirror of the D library - -Field for field, name for name, including where that is not the most -Gleamish choice: - -- `text` is one `String` with the lines joined by newline, not a - `List(String)`. So an object whose only text line is blank is - indistinguishable from an object with no text, *in both languages, in the - same place*. A difference that existed in one and not the other would be - worse than a loss both share, because it would show up in every dump and - mean nothing. -- the properties are typed fields rather than a list of pairs. A writer - wants `obj.parent_ocn`, not a lookup that might miss. - -The point is that the two agree. Making them agree is much cheaper when -they are shaped alike, and the shape is the specification's, not either -language's. - -## What the agreement found - -Four disagreements, none of which a comparison of section counts or of a -handful of fields would have shown. All four are recorded in the D -library's history; the short version: - -1. `.identifier` and `.anchor` are omitted by the text form when they can - be worked out (the identifier is the ocn, the anchor is the segment). - Both loaders now derive them. Over one document that is 1,314 - identifiers. -2. `make` and `doc_has` are one metadata table in the database, - distinguished by a key prefix. Both loaders unprefix them. -3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` - as `\/` and every heading with a slash in it came back wrong. -4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, - which is one language. It is a fact about the file, so neither loader - puts it in the document. - -## Build - -```sh -nix build # sandboxed, no network -nix develop -c gleam test -``` - -Shipped as an Erlang shipment rather than an escript, because sqlite on the -BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. -The dependency hash in `flake.nix` pins what `manifest.toml` pins; change -one and the other must change with it. - -## Format - -Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, -`ssp-format-specification.org` and `ocda-db-format-specification.org` for the -two serialisations. At format **2.x**. -#+END_SRC - -** gleam.toml - -#+HEADER: :tangle ../gleam/gleam.toml -#+HEADER: :noweb yes -#+BEGIN_SRC toml -name = "ocda_lib" -version = "0.1.0" -description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" -licences = ["AGPL-3.0-or-later"] - -[dependencies] -gleam_stdlib = ">= 1.0.0 and < 2.0.0" -argv = ">= 1.1.0 and < 2.0.0" -simplifile = ">= 2.7.0 and < 3.0.0" -sqlight = ">= 1.0.0 and < 2.0.0" -gleam_json = ">= 3.0.0 and < 4.0.0" - -[dev-dependencies] -gleeunit = ">= 1.0.0 and < 2.0.0" -#+END_SRC - * versions GET ** dlang overlays *** ldc diff --git a/org/d-ocda.org b/org/d-ocda.org deleted file mode 100644 index c4510af..0000000 --- a/org/d-ocda.org +++ /dev/null @@ -1,1854 +0,0 @@ --*- mode: org -*- -#+TITLE: dlang load -#+DESCRIPTION: dlang lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* dlang -** model - -#+HEADER: :tangle "../dlang/src/ocda/model.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.model; - -struct OcdaImage { - string name; // image filename as referenced by the object - string sha256; // hex-encoded (uppercase) digest of the file - ulong bytes; // file size - int width; // file's own pixel width (not display width) - int height; // file's own pixel height - bool missing; // file absent or unreadable, no sha256/bytes - string[string] fields; // any further/unrecognised key:value fields -} - -struct OcdaObject { - int ocn; - string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb - string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb - string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) - string identifier; // heading: from the declaration line; otherwise .identifier - string part; // frontmatter, body, backmatter - string is_of_type; // para, block, comment: how the text behaves - string own_section; // .section, when it differs from the block it sits in - string segment_html_is;// the html segment this object sits in - string segment_epub; // the epub segment this heading opens - string segment_epub_is;// the epub segment this object sits in - string table_aligns; // l c r per column - string[] stow_link; // links extracted from the object - string[] anchor_tags; // named targets the markup placed on this object - int parent_ocn; - int last_descendant_ocn; - int ocn_range_first;// a container (a poem): the first and last - int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1) - int[] children; // direct child heading OCNs - int[] ancestors; // markedup_ancestors[8] - bool dummy_heading; - bool ocn_off; - int indent_base; - int indent_hang; - bool bullet; - string lang; - bool has_links; - bool has_notes_reg; - bool has_notes_star; - bool has_images; - string segment; - string segment_prev; - string segment_next; - string anchor; - string heading_lev_anchor; // a heading's own named target - int table_cols; - string table_widths; - bool table_header; - string code_syntax; - bool code_linenumbers; - string heading_ancestors_text; - string sha256; - OcdaImage[] images; // repeatable .image records, in order - string text; // content (lines joined with newline) - - /+ ↓ whether this object is rendered at all. - is_of_type "comment" is markup commentary: carried by the - abstraction so that nothing is lost, and not put into output. An - output that renders it leaks the author notes into the document, - which is what spine itself does not do. - +/ - bool renders() const { - return is_of_type != "comment"; - } - - /+ ↓ the section this object belongs to, which is the block it sits in - unless it says otherwise +/ - string sectionOf() const { - return (own_section.length > 0) ? own_section : section; - } -} - -/+ ↓ what the reader met and did not understand. - A reader that silently drops what it does not know cannot say whether - it understood the document, and cannot be used to check a - specification against an implementation: silence and understanding - look alike. So the parser counts what it ignored, and - --report-unknown prints it. -+/ -struct OcdaUnknown { - size_t[string] property; // .name -> times seen - size_t[string] block; // @name -> times seen - size_t[string] line; // first char -> times seen, lines of no known shape - - bool any() const { - return property.length > 0 || block.length > 0 || line.length > 0; - } -} - -struct OcdaDocument { - string format_version; // as declared on line 1, "" when absent - string source_filename; // from the "% Source:" line - string[string] source; // @source: language, languages, digest - OcdaUnknown unknown; // what was met and not understood - string[string] metadata; // key -> value - string[string] make; // key -> value - string[string] doc_has; // key -> value - OcdaObject[] head; - OcdaObject[] toc; - OcdaObject[] body; - OcdaObject[] endnotes; - OcdaObject[] glossary; - OcdaObject[] bibliography; - OcdaObject[] bookindex; - OcdaObject[] blurb; - OcdaObject[] tail; - - /+ ↓ the major part of the declared version, "" when none was declared +/ - string formatVersionMajor() const { - import std.string : indexOf; - if (format_version.length == 0) { return ""; } - auto dot = format_version.indexOf('.'); - return (dot > 0) ? format_version[0 .. dot] : format_version; - } - - string meta(string key) { - if (auto p = key in metadata) return *p; - return ""; - } - - OcdaObject[]* sectionByName(string name) { - switch (name) { - case "head": return &head; - case "toc": return &toc; - case "body": return &body; - case "endnotes": return &endnotes; - case "glossary": return &glossary; - case "bibliography": return &bibliography; - case "bookindex": return &bookindex; - case "blurb": return &blurb; - case "tail": return &tail; - default: return null; - } - } -} - -/+ ↓ the format major version this reader knows. - The .ssp and the .ocda.db carry one version between them, declared in - band: the .ssp on its first line, the database in its schema.version - row. The minor part is raised when a property is added, and a reader of - the newer format still reads the older one, so only the major part is - checked here. Raise this when the reader has been brought to a new - major, not before: the point of the gate is to refuse what has not been - looked at. -+/ -enum ssp_format_major_read = "2"; -#+END_SRC - -** load -** what to load -*** inventory - -#+HEADER: :tangle "../dlang/src/ocda/inventory.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.inventory; - -/+ ↓ what a source holds, beyond the one document the model is loaded from. - . - The model is one language of one document. Two questions a reader asks - are about the file instead: which languages it holds, and, for a - database, what else it carries. They are here rather than in a reader - because the answer depends on the input form, which is the library's - to know, and so that a reader asking them does not learn which form it - was handed. - . - languages() answers for every form, and answers the same: the - languages in the order the manifest gave them (which every .ssp - records, in @source languages), each with the markup file it was made - from and the sha256 of its .ssp. The database stores that digest; for a - .ssp and a pod it is taken from the bytes, which are the same bytes. - . - carried() is the database's: a .ocda.db holds the markup, images, - configuration and catalogues it was built from, a .ssp holds none of - them, and a pod's are its archive entries rather than anything the - abstraction says. So it answers for a database and says so otherwise. -+/ - -import std.algorithm : endsWith, countUntil; -import std.array : array, split; -import std.conv : to; -import std.digest : toHexString, LetterCase; -import std.digest.sha : sha256Of; -import std.file : read; -import std.string : strip; -import ocda.model; -import ocda.load; -import ocda.load_ssp; -import ocda.load_db; -import ocda.load_pod; -import ocda.sqlite; -import ocda.zip; - -struct OcdaLanguage { - string lang; - string source_filename; // the markup file the abstraction was made from - string ssp_digest; // sha256 of that language's .ssp, upper case hex -} - -struct OcdaCarried { - string role; // source, image, conf, manifest, tools - string compression; // how it is stored: none, zstd - string count; - string bytes; // the files' own size, whatever the storage - string stored; // what they take in the database -} - -OcdaLanguage[] languages(string path) { - final switch (sourceOf(path)) { - case Source.ocda_db: - auto db = Db.open(path); - scope(exit) { db.close(); } - OcdaLanguage[] out_; - foreach (l; dbLanguages(db)) { - out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); - } - return out_; - case Source.ssp: - auto bytes = cast(ubyte[]) read(path); - auto doc = parseSsp(cast(string) bytes.idup); - return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; - case Source.pod: - auto archive = archiveBytes(path); - OcdaLanguage[string] found; - string[] order; - foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { - auto bytes = zipRead(archive, e); - auto doc = parseSsp(cast(string) bytes.idup); - string lang = langOfSspName(e.name); - found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); - if (order.length == 0) { - order = sourceValue(doc, "languages").split; - } - } - /+ ↓ the manifest's order, and anything it did not name after it +/ - OcdaLanguage[] out_; - foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } - foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } - return out_; - case Source.unknown: - throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); - } -} - -/+ ↓ what a database carries, by role and storage; empty for any other form, - which isCarrier() says in advance +/ -bool isCarrier(string path) { - return sourceOf(path) == Source.ocda_db; -} - -OcdaCarried[] carried(string path) { - if (!isCarrier(path)) { return []; } - auto db = Db.open(path); - scope(exit) { db.close(); } - if (!db.has("files")) { return []; } - OcdaCarried[] out_; - foreach (r; db.rows( - "SELECT role, compression, count(*), sum(bytes), sum(length(data)) - FROM files GROUP BY role, compression ORDER BY role")) { - out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); - } - return out_; -} - -private string sourceValue(ref OcdaDocument doc, string key) { - if (auto p = key in doc.source) { return *p; } - return ""; -} - -private string hexOf(const(ubyte)[] bytes) { - return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; -} -#+END_SRC - -*** load -**** load - -#+HEADER: :tangle "../dlang/src/ocda/load.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load; - -/+ ↓ what a path is, and loading it. - . - One place that decides, so that every reader and every writer accepts - the same inputs and a new input type is added once. This mirrors - spine's own ocda/abstraction/load.d, which names the sources it can - enter the pipeline from and tells them apart by suffix. - . - Deliberately *not* here: .sst and .ssm markup. Reading markup means - implementing the parser, which is spine's work and a far larger thing - than a reader or a sample writer needs. This library reads - abstractions. -+/ - -import std.algorithm : endsWith; -import ocda.model; -import ocda.load_ssp; -import ocda.load_db; -import ocda.load_pod; - -enum Source { - ssp, // .ssp, the abstraction as text - ocda_db, // .ocda.db, the abstraction as sqlite - pod, // .sisupod or .zip, an archive carrying a .ssp - unknown, -} - -Source sourceOf(string path) { - if (path.endsWith(".ocda.db")) { return Source.ocda_db; } - if (path.endsWith(".ssp")) { return Source.ssp; } - /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote - before, and every pod already published carries that name, so both - are read. What is inside decides how it is opened, not the name. - +/ - if (path.endsWith(".sisupod")) { return Source.pod; } - if (path.endsWith(".zip")) { return Source.pod; } - if (path.endsWith(".db")) { return Source.ocda_db; } - return Source.unknown; -} - -string sourceName(Source s) { - final switch (s) { - case Source.ssp: return ".ssp (the abstraction as text)"; - case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; - case Source.pod: return "a pod archive carrying a .ssp"; - case Source.unknown: return "not an abstraction this library reads"; - } -} - -class OcdaLoadError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ load whatever the path is into the model. - lang names which language to take from a .ocda.db, which holds - every language of its document, and from a pod, which may carry more - than one .ssp. A .ssp is one language by construction and ignores it. -+/ -OcdaDocument load(string path, string lang = "", bool any_version = false) { - final switch (sourceOf(path)) { - case Source.ssp: return parseSspFile(path); - case Source.ocda_db: return loadDbFile(path, lang, any_version); - case Source.pod: return loadPodFile(path, lang); - case Source.unknown: - throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) - ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); - } -} -#+END_SRC - -**** load db - -#+HEADER: :tangle "../dlang/src/ocda/load_db.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_db; - -/+ ↓ build the model from a .ocda.db. - . - The point of this module is that it returns *the same value* - ocda.load_ssp returns from the same document's .ssp. A writer is then - a function from the model to bytes, and cannot tell which - serialisation it was given, exactly as spine's own writers cannot. - . - Written from the sisudoc-abstraction-format specification, - ocda-db-format-specification.md, whose section 4 gives the column by - column correspondence this module implements. -+/ - -import std.array : join, split; -import std.json : JSONType, parseJSON; -import std.conv : to; -import std.algorithm : startsWith; -import std.string : strip; -import ocda.model; -import ocda.sqlite; - -/+ ↓ the schema major this loader knows (specification section 2) +/ -enum schema_major_read = "2"; - -class OcdaDbError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ every language the database holds, in manifest order +/ -struct DbLanguage { - int doc_id; - string lang; - string source_filename; - string ssp_digest; -} - -DbLanguage[] dbLanguages(ref Db db) { - DbLanguage[] out_; - foreach (r; db.rows( - "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') - FROM documents ORDER BY id")) { - out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); - } - return out_; -} - -/+ ↓ load one language of a database as the model. - lang == "" takes the first language the manifest gave. -+/ -OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { - auto db = Db.open(path); - scope(exit) { db.close(); } - - if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { - throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" - ~ " it does not look like a .ocda.db"); - } - - /+ ↓ specification sections 2 and 3.2: the rows describing the file rather - than a document in it are the only ones with doc_id null +/ - string schema_version = db.one( - "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); - if (schema_version.length > 0 && !any_version) { - auto major = schema_version.split(".")[0]; - if (major != schema_major_read) { - throw new OcdaDbError(path ~ " is schema v" ~ schema_version - ~ ", this loader knows v" ~ schema_major_read ~ ".x"); - } - } - - auto languages = dbLanguages(db); - if (languages.length == 0) { - throw new OcdaDbError(path ~ ": no documents rows"); - } - DbLanguage chosen = languages[0]; - if (lang.length > 0) { - bool found = false; - foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } - if (!found) { - string have; - foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } - throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); - } - } - - OcdaDocument doc; - doc.format_version = schema_version; - doc.source_filename = chosen.source_filename; - - /+ ↓ @source of the .ssp corresponds to the documents row and the - source.digest metadata (specification section 5.1 of the .ssp, and - section 7 here) +/ - doc.source["language"] = chosen.lang; - string languages_joined; - foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } - doc.source["languages"] = languages_joined; - auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " - ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); - if (digest.length > 0) { doc.source["digest"] = digest; } - - /+ ↓ @meta: every metadata row of this language that is not a source.* - property. source.* describes the artefact rather than the document - and lives in @source in the text form. - +/ - foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " - ~ chosen.doc_id.to!string ~ " ORDER BY key")) { - string key = r[0]; - if (key.startsWith("source.")) { continue; } - /+ ↓ translation.ocn_aligned says whether the languages of this - document share one numbering. A .ocda.db can know it because it - holds every language; a .ssp is one language and cannot. So it is - a fact about the *file* rather than about the document, and putting - it in the document's metadata would let a writer tell which - serialisation it was handed, which is the one thing this library - exists to prevent. Left out until the model has a place for - collection-wide facts. - +/ - if (key.startsWith("translation.")) { continue; } - /+ ↓ the database has one metadata table where the text form has three - blocks, and tells them apart by a key prefix. Unprefixing them - here is what makes the two loaders give one model: a writer asks - doc.make["footer"] whichever serialisation it was handed, and - never doc.metadata["make.footer"]. - +/ - if (key.startsWith("make.")) { - doc.make[key["make.".length .. $]] = r[1].strip; - } else if (key.startsWith("doc_has.")) { - doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; - } else { - doc.metadata[key] = r[1].strip; - } - } - - /+ ↓ 2.1 added the container range as two columns. A 2.0 database has - neither, and reads as having no range, which is what 2.0 meant. +/ - bool has_range = false; - foreach (c; db.rows("PRAGMA table_info(objects)")) { - if (c[1] == "ocn_range_first") { has_range = true; } - } - string range_cols = has_range - ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" - : "0, 0"; - - /+ ↓ the objects, in document order within each section +/ - foreach (r; db.rows( - "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), - coalesce(is_of_section,''), coalesce(is_of_type,''), - coalesce(heading_level,-1), coalesce(identifier,''), - coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), - coalesce(children,''), coalesce(ancestors,''), - coalesce(dummy_heading,0), coalesce(object_number_off,0), - coalesce(indent_base,0), coalesce(indent_hang,0), - coalesce(bullet,0), coalesce(lang,''), - coalesce(has_links,0), coalesce(has_notes_reg,0), - coalesce(has_notes_star,0), coalesce(has_images,0), - coalesce(segment,''), coalesce(segment_prev,''), - coalesce(segment_next,''), coalesce(anchor,''), - coalesce(segment_html_is,''), - coalesce(table_cols,0), coalesce(table_widths,''), - coalesce(table_aligns,''), coalesce(table_header,0), - coalesce(code_linenumbers,0), - coalesce(heading_ancestors_text,''), coalesce(sha256,''), - coalesce(text,''), id, coalesce(heading_lev_anchor,''), - coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ " - FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string - ~ " ORDER BY CASE section - WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 - WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 - WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 - WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { - OcdaObject obj; - obj.section = r[0]; - obj.ocn = r[2].to!int; - obj.is_a = r[3]; - obj.part = r[4]; - obj.own_section = (r[5] == r[0]) ? "" : r[5]; - obj.is_of_type = r[6]; - /+ ↓ the database holds the level as a number, the text form as the - letter or digit the markup wrote. Model section 4 is the mapping. +/ - obj.heading_level = levelName(r[7].to!int); - obj.identifier = r[8]; - obj.parent_ocn = r[9].to!int; - obj.last_descendant_ocn = r[10].to!int; - obj.children = jsonInts(r[11]); - obj.ancestors = jsonInts(r[12]); - obj.dummy_heading = (r[13] != "0"); - obj.ocn_off = (r[14] != "0"); - obj.indent_base = r[15].to!int; - obj.indent_hang = r[16].to!int; - obj.bullet = (r[17] != "0"); - obj.lang = r[18]; - obj.has_links = (r[19] != "0"); - obj.has_notes_reg = (r[20] != "0"); - obj.has_notes_star = (r[21] != "0"); - obj.has_images = (r[22] != "0"); - obj.segment = r[23]; - obj.segment_prev = r[24]; - obj.segment_next = r[25]; - obj.anchor = r[26]; - obj.segment_html_is = r[27]; - obj.heading_lev_anchor = r[37]; - obj.segment_epub = r[38]; - obj.segment_epub_is = r[39]; - obj.ocn_range_first = r[40].to!int; - obj.ocn_range_last = r[41].to!int; - obj.table_cols = r[28].to!int; - obj.table_widths = jsonWords(r[29]); - obj.table_aligns = jsonWords(r[30]); - obj.table_header = (r[31] != "0"); - obj.code_linenumbers = (r[32] != "0"); - obj.heading_ancestors_text = jsonJoined(r[33]); - obj.sha256 = r[34]; - obj.text = r[35]; - - /+ ↓ the four repeatable properties of the text form are child tables - here (specification section 4) +/ - string object_id = r[36]; - foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), - coalesce(width,0), coalesce(height,0), - coalesce(missing,0) - FROM object_images WHERE object_id = " ~ object_id - ~ " ORDER BY seq")) { - OcdaImage img; - img.name = i[0]; - img.sha256 = i[1]; - img.bytes = i[2].to!ulong; - img.width = i[3].to!int; - img.height = i[4].to!int; - img.missing = (i[5] != "0"); - obj.images ~= img; - } - foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " - ~ object_id ~ " ORDER BY seq")) { - obj.stow_link ~= l[0]; - } - foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " - ~ object_id ~ " ORDER BY seq")) { - obj.anchor_tags ~= a[0]; - } - - auto section = doc.sectionByName(r[0]); - if (section !is null) { *section ~= obj; } - } - countUnread(db, chosen.doc_id, doc); - return doc; -} - -/+ ↓ what the database holds and the model does not. - . - The text loader meets every property line and counts those it does not - keep, so a reader can say what was read and not understood. This - loader asks for columns by name, and would never see the ones it did - not ask for: it would report that it had understood everything, which - is the silence the count exists to prevent. So it counts them, on the - conditions under which the .ssp writes the same property (specification - section 6.3), and the report is the same from either form: checked, - 36 of 36. -+/ -private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { - string id = doc_id.to!string; - void count(string property, string where) { - auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id - ~ " AND " ~ where).to!size_t; - if (n > 0) { doc.unknown.property[property] = n; } - } - count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); - count("dom_status", "coalesce(dom_status,'') != ''"); - count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); - count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); - count("parent_lev", "coalesce(parent_lev,0) != 0"); - count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); - count("attrib", "coalesce(attrib,'') != ''"); - if (db.has("object_subtoc")) { - auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" - ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; - if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } - } -} - -/+ ↓ model section 4: the eight levels and their indices +/ -private string levelName(int index) { - switch (index) { - case 0: return "A"; - case 1: return "B"; - case 2: return "C"; - case 3: return "D"; - case 4: return "1"; - case 5: return "2"; - case 6: return "3"; - case 7: return "4"; - default: return ""; - } -} - -/+ ↓ the JSON columns, read as JSON. - . - The fixed width arrays are JSON in the database and space separated - (or "|" separated) in the text form. This was first written as a - character scan that threw away brackets and quotes, which is wrong the - moment a value holds an escape: Phobos' JSONValue.toString escapes - "/" as "\/", so every heading containing a slash came back with a - backslash in it. 36 of 36 documents differed on - heading_ancestors_text and nothing else had shown it. - . - So: parse. The values are small, there are few of them, and a real - parser is shorter than the scan it replaces. -+/ -private string[] jsonStrings(string v) { - if (v.length == 0) { return []; } - string[] out_; - try { - auto j = parseJSON(v); - if (j.type != JSONType.array) { return []; } - foreach (e; j.array) { - switch (e.type) { - case JSONType.string: out_ ~= e.str; break; - case JSONType.integer: out_ ~= e.integer.to!string; break; - case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; - case JSONType.float_: out_ ~= e.floating.to!string; break; - case JSONType.true_: out_ ~= "true"; break; - case JSONType.false_: out_ ~= "false"; break; - default: out_ ~= ""; break; - } - } - } catch (Exception) { - return []; - } - return out_; -} - -private int[] jsonInts(string v) { - int[] out_; - foreach (s; jsonStrings(v)) { - if (s.length == 0) { continue; } - try { out_ ~= s.to!int; } catch (Exception) {} - } - return out_; -} - -/+ ↓ space separated in the text form: table_widths, table_aligns +/ -private string jsonWords(string v) { - return jsonStrings(v).join(" "); -} - -/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text - form and a JSON array here. The empty slots are significant: they say - which levels the heading sits under, so the separators carry as much - as the text does. -+/ -private string jsonJoined(string v) { - auto parts = jsonStrings(v); - if (parts.length == 0) { return ""; } - return parts.join("|"); -} -#+END_SRC - -**** load pod - -#+HEADER: :tangle "../dlang/src/ocda/load_pod.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_pod; - -/+ ↓ the abstraction out of a pod archive. - . - A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same - archive unwrapped, which is what spine published before it wrapped them. - Either way, a pod built with --pod2 carries the abstraction inside it, - at media/abstraction/<uid>.<lang>.ssp, and that is what is read. - . - So this is a third input form and *not* a third model: the .ssp inside - is parsed by ocda.load_ssp and comes back as the same value. A writer - over the library gains .sisupod without being touched, which is the - whole reason the dispatch is in one place. - . - A pod built without --pod2 carries markup and no abstraction. That is - refused, and the refusal says which flag makes a pod this reader can - read, because "no abstraction in it" is a fact about the pod rather - than about the reader. -+/ - -import std.algorithm : endsWith, sort; -import std.array : array; -import std.file : read; -import std.string : indexOf; -import ocda.model; -import ocda.load_ssp; -import ocda.zip; -import ocda.zstd; - -/+ ↓ a limit on what a frame may expand to. - A reader of an artefact it did not make declines to be told how much - memory to allocate. 512 MiB is far above the largest pod in the - reference collection (4 MiB) and far below anything that matters. -+/ -enum size_t max_pod_size = 512 * 1024 * 1024; - -class OcdaPodError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ which languages of the abstraction the pod carries, in sorted order +/ -string[] podLanguages(string path) { - auto bytes = archiveBytes(path); - string[] out_; - foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { - string lang = langOfSspName(e.name); - if (lang.length > 0) { out_ ~= lang; } - } - return out_.sort.array; -} - -/+ ↓ load one language of the abstraction a pod carries. - lang == "" takes the only one there is, and where there is a choice it - says so rather than choosing, as the .ocda.db loader does. -+/ -OcdaDocument loadPodFile(string path, string lang = "") { - auto bytes = archiveBytes(path); - auto entries = zipEntries(bytes); - auto ssps = zipFindSuffix(entries, ".ssp"); - if (ssps.length == 0) { - throw new OcdaPodError(path ~ ": carries no .ssp." - ~ " A pod carries the abstraction only when spine was asked for it:" - ~ " spine --pod2 puts it in, spine --source does not."); - } - ZipEntry chosen = ssps[0]; - if (lang.length > 0) { - bool found = false; - foreach (e; ssps) { - if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } - } - if (!found) { - string have; - foreach (i, e; ssps) { - have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); - } - throw new OcdaPodError(path ~ ": no language '" ~ lang - ~ "'; it holds: " ~ have); - } - } else if (ssps.length > 1) { - string have; - foreach (i, e; ssps) { - have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); - } - throw new OcdaPodError(path ~ ": holds more than one language (" - ~ have ~ "); name the one wanted"); - } - auto ssp = zipRead(bytes, chosen); - return parseSsp(cast(string) ssp.idup); -} - -/+ ↓ the archive, unwrapped if it was wrapped. - The bytes decide, not the name: a .sisupod is a zstd frame and a .zip - is not, and a file misnamed either way still reads. -+/ -package ubyte[] archiveBytes(string path) { - auto bytes = cast(ubyte[]) read(path); - return zstdIsFrame(bytes) - ? zstdDecompress(bytes, max_pod_size) - : bytes; -} - -/+ ↓ <uid>.<lang>.ssp -> <lang> +/ -package string langOfSspName(string name) { - if (!name.endsWith(".ssp")) { return ""; } - string stem = name[0 .. $ - 4]; - auto dot = stem.lastIndexOf('.'); - return (dot > 0) ? stem[dot + 1 .. $] : ""; -} - -private ptrdiff_t lastIndexOf(string s, char c) { - for (ptrdiff_t i = s.length - 1; i >= 0; i--) { - if (s[i] == c) { return i; } - } - return -1; -} -#+END_SRC - -**** load ssp - -#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.load_ssp; - -import std.algorithm : splitter, startsWith; -import std.array : array, split; -import std.conv : to, ConvException; -import std.file : readText; -import std.stdio : writeln; -import std.string : strip, stripLeft, indexOf; -import ocda.model; - -OcdaDocument parseSspFile(string filename) { - string content = readText(filename); - return parseSsp(content); -} - -OcdaDocument parseSsp(string content) { - OcdaDocument doc; - OcdaObject current_obj; - bool in_object = false; - /+ ↓ whether a text line has been seen for the object being read. - Not "is the text non-empty": an object whose first text line is blank - (a verse opening on an empty line, which the sample set has seven of) - would then lose it, the next line overwriting rather than appending. - +/ - bool seen_text = false; - string current_section = ""; - string current_block = ""; // meta, make, doc_has, or section name - - foreach (line; content.splitter("\n")) { - /+ ↓ comment. The first line of a .ssp declares the format version, and - it is the one comment that carries meaning: a reader checks the - major part and refuses what it does not know rather than loading - an unfamiliar artefact half populated in silence. - +/ - if (line.length > 0 && line[0] == '%') { - enum _banner = "% SiSU Document Abstraction v"; - enum _source = "% Source: "; - if (doc.format_version.length == 0 && line.startsWith(_banner)) { - doc.format_version = line[_banner.length .. $].strip; - } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { - doc.source_filename = line[_source.length .. $].strip; - } - continue; - } - - /+ ↓ blank line - finalize current object +/ - if (line.length == 0 || line.strip.length == 0) { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = OcdaObject.init; - in_object = false; - seen_text = false; - } - continue; - } - - /+ ↓ section/block open +/ - if (line[0] == '@') { - auto brace_pos = line.indexOf('{'); - if (brace_pos > 0) { - string name = line[1 .. brace_pos].strip; - current_block = name; - if (!isPreambleBlock(name)) { - current_section = name; - /+ ↓ a block that is neither preamble nor a section this reader - knows: counted, so that --report-unknown can say so +/ - if (doc.sectionByName(name) is null) { - doc.unknown.block[name]++; - } - } - } - continue; - } - - /+ ↓ section/block close +/ - if (line[0] == '}') { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = OcdaObject.init; - in_object = false; - seen_text = false; - } - current_block = ""; - continue; - } - - /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ - if (isPreambleBlock(current_block)) { - string stripped = line.stripLeft; - auto colon_pos = stripped.indexOf(':'); - if (colon_pos > 0) { - string key = stripped[0 .. colon_pos].strip; - string value = (colon_pos + 1 < stripped.length) - ? stripped[colon_pos + 1 .. $].strip : ""; - if (current_block == "meta") { doc.metadata[key] = value; } - else if (current_block == "make") { doc.make[key] = value; } - else if (current_block == "source") { doc.source[key] = value; } - else { doc.doc_has[key] = value; } - } - continue; - } - - /+ ↓ object declaration: [N] type ... +/ - if (line[0] == '[') { - if (in_object) { - commitObject(doc, current_section, current_obj); - } - current_obj = OcdaObject.init; - current_obj.section = current_section; - in_object = true; - seen_text = false; - parseObjectDeclaration(line, current_obj); - continue; - } - - /+ ↓ property: .name: value +/ - if (line[0] == '.') { - if (in_object) { - parseProperty(line, current_obj, doc.unknown); - } - continue; - } - - /+ ↓ text content: | content +/ - if (line.length >= 1 && line[0] == '|') { - if (in_object) { - string text_content = (line.length > 2) ? line[2 .. $] : ""; - if (seen_text) { - current_obj.text ~= "\n" ~ text_content; - } else { - current_obj.text = text_content; - seen_text = true; - } - } - continue; - } - - /+ ↓ no known shape. Counted by its first character so that a reader - can be told the file held something this one did not expect - +/ - doc.unknown.line[line[0 .. 1].idup]++; - } - - /+ ↓ finalize last object +/ - if (in_object) { - commitObject(doc, current_section, current_obj); - } - - return doc; -} - -private void parseObjectDeclaration(string line, ref OcdaObject obj) { - /+ parse: [N] type :level identifier +/ - auto bracket_close = line.indexOf(']'); - if (bracket_close < 0) return; - try { - obj.ocn = line[1 .. bracket_close].strip.to!int; - } catch (ConvException) { - obj.ocn = 0; - } - string rest = (bracket_close + 1 < line.length) - ? line[bracket_close + 1 .. $].strip : ""; - if (rest.length == 0) return; - - auto parts = rest.split(" "); - if (parts.length == 0) return; - obj.is_a = parts[0]; - - if (obj.is_a == "heading" && parts.length >= 2) { - /+ parse :A, :B, :1, etc. +/ - string lev = parts[1]; - if (lev.length >= 2 && lev[0] == ':') { - obj.heading_level = lev[1 .. $]; - } - /+ optional identifier after level +/ - if (parts.length >= 3) { - obj.identifier = parts[2]; - } - } -} - -/+ ↓ the blocks that are document preamble rather than sections of objects +/ -private bool isPreambleBlock(string name) { - return name == "source" || name == "meta" || name == "make" || name == "doc_has"; -} - -private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { - /+ parse: .name: value +/ - auto colon_pos = line.indexOf(':'); - if (colon_pos < 1) return; - string name = line[1 .. colon_pos].strip; - string value = (colon_pos + 1 < line.length) - ? line[colon_pos + 1 .. $].strip : ""; - - switch (name) { - case "part": obj.part = value; break; - case "is_of_type": obj.is_of_type = value; break; - case "section": obj.own_section = value; break; - case "identifier": obj.identifier = value; break; - case "segment_html_is": obj.segment_html_is = value; break; - /+ ↓ the epub segments at heading levels 1 to 4 where the html - segments at level 4 alone, so the two are different files and - the abstraction records both +/ - case "segment_epub": obj.segment_epub = value; break; - case "segment_epub_is": obj.segment_epub_is = value; break; - case "table_aligns": obj.table_aligns = value; break; - case "stow_link": obj.stow_link ~= value; break; - case "anchor_tag": obj.anchor_tags ~= value; break; - case "parent": obj.parent_ocn = toIntSafe(value); break; - case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; - case "ocn_range": { - auto r = parseIntArray(value); - if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; } - break; - } - case "children": obj.children = parseIntArray(value); break; - case "ancestors": obj.ancestors = parseIntArray(value); break; - case "dummy": obj.dummy_heading = (value == "true"); break; - case "ocn_off": obj.ocn_off = (value == "true"); break; - case "indent": - auto parts = value.split(" "); - if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); - if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); - break; - case "bullet": obj.bullet = (value == "true"); break; - case "lang": obj.lang = value; break; - case "has": - foreach (flag; value.split(" ")) { - switch (flag) { - case "links": obj.has_links = true; break; - case "notes_reg": obj.has_notes_reg = true; break; - case "notes_star": obj.has_notes_star = true; break; - case "images": obj.has_images = true; break; - default: break; - } - } - break; - case "segment": obj.segment = value; break; - case "segment_prev": obj.segment_prev = value; break; - case "segment_next": obj.segment_next = value; break; - case "anchor": obj.anchor = value; break; - case "heading_lev_anchor": obj.heading_lev_anchor = value; break; - case "table_cols": obj.table_cols = toIntSafe(value); break; - case "table_widths": obj.table_widths = value; break; - case "table_header": obj.table_header = (value == "true"); break; - case "code_syntax": obj.code_syntax = value; break; - case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; - case "heading_ancestors_text": obj.heading_ancestors_text = value; break; - case "sha256": obj.sha256 = value; break; - case "image": obj.images ~= parseImageRecord(value); break; - /+ ↓ a property this reader has no use for, or has never heard of. - Ignored, which is the conforming behaviour within a major - version, but counted rather than passed over in silence - +/ - default: unknown.property[name]++; break; - } -} - -private OcdaImage parseImageRecord(string value) { - /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] - filename first, remaining fields order-independent; unrecognised - fields are kept rather than discarded +/ - OcdaImage img; - bool first = true; - foreach (field; value.split(" ")) { - if (field.length == 0) continue; - if (first) { - img.name = field; - first = false; - continue; - } - auto colon_pos = field.indexOf(':'); - if (colon_pos < 1) continue; - string key = field[0 .. colon_pos]; - string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; - switch (key) { - case "sha256": img.sha256 = val; break; - case "bytes": img.bytes = toUlongSafe(val); break; - case "px": - auto dims = val.split("x"); - if (dims.length == 2) { - img.width = toIntSafe(dims[0]); - img.height = toIntSafe(dims[1]); - } - break; - case "missing": img.missing = (val == "true"); break; - default: img.fields[key] = val; break; - } - } - return img; -} - -/+ ↓ two values the text form leaves out because they can be worked out. - . - The .ssp emits a property only when it carries information, so - .identifier is absent when it is simply the ocn, and .anchor is absent - when it is the segment name. The .ocda.db, being columns rather than - lines, stores both unconditionally. So a loader that takes the text - form literally gives a *different model* from the one the database - gives, and the difference is not small: in the reference collection - the identifier is omitted on all but a handful of objects. - . - The model is the thing both serialisations describe, so the loader - fills them in. Found by dumping both models and diffing them, which no - amount of comparing section counts would have shown. -+/ -private void deriveOmitted(ref OcdaObject obj) { - import std.conv : to; - if (obj.identifier.length == 0 && obj.ocn != 0) { - obj.identifier = obj.ocn.to!string; - } - if (obj.anchor.length == 0 && obj.segment.length > 0) { - obj.anchor = obj.segment; - } -} - -private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { - deriveOmitted(obj); - auto sec = doc.sectionByName(section); - if (sec !is null) { - ,*sec ~= obj; - } -} - -private int toIntSafe(string s) { - try { return s.strip.to!int; } - catch (ConvException) { return 0; } -} - -private ulong toUlongSafe(string s) { - try { return s.strip.to!ulong; } - catch (ConvException) { return 0; } -} - -private int[] parseIntArray(string s) { - int[] result; - foreach (part; s.split(" ")) { - auto v = toIntSafe(part); - result ~= v; - } - return result; -} -#+END_SRC - -*** images - -#+HEADER: :tangle "../dlang/src/ocda/images.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.images; - -/+ ↓ an image's bytes, from whichever source the document came from. - . - The model *names* images, with digest, byte size and pixel size; it does - not hold them. That is right - an abstraction of a document is not a - bundle of its assets - but a writer of any format that shows images - needs the bytes, so something has to know where they are. This is that - something, and it is in the library rather than in a writer because the - answer depends on the *input form* and not on the output. - . - Three answers, and one refusal: - . - .ocda.db the files table, role 'image', by name - .sisupod the archive entry whose path ends in /image/<name> - .zip the same - .ssp beside it: a pod puts the abstraction in - media/abstraction/ and its images in media/image/, so the - path is documented rather than guessed - . - A .ssp that is not in a pod has nowhere to look, and that is reported as - absent rather than as an error: a reader that cannot find an image - should say so and carry on, because the document is still readable. -+/ - -import std.algorithm : endsWith; -import std.array : array; -import std.file : exists, isFile, read; -import std.path : buildPath, dirName; -import ocda.load; -import ocda.sqlite; -import ocda.zip; -import ocda.zstd; - -/+ ↓ the bytes, or an empty slice when the source has no such image +/ -ubyte[] imageBytes(string source, string name) { - final switch (sourceOf(source)) { - case Source.ocda_db: return fromDb(source, name); - case Source.pod: return fromArchive(source, name); - case Source.ssp: return fromBeside(source, name); - case Source.unknown: return []; - } -} - -/+ ↓ can this source give images at all? - A writer asks before it decides whether to emit them, because an <img> - pointing at a file it did not include is worse than no image. -+/ -bool canGiveImages(string source) { - final switch (sourceOf(source)) { - case Source.ocda_db: return true; - case Source.pod: return true; - case Source.ssp: return imageDirBeside(source).length > 0; - case Source.unknown: return false; - } -} - -private ubyte[] fromDb(string path, string name) { - auto db = Db.open(path); - scope(exit) { db.close(); } - if (!db.has("files")) { return []; } - /+ ↓ the name is from the abstraction, not from a caller, and the quote - doubling is what keeps it a value rather than syntax +/ - return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '" - ~ sqlQuote(name) ~ "'"); -} - -private ubyte[] fromArchive(string path, string name) { - auto bytes = cast(ubyte[]) read(path); - if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); } - auto entries = zipEntries(bytes); - foreach (e; entries) { - if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); } - } - return []; -} - -private ubyte[] fromBeside(string path, string name) { - string dir = imageDirBeside(path); - if (dir.length == 0) { return []; } - string file = buildPath(dir, name); - return file.exists && file.isFile ? cast(ubyte[]) read(file) : []; -} - -/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/ -private string imageDirBeside(string path) { - string dir = path.dirName; - if (!dir.endsWith("abstraction")) { return ""; } - string image_dir = buildPath(dir.dirName, "image"); - return image_dir.exists ? image_dir : ""; -} - -/+ ↓ a single quote doubled is how sql says a single quote +/ -private string sqlQuote(string s) { - string out_; - foreach (c; s) { - out_ ~= (c == '\'') ? "''" : [c]; - } - return out_; -} -#+END_SRC - -** sqlite - -#+HEADER: :tangle "../dlang/src/ocda/sqlite.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.sqlite; - -/+ ↓ the sqlite3 C API, declared rather than depended upon. - . - A binding from the dub registry would be a dependency to fetch, and - the point of this reader is that it builds in a sandbox with no - network. The dozen functions a reader needs are declared here and the - library is linked directly, which costs a page of extern(C) and saves - a dependency. -+/ - -import std.conv : to; -import std.string : fromStringz, toStringz; - -extern (C) { - struct sqlite3; - struct sqlite3_stmt; - - int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs); - int sqlite3_close(sqlite3* db); - int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte, - sqlite3_stmt** stmt, const(char)** tail); - int sqlite3_step(sqlite3_stmt* stmt); - int sqlite3_finalize(sqlite3_stmt* stmt); - int sqlite3_column_count(sqlite3_stmt* stmt); - const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col); - const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col); - int sqlite3_column_int(sqlite3_stmt* stmt, int col); - int sqlite3_column_type(sqlite3_stmt* stmt, int col); - const(char)* sqlite3_errmsg(sqlite3* db); - const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col); - int sqlite3_column_bytes(sqlite3_stmt* stmt, int col); -} - -enum SQLITE_OK = 0; -enum SQLITE_ROW = 100; -enum SQLITE_DONE = 101; -enum SQLITE_NULL = 5; -enum SQLITE_OPEN_READONLY = 0x00000001; - -class SqliteError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ a database, opened read only. - Read only is deliberate: a reader has no business writing, and a - request that opens a database read-write can create an empty one where - a path is mistyped. -+/ -struct Db { - private sqlite3* _handle; - - static Db open(string path) { - sqlite3* handle; - auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null); - if (rc != SQLITE_OK) { - string msg = (handle is null) - ? "cannot open " ~ path - : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup; - if (handle !is null) { sqlite3_close(handle); } - throw new SqliteError(msg); - } - Db db; - db._handle = handle; - return db; - } - - void close() { - if (_handle !is null) { sqlite3_close(_handle); _handle = null; } - } - - /+ ↓ every row of a query, each row as its columns in order +/ - string[][] rows(string sql) { - sqlite3_stmt* stmt; - auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null); - if (rc != SQLITE_OK) { - throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql); - } - scope(exit) { sqlite3_finalize(stmt); } - string[][] out_rows; - int cols = sqlite3_column_count(stmt); - while (sqlite3_step(stmt) == SQLITE_ROW) { - string[] row; - foreach (c; 0 .. cols) { - row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL) - ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup; - } - out_rows ~= row; - } - return out_rows; - } - - /+ ↓ the first column of the first row, "" when there is none +/ - string one(string sql) { - auto r = rows(sql); - return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; - } - - /+ ↓ the first column of the first row as bytes, for a blob. - Separate from one(), because a blob is not text: an image read as text - stops at the first zero byte, and a png's ninth byte is zero. - +/ - ubyte[] blob(string sql) { - sqlite3_stmt* stmt; - if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) { - throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup - ~ " [" ~ sql ~ "]"); - } - scope(exit) { sqlite3_finalize(stmt); } - if (sqlite3_step(stmt) != SQLITE_ROW) { return []; } - int n = sqlite3_column_bytes(stmt, 0); - if (n <= 0) { return []; } - auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0); - if (p is null) { return []; } - return p[0 .. n].dup; - } - - /+ ↓ does a table or view of this name exist +/ - bool has(string name) { - return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; - } -} -#+END_SRC - -** compression -*** dlang read zip - -#+HEADER: :tangle "../dlang/src/ocda/zip.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.zip; - -/+ ↓ enough of the zip format to find one file in a pod and read it. - . - Not a zip library. A pod archive holds a handful of entries, spine - writes them **stored** (uncompressed) so that the outer zstd frame does - the compressing once over the whole thing, and what is wanted from it - here is one named entry. So: read the central directory, find the - entry, return its bytes. - . - Deflate is *refused rather than mishandled*. Phobos can inflate, but a - library that says it read an entry it in fact skipped is worse than one - that says it cannot, and spine's own archives do not need it. -+/ - -import std.algorithm : endsWith; -import std.conv : to; - -class OcdaZipError : Exception { - this(string msg) { super(msg); } -} - -struct ZipEntry { - string name; - uint method; // 0 stored, 8 deflate - uint compressed; - uint uncompressed; - uint local_offset; -} - -/+ ↓ the entries, from the central directory at the end of the file +/ -ZipEntry[] zipEntries(const(ubyte)[] bytes) { - enum uint eocd_sig = 0x06054b50; - enum uint cd_sig = 0x02014b50; - if (bytes.length < 22) { - throw new OcdaZipError("too short to be a zip archive"); - } - /+ ↓ the end of central directory record is last, and may be followed by a - comment of up to 65535 bytes, so it is searched for backwards +/ - ptrdiff_t eocd = -1; - ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0; - for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) { - if (u32(bytes, i) == eocd_sig) { eocd = i; break; } - } - if (eocd < 0) { - throw new OcdaZipError("no end of central directory record;" - ~ " it does not look like a zip archive"); - } - uint count = u16(bytes, eocd + 10); - uint offset = u32(bytes, eocd + 16); - ZipEntry[] out_; - size_t p = offset; - foreach (_; 0 .. count) { - if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) { - throw new OcdaZipError("central directory entry " - ~ out_.length.to!string ~ " is malformed"); - } - ZipEntry e; - e.method = u16(bytes, p + 10); - e.compressed = u32(bytes, p + 20); - e.uncompressed = u32(bytes, p + 24); - uint name_len = u16(bytes, p + 28); - uint extra_len = u16(bytes, p + 30); - uint cmnt_len = u16(bytes, p + 32); - e.local_offset = u32(bytes, p + 42); - e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup; - out_ ~= e; - p += 46 + name_len + extra_len + cmnt_len; - } - return out_; -} - -/+ ↓ one entry's bytes, from its local header +/ -ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) { - enum uint local_sig = 0x04034b50; - size_t p = e.local_offset; - if (p + 30 > bytes.length || u32(bytes, p) != local_sig) { - throw new OcdaZipError(e.name ~ ": local header is malformed"); - } - uint name_len = u16(bytes, p + 26); - uint extra_len = u16(bytes, p + 28); - size_t start = p + 30 + name_len + extra_len; - size_t end = start + e.compressed; - if (end > bytes.length) { - throw new OcdaZipError(e.name ~ ": runs past the end of the archive"); - } - if (e.method != 0) { - throw new OcdaZipError(e.name ~ ": compression method " - ~ e.method.to!string ~ "; this reader takes only stored entries," - ~ " which is what spine writes"); - } - return bytes[start .. end].dup; -} - -/+ ↓ the first entry whose name ends this way, or none. - Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid. -+/ -ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) { - ZipEntry[] out_; - foreach (e; entries) { - if (e.name.endsWith(suffix)) { out_ ~= e; } - } - return out_; -} - -private uint u16(const(ubyte)[] b, size_t i) { - return cast(uint) b[i] | (cast(uint) b[i + 1] << 8); -} - -private uint u32(const(ubyte)[] b, size_t i) { - return cast(uint) b[i] - | (cast(uint) b[i + 1] << 8) - | (cast(uint) b[i + 2] << 16) - | (cast(uint) b[i + 3] << 24); -} -#+END_SRC - -*** dlang read zstd - -#+HEADER: :tangle "../dlang/src/ocda/zstd.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.zstd; - -/+ ↓ libzstd, declared rather than bound. - . - A .sisupod is one zstd frame wrapping a zip archive, so reading one - needs exactly two things from the library: is this a frame, and what is - in it. Both are declared here and the C library is linked, which is the - same arrangement ocda.sqlite uses: no vendored tree, no generated - header, and the prototypes say what is being called. -+/ - -extern (C) @nogc nothrow private { - size_t ZSTD_decompress(void* dst, size_t dstCapacity, - const(void)* src, size_t compressedSize); - ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); - uint ZSTD_isError(size_t code); - const(char)* ZSTD_getErrorName(size_t code); -} - -/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/ -private enum ulong contentSizeUnknown = ulong.max; -private enum ulong contentSizeError = ulong.max - 1; - -/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/ -private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD]; - -class OcdaZstdError : Exception { - this(string msg) { super(msg); } -} - -/+ ↓ does this begin with a zstd frame? - Asked of the bytes rather than of the filename, because what is inside - decides how a thing is opened. A .sisupod is a frame; a .zip published - before spine wrapped them is not, and both are read. -+/ -bool zstdIsFrame(const(ubyte)[] bytes) { - if (bytes.length < zstd_magic.length) { return false; } - foreach (i, b; zstd_magic) { - if (bytes[i] != b) { return false; } - } - return true; -} - -/+ ↓ one frame in, its contents out. - max_size is a refusal rather than a buffer size: a frame declares how - much it expands to, and a frame that claims more than the caller is - willing to hold is not decompressed at all. A reader of an artefact it - did not make wants that limit. -+/ -ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) { - if (!zstdIsFrame(bytes)) { - throw new OcdaZstdError("not a zstd frame"); - } - ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length); - if (declared == contentSizeError) { - throw new OcdaZstdError("zstd frame: could not read its content size"); - } - if (declared == contentSizeUnknown) { - throw new OcdaZstdError("zstd frame declares no content size;" - ~ " this reader takes only frames that do"); - } - if (declared > max_size) { - import std.conv : to; - throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string - ~ " bytes, over the " ~ max_size.to!string ~ " byte limit"); - } - auto out_ = new ubyte[](cast(size_t) declared); - size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length); - if (ZSTD_isError(got)) { - import std.string : fromStringz; - throw new OcdaZstdError("zstd: " - ~ ZSTD_getErrorName(got).fromStringz.idup); - } - if (got != declared) { - throw new OcdaZstdError("zstd frame expanded to a different size" - ~ " than it declared"); - } - return out_; -} -#+END_SRC - -** dump - -#+HEADER: :tangle "../dlang/src/ocda/dump.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.dump; - -/+ ↓ the model, written out canonically. - . - Not an output format: an *oracle*. Two libraries in two languages - implement the same model from the same specification, and the cheap - way to ask whether they agree is to have each write the whole loaded - document as text and diff the two. That is only useful if the text is - fully determined by the model, so: - . - - every line is "<key> <value>", the value running to end of line, - and only one free-form value per line; - - a field at its default is *omitted*, so the output says what the - document has rather than what the struct declares; - - dictionaries are emitted in sorted key order, because neither - language's hash map has an order worth relying on; - - sections come in the fixed order of the format, not - alphabetically. - . - Anything that reads this to reconstruct a document has misunderstood - it. Load the .ssp or the .ocda.db. -+/ - -import std.algorithm : sort, splitter; -import std.array : appender, array, join; -import std.conv : to; -import ocda.model; - -/+ ↓ the nine sections, in format order +/ -static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", "tail"]; - -string dumpDocument(ref OcdaDocument doc) { - auto out_ = appender!string; - - void kv(string key, string value) { - if (value.length == 0) { return; } - out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; - } - void kn(string key, long value) { - if (value == 0) { return; } - kv(key, value.to!string); - } - void kb(string key, bool value) { - if (!value) { return; } - kv(key, "1"); - } - void dict(string prefix, string[string] d) { - foreach (key; d.keys.dup.sort) { - out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; - out_ ~= d[key]; out_ ~= "\n"; - } - } - - kv("version", doc.format_version); - kv("source_filename", doc.source_filename); - dict("source", doc.source); - dict("meta", doc.metadata); - dict("make", doc.make); - dict("doc_has", doc.doc_has); - - foreach (name; dump_sections) { - auto sec = doc.sectionByName(name); - if (sec is null || (*sec).length == 0) { continue; } - kv("section", name ~ " " ~ (*sec).length.to!string); - } - - foreach (name; dump_sections) { - auto sec = doc.sectionByName(name); - if (sec is null) { continue; } - foreach (i, obj; *sec) { - out_ ~= "object "; out_ ~= name; out_ ~= " "; - out_ ~= i.to!string; out_ ~= "\n"; - kn("ocn", obj.ocn); - kv("is_a", obj.is_a); - kv("own_section", obj.own_section); - kv("level", obj.heading_level); - kv("identifier", obj.identifier); - kv("part", obj.part); - kv("is_of_type", obj.is_of_type); - kv("segment_html_is", obj.segment_html_is); - kv("segment_epub", obj.segment_epub); - kv("segment_epub_is", obj.segment_epub_is); - kn("parent", obj.parent_ocn); - kn("last_descendant", obj.last_descendant_ocn); - kn("ocn_range_first", obj.ocn_range_first); - kn("ocn_range_last", obj.ocn_range_last); - kv("children", ints(obj.children)); - kv("ancestors", ints(obj.ancestors)); - kb("dummy", obj.dummy_heading); - kb("ocn_off", obj.ocn_off); - kn("indent_base", obj.indent_base); - kn("indent_hang", obj.indent_hang); - kb("bullet", obj.bullet); - kv("lang", obj.lang); - kb("has_links", obj.has_links); - kb("has_notes_reg", obj.has_notes_reg); - kb("has_notes_star", obj.has_notes_star); - kb("has_images", obj.has_images); - kv("segment", obj.segment); - kv("segment_prev", obj.segment_prev); - kv("segment_next", obj.segment_next); - kv("anchor", obj.anchor); - kv("heading_lev_anchor", obj.heading_lev_anchor); - kn("table_cols", obj.table_cols); - kv("table_widths", obj.table_widths); - kv("table_aligns", obj.table_aligns); - kb("table_header", obj.table_header); - kv("code_syntax", obj.code_syntax); - kb("code_linenumbers", obj.code_linenumbers); - kv("heading_ancestors_text", obj.heading_ancestors_text); - kv("sha256", obj.sha256); - foreach (url; obj.stow_link) { kv("stow_link", url); } - foreach (a; obj.anchor_tags) { kv("anchor_tag", a); } - foreach (img; obj.images) { - /+ ↓ the name last, because a filename may hold a space where - none of the other fields can +/ - kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " - ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " - ~ (img.missing ? "1" : "0") ~ " " ~ img.name); - foreach (key; img.fields.keys.dup.sort) { - kv("image_field", key ~ " " ~ img.fields[key]); - } - } - /+ ↓ text, when there is any. An object whose only text line is - blank holds "" here, indistinguishable from no text at all, - because the model joins the lines into one string. Both - libraries lose the same distinction in the same place, which is - what makes them comparable; it is recorded rather than hidden. - +/ - if (obj.text.length > 0) { - auto lines = obj.text.splitter("\n").array; - kv("text", lines.length.to!string); - foreach (line; lines) { - out_ ~= "| "; out_ ~= line; out_ ~= "\n"; - } - } - } - } - return out_.data; -} - -/+ ↓ what the reader met and did not understand, sorted. Separate from the - document dump: it describes the reader, not the document, and the - .ocda.db loader has nothing to report where the .ssp loader does. -+/ -string dumpUnknown(ref OcdaDocument doc) { - auto out_ = appender!string; - void counts(string prefix, size_t[string] d) { - foreach (key; d.keys.dup.sort) { - out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; - out_ ~= d[key].to!string; out_ ~= "\n"; - } - } - counts("property", doc.unknown.property); - counts("block", doc.unknown.block); - counts("line", doc.unknown.line); - return out_.data; -} - -private string ints(int[] xs) { - if (xs.length == 0) { return ""; } - string[] parts; - foreach (x; xs) { parts ~= x.to!string; } - return parts.join(","); -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/d-read.org b/org/d-read.org deleted file mode 100644 index 9652e21..0000000 --- a/org/d-read.org +++ /dev/null @@ -1,1732 +0,0 @@ --*- mode: org -*- -#+TITLE: dlang load -#+DESCRIPTION: dlang lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* dlang :dlang: -** main - -#+HEADER: :tangle "../dlang/src/read/main.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -/++ - ocda-read - reads the object-centric document abstraction, in any of the - forms it is written in, and says what is in it. - - ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] - - One reader where there were two: sisudoc-ssp-reader read a .ssp and - sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is - built on the library, so every mode reads every form, and a mode given the - .ssp, the .ocda.db and the .sisupod of a document says the same thing of - each: it is told what the document is, not which file it came from. - - Two things are about the file rather than the document, and say so: - --files, what a .ocda.db carries besides the abstraction, and --validate, - which checks a .ssp as text. Everything else is the model's. - - The output modules are the .ssp reader's, moved onto the library's model. -+/ -module read.main; - -import std.algorithm : canFind, map; -import std.array : array, join; -import std.getopt; -import std.stdio : writeln, writefln, stderr; -import ocda.model; -import ocda.load; -import ocda.inventory; -import read.tree; -import read.html; -import read.llm_text; -import read.llm_jsonl; -import read.images; -import read.verify; -import read.report_unknown; -import read.validate; - -int main(string[] args) { - bool do_tree = false; - bool do_html = false; - bool do_llm_text = false; - bool do_llm_jsonl = false; - bool do_images = false; - bool do_verify = false; - bool do_report_unknown = false; - bool do_validate = false; - bool do_languages = false; - bool do_files = false; - bool any_version = false; - string image_dir = ""; - string output_file = ""; - string lang = ""; - - GetoptResult helpInfo; - try { - helpInfo = getopt(args, - "tree", "the headings, as a tree", &do_tree, - "outline", "the same as --tree", &do_tree, - "html", "the document as one html page", &do_html, - "llm-text", "plain text with heading breadcrumbs", &do_llm_text, - "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, - "images", "the image records: digest, size, pixels", &do_images, - "verify", "check the image records against --image-dir", &do_verify, - "image-dir", "the images to verify against", &image_dir, - "languages", "the languages the file holds", &do_languages, - "files", "what a .ocda.db carries besides the abstraction", &do_files, - "report-unknown", "what was read and not understood", &do_report_unknown, - "validate", "check a .ssp against its specification, as text", &do_validate, - "lang", "=<code> the language to read (default: the first)", &lang, - "output", "write the mode's output to a file", &output_file, - "any-version", "read a format major this reader does not know", &any_version, - ); - } catch (GetOptException e) { - stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); - return 2; - } - - if (helpInfo.helpWanted || args.length < 2) { - defaultGetoptPrinter( - "ocda-read - reads the document abstraction, in any of its forms\n\n" - ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n", - helpInfo.options, - ); - return 0; - } - - string input_file = args[1]; - if (lang.length == 0 && args.length > 2) { lang = args[2]; } - bool lang_named = (lang.length > 0); - - /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not - at all: there is no text form of a database to check, and a pod's - .ssp is checked by taking it out +/ - if (do_validate && sourceOf(input_file) != Source.ssp) { - stderr.writeln("--validate checks a .ssp as text, against the .ssp", - " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); - return 2; - } - - /+ ↓ which language. A .ocda.db and a pod may hold several; where none is - named, every form reads the first in the manifest's order and says - so, rather than the database taking one and the pod refusing +/ - OcdaLanguage[] held; - OcdaDocument doc; - try { - held = languages(input_file); - if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } - if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { - stderr.writeln("error: no language '", lang, "' in ", input_file, - "; it holds: ", held.map!(l => l.lang).array.join(" ")); - return 2; - } - doc = load(input_file, lang, any_version); - } catch (Exception e) { - stderr.writeln("Error reading ", input_file, ": ", e.msg); - return 2; - } - - /+ ↓ the version gate: the major part, and refuse what is not known +/ - if (doc.format_version.length == 0) { - stderr.writeln("warning: ", input_file, " declares no format version"); - } else if (doc.formatVersionMajor != ssp_format_major_read) { - if (any_version) { - stderr.writeln("warning: ", input_file, " is format v", doc.format_version, - ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); - } else { - stderr.writeln("error: ", input_file, " is format v", doc.format_version, - ", this reader knows v", ssp_format_major_read, ".x"); - stderr.writeln(" a major version differs: a property may have", - " changed meaning or gone away."); - stderr.writeln(" --any-version reads it regardless."); - return 3; - } - } - - writeln("Read: ", input_file); - if (doc.format_version.length > 0) { - writeln(" Format: v", doc.format_version); - } - if (auto _lang = "language" in doc.source) { - writeln(" Source: ", *_lang, - ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", - ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); - } - if (held.length > 1 && !lang_named) { - writeln(" (reading ", lang, "; --lang=<code> for another)"); - } - writeln(" Title: ", doc.meta("title.main")); - writeln(" Author: ", doc.meta("creator.author")); - writeln(" Sections: head=", doc.head.length, - " toc=", doc.toc.length, - " body=", doc.body.length, - " endnotes=", doc.endnotes.length, - " glossary=", doc.glossary.length, - " bibliography=", doc.bibliography.length, - " bookindex=", doc.bookindex.length, - " blurb=", doc.blurb.length, - " tail=", doc.tail.length); - auto image_records = imageRecordCount(doc); - if (image_records > 0) { - writeln(" Images: ", image_records, " record", - (image_records == 1) ? "" : "s"); - } - - if (do_languages) { printLanguages(held); } - if (do_files) { printFiles(input_file); } - if (do_tree) { outputTree(doc, output_file); } - if (do_html) { outputHtmlScroll(doc, output_file); } - if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } - if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } - if (do_images) { outputImages(doc, output_file); } - - int _verify_failures = 0; - if (do_verify) { - if (image_dir.length == 0) { - stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); - return 2; - } - _verify_failures = verifyImages(doc, image_dir, output_file); - } - - if (do_report_unknown) { reportUnknown(doc); } - - int _validation_findings = 0; - if (do_validate) { - auto _found = validateSspFile(input_file, ssp_format_major_read); - reportValidation(input_file, _found); - _validation_findings = cast(int) _found.length; - } - - if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images - && !do_verify && !do_report_unknown && !do_validate - && !do_languages && !do_files) { - writeln("\n(this is the summary: --help lists the modes, each adding its output)"); - } - - return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; -} - -/+ ↓ the languages, each with the markup it was made from and its .ssp's - digest: the same from every form, the database storing the digest and - the others taking it from the bytes +/ -void printLanguages(OcdaLanguage[] held) { - writeln(); - writeln(" lang source ssp digest"); - foreach (l; held) { - writefln(" %-7s %-31s %s", l.lang, l.source_filename, - (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); - } -} - -/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries - nothing, and a pod's files are its archive entries rather than anything - the abstraction records, so for those it says so +/ -void printFiles(string path) { - writeln(); - if (!isCarrier(path)) { - writeln(" carried files: only a .ocda.db carries files; this is ", - sourceName(sourceOf(path))); - return; - } - auto rows = carried(path); - if (rows.length == 0) { writeln(" carries no files"); return; } - writeln(" carried files:"); - writeln(" role stored n original in the file"); - foreach (r; rows) { - writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); - } -} -#+END_SRC - -** check -*** read validate =.ssp= - -#+HEADER: :tangle "../dlang/src/read/validate.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.validate; - -/+ ↓ check a .ssp against the format specification. - . - Written from specs/sisudoc-abstraction-format/ssp-format-specification.md - and from nothing else: the point of it is to be what the specification - says, executed, so that a file can be held against the document rather - than against an implementation. Each check names the section it comes - from, so a disagreement can be taken back to the text. -+/ - -import std.algorithm : canFind, startsWith; -import std.array : array, split; -import std.conv : to; -import std.file : readText; -import std.stdio : writeln, writefln; -import std.string : indexOf, strip; - -struct SspFinding { - size_t line; // 1-based - string section; // the specification section the rule is from - string detail; -} - -/+ ↓ specification section 6.3: the properties, in the order they are - emitted. A reader may rely on the order, so a validator can check it +/ -private immutable string[] property_order = [ - "identifier", "part", "section", "parent", "last_descendant", "ocn_range", - "children", - "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", - "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", - "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", - "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", - "table_header", "code_linenumbers", "stow_link", "segment", "anchor", - "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", - "segment_html_is", "segment_epub_is", "segment_lv4_is", - "heading_ancestors_text", "lev4_subtoc", "anchor_tag", -]; - -/+ ↓ section 6.1: the sixteen kinds +/ -private immutable string[] kinds = [ - "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", - "comment", "verse", "glossary", "quote", "poem", "block", "table", - "blurb", "group", -]; - -/+ ↓ section 4: the nine sections, in the order they appear +/ -private immutable string[] section_order = [ - "head", "toc", "body", "endnotes", "glossary", "bibliography", - "bookindex", "blurb", "tail", -]; - -/+ ↓ section 4: the four preamble blocks, in the order they appear +/ -private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; - -/+ ↓ section 6.1: the heading levels +/ -private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; - -private int propertyRank(string name) { - foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } - return -1; -} - -SspFinding[] validateSspFile(string filename, string major_known) { - SspFinding[] found; - auto lines = readText(filename).split("\n"); - - bool seen_banner = false; - bool seen_source_line = false; - string block = ""; // the block currently open, "" when none - int preamble_at = -1; // how far through the preamble order - int section_at = -1; // how far through the section order - bool in_object = false; - int prev_rank = -1; - size_t doc_has_keys = 0; - bool in_text = false; // text lines come after properties - - foreach (i, line; lines) { - size_t no = i + 1; - - /+ ↓ section 2: the version, on the first line +/ - if (no == 1) { - enum banner = "% SiSU Document Abstraction v"; - if (!line.startsWith(banner)) { - found ~= SspFinding(no, "2", "first line is not the version banner"); - } else { - seen_banner = true; - string v = line[banner.length .. $].strip; - auto dot = v.indexOf('.'); - string major = (dot > 0) ? v[0 .. dot] : v; - if (major != major_known) { - found ~= SspFinding(no, "2.1", - "format major v" ~ major ~ ", this validator knows v" ~ major_known); - } - } - continue; - } - if (no == 2) { - if (!line.startsWith("% Source: ")) { - found ~= SspFinding(no, "4", "second line is not \"% Source: \""); - } else { seen_source_line = true; } - continue; - } - - if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } - - /+ ↓ section 3: a comment is the file header only +/ - if (line[0] == '%') { - found ~= SspFinding(no, "3", "a comment outside the two header lines"); - continue; - } - - /+ ↓ block open +/ - if (line[0] == '@') { - auto brace = line.indexOf(" {"); - if (brace < 1 || line[$ - 1] != '{') { - found ~= SspFinding(no, "3", "block open is not \"@name {\""); - continue; - } - string name = line[1 .. brace]; - if (preamble_order.canFind(name)) { - int at = 0; - foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } - if (at <= preamble_at) { - found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); - } - preamble_at = at; - if (section_at >= 0) { - found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); - } - } else if (section_order.canFind(name)) { - int at = 0; - foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } - if (at <= section_at) { - found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); - } - section_at = at; - } else { - found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); - } - block = name; - doc_has_keys = 0; - continue; - } - - if (line[0] == '}') { - if (line.length != 1) { - found ~= SspFinding(no, "3", "block close carries more than \"}\""); - } - /+ ↓ section 5.4: ten keys, always +/ - if (block == "doc_has" && doc_has_keys != 10) { - found ~= SspFinding(no, "5.4", - "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); - } - block = ""; - in_object = false; - prev_rank = -1; - continue; - } - - /+ ↓ section 5: inside a preamble block, " key: value" +/ - if (preamble_order.canFind(block)) { - if (!line.startsWith(" ") || line.indexOf(": ") < 3) { - found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); - } else if (block == "doc_has") { - doc_has_keys++; - } - continue; - } - - /+ ↓ section 6.1: the object declaration line +/ - if (line[0] == '[') { - auto close = line.indexOf("] "); - if (close < 2) { - found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); - continue; - } - string ocn = line[1 .. close]; - foreach (c; ocn) { - if (c < '0' || c > '9') { - found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); - break; - } - } - auto rest = line[close + 2 .. $].split(" "); - if (rest.length == 0 || !kinds.canFind(rest[0])) { - found ~= SspFinding(no, "6.1", - "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); - } else if (rest[0] == "heading") { - if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { - found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); - } else if (!levels.canFind(rest[1][1 .. $])) { - found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); - } - } - in_object = true; - in_text = false; - prev_rank = -1; - continue; - } - - /+ ↓ section 6.3: properties, known and in order +/ - if (line[0] == '.') { - if (!in_object) { - found ~= SspFinding(no, "6", "a property outside an object record"); - continue; - } - if (in_text) { - found ~= SspFinding(no, "6", "a property after the object text began"); - } - auto colon = line.indexOf(": "); - if (colon < 2) { - found ~= SspFinding(no, "3", "property is not \".name: value\""); - continue; - } - string name = line[1 .. colon]; - int rank = propertyRank(name); - if (rank < 0) { - found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); - } else { - /+ ↓ the four repeatable properties may recur at the same rank +/ - if (rank < prev_rank) { - found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); - } - prev_rank = rank; - } - continue; - } - - /+ ↓ section 6.2: text lines +/ - if (line[0] == '|') { - if (!in_object) { - found ~= SspFinding(no, "6.2", "a text line outside an object record"); - } - if (line.length < 2 || line[1] != ' ') { - found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); - } - in_text = true; - continue; - } - - found ~= SspFinding(no, "3", "line of no shape the specification defines"); - } - - if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } - if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } - return found; -} - -void reportValidation(string filename, SspFinding[] found) { - if (found.length == 0) { - writeln("\nvalid: ", filename, - " conforms to the .ssp specification as this validator reads it"); - return; - } - writefln("\n%d finding%s against the specification:", - found.length, (found.length == 1) ? "" : "s"); - foreach (f; found) { - if (f.line > 0) { - writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); - } else { - writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); - } - } -} -#+END_SRC - -*** read verify images - -#+HEADER: :tangle "../dlang/src/read/verify.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.verify; - -import std.algorithm : sort; -import std.array : array, join; -import std.conv : to; -import std.digest : toHexString; -import std.digest.sha : SHA256, digest; -import std.file : dirEntries, exists, isDir, SpanMode, getSize; -import std.path : baseName, buildPath, extension; -import std.stdio : writeln, File; -import std.string : toLower; -import ocda.model; -import read.images : gatherImageRecords; - -/+ ↓ verification: recompute what the .image records claim, against the - images as they stand in a directory (a pod's media/image). - - ssp-reader --verify --image-dir=<pod>/media/image doc.ssp - - Reports one line per distinct file: - - OK won_benkler_2_1.png - MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... - ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) - PRESENT ffa.png (recorded missing, but the file is there) - EXTRA unused.png (in directory, no record in this .ssp) - - Returns the number of failures. EXTRA is reported but is not a failure: - a pod carries the images of every language version of a document, while - one .ssp covers one language. -+/ -int verifyImages(OcdaDocument doc, string image_dir, string output_file) { - string[] lines; - int failures = 0; - int ok = 0; - int mismatch = 0; - int absent = 0; - int extra = 0; - - if (!(image_dir.exists) || !(image_dir.isDir)) { - writeln("ERROR: image directory not found: ", image_dir); - return 1; - } - auto gathered = gatherImageRecords(doc); - bool[string] recorded; - - foreach (e; gathered.entries) { - recorded[e.name] = true; - string pth = buildPath(image_dir, e.name); - if (!(pth.exists)) { - if (e.missing) { - /+ recorded missing, and missing here too: consistent +/ - ok += 1; - lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; - } else { - absent += 1; - failures += 1; - lines ~= "ABSENT " ~ e.name - ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; - } - continue; - } - if (e.missing) { - mismatch += 1; - failures += 1; - lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; - continue; - } - auto file_ = readFileStat(pth); - string[] differs; - if (file_.sha256 != e.sha256) { - differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; - } - if (file_.bytes != e.bytes) { - differs ~= "bytes recorded " ~ e.bytes.to!string - ~ " file " ~ file_.bytes.to!string; - } - /+ px is written only where the dimensions could be read, and read here - only for the formats below, so compare only when both are known +/ - if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 - && (file_.width != e.width || file_.height != e.height)) { - differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string - ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; - } - if (differs.length > 0) { - mismatch += 1; - failures += 1; - lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); - } else { - ok += 1; - lines ~= "OK " ~ e.name; - } - } - - /+ ↓ files in the directory that no record names +/ - string[] in_dir; - foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { - if (!(entry.isFile)) continue; - string nm = entry.name.baseName; - switch (nm.extension.toLower) { - case ".png": case ".jpg": case ".jpeg": case ".gif": - in_dir ~= nm; - break; - default: break; - } - } - in_dir.sort; - foreach (nm; in_dir) { - if (nm !in recorded) { - extra += 1; - lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; - } - } - - string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" - ~ ((gathered.entries.length == 1) ? "" : "s") - ~ " in " ~ gathered.records.to!string ~ " record" - ~ ((gathered.records == 1) ? "" : "s") - ~ ", against " ~ image_dir; - lines = [head, ""] ~ lines; - lines ~= ""; - lines ~= "verify: " ~ ok.to!string ~ " ok, " - ~ mismatch.to!string ~ " mismatch, " - ~ absent.to!string ~ " absent, " - ~ extra.to!string ~ " extra" - ~ ((failures == 0) ? " PASS" : " FAIL"); - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } - return failures; -} - -private struct FileStat_ { - string sha256; - ulong bytes; - int width; - int height; -} - -private FileStat_ readFileStat(string pth) { - FileStat_ ret; - auto f = File(pth, "rb"); - ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); - ret.sha256 = sha.toHexString.to!string; - ret.bytes = pth.getSize; - auto dim = readImageDimensions(pth); - ret.width = dim[0]; - ret.height = dim[1]; - return ret; -} - -/+ ↓ pixel dimensions from the file header, for the formats spine's markup - admits (png, gif, jpg). Header read only, no decoding, and no dependency - beyond phobos. Returns [0, 0] where the header cannot be read -+/ -private int[2] readImageDimensions(string pth) { - int[2] ret = [0, 0]; - ubyte[] head; - try { - auto f = File(pth, "rb"); - head = new ubyte[](64 * 1024); - head = f.rawRead(head); - } catch (Exception ex) { - return ret; - } - if (head.length >= 24 - && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { - /+ png: IHDR width & height, big endian, at 16 and 20 +/ - ret[0] = beInt(head[16 .. 20]); - ret[1] = beInt(head[20 .. 24]); - return ret; - } - if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { - /+ gif: logical screen width & height, little endian, at 6 and 8 +/ - ret[0] = head[6] | (head[7] << 8); - ret[1] = head[8] | (head[9] << 8); - return ret; - } - if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { - /+ jpeg: walk the segments to a start of frame, height then width, - big endian, at offsets 5 and 7 of the segment - +/ - size_t i = 2; - while (i + 9 < head.length) { - if (head[i] != 0xff) { i += 1; continue; } - ubyte marker = head[i + 1]; - if (marker == 0xd8 || marker == 0x01 - || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } - if (marker == 0xff) { i += 1; continue; } - size_t seg_len = (head[i + 2] << 8) | head[i + 3]; - bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) - || (marker >= 0xc5 && marker <= 0xc7) - || (marker >= 0xc9 && marker <= 0xcb) - || (marker >= 0xcd && marker <= 0xcf)); - if (is_sof) { - ret[1] = (head[i + 5] << 8) | head[i + 6]; - ret[0] = (head[i + 7] << 8) | head[i + 8]; - return ret; - } - if (seg_len < 2) break; - i += 2 + seg_len; - } - } - return ret; -} - -private int beInt(const(ubyte)[] b) { - return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); -} -#+END_SRC - -*** read report unknown - -#+HEADER: :tangle "../dlang/src/read/report_unknown.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.report_unknown; - -import std.algorithm : sort; -import std.array : array; -import std.stdio : writeln, writefln; -import ocda.model; - -/+ ↓ say what was read and not understood. - This is what makes the reader usable as a check on the format - specification: a reader that ignores what it does not know is - conforming, but one that cannot say what it ignored cannot tell you - whether the specification it was written from was complete. Silence - and understanding look the same from outside. -+/ -void reportUnknown(ref OcdaDocument doc) { - if (!doc.unknown.any) { - writeln("\nread and understood: every property, block and line was known"); - return; - } - writeln("\nread and NOT understood:"); - if (doc.unknown.property.length > 0) { - writeln(" properties ignored (", doc.unknown.property.length, " distinct):"); - foreach (name; doc.unknown.property.keys.sort) { - writefln(" .%-26s %8d", name, doc.unknown.property[name]); - } - } - if (doc.unknown.block.length > 0) { - writeln(" blocks ignored (", doc.unknown.block.length, " distinct):"); - foreach (name; doc.unknown.block.keys.sort) { - writefln(" @%-26s %8d", name, doc.unknown.block[name]); - } - } - if (doc.unknown.line.length > 0) { - writeln(" lines of no known shape, by first character:"); - foreach (c; doc.unknown.line.keys.sort) { - writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]); - } - } - writeln(" (ignoring these is conforming within a major version;", - " the count is so that it is not silent)"); -} -#+END_SRC - -** misc -*** read tree - -#+HEADER: :tangle "../dlang/src/read/tree.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.tree; - -import std.conv : to; -import std.stdio : writeln, writef, writefln, File; -import std.array : replicate; -import ocda.model; - -void outputTree(OcdaDocument doc, string output_file) { - string[] lines; - - lines ~= "Document: " ~ doc.meta("title.main"); - if (doc.meta("title.subtitle").length > 0) - lines ~= " " ~ doc.meta("title.subtitle"); - lines ~= "Author: " ~ doc.meta("creator.author"); - lines ~= ""; - - /+ ↓ build OCN -> object lookup for body headings +/ - OcdaObject[int] headings; - foreach (obj; doc.body) { - if (obj.is_a == "heading" && obj.ocn > 0) { - headings[obj.ocn] = obj; - } - } - - /+ ↓ find root heading (from head section, or first body heading) +/ - if (doc.head.length > 0 && doc.head[0].is_a == "heading") { - auto root = doc.head[0]; - lines ~= formatHeading(root, 0); - walkChildren(root, headings, lines, 1); - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { - foreach (child_ocn; parent.children) { - if (auto p = child_ocn in headings) { - lines ~= formatHeading(*p, depth); - walkChildren(*p, headings, lines, depth + 1); - } - } -} - -private string formatHeading(OcdaObject obj, int depth) { - string indent = replicate(" ", depth); - string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; - string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; - /+ ↓ strip internal markup from heading text for clean display +/ - string text = stripInternalMarkup(obj.text); - return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; -} - -private string stripInternalMarkup(string text) { - /+ ↓ simple removal of common internal markers for display +/ - import std.regex : replaceAll, regex; - import std.array : replace; - string result = text; - result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") - .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") - .replace("┨", "").replace("┣", "") - .replace("┙", " ").replace("┘", "\n"); - return result; -} -#+END_SRC - -*** read html - -#+HEADER: :tangle "../dlang/src/read/html.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.html; - -import std.array : replace, split, join, appender; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, startsWith; -import ocda.model; - -void outputHtmlScroll(OcdaDocument doc, string output_file) { - auto buf = appender!(string[]); - - buf ~= htmlHead(doc); - - /+ ↓ title +/ - if (doc.head.length > 0) { - auto title_obj = doc.head[0]; - buf ~= `<div class="title">`; - buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`; - if (doc.meta("title.subtitle").length > 0) - buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`; - buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`; - buf ~= `</div>`; - } - - /+ ↓ table of contents +/ - if (doc.toc.length > 0) { - buf ~= `<div class="toc">`; - buf ~= `<h2>Table of Contents</h2>`; - foreach (obj; doc.toc) { - if (obj.is_a == "toc") { - string text = convertInlineMarkup(obj.text); - int indent = obj.indent_base; - string cls = (indent > 4) ? "toc-sub" : "toc-entry"; - buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`; - } - } - buf ~= `</div>`; - buf ~= `<hr/>`; - } - - /+ ↓ body +/ - buf ~= `<div class="body">`; - foreach (obj; doc.body) { - buf ~= renderObject(obj); - } - buf ~= `</div>`; - - /+ ↓ endnotes +/ - bool has_endnotes = false; - foreach (obj; doc.endnotes) { - if (obj.is_a == "endnote") { has_endnotes = true; break; } - } - if (has_endnotes) { - buf ~= `<div class="endnotes">`; - buf ~= `<hr/>`; - buf ~= `<h2>Endnotes</h2>`; - foreach (obj; doc.endnotes) { - if (obj.is_a == "endnote") { - string text = convertInlineMarkup(obj.text); - string parent_ref = (obj.parent_ocn > 0) - ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>` - : ""; - buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`; - } - } - buf ~= `</div>`; - } - - /+ ↓ glossary +/ - if (doc.glossary.length > 0) { - buf ~= `<div class="glossary">`; - buf ~= `<hr/>`; - buf ~= `<h2>Glossary</h2>`; - foreach (obj; doc.glossary) { - string text = convertInlineMarkup(obj.text); - buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`; - } - buf ~= `</div>`; - } - - /+ ↓ bookindex +/ - bool has_bookindex = false; - foreach (obj; doc.bookindex) { - if (obj.is_a == "bookindex") { has_bookindex = true; break; } - } - if (has_bookindex) { - buf ~= `<div class="bookindex">`; - buf ~= `<hr/>`; - buf ~= `<h2>Book Index</h2>`; - foreach (obj; doc.bookindex) { - if (obj.is_a == "bookindex") { - string text = convertInlineMarkup(obj.text); - buf ~= `<p class="index-entry">` ~ text ~ `</p>`; - } - } - buf ~= `</div>`; - } - - buf ~= htmlTail(); - - /+ ↓ write output +/ - string[] output = buf.data; - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; output) f.writeln(line); - } else { - foreach (line; output) writeln(line); - } -} - -private string[] renderObject(OcdaObject obj) { - /+ ↓ markup commentary is carried by the abstraction and is not output. - Rendering it would put the author notes into the document, which - spine itself does not do - +/ - if (!obj.renders) { return []; } - string[] lines; - string text = convertInlineMarkup(obj.text); - string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) - ? ` id="o` ~ obj.ocn.to!string ~ `"` - : ""; - string ocn_display = (obj.ocn > 0 && !obj.ocn_off) - ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>` - : ""; - - switch (obj.is_a) { - case "heading": - string tag = headingTag(obj.heading_level); - string seg_anchor = (obj.segment.length > 0) - ? ` id="` ~ htmlEscape(obj.segment) ~ `"` - : ocn_anchor; - lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`; - break; - case "para": - string indent_cls = ""; - if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`; - else if (obj.bullet) indent_cls = ` class="bullet"`; - lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`; - break; - case "code": - lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`; - break; - case "table": - lines ~= renderTable(obj, ocn_anchor); - break; - case "group": - lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`; - break; - case "block": - lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`; - break; - case "verse": - lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`; - break; - case "quote": - lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`; - break; - default: - if (text.length > 0) - lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`; - break; - } - return lines; -} - -private string headingTag(string level) { - switch (level) { - case "A": return "h1"; - case "B": return "h2"; - case "C": return "h3"; - case "D": return "h4"; - case "1": return "h3"; - case "2": return "h4"; - case "3": return "h5"; - case "4": return "h6"; - default: return "h4"; - } -} - -private string renderTable(OcdaObject obj, string ocn_anchor) { - string[] lines; - lines ~= `<table` ~ ocn_anchor ~ `>`; - /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/ - auto rows = obj.text.split("\n"); - bool first_row = obj.table_header; - foreach (row; rows) { - if (row.strip.length == 0) continue; - string tag = first_row ? "th" : "td"; - lines ~= `<tr>`; - foreach (cell; row.split("┊")) { - lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`; - } - lines ~= `</tr>`; - first_row = false; - } - lines ~= `</table>`; - return lines.join("\n"); -} - -/+ ↓ convert spine internal Unicode markup to HTML +/ -string convertInlineMarkup(string text) { - if (text.length == 0) return ""; - string result = htmlEscape(text); - - /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc. - Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X - The ┨┣ form is used within heading content +/ - - /+ ↓ bold +/ - result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>"); - result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>"); - - /+ ↓ emphasis +/ - result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>"); - result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>"); - - /+ ↓ italic +/ - result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>"); - result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>"); - - /+ ↓ underscore +/ - result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>"); - result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>"); - - /+ ↓ superscript +/ - result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>"); - result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>"); - - /+ ↓ subscript +/ - result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>"); - result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>"); - - /+ ↓ monospace +/ - result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>"); - - /+ ↓ links: ┥link text┝┤url├ +/ - result = replaceLinks(result); - - /+ ↓ footnote markers: 【N text】 +/ - result = replaceFootnotes(result); - - /+ ↓ line breaks +/ - result = result.replace("┘", "<br/>"); - result = result.replace("┙", "<br/>"); - - /+ ↓ non-breaking spaces +/ - result = result.replace("░", " "); - - /+ ↓ bullet markers +/ - result = result.replace("● ", "• "); - - /+ ↓ page break markers +/ - result = result.replace("┼", "").replace("┿", "").replace("╂", ""); - - return result; -} - -private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) { - /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/ - string result = text; - while (true) { - auto open_pos = result.indexOf(open_mark); - if (open_pos < 0) break; - auto after_open = open_pos + open_mark.length; - /+ ↓ look for closing ⑆X +/ - string close_mark = "⑆" ~ close_suffix; - auto close_pos = indexOf(result[after_open .. $], close_mark); - if (close_pos < 0) break; - close_pos += after_open; - string content = result[after_open .. close_pos]; - result = result[0 .. open_pos] ~ html_open ~ content ~ html_close - ~ result[close_pos + close_mark.length .. $]; - } - return result; -} - -private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) { - /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/ - string result = text; - string full_open = open_mark ~ bracket_open; - while (true) { - auto open_pos = result.indexOf(full_open); - if (open_pos < 0) break; - auto after_open = open_pos + full_open.length; - auto close_pos = indexOf(result[after_open .. $], bracket_close); - if (close_pos < 0) break; - close_pos += after_open; - string content = result[after_open .. close_pos]; - result = result[0 .. open_pos] ~ html_open ~ content ~ html_close - ~ result[close_pos + bracket_close.length .. $]; - } - return result; -} - -private string replaceLinks(string text) { - /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/ - string result = text; - while (true) { - auto link_open = result.indexOf("┥"); - if (link_open < 0) break; - auto link_mid = indexOf(result[link_open .. $], "┝┤"); - if (link_mid < 0) break; - link_mid += link_open; - auto link_close = indexOf(result[link_mid .. $], "├"); - if (link_close < 0) break; - link_close += link_mid; - string link_text = result[link_open + "┥".length .. link_mid]; - string url = result[link_mid + "┝┤".length .. link_close]; - /+ ↓ handle internal links (¤ prefix) +/ - if (url.length > 0 && url[0] == '¤') { - url = url[("¤").length .. $]; - } - /+ ↓ handle OCN-only links (#N) +/ - if (url.length > 0 && url[0] == '#') { - url = "#o" ~ url[1 .. $]; - } - result = result[0 .. link_open] - ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>` - ~ result[link_close + "├".length .. $]; - } - return result; -} - -private string replaceFootnotes(string text) { - /+ ↓ replace 【N content】 with superscript note number +/ - string result = text; - while (true) { - auto fn_open = result.indexOf("【"); - if (fn_open < 0) break; - auto fn_close = indexOf(result[fn_open .. $], "】"); - if (fn_close < 0) break; - fn_close += fn_open; - string fn_content = result[fn_open + "【".length .. fn_close]; - /+ ↓ extract note number (first token before space) +/ - auto space_pos = fn_content.indexOf(' '); - string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content; - result = result[0 .. fn_open] - ~ `<sup class="fn">` ~ note_num ~ `</sup>` - ~ result[fn_close + "】".length .. $]; - } - return result; -} - -string htmlEscape(string text) { - return text.replace("&", "&").replace("<", "<").replace(">", ">") - .replace("\"", """); -} - -private string[] htmlHead(OcdaDocument doc) { - string[] lines; - lines ~= `<!DOCTYPE html>`; - lines ~= `<html>`; - lines ~= `<head>`; - lines ~= `<meta charset="utf-8"/>`; - lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`; - lines ~= `<style>`; - lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`; - lines ~= ` .title { text-align: center; margin-bottom: 2em; }`; - lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`; - lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`; - lines ~= ` .toc { margin-bottom: 2em; }`; - lines ~= ` .toc-entry { margin: 0.2em 0; }`; - lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`; - lines ~= ` .indent-1 { margin-left: 2em; }`; - lines ~= ` .indent-2 { margin-left: 4em; }`; - lines ~= ` .indent-3 { margin-left: 6em; }`; - lines ~= ` .bullet { margin-left: 2em; }`; - lines ~= ` .bullet:before { content: "\2022 "; }`; - lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`; - lines ~= ` .group { margin: 1em 2em; }`; - lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`; - lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`; - lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`; - lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`; - lines ~= ` th { background: #f0f0f0; }`; - lines ~= ` .endnote { font-size: 0.9em; }`; - lines ~= ` .fn { color: #666; }`; - lines ~= ` .index-entry { font-size: 0.9em; }`; - lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`; - lines ~= `</style>`; - lines ~= `</head>`; - lines ~= `<body>`; - return lines; -} - -private string[] htmlTail() { - return [`</body>`, `</html>`]; -} -#+END_SRC - -*** read images - -#+HEADER: :tangle "../dlang/src/read/images.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.images; - -import std.array : join; -import std.conv : to; -import std.stdio : writeln, File; -import ocda.model; - -/+ ↓ the .image records of a document, gathered per distinct file, with the - objects referencing each file, in order of first appearance +/ -struct ImageEntry { - string name; - string sha256; - ulong bytes; - int width; - int height; - bool missing; - string[] refs; - string[] other_sha256; // same filename, differing digest -} - -struct ImageRecords { - int records; // .image lines, an image referenced twice counts twice - ImageEntry[] entries; // distinct files -} - -ImageRecords gatherImageRecords(OcdaDocument doc) { - ImageRecords ret; - size_t[string] seen; - foreach (section; sectionNames) { - auto objs = doc.sectionByName(section); - if (objs is null) continue; - foreach (obj; *objs) { - foreach (img; obj.images) { - ret.records += 1; - string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]"; - if (auto at = img.name in seen) { - if (ret.entries[*at].sha256 != img.sha256 - && img.sha256.length > 0 - && ret.entries[*at].sha256.length > 0) { - ret.entries[*at].other_sha256 ~= img.sha256; - } - ret.entries[*at].refs ~= _ref; - } else { - ImageEntry e_; - e_.name = img.name; - e_.sha256 = img.sha256; - e_.bytes = img.bytes; - e_.width = img.width; - e_.height = img.height; - e_.missing = img.missing; - e_.refs = [_ref]; - seen[img.name] = ret.entries.length; - ret.entries ~= e_; - } - } - } - } - return ret; -} - -/+ ↓ image manifest, line oriented and greppable: - - <sha256> <bytes> <px> <filename> refs: body[97] body[412] - MISSING <filename> refs: body[840] - - The point of the record in the .ssp is verification (is the image - shipped in this pod the image the abstraction was built from), so this - output is the shape a verifier wants: one line per distinct file. -+/ -void outputImages(OcdaDocument doc, string output_file) { - auto gathered = gatherImageRecords(doc); - string[] lines; - int missing = 0; - foreach (e; gathered.entries) { - if (e.missing) missing += 1; - } - lines ~= "Images: " ~ gathered.records.to!string ~ " record" - ~ ((gathered.records == 1) ? "" : "s") - ~ ", " ~ gathered.entries.length.to!string ~ " distinct file" - ~ ((gathered.entries.length == 1) ? "" : "s") - ~ ", " ~ missing.to!string ~ " missing"; - lines ~= ""; - foreach (e; gathered.entries) { - string px = (e.width > 0 && e.height > 0) - ? e.width.to!string ~ "x" ~ e.height.to!string : "-"; - lines ~= (e.missing) - ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ") - : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px - ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" "); - /+ ↓ one filename, more than one digest: the .ssp is inconsistent, - or two distinct files share a name +/ - foreach (other; e.other_sha256) { - lines ~= " WARNING differing digest for same filename: " ~ other; - } - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -/+ ↓ count of .image records in a document, for the parse summary +/ -int imageRecordCount(OcdaDocument doc) { - return gatherImageRecords(doc).records; -} - -package immutable string[] sectionNames = [ - "head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", -]; -#+END_SRC - -** llm -*** read llm text - -#+HEADER: :tangle "../dlang/src/read/llm_text.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.llm_text; - -import std.array : replace, split, join; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, stripRight; -import ocda.model; - -/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption - format: - ## Document Title > Chapter > Section > [OCN] - - Clean paragraph text with inline markup stripped. -+/ -void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { - string[] lines; - string doc_title = doc.meta("title.main"); - - /+ ↓ document header +/ - lines ~= "# " ~ doc_title; - if (doc.meta("title.subtitle").length > 0) - lines ~= "# " ~ doc.meta("title.subtitle"); - lines ~= "# " ~ doc.meta("creator.author"); - lines ~= ""; - - /+ ↓ build heading text lookup from ancestors +/ - string[int] heading_text; - foreach (obj; doc.head) { - if (obj.is_a == "heading" && obj.ocn > 0) - heading_text[obj.ocn] = stripMarkup(obj.text); - } - foreach (obj; doc.body) { - if (obj.is_a == "heading") - heading_text[obj.ocn] = stripMarkup(obj.text); - } - - /+ ↓ body objects +/ - string current_breadcrumb = ""; - foreach (obj; doc.body) { - if (obj.is_a == "heading") { - current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); - lines ~= ""; - lines ~= "## " ~ current_breadcrumb; - lines ~= ""; - continue; - } - - /+ ↓ markup commentary is not content +/ - if (!obj.renders) { continue; } - - string text = stripMarkup(obj.text); - if (text.strip.length == 0) continue; - - string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) - ? " [" ~ obj.ocn.to!string ~ "]" - : ""; - - switch (obj.is_a) { - case "para": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "code": - lines ~= "```"; - lines ~= obj.text; /+ preserve raw text for code +/ - lines ~= "```"; - lines ~= ""; - break; - case "table": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "verse": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "quote": - lines ~= "> " ~ text ~ ocn_ref; - lines ~= ""; - break; - case "group": - case "block": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - default: - if (text.length > 0) { - lines ~= text ~ ocn_ref; - lines ~= ""; - } - break; - } - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { - /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ - string[] parts; - if (ancestors_text.length > 0) { - foreach (part; ancestors_text.split("|")) { - string clean = stripMarkup(part).strip; - if (clean.length > 0 && clean != doc_title) { - /+ ↓ truncate very long heading text for breadcrumb +/ - if (clean.length > 60) - clean = clean[0 .. 57] ~ "..."; - parts ~= clean; - } - } - } - /+ ↓ add the current heading itself +/ - string heading_text = stripMarkup(obj.text).strip; - if (heading_text.length > 60) - heading_text = heading_text[0 .. 57] ~ "..."; - - string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; - - if (parts.length > 0) - return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; - else - return doc_title ~ " > " ~ heading_text ~ ocn_ref; -} - -/+ ↓ strip all internal Unicode markup to produce clean plain text +/ -string stripMarkup(string text) { - if (text.length == 0) return ""; - string r = text; - - /+ ↓ links: ┥link text┝┤url├ -> link text +/ - while (true) { - auto lo = r.indexOf("┥"); - if (lo < 0) break; - auto lm = indexOf(r[lo .. $], "┝┤"); - if (lm < 0) break; - lm += lo; - auto lc = indexOf(r[lm .. $], "├"); - if (lc < 0) break; - lc += lm; - string link_text = r[lo + "┥".length .. lm]; - r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; - } - - /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ - while (true) { - auto fo = r.indexOf("【"); - if (fo < 0) break; - auto fc = indexOf(r[fo .. $], "】"); - if (fc < 0) break; - fc += fo; - r = r[0 .. fo] ~ r[fc + "】".length .. $]; - } - - /+ ↓ font face markers +/ - r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") - .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") - .replace("⑆■", "") - .replace("┨", "").replace("┣*", "").replace("┣!", "") - .replace("┣/", "").replace("┣_", "").replace("┣^", "") - .replace("┣,", "").replace("┣", ""); - - /+ ↓ line breaks and special chars +/ - r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); - r = r.replace("░", " "); - r = r.replace("● ", "- "); - r = r.replace("┼", "").replace("┿", "").replace("╂", ""); - - /+ ↓ table cell separators +/ - r = r.replace("┊", " | "); - - /+ ↓ clean up multiple spaces +/ - while (r.indexOf(" ") >= 0) - r = r.replace(" ", " "); - - return r.strip; -} -#+END_SRC - -*** read llm jsonl - -#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.llm_jsonl; - -import std.array : split, join, replace; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, stripRight; -import ocda.model; -import read.llm_text : stripMarkup; - -/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines - one JSON object per line, each containing: - doc, ocn, path (breadcrumb), type, text (clean), - images (only where the object embeds one, so that a chunk whose - text reads around a figure carries the figure's identity), - index_terms (from bookindex entries associated with this object) -+/ -void outputLlmJsonl(OcdaDocument doc, string output_file) { - string[] lines; - string doc_id = doc.meta("title.main"); - string doc_author = doc.meta("creator.author"); - - /+ ↓ build bookindex -> OCN mapping for index_terms - bookindex objects in the .ssp have parent_ocn linking to body objects - but the index terms are in the text of bookindex objects, not - directly attached to body objects. We use a simpler approach: - headings carry heading_ancestors_text which gives context. +/ - - /+ ↓ track current heading path +/ - string current_path = ""; - - foreach (obj; doc.body) { - if (obj.is_a == "heading") { - current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); - /+ ↓ emit heading as a chunk too +/ - string heading_text = stripMarkup(obj.text); - if (heading_text.strip.length > 0) { - lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); - } - continue; - } - - /+ ↓ markup commentary is not content, and least of all is it a chunk - to embed: it would be indexed as though the author had written it - into the document - +/ - if (!obj.renders) { continue; } - - string text = stripMarkup(obj.text); - if (text.strip.length == 0) continue; - - lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { - return `{"doc":` ~ jsonStr(doc_id) - ~ `,"author":` ~ jsonStr(doc_author) - ~ `,"ocn":` ~ ocn.to!string - ~ `,"path":` ~ jsonStr(path) - ~ `,"type":` ~ jsonStr(type) - ~ `,"text":` ~ jsonStr(text) - ~ jsonImages(images) - ~ `}`; -} - -private string jsonImages(OcdaImage[] images) { - /+ ↓ omitted entirely where the object embeds no image +/ - if (images.length == 0) return ""; - string[] items; - foreach (img; images) { - string item = `{"name":` ~ jsonStr(img.name); - if (img.missing) { - item ~= `,"missing":true`; - } else { - item ~= `,"sha256":` ~ jsonStr(img.sha256) - ~ `,"bytes":` ~ img.bytes.to!string; - if (img.width > 0 && img.height > 0) { - item ~= `,"width":` ~ img.width.to!string - ~ `,"height":` ~ img.height.to!string; - } - } - items ~= item ~ `}`; - } - return `,"images":[` ~ items.join(",") ~ `]`; -} - -private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { - string[] parts; - if (ancestors_text.length > 0) { - foreach (part; ancestors_text.split("|")) { - string clean = stripMarkup(part).strip; - if (clean.length > 0 && clean != doc_title) { - parts ~= clean; - } - } - } - string heading_text = stripMarkup(obj.text).strip; - if (parts.length > 0) - return parts.join(" > ") ~ " > " ~ heading_text; - else - return heading_text; -} - -private string jsonStr(string s) { - /+ ↓ JSON string escaping +/ - string r = s; - r = r.replace(`\`, `\\`); - r = r.replace(`"`, `\"`); - r = r.replace("\n", `\n`); - r = r.replace("\r", `\r`); - r = r.replace("\t", `\t`); - return `"` ~ r ~ `"`; -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/d-test.org b/org/d-test.org deleted file mode 100644 index d159819..0000000 --- a/org/d-test.org +++ /dev/null @@ -1,178 +0,0 @@ --*- mode: org -*- -#+TITLE: dlang load -#+DESCRIPTION: dlang lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* dlang :dlang: -** tests -*** test agree - -#+HEADER: :tangle "../dlang/test/agree.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -/++ - Do the two loaders agree? - - The library's whole purpose is that a writer cannot tell which - serialisation it was handed. That is only true if loading a document's - .ssp and loading its .ocda.db give the same model, so this asks. - - Usage: agree <doc>.ssp <doc>.ocda.db [lang] -+/ -import std.conv : to; -import std.stdio; -import ocda.model; -import ocda.load; - -int main(string[] args) { - if (args.length < 3) { - stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]"); - return 2; - } - string lang = (args.length > 3) ? args[3] : ""; - OcdaDocument a, b; - try { - a = load(args[1]); - b = load(args[2], lang); - } catch (Exception e) { - stderr.writeln("error: ", e.msg); - return 2; - } - - int differences = 0; - void differ(string what, string from_ssp, string from_db) { - writeln(" differ: ", what); - writeln(" .ssp: ", from_ssp); - writeln(" .ocda.db: ", from_db); - differences++; - } - - if (a.format_version != b.format_version) { - differ("format_version", a.format_version, b.format_version); - } - if (a.source_filename != b.source_filename) { - differ("source_filename", a.source_filename, b.source_filename); - } - - /+ ↓ the sections, object by object. Position within a section is the - identity to compare on: an ocn is not unique and 0 is shared +/ - static immutable sections = ["head", "toc", "body", "endnotes", "glossary", - "bibliography", "bookindex", "blurb", "tail"]; - foreach (name; sections) { - auto sa = a.sectionByName(name); - auto sb = b.sectionByName(name); - if (sa is null || sb is null) { continue; } - if ((*sa).length != (*sb).length) { - differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string); - continue; - } - foreach (i, oa; *sa) { - auto ob = (*sb)[i]; - string at = name ~ "[" ~ i.to!string ~ "]"; - if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); } - if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); } - if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); } - if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); } - if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); } - if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); } - if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); } - if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); } - if (oa.ancestors != ob.ancestors) { - differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string); - } - if (oa.images.length != ob.images.length) { - differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string); - } - if (differences > 12) { writeln(" ... stopping after 12"); return 1; } - } - } - - if (differences == 0) { - writeln("agree: the two loaders give the same model"); - return 0; - } - writeln(differences, " difference(s)"); - return 1; -} -#+END_SRC - -*** test dump - -#+HEADER: :tangle "../dlang/test/dump.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -/++ - Write a loaded abstraction out canonically. - - The oracle for everything that claims to implement this model: load a - .ssp or a .ocda.db and print the model, deterministically, so that two - implementations can be compared with diff. - - Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang] -+/ -import std.stdio; -import ocda.model; -import ocda.load; -import ocda.dump; - -int main(string[] args) { - bool unknown = false; - string[] rest; - foreach (a; args[1 .. $]) { - if (a == "--unknown") { unknown = true; } else { rest ~= a; } - } - if (rest.length < 1) { - stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]"); - return 2; - } - string lang = (rest.length > 1) ? rest[1] : ""; - OcdaDocument doc; - try { - doc = load(rest[0], lang); - } catch (Exception e) { - stderr.writeln("error: ", e.msg); - return 2; - } - write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); - return 0; -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/d.org b/org/d.org new file mode 100644 index 0000000..1eade53 --- /dev/null +++ b/org/d.org @@ -0,0 +1,3895 @@ +-*- mode: org -*- +#+TITLE: dlang load +#+DESCRIPTION: dlang lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* ocda +** model + +#+HEADER: :tangle "../dlang/src/ocda/model.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.model; + +struct OcdaImage { + string name; // image filename as referenced by the object + string sha256; // hex-encoded (uppercase) digest of the file + ulong bytes; // file size + int width; // file's own pixel width (not display width) + int height; // file's own pixel height + bool missing; // file absent or unreadable, no sha256/bytes + string[string] fields; // any further/unrecognised key:value fields +} + +struct OcdaObject { + int ocn; + string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb + string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb + string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) + string identifier; // heading: from the declaration line; otherwise .identifier + string part; // frontmatter, body, backmatter + string is_of_type; // para, block, comment: how the text behaves + string own_section; // .section, when it differs from the block it sits in + string segment_html_is;// the html segment this object sits in + string segment_epub; // the epub segment this heading opens + string segment_epub_is;// the epub segment this object sits in + string table_aligns; // l c r per column + string[] stow_link; // links extracted from the object + string[] anchor_tags; // named targets the markup placed on this object + int parent_ocn; + int last_descendant_ocn; + int ocn_range_first;// a container (a poem): the first and last + int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1) + int[] children; // direct child heading OCNs + int[] ancestors; // markedup_ancestors[8] + bool dummy_heading; + bool ocn_off; + int indent_base; + int indent_hang; + bool bullet; + string lang; + bool has_links; + bool has_notes_reg; + bool has_notes_star; + bool has_images; + string segment; + string segment_prev; + string segment_next; + string anchor; + string heading_lev_anchor; // a heading's own named target + int table_cols; + string table_widths; + bool table_header; + string code_syntax; + bool code_linenumbers; + string heading_ancestors_text; + string sha256; + OcdaImage[] images; // repeatable .image records, in order + string text; // content (lines joined with newline) + + /+ ↓ whether this object is rendered at all. + is_of_type "comment" is markup commentary: carried by the + abstraction so that nothing is lost, and not put into output. An + output that renders it leaks the author notes into the document, + which is what spine itself does not do. + +/ + bool renders() const { + return is_of_type != "comment"; + } + + /+ ↓ the section this object belongs to, which is the block it sits in + unless it says otherwise +/ + string sectionOf() const { + return (own_section.length > 0) ? own_section : section; + } +} + +/+ ↓ what the reader met and did not understand. + A reader that silently drops what it does not know cannot say whether + it understood the document, and cannot be used to check a + specification against an implementation: silence and understanding + look alike. So the parser counts what it ignored, and + --report-unknown prints it. ++/ +struct OcdaUnknown { + size_t[string] property; // .name -> times seen + size_t[string] block; // @name -> times seen + size_t[string] line; // first char -> times seen, lines of no known shape + + bool any() const { + return property.length > 0 || block.length > 0 || line.length > 0; + } +} + +struct OcdaDocument { + string format_version; // as declared on line 1, "" when absent + string source_filename; // from the "% Source:" line + string[string] source; // @source: language, languages, digest + OcdaUnknown unknown; // what was met and not understood + string[string] metadata; // key -> value + string[string] make; // key -> value + string[string] doc_has; // key -> value + OcdaObject[] head; + OcdaObject[] toc; + OcdaObject[] body; + OcdaObject[] endnotes; + OcdaObject[] glossary; + OcdaObject[] bibliography; + OcdaObject[] bookindex; + OcdaObject[] blurb; + OcdaObject[] tail; + + /+ ↓ the major part of the declared version, "" when none was declared +/ + string formatVersionMajor() const { + import std.string : indexOf; + if (format_version.length == 0) { return ""; } + auto dot = format_version.indexOf('.'); + return (dot > 0) ? format_version[0 .. dot] : format_version; + } + + string meta(string key) { + if (auto p = key in metadata) return *p; + return ""; + } + + OcdaObject[]* sectionByName(string name) { + switch (name) { + case "head": return &head; + case "toc": return &toc; + case "body": return &body; + case "endnotes": return &endnotes; + case "glossary": return &glossary; + case "bibliography": return &bibliography; + case "bookindex": return &bookindex; + case "blurb": return &blurb; + case "tail": return &tail; + default: return null; + } + } +} + +/+ ↓ the format major version this reader knows. + The .ssp and the .ocda.db carry one version between them, declared in + band: the .ssp on its first line, the database in its schema.version + row. The minor part is raised when a property is added, and a reader of + the newer format still reads the older one, so only the major part is + checked here. Raise this when the reader has been brought to a new + major, not before: the point of the gate is to refuse what has not been + looked at. ++/ +enum ssp_format_major_read = "2"; +#+END_SRC + +** load +** what to load +*** inventory + +#+HEADER: :tangle "../dlang/src/ocda/inventory.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} +#+END_SRC + +*** load +**** load + +#+HEADER: :tangle "../dlang/src/ocda/load.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load; + +/+ ↓ what a path is, and loading it. + . + One place that decides, so that every reader and every writer accepts + the same inputs and a new input type is added once. This mirrors + spine's own ocda/abstraction/load.d, which names the sources it can + enter the pipeline from and tells them apart by suffix. + . + Deliberately *not* here: .sst and .ssm markup. Reading markup means + implementing the parser, which is spine's work and a far larger thing + than a reader or a sample writer needs. This library reads + abstractions. ++/ + +import std.algorithm : endsWith; +import ocda.model; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; + +enum Source { + ssp, // .ssp, the abstraction as text + ocda_db, // .ocda.db, the abstraction as sqlite + pod, // .sisupod or .zip, an archive carrying a .ssp + unknown, +} + +Source sourceOf(string path) { + if (path.endsWith(".ocda.db")) { return Source.ocda_db; } + if (path.endsWith(".ssp")) { return Source.ssp; } + /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote + before, and every pod already published carries that name, so both + are read. What is inside decides how it is opened, not the name. + +/ + if (path.endsWith(".sisupod")) { return Source.pod; } + if (path.endsWith(".zip")) { return Source.pod; } + if (path.endsWith(".db")) { return Source.ocda_db; } + return Source.unknown; +} + +string sourceName(Source s) { + final switch (s) { + case Source.ssp: return ".ssp (the abstraction as text)"; + case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; + case Source.pod: return "a pod archive carrying a .ssp"; + case Source.unknown: return "not an abstraction this library reads"; + } +} + +class OcdaLoadError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ load whatever the path is into the model. + lang names which language to take from a .ocda.db, which holds + every language of its document, and from a pod, which may carry more + than one .ssp. A .ssp is one language by construction and ignores it. ++/ +OcdaDocument load(string path, string lang = "", bool any_version = false) { + final switch (sourceOf(path)) { + case Source.ssp: return parseSspFile(path); + case Source.ocda_db: return loadDbFile(path, lang, any_version); + case Source.pod: return loadPodFile(path, lang); + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) + ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); + } +} +#+END_SRC + +**** load db + +#+HEADER: :tangle "../dlang/src/ocda/load_db.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_db; + +/+ ↓ build the model from a .ocda.db. + . + The point of this module is that it returns *the same value* + ocda.load_ssp returns from the same document's .ssp. A writer is then + a function from the model to bytes, and cannot tell which + serialisation it was given, exactly as spine's own writers cannot. + . + Written from the sisudoc-abstraction-format specification, + ocda-db-format-specification.md, whose section 4 gives the column by + column correspondence this module implements. ++/ + +import std.array : join, split; +import std.json : JSONType, parseJSON; +import std.conv : to; +import std.algorithm : startsWith; +import std.string : strip; +import ocda.model; +import ocda.sqlite; + +/+ ↓ the schema major this loader knows (specification section 2) +/ +enum schema_major_read = "2"; + +class OcdaDbError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ every language the database holds, in manifest order +/ +struct DbLanguage { + int doc_id; + string lang; + string source_filename; + string ssp_digest; +} + +DbLanguage[] dbLanguages(ref Db db) { + DbLanguage[] out_; + foreach (r; db.rows( + "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') + FROM documents ORDER BY id")) { + out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); + } + return out_; +} + +/+ ↓ load one language of a database as the model. + lang == "" takes the first language the manifest gave. ++/ +OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { + auto db = Db.open(path); + scope(exit) { db.close(); } + + if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { + throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" + ~ " it does not look like a .ocda.db"); + } + + /+ ↓ specification sections 2 and 3.2: the rows describing the file rather + than a document in it are the only ones with doc_id null +/ + string schema_version = db.one( + "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); + if (schema_version.length > 0 && !any_version) { + auto major = schema_version.split(".")[0]; + if (major != schema_major_read) { + throw new OcdaDbError(path ~ " is schema v" ~ schema_version + ~ ", this loader knows v" ~ schema_major_read ~ ".x"); + } + } + + auto languages = dbLanguages(db); + if (languages.length == 0) { + throw new OcdaDbError(path ~ ": no documents rows"); + } + DbLanguage chosen = languages[0]; + if (lang.length > 0) { + bool found = false; + foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } + if (!found) { + string have; + foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } + throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); + } + } + + OcdaDocument doc; + doc.format_version = schema_version; + doc.source_filename = chosen.source_filename; + + /+ ↓ @source of the .ssp corresponds to the documents row and the + source.digest metadata (specification section 5.1 of the .ssp, and + section 7 here) +/ + doc.source["language"] = chosen.lang; + string languages_joined; + foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } + doc.source["languages"] = languages_joined; + auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); + if (digest.length > 0) { doc.source["digest"] = digest; } + + /+ ↓ @meta: every metadata row of this language that is not a source.* + property. source.* describes the artefact rather than the document + and lives in @source in the text form. + +/ + foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " ORDER BY key")) { + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } + } + + /+ ↓ 2.1 added the container range as two columns. A 2.0 database has + neither, and reads as having no range, which is what 2.0 meant. +/ + bool has_range = false; + foreach (c; db.rows("PRAGMA table_info(objects)")) { + if (c[1] == "ocn_range_first") { has_range = true; } + } + string range_cols = has_range + ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" + : "0, 0"; + + /+ ↓ the objects, in document order within each section +/ + foreach (r; db.rows( + "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), + coalesce(is_of_section,''), coalesce(is_of_type,''), + coalesce(heading_level,-1), coalesce(identifier,''), + coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), + coalesce(children,''), coalesce(ancestors,''), + coalesce(dummy_heading,0), coalesce(object_number_off,0), + coalesce(indent_base,0), coalesce(indent_hang,0), + coalesce(bullet,0), coalesce(lang,''), + coalesce(has_links,0), coalesce(has_notes_reg,0), + coalesce(has_notes_star,0), coalesce(has_images,0), + coalesce(segment,''), coalesce(segment_prev,''), + coalesce(segment_next,''), coalesce(anchor,''), + coalesce(segment_html_is,''), + coalesce(table_cols,0), coalesce(table_widths,''), + coalesce(table_aligns,''), coalesce(table_header,0), + coalesce(code_linenumbers,0), + coalesce(heading_ancestors_text,''), coalesce(sha256,''), + coalesce(text,''), id, coalesce(heading_lev_anchor,''), + coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ " + FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string + ~ " ORDER BY CASE section + WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 + WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 + WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 + WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { + OcdaObject obj; + obj.section = r[0]; + obj.ocn = r[2].to!int; + obj.is_a = r[3]; + obj.part = r[4]; + obj.own_section = (r[5] == r[0]) ? "" : r[5]; + obj.is_of_type = r[6]; + /+ ↓ the database holds the level as a number, the text form as the + letter or digit the markup wrote. Model section 4 is the mapping. +/ + obj.heading_level = levelName(r[7].to!int); + obj.identifier = r[8]; + obj.parent_ocn = r[9].to!int; + obj.last_descendant_ocn = r[10].to!int; + obj.children = jsonInts(r[11]); + obj.ancestors = jsonInts(r[12]); + obj.dummy_heading = (r[13] != "0"); + obj.ocn_off = (r[14] != "0"); + obj.indent_base = r[15].to!int; + obj.indent_hang = r[16].to!int; + obj.bullet = (r[17] != "0"); + obj.lang = r[18]; + obj.has_links = (r[19] != "0"); + obj.has_notes_reg = (r[20] != "0"); + obj.has_notes_star = (r[21] != "0"); + obj.has_images = (r[22] != "0"); + obj.segment = r[23]; + obj.segment_prev = r[24]; + obj.segment_next = r[25]; + obj.anchor = r[26]; + obj.segment_html_is = r[27]; + obj.heading_lev_anchor = r[37]; + obj.segment_epub = r[38]; + obj.segment_epub_is = r[39]; + obj.ocn_range_first = r[40].to!int; + obj.ocn_range_last = r[41].to!int; + obj.table_cols = r[28].to!int; + obj.table_widths = jsonWords(r[29]); + obj.table_aligns = jsonWords(r[30]); + obj.table_header = (r[31] != "0"); + obj.code_linenumbers = (r[32] != "0"); + obj.heading_ancestors_text = jsonJoined(r[33]); + obj.sha256 = r[34]; + obj.text = r[35]; + + /+ ↓ the four repeatable properties of the text form are child tables + here (specification section 4) +/ + string object_id = r[36]; + foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), + coalesce(width,0), coalesce(height,0), + coalesce(missing,0) + FROM object_images WHERE object_id = " ~ object_id + ~ " ORDER BY seq")) { + OcdaImage img; + img.name = i[0]; + img.sha256 = i[1]; + img.bytes = i[2].to!ulong; + img.width = i[3].to!int; + img.height = i[4].to!int; + img.missing = (i[5] != "0"); + obj.images ~= img; + } + foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.stow_link ~= l[0]; + } + foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.anchor_tags ~= a[0]; + } + + auto section = doc.sectionByName(r[0]); + if (section !is null) { *section ~= obj; } + } + countUnread(db, chosen.doc_id, doc); + return doc; +} + +/+ ↓ what the database holds and the model does not. + . + The text loader meets every property line and counts those it does not + keep, so a reader can say what was read and not understood. This + loader asks for columns by name, and would never see the ones it did + not ask for: it would report that it had understood everything, which + is the silence the count exists to prevent. So it counts them, on the + conditions under which the .ssp writes the same property (specification + section 6.3), and the report is the same from either form: checked, + 36 of 36. ++/ +private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { + string id = doc_id.to!string; + void count(string property, string where) { + auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id + ~ " AND " ~ where).to!size_t; + if (n > 0) { doc.unknown.property[property] = n; } + } + count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); + count("dom_status", "coalesce(dom_status,'') != ''"); + count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); + count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); + count("parent_lev", "coalesce(parent_lev,0) != 0"); + count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); + count("attrib", "coalesce(attrib,'') != ''"); + if (db.has("object_subtoc")) { + auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" + ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; + if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } + } +} + +/+ ↓ model section 4: the eight levels and their indices +/ +private string levelName(int index) { + switch (index) { + case 0: return "A"; + case 1: return "B"; + case 2: return "C"; + case 3: return "D"; + case 4: return "1"; + case 5: return "2"; + case 6: return "3"; + case 7: return "4"; + default: return ""; + } +} + +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + +private int[] jsonInts(string v) { + int[] out_; + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} + } + return out_; +} + +/+ ↓ space separated in the text form: table_widths, table_aligns +/ +private string jsonWords(string v) { + return jsonStrings(v).join(" "); +} + +/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ +private string jsonJoined(string v) { + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); +} +#+END_SRC + +**** load pod + +#+HEADER: :tangle "../dlang/src/ocda/load_pod.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_pod; + +/+ ↓ the abstraction out of a pod archive. + . + A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same + archive unwrapped, which is what spine published before it wrapped them. + Either way, a pod built with --pod2 carries the abstraction inside it, + at media/abstraction/<uid>.<lang>.ssp, and that is what is read. + . + So this is a third input form and *not* a third model: the .ssp inside + is parsed by ocda.load_ssp and comes back as the same value. A writer + over the library gains .sisupod without being touched, which is the + whole reason the dispatch is in one place. + . + A pod built without --pod2 carries markup and no abstraction. That is + refused, and the refusal says which flag makes a pod this reader can + read, because "no abstraction in it" is a fact about the pod rather + than about the reader. ++/ + +import std.algorithm : endsWith, sort; +import std.array : array; +import std.file : read; +import std.string : indexOf; +import ocda.model; +import ocda.load_ssp; +import ocda.zip; +import ocda.zstd; + +/+ ↓ a limit on what a frame may expand to. + A reader of an artefact it did not make declines to be told how much + memory to allocate. 512 MiB is far above the largest pod in the + reference collection (4 MiB) and far below anything that matters. ++/ +enum size_t max_pod_size = 512 * 1024 * 1024; + +class OcdaPodError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ which languages of the abstraction the pod carries, in sorted order +/ +string[] podLanguages(string path) { + auto bytes = archiveBytes(path); + string[] out_; + foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { + string lang = langOfSspName(e.name); + if (lang.length > 0) { out_ ~= lang; } + } + return out_.sort.array; +} + +/+ ↓ load one language of the abstraction a pod carries. + lang == "" takes the only one there is, and where there is a choice it + says so rather than choosing, as the .ocda.db loader does. ++/ +OcdaDocument loadPodFile(string path, string lang = "") { + auto bytes = archiveBytes(path); + auto entries = zipEntries(bytes); + auto ssps = zipFindSuffix(entries, ".ssp"); + if (ssps.length == 0) { + throw new OcdaPodError(path ~ ": carries no .ssp." + ~ " A pod carries the abstraction only when spine was asked for it:" + ~ " spine --pod2 puts it in, spine --source does not."); + } + ZipEntry chosen = ssps[0]; + if (lang.length > 0) { + bool found = false; + foreach (e; ssps) { + if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } + } + if (!found) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": no language '" ~ lang + ~ "'; it holds: " ~ have); + } + } else if (ssps.length > 1) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": holds more than one language (" + ~ have ~ "); name the one wanted"); + } + auto ssp = zipRead(bytes, chosen); + return parseSsp(cast(string) ssp.idup); +} + +/+ ↓ the archive, unwrapped if it was wrapped. + The bytes decide, not the name: a .sisupod is a zstd frame and a .zip + is not, and a file misnamed either way still reads. ++/ +package ubyte[] archiveBytes(string path) { + auto bytes = cast(ubyte[]) read(path); + return zstdIsFrame(bytes) + ? zstdDecompress(bytes, max_pod_size) + : bytes; +} + +/+ ↓ <uid>.<lang>.ssp -> <lang> +/ +package string langOfSspName(string name) { + if (!name.endsWith(".ssp")) { return ""; } + string stem = name[0 .. $ - 4]; + auto dot = stem.lastIndexOf('.'); + return (dot > 0) ? stem[dot + 1 .. $] : ""; +} + +private ptrdiff_t lastIndexOf(string s, char c) { + for (ptrdiff_t i = s.length - 1; i >= 0; i--) { + if (s[i] == c) { return i; } + } + return -1; +} +#+END_SRC + +**** load ssp + +#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_ssp; + +import std.algorithm : splitter, startsWith; +import std.array : array, split; +import std.conv : to, ConvException; +import std.file : readText; +import std.stdio : writeln; +import std.string : strip, stripLeft, indexOf; +import ocda.model; + +OcdaDocument parseSspFile(string filename) { + string content = readText(filename); + return parseSsp(content); +} + +OcdaDocument parseSsp(string content) { + OcdaDocument doc; + OcdaObject current_obj; + bool in_object = false; + /+ ↓ whether a text line has been seen for the object being read. + Not "is the text non-empty": an object whose first text line is blank + (a verse opening on an empty line, which the sample set has seven of) + would then lose it, the next line overwriting rather than appending. + +/ + bool seen_text = false; + string current_section = ""; + string current_block = ""; // meta, make, doc_has, or section name + + foreach (line; content.splitter("\n")) { + /+ ↓ comment. The first line of a .ssp declares the format version, and + it is the one comment that carries meaning: a reader checks the + major part and refuses what it does not know rather than loading + an unfamiliar artefact half populated in silence. + +/ + if (line.length > 0 && line[0] == '%') { + enum _banner = "% SiSU Document Abstraction v"; + enum _source = "% Source: "; + if (doc.format_version.length == 0 && line.startsWith(_banner)) { + doc.format_version = line[_banner.length .. $].strip; + } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { + doc.source_filename = line[_source.length .. $].strip; + } + continue; + } + + /+ ↓ blank line - finalize current object +/ + if (line.length == 0 || line.strip.length == 0) { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + continue; + } + + /+ ↓ section/block open +/ + if (line[0] == '@') { + auto brace_pos = line.indexOf('{'); + if (brace_pos > 0) { + string name = line[1 .. brace_pos].strip; + current_block = name; + if (!isPreambleBlock(name)) { + current_section = name; + /+ ↓ a block that is neither preamble nor a section this reader + knows: counted, so that --report-unknown can say so +/ + if (doc.sectionByName(name) is null) { + doc.unknown.block[name]++; + } + } + } + continue; + } + + /+ ↓ section/block close +/ + if (line[0] == '}') { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + current_block = ""; + continue; + } + + /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ + if (isPreambleBlock(current_block)) { + string stripped = line.stripLeft; + auto colon_pos = stripped.indexOf(':'); + if (colon_pos > 0) { + string key = stripped[0 .. colon_pos].strip; + string value = (colon_pos + 1 < stripped.length) + ? stripped[colon_pos + 1 .. $].strip : ""; + if (current_block == "meta") { doc.metadata[key] = value; } + else if (current_block == "make") { doc.make[key] = value; } + else if (current_block == "source") { doc.source[key] = value; } + else { doc.doc_has[key] = value; } + } + continue; + } + + /+ ↓ object declaration: [N] type ... +/ + if (line[0] == '[') { + if (in_object) { + commitObject(doc, current_section, current_obj); + } + current_obj = OcdaObject.init; + current_obj.section = current_section; + in_object = true; + seen_text = false; + parseObjectDeclaration(line, current_obj); + continue; + } + + /+ ↓ property: .name: value +/ + if (line[0] == '.') { + if (in_object) { + parseProperty(line, current_obj, doc.unknown); + } + continue; + } + + /+ ↓ text content: | content +/ + if (line.length >= 1 && line[0] == '|') { + if (in_object) { + string text_content = (line.length > 2) ? line[2 .. $] : ""; + if (seen_text) { + current_obj.text ~= "\n" ~ text_content; + } else { + current_obj.text = text_content; + seen_text = true; + } + } + continue; + } + + /+ ↓ no known shape. Counted by its first character so that a reader + can be told the file held something this one did not expect + +/ + doc.unknown.line[line[0 .. 1].idup]++; + } + + /+ ↓ finalize last object +/ + if (in_object) { + commitObject(doc, current_section, current_obj); + } + + return doc; +} + +private void parseObjectDeclaration(string line, ref OcdaObject obj) { + /+ parse: [N] type :level identifier +/ + auto bracket_close = line.indexOf(']'); + if (bracket_close < 0) return; + try { + obj.ocn = line[1 .. bracket_close].strip.to!int; + } catch (ConvException) { + obj.ocn = 0; + } + string rest = (bracket_close + 1 < line.length) + ? line[bracket_close + 1 .. $].strip : ""; + if (rest.length == 0) return; + + auto parts = rest.split(" "); + if (parts.length == 0) return; + obj.is_a = parts[0]; + + if (obj.is_a == "heading" && parts.length >= 2) { + /+ parse :A, :B, :1, etc. +/ + string lev = parts[1]; + if (lev.length >= 2 && lev[0] == ':') { + obj.heading_level = lev[1 .. $]; + } + /+ optional identifier after level +/ + if (parts.length >= 3) { + obj.identifier = parts[2]; + } + } +} + +/+ ↓ the blocks that are document preamble rather than sections of objects +/ +private bool isPreambleBlock(string name) { + return name == "source" || name == "meta" || name == "make" || name == "doc_has"; +} + +private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { + /+ parse: .name: value +/ + auto colon_pos = line.indexOf(':'); + if (colon_pos < 1) return; + string name = line[1 .. colon_pos].strip; + string value = (colon_pos + 1 < line.length) + ? line[colon_pos + 1 .. $].strip : ""; + + switch (name) { + case "part": obj.part = value; break; + case "is_of_type": obj.is_of_type = value; break; + case "section": obj.own_section = value; break; + case "identifier": obj.identifier = value; break; + case "segment_html_is": obj.segment_html_is = value; break; + /+ ↓ the epub segments at heading levels 1 to 4 where the html + segments at level 4 alone, so the two are different files and + the abstraction records both +/ + case "segment_epub": obj.segment_epub = value; break; + case "segment_epub_is": obj.segment_epub_is = value; break; + case "table_aligns": obj.table_aligns = value; break; + case "stow_link": obj.stow_link ~= value; break; + case "anchor_tag": obj.anchor_tags ~= value; break; + case "parent": obj.parent_ocn = toIntSafe(value); break; + case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; + case "ocn_range": { + auto r = parseIntArray(value); + if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; } + break; + } + case "children": obj.children = parseIntArray(value); break; + case "ancestors": obj.ancestors = parseIntArray(value); break; + case "dummy": obj.dummy_heading = (value == "true"); break; + case "ocn_off": obj.ocn_off = (value == "true"); break; + case "indent": + auto parts = value.split(" "); + if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); + if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); + break; + case "bullet": obj.bullet = (value == "true"); break; + case "lang": obj.lang = value; break; + case "has": + foreach (flag; value.split(" ")) { + switch (flag) { + case "links": obj.has_links = true; break; + case "notes_reg": obj.has_notes_reg = true; break; + case "notes_star": obj.has_notes_star = true; break; + case "images": obj.has_images = true; break; + default: break; + } + } + break; + case "segment": obj.segment = value; break; + case "segment_prev": obj.segment_prev = value; break; + case "segment_next": obj.segment_next = value; break; + case "anchor": obj.anchor = value; break; + case "heading_lev_anchor": obj.heading_lev_anchor = value; break; + case "table_cols": obj.table_cols = toIntSafe(value); break; + case "table_widths": obj.table_widths = value; break; + case "table_header": obj.table_header = (value == "true"); break; + case "code_syntax": obj.code_syntax = value; break; + case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; + case "heading_ancestors_text": obj.heading_ancestors_text = value; break; + case "sha256": obj.sha256 = value; break; + case "image": obj.images ~= parseImageRecord(value); break; + /+ ↓ a property this reader has no use for, or has never heard of. + Ignored, which is the conforming behaviour within a major + version, but counted rather than passed over in silence + +/ + default: unknown.property[name]++; break; + } +} + +private OcdaImage parseImageRecord(string value) { + /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] + filename first, remaining fields order-independent; unrecognised + fields are kept rather than discarded +/ + OcdaImage img; + bool first = true; + foreach (field; value.split(" ")) { + if (field.length == 0) continue; + if (first) { + img.name = field; + first = false; + continue; + } + auto colon_pos = field.indexOf(':'); + if (colon_pos < 1) continue; + string key = field[0 .. colon_pos]; + string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; + switch (key) { + case "sha256": img.sha256 = val; break; + case "bytes": img.bytes = toUlongSafe(val); break; + case "px": + auto dims = val.split("x"); + if (dims.length == 2) { + img.width = toIntSafe(dims[0]); + img.height = toIntSafe(dims[1]); + } + break; + case "missing": img.missing = (val == "true"); break; + default: img.fields[key] = val; break; + } + } + return img; +} + +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + +private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); + auto sec = doc.sectionByName(section); + if (sec !is null) { + ,*sec ~= obj; + } +} + +private int toIntSafe(string s) { + try { return s.strip.to!int; } + catch (ConvException) { return 0; } +} + +private ulong toUlongSafe(string s) { + try { return s.strip.to!ulong; } + catch (ConvException) { return 0; } +} + +private int[] parseIntArray(string s) { + int[] result; + foreach (part; s.split(" ")) { + auto v = toIntSafe(part); + result ~= v; + } + return result; +} +#+END_SRC + +*** images + +#+HEADER: :tangle "../dlang/src/ocda/images.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.images; + +/+ ↓ an image's bytes, from whichever source the document came from. + . + The model *names* images, with digest, byte size and pixel size; it does + not hold them. That is right - an abstraction of a document is not a + bundle of its assets - but a writer of any format that shows images + needs the bytes, so something has to know where they are. This is that + something, and it is in the library rather than in a writer because the + answer depends on the *input form* and not on the output. + . + Three answers, and one refusal: + . + .ocda.db the files table, role 'image', by name + .sisupod the archive entry whose path ends in /image/<name> + .zip the same + .ssp beside it: a pod puts the abstraction in + media/abstraction/ and its images in media/image/, so the + path is documented rather than guessed + . + A .ssp that is not in a pod has nowhere to look, and that is reported as + absent rather than as an error: a reader that cannot find an image + should say so and carry on, because the document is still readable. ++/ + +import std.algorithm : endsWith; +import std.array : array; +import std.file : exists, isFile, read; +import std.path : buildPath, dirName; +import ocda.load; +import ocda.sqlite; +import ocda.zip; +import ocda.zstd; + +/+ ↓ the bytes, or an empty slice when the source has no such image +/ +ubyte[] imageBytes(string source, string name) { + final switch (sourceOf(source)) { + case Source.ocda_db: return fromDb(source, name); + case Source.pod: return fromArchive(source, name); + case Source.ssp: return fromBeside(source, name); + case Source.unknown: return []; + } +} + +/+ ↓ can this source give images at all? + A writer asks before it decides whether to emit them, because an <img> + pointing at a file it did not include is worse than no image. ++/ +bool canGiveImages(string source) { + final switch (sourceOf(source)) { + case Source.ocda_db: return true; + case Source.pod: return true; + case Source.ssp: return imageDirBeside(source).length > 0; + case Source.unknown: return false; + } +} + +private ubyte[] fromDb(string path, string name) { + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + /+ ↓ the name is from the abstraction, not from a caller, and the quote + doubling is what keeps it a value rather than syntax +/ + return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '" + ~ sqlQuote(name) ~ "'"); +} + +private ubyte[] fromArchive(string path, string name) { + auto bytes = cast(ubyte[]) read(path); + if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); } + auto entries = zipEntries(bytes); + foreach (e; entries) { + if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); } + } + return []; +} + +private ubyte[] fromBeside(string path, string name) { + string dir = imageDirBeside(path); + if (dir.length == 0) { return []; } + string file = buildPath(dir, name); + return file.exists && file.isFile ? cast(ubyte[]) read(file) : []; +} + +/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/ +private string imageDirBeside(string path) { + string dir = path.dirName; + if (!dir.endsWith("abstraction")) { return ""; } + string image_dir = buildPath(dir.dirName, "image"); + return image_dir.exists ? image_dir : ""; +} + +/+ ↓ a single quote doubled is how sql says a single quote +/ +private string sqlQuote(string s) { + string out_; + foreach (c; s) { + out_ ~= (c == '\'') ? "''" : [c]; + } + return out_; +} +#+END_SRC + +** sqlite + +#+HEADER: :tangle "../dlang/src/ocda/sqlite.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.sqlite; + +/+ ↓ the sqlite3 C API, declared rather than depended upon. + . + A binding from the dub registry would be a dependency to fetch, and + the point of this reader is that it builds in a sandbox with no + network. The dozen functions a reader needs are declared here and the + library is linked directly, which costs a page of extern(C) and saves + a dependency. ++/ + +import std.conv : to; +import std.string : fromStringz, toStringz; + +extern (C) { + struct sqlite3; + struct sqlite3_stmt; + + int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs); + int sqlite3_close(sqlite3* db); + int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte, + sqlite3_stmt** stmt, const(char)** tail); + int sqlite3_step(sqlite3_stmt* stmt); + int sqlite3_finalize(sqlite3_stmt* stmt); + int sqlite3_column_count(sqlite3_stmt* stmt); + const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col); + const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col); + int sqlite3_column_int(sqlite3_stmt* stmt, int col); + int sqlite3_column_type(sqlite3_stmt* stmt, int col); + const(char)* sqlite3_errmsg(sqlite3* db); + const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col); + int sqlite3_column_bytes(sqlite3_stmt* stmt, int col); +} + +enum SQLITE_OK = 0; +enum SQLITE_ROW = 100; +enum SQLITE_DONE = 101; +enum SQLITE_NULL = 5; +enum SQLITE_OPEN_READONLY = 0x00000001; + +class SqliteError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ a database, opened read only. + Read only is deliberate: a reader has no business writing, and a + request that opens a database read-write can create an empty one where + a path is mistyped. ++/ +struct Db { + private sqlite3* _handle; + + static Db open(string path) { + sqlite3* handle; + auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null); + if (rc != SQLITE_OK) { + string msg = (handle is null) + ? "cannot open " ~ path + : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup; + if (handle !is null) { sqlite3_close(handle); } + throw new SqliteError(msg); + } + Db db; + db._handle = handle; + return db; + } + + void close() { + if (_handle !is null) { sqlite3_close(_handle); _handle = null; } + } + + /+ ↓ every row of a query, each row as its columns in order +/ + string[][] rows(string sql) { + sqlite3_stmt* stmt; + auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null); + if (rc != SQLITE_OK) { + throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql); + } + scope(exit) { sqlite3_finalize(stmt); } + string[][] out_rows; + int cols = sqlite3_column_count(stmt); + while (sqlite3_step(stmt) == SQLITE_ROW) { + string[] row; + foreach (c; 0 .. cols) { + row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL) + ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup; + } + out_rows ~= row; + } + return out_rows; + } + + /+ ↓ the first column of the first row, "" when there is none +/ + string one(string sql) { + auto r = rows(sql); + return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; + } + + /+ ↓ the first column of the first row as bytes, for a blob. + Separate from one(), because a blob is not text: an image read as text + stops at the first zero byte, and a png's ninth byte is zero. + +/ + ubyte[] blob(string sql) { + sqlite3_stmt* stmt; + if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) { + throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup + ~ " [" ~ sql ~ "]"); + } + scope(exit) { sqlite3_finalize(stmt); } + if (sqlite3_step(stmt) != SQLITE_ROW) { return []; } + int n = sqlite3_column_bytes(stmt, 0); + if (n <= 0) { return []; } + auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0); + if (p is null) { return []; } + return p[0 .. n].dup; + } + + /+ ↓ does a table or view of this name exist +/ + bool has(string name) { + return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; + } +} +#+END_SRC + +** compression +*** dlang read zip + +#+HEADER: :tangle "../dlang/src/ocda/zip.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.zip; + +/+ ↓ enough of the zip format to find one file in a pod and read it. + . + Not a zip library. A pod archive holds a handful of entries, spine + writes them **stored** (uncompressed) so that the outer zstd frame does + the compressing once over the whole thing, and what is wanted from it + here is one named entry. So: read the central directory, find the + entry, return its bytes. + . + Deflate is *refused rather than mishandled*. Phobos can inflate, but a + library that says it read an entry it in fact skipped is worse than one + that says it cannot, and spine's own archives do not need it. ++/ + +import std.algorithm : endsWith; +import std.conv : to; + +class OcdaZipError : Exception { + this(string msg) { super(msg); } +} + +struct ZipEntry { + string name; + uint method; // 0 stored, 8 deflate + uint compressed; + uint uncompressed; + uint local_offset; +} + +/+ ↓ the entries, from the central directory at the end of the file +/ +ZipEntry[] zipEntries(const(ubyte)[] bytes) { + enum uint eocd_sig = 0x06054b50; + enum uint cd_sig = 0x02014b50; + if (bytes.length < 22) { + throw new OcdaZipError("too short to be a zip archive"); + } + /+ ↓ the end of central directory record is last, and may be followed by a + comment of up to 65535 bytes, so it is searched for backwards +/ + ptrdiff_t eocd = -1; + ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0; + for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) { + if (u32(bytes, i) == eocd_sig) { eocd = i; break; } + } + if (eocd < 0) { + throw new OcdaZipError("no end of central directory record;" + ~ " it does not look like a zip archive"); + } + uint count = u16(bytes, eocd + 10); + uint offset = u32(bytes, eocd + 16); + ZipEntry[] out_; + size_t p = offset; + foreach (_; 0 .. count) { + if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) { + throw new OcdaZipError("central directory entry " + ~ out_.length.to!string ~ " is malformed"); + } + ZipEntry e; + e.method = u16(bytes, p + 10); + e.compressed = u32(bytes, p + 20); + e.uncompressed = u32(bytes, p + 24); + uint name_len = u16(bytes, p + 28); + uint extra_len = u16(bytes, p + 30); + uint cmnt_len = u16(bytes, p + 32); + e.local_offset = u32(bytes, p + 42); + e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup; + out_ ~= e; + p += 46 + name_len + extra_len + cmnt_len; + } + return out_; +} + +/+ ↓ one entry's bytes, from its local header +/ +ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) { + enum uint local_sig = 0x04034b50; + size_t p = e.local_offset; + if (p + 30 > bytes.length || u32(bytes, p) != local_sig) { + throw new OcdaZipError(e.name ~ ": local header is malformed"); + } + uint name_len = u16(bytes, p + 26); + uint extra_len = u16(bytes, p + 28); + size_t start = p + 30 + name_len + extra_len; + size_t end = start + e.compressed; + if (end > bytes.length) { + throw new OcdaZipError(e.name ~ ": runs past the end of the archive"); + } + if (e.method != 0) { + throw new OcdaZipError(e.name ~ ": compression method " + ~ e.method.to!string ~ "; this reader takes only stored entries," + ~ " which is what spine writes"); + } + return bytes[start .. end].dup; +} + +/+ ↓ the first entry whose name ends this way, or none. + Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid. ++/ +ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) { + ZipEntry[] out_; + foreach (e; entries) { + if (e.name.endsWith(suffix)) { out_ ~= e; } + } + return out_; +} + +private uint u16(const(ubyte)[] b, size_t i) { + return cast(uint) b[i] | (cast(uint) b[i + 1] << 8); +} + +private uint u32(const(ubyte)[] b, size_t i) { + return cast(uint) b[i] + | (cast(uint) b[i + 1] << 8) + | (cast(uint) b[i + 2] << 16) + | (cast(uint) b[i + 3] << 24); +} +#+END_SRC + +*** dlang read zstd + +#+HEADER: :tangle "../dlang/src/ocda/zstd.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.zstd; + +/+ ↓ libzstd, declared rather than bound. + . + A .sisupod is one zstd frame wrapping a zip archive, so reading one + needs exactly two things from the library: is this a frame, and what is + in it. Both are declared here and the C library is linked, which is the + same arrangement ocda.sqlite uses: no vendored tree, no generated + header, and the prototypes say what is being called. ++/ + +extern (C) @nogc nothrow private { + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); +} + +/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/ +private enum ulong contentSizeUnknown = ulong.max; +private enum ulong contentSizeError = ulong.max - 1; + +/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/ +private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD]; + +class OcdaZstdError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ does this begin with a zstd frame? + Asked of the bytes rather than of the filename, because what is inside + decides how a thing is opened. A .sisupod is a frame; a .zip published + before spine wrapped them is not, and both are read. ++/ +bool zstdIsFrame(const(ubyte)[] bytes) { + if (bytes.length < zstd_magic.length) { return false; } + foreach (i, b; zstd_magic) { + if (bytes[i] != b) { return false; } + } + return true; +} + +/+ ↓ one frame in, its contents out. + max_size is a refusal rather than a buffer size: a frame declares how + much it expands to, and a frame that claims more than the caller is + willing to hold is not decompressed at all. A reader of an artefact it + did not make wants that limit. ++/ +ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) { + if (!zstdIsFrame(bytes)) { + throw new OcdaZstdError("not a zstd frame"); + } + ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length); + if (declared == contentSizeError) { + throw new OcdaZstdError("zstd frame: could not read its content size"); + } + if (declared == contentSizeUnknown) { + throw new OcdaZstdError("zstd frame declares no content size;" + ~ " this reader takes only frames that do"); + } + if (declared > max_size) { + import std.conv : to; + throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string + ~ " bytes, over the " ~ max_size.to!string ~ " byte limit"); + } + auto out_ = new ubyte[](cast(size_t) declared); + size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length); + if (ZSTD_isError(got)) { + import std.string : fromStringz; + throw new OcdaZstdError("zstd: " + ~ ZSTD_getErrorName(got).fromStringz.idup); + } + if (got != declared) { + throw new OcdaZstdError("zstd frame expanded to a different size" + ~ " than it declared"); + } + return out_; +} +#+END_SRC + +** dump + +#+HEADER: :tangle "../dlang/src/ocda/dump.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.dump; + +/+ ↓ the model, written out canonically. + . + Not an output format: an *oracle*. Two libraries in two languages + implement the same model from the same specification, and the cheap + way to ask whether they agree is to have each write the whole loaded + document as text and diff the two. That is only useful if the text is + fully determined by the model, so: + . + - every line is "<key> <value>", the value running to end of line, + and only one free-form value per line; + - a field at its default is *omitted*, so the output says what the + document has rather than what the struct declares; + - dictionaries are emitted in sorted key order, because neither + language's hash map has an order worth relying on; + - sections come in the fixed order of the format, not + alphabetically. + . + Anything that reads this to reconstruct a document has misunderstood + it. Load the .ssp or the .ocda.db. ++/ + +import std.algorithm : sort, splitter; +import std.array : appender, array, join; +import std.conv : to; +import ocda.model; + +/+ ↓ the nine sections, in format order +/ +static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + +string dumpDocument(ref OcdaDocument doc) { + auto out_ = appender!string; + + void kv(string key, string value) { + if (value.length == 0) { return; } + out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; + } + void kn(string key, long value) { + if (value == 0) { return; } + kv(key, value.to!string); + } + void kb(string key, bool value) { + if (!value) { return; } + kv(key, "1"); + } + void dict(string prefix, string[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key]; out_ ~= "\n"; + } + } + + kv("version", doc.format_version); + kv("source_filename", doc.source_filename); + dict("source", doc.source); + dict("meta", doc.metadata); + dict("make", doc.make); + dict("doc_has", doc.doc_has); + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null || (*sec).length == 0) { continue; } + kv("section", name ~ " " ~ (*sec).length.to!string); + } + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null) { continue; } + foreach (i, obj; *sec) { + out_ ~= "object "; out_ ~= name; out_ ~= " "; + out_ ~= i.to!string; out_ ~= "\n"; + kn("ocn", obj.ocn); + kv("is_a", obj.is_a); + kv("own_section", obj.own_section); + kv("level", obj.heading_level); + kv("identifier", obj.identifier); + kv("part", obj.part); + kv("is_of_type", obj.is_of_type); + kv("segment_html_is", obj.segment_html_is); + kv("segment_epub", obj.segment_epub); + kv("segment_epub_is", obj.segment_epub_is); + kn("parent", obj.parent_ocn); + kn("last_descendant", obj.last_descendant_ocn); + kn("ocn_range_first", obj.ocn_range_first); + kn("ocn_range_last", obj.ocn_range_last); + kv("children", ints(obj.children)); + kv("ancestors", ints(obj.ancestors)); + kb("dummy", obj.dummy_heading); + kb("ocn_off", obj.ocn_off); + kn("indent_base", obj.indent_base); + kn("indent_hang", obj.indent_hang); + kb("bullet", obj.bullet); + kv("lang", obj.lang); + kb("has_links", obj.has_links); + kb("has_notes_reg", obj.has_notes_reg); + kb("has_notes_star", obj.has_notes_star); + kb("has_images", obj.has_images); + kv("segment", obj.segment); + kv("segment_prev", obj.segment_prev); + kv("segment_next", obj.segment_next); + kv("anchor", obj.anchor); + kv("heading_lev_anchor", obj.heading_lev_anchor); + kn("table_cols", obj.table_cols); + kv("table_widths", obj.table_widths); + kv("table_aligns", obj.table_aligns); + kb("table_header", obj.table_header); + kv("code_syntax", obj.code_syntax); + kb("code_linenumbers", obj.code_linenumbers); + kv("heading_ancestors_text", obj.heading_ancestors_text); + kv("sha256", obj.sha256); + foreach (url; obj.stow_link) { kv("stow_link", url); } + foreach (a; obj.anchor_tags) { kv("anchor_tag", a); } + foreach (img; obj.images) { + /+ ↓ the name last, because a filename may hold a space where + none of the other fields can +/ + kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " + ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " + ~ (img.missing ? "1" : "0") ~ " " ~ img.name); + foreach (key; img.fields.keys.dup.sort) { + kv("image_field", key ~ " " ~ img.fields[key]); + } + } + /+ ↓ text, when there is any. An object whose only text line is + blank holds "" here, indistinguishable from no text at all, + because the model joins the lines into one string. Both + libraries lose the same distinction in the same place, which is + what makes them comparable; it is recorded rather than hidden. + +/ + if (obj.text.length > 0) { + auto lines = obj.text.splitter("\n").array; + kv("text", lines.length.to!string); + foreach (line; lines) { + out_ ~= "| "; out_ ~= line; out_ ~= "\n"; + } + } + } + } + return out_.data; +} + +/+ ↓ what the reader met and did not understand, sorted. Separate from the + document dump: it describes the reader, not the document, and the + .ocda.db loader has nothing to report where the .ssp loader does. ++/ +string dumpUnknown(ref OcdaDocument doc) { + auto out_ = appender!string; + void counts(string prefix, size_t[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key].to!string; out_ ~= "\n"; + } + } + counts("property", doc.unknown.property); + counts("block", doc.unknown.block); + counts("line", doc.unknown.line); + return out_.data; +} + +private string ints(int[] xs) { + if (xs.length == 0) { return ""; } + string[] parts; + foreach (x; xs) { parts ~= x.to!string; } + return parts.join(","); +} +#+END_SRC + +* read +** main + +#+HEADER: :tangle "../dlang/src/read/main.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + ocda-read - reads the object-centric document abstraction, in any of the + forms it is written in, and says what is in it. + + ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] + + One reader where there were two: sisudoc-ssp-reader read a .ssp and + sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is + built on the library, so every mode reads every form, and a mode given the + .ssp, the .ocda.db and the .sisupod of a document says the same thing of + each: it is told what the document is, not which file it came from. + + Two things are about the file rather than the document, and say so: + --files, what a .ocda.db carries besides the abstraction, and --validate, + which checks a .ssp as text. Everything else is the model's. + + The output modules are the .ssp reader's, moved onto the library's model. ++/ +module read.main; + +import std.algorithm : canFind, map; +import std.array : array, join; +import std.getopt; +import std.stdio : writeln, writefln, stderr; +import ocda.model; +import ocda.load; +import ocda.inventory; +import read.tree; +import read.html; +import read.llm_text; +import read.llm_jsonl; +import read.images; +import read.verify; +import read.report_unknown; +import read.validate; + +int main(string[] args) { + bool do_tree = false; + bool do_html = false; + bool do_llm_text = false; + bool do_llm_jsonl = false; + bool do_images = false; + bool do_verify = false; + bool do_report_unknown = false; + bool do_validate = false; + bool do_languages = false; + bool do_files = false; + bool any_version = false; + string image_dir = ""; + string output_file = ""; + string lang = ""; + + GetoptResult helpInfo; + try { + helpInfo = getopt(args, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, + "html", "the document as one html page", &do_html, + "llm-text", "plain text with heading breadcrumbs", &do_llm_text, + "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, + "images", "the image records: digest, size, pixels", &do_images, + "verify", "check the image records against --image-dir", &do_verify, + "image-dir", "the images to verify against", &image_dir, + "languages", "the languages the file holds", &do_languages, + "files", "what a .ocda.db carries besides the abstraction", &do_files, + "report-unknown", "what was read and not understood", &do_report_unknown, + "validate", "check a .ssp against its specification, as text", &do_validate, + "lang", "=<code> the language to read (default: the first)", &lang, + "output", "write the mode's output to a file", &output_file, + "any-version", "read a format major this reader does not know", &any_version, + ); + } catch (GetOptException e) { + stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); + return 2; + } + + if (helpInfo.helpWanted || args.length < 2) { + defaultGetoptPrinter( + "ocda-read - reads the document abstraction, in any of its forms\n\n" + ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n", + helpInfo.options, + ); + return 0; + } + + string input_file = args[1]; + if (lang.length == 0 && args.length > 2) { lang = args[2]; } + bool lang_named = (lang.length > 0); + + /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not + at all: there is no text form of a database to check, and a pod's + .ssp is checked by taking it out +/ + if (do_validate && sourceOf(input_file) != Source.ssp) { + stderr.writeln("--validate checks a .ssp as text, against the .ssp", + " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); + return 2; + } + + /+ ↓ which language. A .ocda.db and a pod may hold several; where none is + named, every form reads the first in the manifest's order and says + so, rather than the database taking one and the pod refusing +/ + OcdaLanguage[] held; + OcdaDocument doc; + try { + held = languages(input_file); + if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } + if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { + stderr.writeln("error: no language '", lang, "' in ", input_file, + "; it holds: ", held.map!(l => l.lang).array.join(" ")); + return 2; + } + doc = load(input_file, lang, any_version); + } catch (Exception e) { + stderr.writeln("Error reading ", input_file, ": ", e.msg); + return 2; + } + + /+ ↓ the version gate: the major part, and refuse what is not known +/ + if (doc.format_version.length == 0) { + stderr.writeln("warning: ", input_file, " declares no format version"); + } else if (doc.formatVersionMajor != ssp_format_major_read) { + if (any_version) { + stderr.writeln("warning: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); + } else { + stderr.writeln("error: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x"); + stderr.writeln(" a major version differs: a property may have", + " changed meaning or gone away."); + stderr.writeln(" --any-version reads it regardless."); + return 3; + } + } + + writeln("Read: ", input_file); + if (doc.format_version.length > 0) { + writeln(" Format: v", doc.format_version); + } + if (auto _lang = "language" in doc.source) { + writeln(" Source: ", *_lang, + ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", + ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); + } + if (held.length > 1 && !lang_named) { + writeln(" (reading ", lang, "; --lang=<code> for another)"); + } + writeln(" Title: ", doc.meta("title.main")); + writeln(" Author: ", doc.meta("creator.author")); + writeln(" Sections: head=", doc.head.length, + " toc=", doc.toc.length, + " body=", doc.body.length, + " endnotes=", doc.endnotes.length, + " glossary=", doc.glossary.length, + " bibliography=", doc.bibliography.length, + " bookindex=", doc.bookindex.length, + " blurb=", doc.blurb.length, + " tail=", doc.tail.length); + auto image_records = imageRecordCount(doc); + if (image_records > 0) { + writeln(" Images: ", image_records, " record", + (image_records == 1) ? "" : "s"); + } + + if (do_languages) { printLanguages(held); } + if (do_files) { printFiles(input_file); } + if (do_tree) { outputTree(doc, output_file); } + if (do_html) { outputHtmlScroll(doc, output_file); } + if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } + if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } + if (do_images) { outputImages(doc, output_file); } + + int _verify_failures = 0; + if (do_verify) { + if (image_dir.length == 0) { + stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); + return 2; + } + _verify_failures = verifyImages(doc, image_dir, output_file); + } + + if (do_report_unknown) { reportUnknown(doc); } + + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images + && !do_verify && !do_report_unknown && !do_validate + && !do_languages && !do_files) { + writeln("\n(this is the summary: --help lists the modes, each adding its output)"); + } + + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; +} + +/+ ↓ the languages, each with the markup it was made from and its .ssp's + digest: the same from every form, the database storing the digest and + the others taking it from the bytes +/ +void printLanguages(OcdaLanguage[] held) { + writeln(); + writeln(" lang source ssp digest"); + foreach (l; held) { + writefln(" %-7s %-31s %s", l.lang, l.source_filename, + (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); + } +} + +/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries + nothing, and a pod's files are its archive entries rather than anything + the abstraction records, so for those it says so +/ +void printFiles(string path) { + writeln(); + if (!isCarrier(path)) { + writeln(" carried files: only a .ocda.db carries files; this is ", + sourceName(sourceOf(path))); + return; + } + auto rows = carried(path); + if (rows.length == 0) { writeln(" carries no files"); return; } + writeln(" carried files:"); + writeln(" role stored n original in the file"); + foreach (r; rows) { + writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); + } +} +#+END_SRC + +** check +*** read validate =.ssp= + +#+HEADER: :tangle "../dlang/src/read/validate.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.validate; + +/+ ↓ check a .ssp against the format specification. + . + Written from specs/sisudoc-abstraction-format/ssp-format-specification.md + and from nothing else: the point of it is to be what the specification + says, executed, so that a file can be held against the document rather + than against an implementation. Each check names the section it comes + from, so a disagreement can be taken back to the text. ++/ + +import std.algorithm : canFind, startsWith; +import std.array : array, split; +import std.conv : to; +import std.file : readText; +import std.stdio : writeln, writefln; +import std.string : indexOf, strip; + +struct SspFinding { + size_t line; // 1-based + string section; // the specification section the rule is from + string detail; +} + +/+ ↓ specification section 6.3: the properties, in the order they are + emitted. A reader may rely on the order, so a validator can check it +/ +private immutable string[] property_order = [ + "identifier", "part", "section", "parent", "last_descendant", "ocn_range", + "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", + "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", + "table_header", "code_linenumbers", "stow_link", "segment", "anchor", + "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", + "segment_html_is", "segment_epub_is", "segment_lv4_is", + "heading_ancestors_text", "lev4_subtoc", "anchor_tag", +]; + +/+ ↓ section 6.1: the sixteen kinds +/ +private immutable string[] kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", + "blurb", "group", +]; + +/+ ↓ section 4: the nine sections, in the order they appear +/ +private immutable string[] section_order = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", + "bookindex", "blurb", "tail", +]; + +/+ ↓ section 4: the four preamble blocks, in the order they appear +/ +private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; + +/+ ↓ section 6.1: the heading levels +/ +private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; + +private int propertyRank(string name) { + foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } + return -1; +} + +SspFinding[] validateSspFile(string filename, string major_known) { + SspFinding[] found; + auto lines = readText(filename).split("\n"); + + bool seen_banner = false; + bool seen_source_line = false; + string block = ""; // the block currently open, "" when none + int preamble_at = -1; // how far through the preamble order + int section_at = -1; // how far through the section order + bool in_object = false; + int prev_rank = -1; + size_t doc_has_keys = 0; + bool in_text = false; // text lines come after properties + + foreach (i, line; lines) { + size_t no = i + 1; + + /+ ↓ section 2: the version, on the first line +/ + if (no == 1) { + enum banner = "% SiSU Document Abstraction v"; + if (!line.startsWith(banner)) { + found ~= SspFinding(no, "2", "first line is not the version banner"); + } else { + seen_banner = true; + string v = line[banner.length .. $].strip; + auto dot = v.indexOf('.'); + string major = (dot > 0) ? v[0 .. dot] : v; + if (major != major_known) { + found ~= SspFinding(no, "2.1", + "format major v" ~ major ~ ", this validator knows v" ~ major_known); + } + } + continue; + } + if (no == 2) { + if (!line.startsWith("% Source: ")) { + found ~= SspFinding(no, "4", "second line is not \"% Source: \""); + } else { seen_source_line = true; } + continue; + } + + if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } + + /+ ↓ section 3: a comment is the file header only +/ + if (line[0] == '%') { + found ~= SspFinding(no, "3", "a comment outside the two header lines"); + continue; + } + + /+ ↓ block open +/ + if (line[0] == '@') { + auto brace = line.indexOf(" {"); + if (brace < 1 || line[$ - 1] != '{') { + found ~= SspFinding(no, "3", "block open is not \"@name {\""); + continue; + } + string name = line[1 .. brace]; + if (preamble_order.canFind(name)) { + int at = 0; + foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } + if (at <= preamble_at) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); + } + preamble_at = at; + if (section_at >= 0) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); + } + } else if (section_order.canFind(name)) { + int at = 0; + foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } + if (at <= section_at) { + found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); + } + section_at = at; + } else { + found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); + } + block = name; + doc_has_keys = 0; + continue; + } + + if (line[0] == '}') { + if (line.length != 1) { + found ~= SspFinding(no, "3", "block close carries more than \"}\""); + } + /+ ↓ section 5.4: ten keys, always +/ + if (block == "doc_has" && doc_has_keys != 10) { + found ~= SspFinding(no, "5.4", + "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); + } + block = ""; + in_object = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 5: inside a preamble block, " key: value" +/ + if (preamble_order.canFind(block)) { + if (!line.startsWith(" ") || line.indexOf(": ") < 3) { + found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); + } else if (block == "doc_has") { + doc_has_keys++; + } + continue; + } + + /+ ↓ section 6.1: the object declaration line +/ + if (line[0] == '[') { + auto close = line.indexOf("] "); + if (close < 2) { + found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); + continue; + } + string ocn = line[1 .. close]; + foreach (c; ocn) { + if (c < '0' || c > '9') { + found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); + break; + } + } + auto rest = line[close + 2 .. $].split(" "); + if (rest.length == 0 || !kinds.canFind(rest[0])) { + found ~= SspFinding(no, "6.1", + "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); + } else if (rest[0] == "heading") { + if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { + found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); + } else if (!levels.canFind(rest[1][1 .. $])) { + found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); + } + } + in_object = true; + in_text = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 6.3: properties, known and in order +/ + if (line[0] == '.') { + if (!in_object) { + found ~= SspFinding(no, "6", "a property outside an object record"); + continue; + } + if (in_text) { + found ~= SspFinding(no, "6", "a property after the object text began"); + } + auto colon = line.indexOf(": "); + if (colon < 2) { + found ~= SspFinding(no, "3", "property is not \".name: value\""); + continue; + } + string name = line[1 .. colon]; + int rank = propertyRank(name); + if (rank < 0) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); + } else { + /+ ↓ the four repeatable properties may recur at the same rank +/ + if (rank < prev_rank) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); + } + prev_rank = rank; + } + continue; + } + + /+ ↓ section 6.2: text lines +/ + if (line[0] == '|') { + if (!in_object) { + found ~= SspFinding(no, "6.2", "a text line outside an object record"); + } + if (line.length < 2 || line[1] != ' ') { + found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); + } + in_text = true; + continue; + } + + found ~= SspFinding(no, "3", "line of no shape the specification defines"); + } + + if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } + if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } + return found; +} + +void reportValidation(string filename, SspFinding[] found) { + if (found.length == 0) { + writeln("\nvalid: ", filename, + " conforms to the .ssp specification as this validator reads it"); + return; + } + writefln("\n%d finding%s against the specification:", + found.length, (found.length == 1) ? "" : "s"); + foreach (f; found) { + if (f.line > 0) { + writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); + } else { + writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); + } + } +} +#+END_SRC + +*** read verify images + +#+HEADER: :tangle "../dlang/src/read/verify.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.verify; + +import std.algorithm : sort; +import std.array : array, join; +import std.conv : to; +import std.digest : toHexString; +import std.digest.sha : SHA256, digest; +import std.file : dirEntries, exists, isDir, SpanMode, getSize; +import std.path : baseName, buildPath, extension; +import std.stdio : writeln, File; +import std.string : toLower; +import ocda.model; +import read.images : gatherImageRecords; + +/+ ↓ verification: recompute what the .image records claim, against the + images as they stand in a directory (a pod's media/image). + + ssp-reader --verify --image-dir=<pod>/media/image doc.ssp + + Reports one line per distinct file: + + OK won_benkler_2_1.png + MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... + ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) + PRESENT ffa.png (recorded missing, but the file is there) + EXTRA unused.png (in directory, no record in this .ssp) + + Returns the number of failures. EXTRA is reported but is not a failure: + a pod carries the images of every language version of a document, while + one .ssp covers one language. ++/ +int verifyImages(OcdaDocument doc, string image_dir, string output_file) { + string[] lines; + int failures = 0; + int ok = 0; + int mismatch = 0; + int absent = 0; + int extra = 0; + + if (!(image_dir.exists) || !(image_dir.isDir)) { + writeln("ERROR: image directory not found: ", image_dir); + return 1; + } + auto gathered = gatherImageRecords(doc); + bool[string] recorded; + + foreach (e; gathered.entries) { + recorded[e.name] = true; + string pth = buildPath(image_dir, e.name); + if (!(pth.exists)) { + if (e.missing) { + /+ recorded missing, and missing here too: consistent +/ + ok += 1; + lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; + } else { + absent += 1; + failures += 1; + lines ~= "ABSENT " ~ e.name + ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; + } + continue; + } + if (e.missing) { + mismatch += 1; + failures += 1; + lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; + continue; + } + auto file_ = readFileStat(pth); + string[] differs; + if (file_.sha256 != e.sha256) { + differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; + } + if (file_.bytes != e.bytes) { + differs ~= "bytes recorded " ~ e.bytes.to!string + ~ " file " ~ file_.bytes.to!string; + } + /+ px is written only where the dimensions could be read, and read here + only for the formats below, so compare only when both are known +/ + if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 + && (file_.width != e.width || file_.height != e.height)) { + differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string + ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; + } + if (differs.length > 0) { + mismatch += 1; + failures += 1; + lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); + } else { + ok += 1; + lines ~= "OK " ~ e.name; + } + } + + /+ ↓ files in the directory that no record names +/ + string[] in_dir; + foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { + if (!(entry.isFile)) continue; + string nm = entry.name.baseName; + switch (nm.extension.toLower) { + case ".png": case ".jpg": case ".jpeg": case ".gif": + in_dir ~= nm; + break; + default: break; + } + } + in_dir.sort; + foreach (nm; in_dir) { + if (nm !in recorded) { + extra += 1; + lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; + } + } + + string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ " in " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", against " ~ image_dir; + lines = [head, ""] ~ lines; + lines ~= ""; + lines ~= "verify: " ~ ok.to!string ~ " ok, " + ~ mismatch.to!string ~ " mismatch, " + ~ absent.to!string ~ " absent, " + ~ extra.to!string ~ " extra" + ~ ((failures == 0) ? " PASS" : " FAIL"); + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } + return failures; +} + +private struct FileStat_ { + string sha256; + ulong bytes; + int width; + int height; +} + +private FileStat_ readFileStat(string pth) { + FileStat_ ret; + auto f = File(pth, "rb"); + ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); + ret.sha256 = sha.toHexString.to!string; + ret.bytes = pth.getSize; + auto dim = readImageDimensions(pth); + ret.width = dim[0]; + ret.height = dim[1]; + return ret; +} + +/+ ↓ pixel dimensions from the file header, for the formats spine's markup + admits (png, gif, jpg). Header read only, no decoding, and no dependency + beyond phobos. Returns [0, 0] where the header cannot be read ++/ +private int[2] readImageDimensions(string pth) { + int[2] ret = [0, 0]; + ubyte[] head; + try { + auto f = File(pth, "rb"); + head = new ubyte[](64 * 1024); + head = f.rawRead(head); + } catch (Exception ex) { + return ret; + } + if (head.length >= 24 + && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { + /+ png: IHDR width & height, big endian, at 16 and 20 +/ + ret[0] = beInt(head[16 .. 20]); + ret[1] = beInt(head[20 .. 24]); + return ret; + } + if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { + /+ gif: logical screen width & height, little endian, at 6 and 8 +/ + ret[0] = head[6] | (head[7] << 8); + ret[1] = head[8] | (head[9] << 8); + return ret; + } + if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { + /+ jpeg: walk the segments to a start of frame, height then width, + big endian, at offsets 5 and 7 of the segment + +/ + size_t i = 2; + while (i + 9 < head.length) { + if (head[i] != 0xff) { i += 1; continue; } + ubyte marker = head[i + 1]; + if (marker == 0xd8 || marker == 0x01 + || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } + if (marker == 0xff) { i += 1; continue; } + size_t seg_len = (head[i + 2] << 8) | head[i + 3]; + bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf)); + if (is_sof) { + ret[1] = (head[i + 5] << 8) | head[i + 6]; + ret[0] = (head[i + 7] << 8) | head[i + 8]; + return ret; + } + if (seg_len < 2) break; + i += 2 + seg_len; + } + } + return ret; +} + +private int beInt(const(ubyte)[] b) { + return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); +} +#+END_SRC + +*** read report unknown + +#+HEADER: :tangle "../dlang/src/read/report_unknown.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.report_unknown; + +import std.algorithm : sort; +import std.array : array; +import std.stdio : writeln, writefln; +import ocda.model; + +/+ ↓ say what was read and not understood. + This is what makes the reader usable as a check on the format + specification: a reader that ignores what it does not know is + conforming, but one that cannot say what it ignored cannot tell you + whether the specification it was written from was complete. Silence + and understanding look the same from outside. ++/ +void reportUnknown(ref OcdaDocument doc) { + if (!doc.unknown.any) { + writeln("\nread and understood: every property, block and line was known"); + return; + } + writeln("\nread and NOT understood:"); + if (doc.unknown.property.length > 0) { + writeln(" properties ignored (", doc.unknown.property.length, " distinct):"); + foreach (name; doc.unknown.property.keys.sort) { + writefln(" .%-26s %8d", name, doc.unknown.property[name]); + } + } + if (doc.unknown.block.length > 0) { + writeln(" blocks ignored (", doc.unknown.block.length, " distinct):"); + foreach (name; doc.unknown.block.keys.sort) { + writefln(" @%-26s %8d", name, doc.unknown.block[name]); + } + } + if (doc.unknown.line.length > 0) { + writeln(" lines of no known shape, by first character:"); + foreach (c; doc.unknown.line.keys.sort) { + writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]); + } + } + writeln(" (ignoring these is conforming within a major version;", + " the count is so that it is not silent)"); +} +#+END_SRC + +** misc +*** read tree + +#+HEADER: :tangle "../dlang/src/read/tree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.tree; + +import std.conv : to; +import std.stdio : writeln, writef, writefln, File; +import std.array : replicate; +import ocda.model; + +void outputTree(OcdaDocument doc, string output_file) { + string[] lines; + + lines ~= "Document: " ~ doc.meta("title.main"); + if (doc.meta("title.subtitle").length > 0) + lines ~= " " ~ doc.meta("title.subtitle"); + lines ~= "Author: " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build OCN -> object lookup for body headings +/ + OcdaObject[int] headings; + foreach (obj; doc.body) { + if (obj.is_a == "heading" && obj.ocn > 0) { + headings[obj.ocn] = obj; + } + } + + /+ ↓ find root heading (from head section, or first body heading) +/ + if (doc.head.length > 0 && doc.head[0].is_a == "heading") { + auto root = doc.head[0]; + lines ~= formatHeading(root, 0); + walkChildren(root, headings, lines, 1); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { + foreach (child_ocn; parent.children) { + if (auto p = child_ocn in headings) { + lines ~= formatHeading(*p, depth); + walkChildren(*p, headings, lines, depth + 1); + } + } +} + +private string formatHeading(OcdaObject obj, int depth) { + string indent = replicate(" ", depth); + string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; + string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + /+ ↓ strip internal markup from heading text for clean display +/ + string text = stripInternalMarkup(obj.text); + return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; +} + +private string stripInternalMarkup(string text) { + /+ ↓ simple removal of common internal markers for display +/ + import std.regex : replaceAll, regex; + import std.array : replace; + string result = text; + result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("┨", "").replace("┣", "") + .replace("┙", " ").replace("┘", "\n"); + return result; +} +#+END_SRC + +*** read html + +#+HEADER: :tangle "../dlang/src/read/html.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.html; + +import std.array : replace, split, join, appender; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, startsWith; +import ocda.model; + +void outputHtmlScroll(OcdaDocument doc, string output_file) { + auto buf = appender!(string[]); + + buf ~= htmlHead(doc); + + /+ ↓ title +/ + if (doc.head.length > 0) { + auto title_obj = doc.head[0]; + buf ~= `<div class="title">`; + buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`; + if (doc.meta("title.subtitle").length > 0) + buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`; + buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`; + buf ~= `</div>`; + } + + /+ ↓ table of contents +/ + if (doc.toc.length > 0) { + buf ~= `<div class="toc">`; + buf ~= `<h2>Table of Contents</h2>`; + foreach (obj; doc.toc) { + if (obj.is_a == "toc") { + string text = convertInlineMarkup(obj.text); + int indent = obj.indent_base; + string cls = (indent > 4) ? "toc-sub" : "toc-entry"; + buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`; + } + } + buf ~= `</div>`; + buf ~= `<hr/>`; + } + + /+ ↓ body +/ + buf ~= `<div class="body">`; + foreach (obj; doc.body) { + buf ~= renderObject(obj); + } + buf ~= `</div>`; + + /+ ↓ endnotes +/ + bool has_endnotes = false; + foreach (obj; doc.endnotes) { + if (obj.is_a == "endnote") { has_endnotes = true; break; } + } + if (has_endnotes) { + buf ~= `<div class="endnotes">`; + buf ~= `<hr/>`; + buf ~= `<h2>Endnotes</h2>`; + foreach (obj; doc.endnotes) { + if (obj.is_a == "endnote") { + string text = convertInlineMarkup(obj.text); + string parent_ref = (obj.parent_ocn > 0) + ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>` + : ""; + buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`; + } + } + buf ~= `</div>`; + } + + /+ ↓ glossary +/ + if (doc.glossary.length > 0) { + buf ~= `<div class="glossary">`; + buf ~= `<hr/>`; + buf ~= `<h2>Glossary</h2>`; + foreach (obj; doc.glossary) { + string text = convertInlineMarkup(obj.text); + buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`; + } + buf ~= `</div>`; + } + + /+ ↓ bookindex +/ + bool has_bookindex = false; + foreach (obj; doc.bookindex) { + if (obj.is_a == "bookindex") { has_bookindex = true; break; } + } + if (has_bookindex) { + buf ~= `<div class="bookindex">`; + buf ~= `<hr/>`; + buf ~= `<h2>Book Index</h2>`; + foreach (obj; doc.bookindex) { + if (obj.is_a == "bookindex") { + string text = convertInlineMarkup(obj.text); + buf ~= `<p class="index-entry">` ~ text ~ `</p>`; + } + } + buf ~= `</div>`; + } + + buf ~= htmlTail(); + + /+ ↓ write output +/ + string[] output = buf.data; + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; output) f.writeln(line); + } else { + foreach (line; output) writeln(line); + } +} + +private string[] renderObject(OcdaObject obj) { + /+ ↓ markup commentary is carried by the abstraction and is not output. + Rendering it would put the author notes into the document, which + spine itself does not do + +/ + if (!obj.renders) { return []; } + string[] lines; + string text = convertInlineMarkup(obj.text); + string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) + ? ` id="o` ~ obj.ocn.to!string ~ `"` + : ""; + string ocn_display = (obj.ocn > 0 && !obj.ocn_off) + ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>` + : ""; + + switch (obj.is_a) { + case "heading": + string tag = headingTag(obj.heading_level); + string seg_anchor = (obj.segment.length > 0) + ? ` id="` ~ htmlEscape(obj.segment) ~ `"` + : ocn_anchor; + lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`; + break; + case "para": + string indent_cls = ""; + if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`; + else if (obj.bullet) indent_cls = ` class="bullet"`; + lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`; + break; + case "code": + lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`; + break; + case "table": + lines ~= renderTable(obj, ocn_anchor); + break; + case "group": + lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`; + break; + case "block": + lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`; + break; + case "verse": + lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`; + break; + case "quote": + lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`; + break; + default: + if (text.length > 0) + lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`; + break; + } + return lines; +} + +private string headingTag(string level) { + switch (level) { + case "A": return "h1"; + case "B": return "h2"; + case "C": return "h3"; + case "D": return "h4"; + case "1": return "h3"; + case "2": return "h4"; + case "3": return "h5"; + case "4": return "h6"; + default: return "h4"; + } +} + +private string renderTable(OcdaObject obj, string ocn_anchor) { + string[] lines; + lines ~= `<table` ~ ocn_anchor ~ `>`; + /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/ + auto rows = obj.text.split("\n"); + bool first_row = obj.table_header; + foreach (row; rows) { + if (row.strip.length == 0) continue; + string tag = first_row ? "th" : "td"; + lines ~= `<tr>`; + foreach (cell; row.split("┊")) { + lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`; + } + lines ~= `</tr>`; + first_row = false; + } + lines ~= `</table>`; + return lines.join("\n"); +} + +/+ ↓ convert spine internal Unicode markup to HTML +/ +string convertInlineMarkup(string text) { + if (text.length == 0) return ""; + string result = htmlEscape(text); + + /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc. + Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X + The ┨┣ form is used within heading content +/ + + /+ ↓ bold +/ + result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>"); + result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>"); + + /+ ↓ emphasis +/ + result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>"); + result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>"); + + /+ ↓ italic +/ + result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>"); + result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>"); + + /+ ↓ underscore +/ + result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>"); + result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>"); + + /+ ↓ superscript +/ + result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>"); + result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>"); + + /+ ↓ subscript +/ + result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>"); + result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>"); + + /+ ↓ monospace +/ + result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>"); + + /+ ↓ links: ┥link text┝┤url├ +/ + result = replaceLinks(result); + + /+ ↓ footnote markers: 【N text】 +/ + result = replaceFootnotes(result); + + /+ ↓ line breaks +/ + result = result.replace("┘", "<br/>"); + result = result.replace("┙", "<br/>"); + + /+ ↓ non-breaking spaces +/ + result = result.replace("░", " "); + + /+ ↓ bullet markers +/ + result = result.replace("● ", "• "); + + /+ ↓ page break markers +/ + result = result.replace("┼", "").replace("┿", "").replace("╂", ""); + + return result; +} + +private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) { + /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/ + string result = text; + while (true) { + auto open_pos = result.indexOf(open_mark); + if (open_pos < 0) break; + auto after_open = open_pos + open_mark.length; + /+ ↓ look for closing ⑆X +/ + string close_mark = "⑆" ~ close_suffix; + auto close_pos = indexOf(result[after_open .. $], close_mark); + if (close_pos < 0) break; + close_pos += after_open; + string content = result[after_open .. close_pos]; + result = result[0 .. open_pos] ~ html_open ~ content ~ html_close + ~ result[close_pos + close_mark.length .. $]; + } + return result; +} + +private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) { + /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/ + string result = text; + string full_open = open_mark ~ bracket_open; + while (true) { + auto open_pos = result.indexOf(full_open); + if (open_pos < 0) break; + auto after_open = open_pos + full_open.length; + auto close_pos = indexOf(result[after_open .. $], bracket_close); + if (close_pos < 0) break; + close_pos += after_open; + string content = result[after_open .. close_pos]; + result = result[0 .. open_pos] ~ html_open ~ content ~ html_close + ~ result[close_pos + bracket_close.length .. $]; + } + return result; +} + +private string replaceLinks(string text) { + /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/ + string result = text; + while (true) { + auto link_open = result.indexOf("┥"); + if (link_open < 0) break; + auto link_mid = indexOf(result[link_open .. $], "┝┤"); + if (link_mid < 0) break; + link_mid += link_open; + auto link_close = indexOf(result[link_mid .. $], "├"); + if (link_close < 0) break; + link_close += link_mid; + string link_text = result[link_open + "┥".length .. link_mid]; + string url = result[link_mid + "┝┤".length .. link_close]; + /+ ↓ handle internal links (¤ prefix) +/ + if (url.length > 0 && url[0] == '¤') { + url = url[("¤").length .. $]; + } + /+ ↓ handle OCN-only links (#N) +/ + if (url.length > 0 && url[0] == '#') { + url = "#o" ~ url[1 .. $]; + } + result = result[0 .. link_open] + ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>` + ~ result[link_close + "├".length .. $]; + } + return result; +} + +private string replaceFootnotes(string text) { + /+ ↓ replace 【N content】 with superscript note number +/ + string result = text; + while (true) { + auto fn_open = result.indexOf("【"); + if (fn_open < 0) break; + auto fn_close = indexOf(result[fn_open .. $], "】"); + if (fn_close < 0) break; + fn_close += fn_open; + string fn_content = result[fn_open + "【".length .. fn_close]; + /+ ↓ extract note number (first token before space) +/ + auto space_pos = fn_content.indexOf(' '); + string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content; + result = result[0 .. fn_open] + ~ `<sup class="fn">` ~ note_num ~ `</sup>` + ~ result[fn_close + "】".length .. $]; + } + return result; +} + +string htmlEscape(string text) { + return text.replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\"", """); +} + +private string[] htmlHead(OcdaDocument doc) { + string[] lines; + lines ~= `<!DOCTYPE html>`; + lines ~= `<html>`; + lines ~= `<head>`; + lines ~= `<meta charset="utf-8"/>`; + lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`; + lines ~= `<style>`; + lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`; + lines ~= ` .title { text-align: center; margin-bottom: 2em; }`; + lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`; + lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`; + lines ~= ` .toc { margin-bottom: 2em; }`; + lines ~= ` .toc-entry { margin: 0.2em 0; }`; + lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`; + lines ~= ` .indent-1 { margin-left: 2em; }`; + lines ~= ` .indent-2 { margin-left: 4em; }`; + lines ~= ` .indent-3 { margin-left: 6em; }`; + lines ~= ` .bullet { margin-left: 2em; }`; + lines ~= ` .bullet:before { content: "\2022 "; }`; + lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`; + lines ~= ` .group { margin: 1em 2em; }`; + lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`; + lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`; + lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`; + lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`; + lines ~= ` th { background: #f0f0f0; }`; + lines ~= ` .endnote { font-size: 0.9em; }`; + lines ~= ` .fn { color: #666; }`; + lines ~= ` .index-entry { font-size: 0.9em; }`; + lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`; + lines ~= `</style>`; + lines ~= `</head>`; + lines ~= `<body>`; + return lines; +} + +private string[] htmlTail() { + return [`</body>`, `</html>`]; +} +#+END_SRC + +*** read images + +#+HEADER: :tangle "../dlang/src/read/images.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.images; + +import std.array : join; +import std.conv : to; +import std.stdio : writeln, File; +import ocda.model; + +/+ ↓ the .image records of a document, gathered per distinct file, with the + objects referencing each file, in order of first appearance +/ +struct ImageEntry { + string name; + string sha256; + ulong bytes; + int width; + int height; + bool missing; + string[] refs; + string[] other_sha256; // same filename, differing digest +} + +struct ImageRecords { + int records; // .image lines, an image referenced twice counts twice + ImageEntry[] entries; // distinct files +} + +ImageRecords gatherImageRecords(OcdaDocument doc) { + ImageRecords ret; + size_t[string] seen; + foreach (section; sectionNames) { + auto objs = doc.sectionByName(section); + if (objs is null) continue; + foreach (obj; *objs) { + foreach (img; obj.images) { + ret.records += 1; + string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]"; + if (auto at = img.name in seen) { + if (ret.entries[*at].sha256 != img.sha256 + && img.sha256.length > 0 + && ret.entries[*at].sha256.length > 0) { + ret.entries[*at].other_sha256 ~= img.sha256; + } + ret.entries[*at].refs ~= _ref; + } else { + ImageEntry e_; + e_.name = img.name; + e_.sha256 = img.sha256; + e_.bytes = img.bytes; + e_.width = img.width; + e_.height = img.height; + e_.missing = img.missing; + e_.refs = [_ref]; + seen[img.name] = ret.entries.length; + ret.entries ~= e_; + } + } + } + } + return ret; +} + +/+ ↓ image manifest, line oriented and greppable: + + <sha256> <bytes> <px> <filename> refs: body[97] body[412] + MISSING <filename> refs: body[840] + + The point of the record in the .ssp is verification (is the image + shipped in this pod the image the abstraction was built from), so this + output is the shape a verifier wants: one line per distinct file. ++/ +void outputImages(OcdaDocument doc, string output_file) { + auto gathered = gatherImageRecords(doc); + string[] lines; + int missing = 0; + foreach (e; gathered.entries) { + if (e.missing) missing += 1; + } + lines ~= "Images: " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ ", " ~ missing.to!string ~ " missing"; + lines ~= ""; + foreach (e; gathered.entries) { + string px = (e.width > 0 && e.height > 0) + ? e.width.to!string ~ "x" ~ e.height.to!string : "-"; + lines ~= (e.missing) + ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ") + : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px + ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" "); + /+ ↓ one filename, more than one digest: the .ssp is inconsistent, + or two distinct files share a name +/ + foreach (other; e.other_sha256) { + lines ~= " WARNING differing digest for same filename: " ~ other; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +/+ ↓ count of .image records in a document, for the parse summary +/ +int imageRecordCount(OcdaDocument doc) { + return gatherImageRecords(doc).records; +} + +package immutable string[] sectionNames = [ + "head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", +]; +#+END_SRC + +** llm +*** read llm text + +#+HEADER: :tangle "../dlang/src/read/llm_text.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_text; + +import std.array : replace, split, join; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; + +/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption + format: + ## Document Title > Chapter > Section > [OCN] + + Clean paragraph text with inline markup stripped. ++/ +void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { + string[] lines; + string doc_title = doc.meta("title.main"); + + /+ ↓ document header +/ + lines ~= "# " ~ doc_title; + if (doc.meta("title.subtitle").length > 0) + lines ~= "# " ~ doc.meta("title.subtitle"); + lines ~= "# " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build heading text lookup from ancestors +/ + string[int] heading_text; + foreach (obj; doc.head) { + if (obj.is_a == "heading" && obj.ocn > 0) + heading_text[obj.ocn] = stripMarkup(obj.text); + } + foreach (obj; doc.body) { + if (obj.is_a == "heading") + heading_text[obj.ocn] = stripMarkup(obj.text); + } + + /+ ↓ body objects +/ + string current_breadcrumb = ""; + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); + lines ~= ""; + lines ~= "## " ~ current_breadcrumb; + lines ~= ""; + continue; + } + + /+ ↓ markup commentary is not content +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) + ? " [" ~ obj.ocn.to!string ~ "]" + : ""; + + switch (obj.is_a) { + case "para": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "code": + lines ~= "```"; + lines ~= obj.text; /+ preserve raw text for code +/ + lines ~= "```"; + lines ~= ""; + break; + case "table": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "verse": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "quote": + lines ~= "> " ~ text ~ ocn_ref; + lines ~= ""; + break; + case "group": + case "block": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + default: + if (text.length > 0) { + lines ~= text ~ ocn_ref; + lines ~= ""; + } + break; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { + /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + /+ ↓ truncate very long heading text for breadcrumb +/ + if (clean.length > 60) + clean = clean[0 .. 57] ~ "..."; + parts ~= clean; + } + } + } + /+ ↓ add the current heading itself +/ + string heading_text = stripMarkup(obj.text).strip; + if (heading_text.length > 60) + heading_text = heading_text[0 .. 57] ~ "..."; + + string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + + if (parts.length > 0) + return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; + else + return doc_title ~ " > " ~ heading_text ~ ocn_ref; +} + +/+ ↓ strip all internal Unicode markup to produce clean plain text +/ +string stripMarkup(string text) { + if (text.length == 0) return ""; + string r = text; + + /+ ↓ links: ┥link text┝┤url├ -> link text +/ + while (true) { + auto lo = r.indexOf("┥"); + if (lo < 0) break; + auto lm = indexOf(r[lo .. $], "┝┤"); + if (lm < 0) break; + lm += lo; + auto lc = indexOf(r[lm .. $], "├"); + if (lc < 0) break; + lc += lm; + string link_text = r[lo + "┥".length .. lm]; + r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; + } + + /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ + while (true) { + auto fo = r.indexOf("【"); + if (fo < 0) break; + auto fc = indexOf(r[fo .. $], "】"); + if (fc < 0) break; + fc += fo; + r = r[0 .. fo] ~ r[fc + "】".length .. $]; + } + + /+ ↓ font face markers +/ + r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("⑆■", "") + .replace("┨", "").replace("┣*", "").replace("┣!", "") + .replace("┣/", "").replace("┣_", "").replace("┣^", "") + .replace("┣,", "").replace("┣", ""); + + /+ ↓ line breaks and special chars +/ + r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); + r = r.replace("░", " "); + r = r.replace("● ", "- "); + r = r.replace("┼", "").replace("┿", "").replace("╂", ""); + + /+ ↓ table cell separators +/ + r = r.replace("┊", " | "); + + /+ ↓ clean up multiple spaces +/ + while (r.indexOf(" ") >= 0) + r = r.replace(" ", " "); + + return r.strip; +} +#+END_SRC + +*** read llm jsonl + +#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} +#+END_SRC + +* test +** tests +*** test agree + +#+HEADER: :tangle "../dlang/test/agree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + Do the two loaders agree? + + The library's whole purpose is that a writer cannot tell which + serialisation it was handed. That is only true if loading a document's + .ssp and loading its .ocda.db give the same model, so this asks. + + Usage: agree <doc>.ssp <doc>.ocda.db [lang] ++/ +import std.conv : to; +import std.stdio; +import ocda.model; +import ocda.load; + +int main(string[] args) { + if (args.length < 3) { + stderr.writeln("usage: agree <doc>.ssp <doc>.ocda.db [lang]"); + return 2; + } + string lang = (args.length > 3) ? args[3] : ""; + OcdaDocument a, b; + try { + a = load(args[1]); + b = load(args[2], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + + int differences = 0; + void differ(string what, string from_ssp, string from_db) { + writeln(" differ: ", what); + writeln(" .ssp: ", from_ssp); + writeln(" .ocda.db: ", from_db); + differences++; + } + + if (a.format_version != b.format_version) { + differ("format_version", a.format_version, b.format_version); + } + if (a.source_filename != b.source_filename) { + differ("source_filename", a.source_filename, b.source_filename); + } + + /+ ↓ the sections, object by object. Position within a section is the + identity to compare on: an ocn is not unique and 0 is shared +/ + static immutable sections = ["head", "toc", "body", "endnotes", "glossary", + "bibliography", "bookindex", "blurb", "tail"]; + foreach (name; sections) { + auto sa = a.sectionByName(name); + auto sb = b.sectionByName(name); + if (sa is null || sb is null) { continue; } + if ((*sa).length != (*sb).length) { + differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string); + continue; + } + foreach (i, oa; *sa) { + auto ob = (*sb)[i]; + string at = name ~ "[" ~ i.to!string ~ "]"; + if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); } + if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); } + if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); } + if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); } + if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); } + if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); } + if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); } + if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); } + if (oa.ancestors != ob.ancestors) { + differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string); + } + if (oa.images.length != ob.images.length) { + differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string); + } + if (differences > 12) { writeln(" ... stopping after 12"); return 1; } + } + } + + if (differences == 0) { + writeln("agree: the two loaders give the same model"); + return 0; + } + writeln(differences, " difference(s)"); + return 1; +} +#+END_SRC + +*** test dump + +#+HEADER: :tangle "../dlang/test/dump.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + Write a loaded abstraction out canonically. + + The oracle for everything that claims to implement this model: load a + .ssp or a .ocda.db and print the model, deterministically, so that two + implementations can be compared with diff. + + Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang] ++/ +import std.stdio; +import ocda.model; +import ocda.load; +import ocda.dump; + +int main(string[] args) { + bool unknown = false; + string[] rest; + foreach (a; args[1 .. $]) { + if (a == "--unknown") { unknown = true; } else { rest ~= a; } + } + if (rest.length < 1) { + stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]"); + return 2; + } + string lang = (rest.length > 1) ? rest[1] : ""; + OcdaDocument doc; + try { + doc = load(rest[0], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); + return 0; +} +#+END_SRC + +* config +** nix flake.nix :flake: + +#+HEADER: :tangle ../dlang/flake.nix +#+HEADER: :noweb yes +#+BEGIN_SRC nix +{ + description = "ocda-lib: the object-centric document abstraction as a D library"; + inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + /* A library, so the library itself is not a build product: a program + that uses it compiles its modules alongside its own, as the writer's + flake does, or takes it as a dub dependency. What is built here are + programs over it: the reader, and the two that put the library to + the question - the agreement check, which loads a document's .ssp + and its .ocda.db and compares the two models, and the dump, which + writes a loaded model out canonically so that an implementation in + another language can be diffed against this one. + . + `mains` is what is compiled besides the library: the checks are one + file each in test/, the reader is src/read/. + */ + mkTool = { tool, binary, what, mains, compiler, compilerBin }: + pkgs.stdenv.mkDerivation { + pname = "ocda-lib-${tool}"; + version = "0.1.0"; + src = self; + nativeBuildInputs = [ compiler ]; + buildInputs = [ pkgs.sqlite pkgs.zstd ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ${tool} $out/bin/${binary} + runHook postInstall + ''; + meta = { + description = what; + mainProgram = binary; + }; + }; + ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; + ocda-read = mkTool (ldc // { + tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; + what = "reads the abstraction in any of its forms and says what is in it"; + }); + ocda-agree = mkTool (ldc // { + tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; + what = "checks that the .ssp and .ocda.db loaders give one model"; + }); + ocda-dump = mkTool (ldc // { + tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; + what = "writes a loaded abstraction out canonically, as an oracle"; + }); + in { + packages.${system} = { + /* All three in one bin/, so that one build leaves every program in + one result link. Built separately into the same link, each build + replaced the one before, and a script looking for ocda-dump beside + ocda-agree found nothing. + */ + default = pkgs.symlinkJoin { + name = "ocda-lib-programs-0.1.0"; + paths = [ ocda-read ocda-agree ocda-dump ]; + }; + inherit ocda-read ocda-agree ocda-dump; + }; + + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; + }; + }; +} +#+END_SRC + +** README.md + +#+HEADER: :tangle ../dlang/README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# ocda-lib, in D + +The abstraction's model and both its loaders, as a dub library. + +## Modules + +| | | +|---|---| +| `ocda.model` | the document, the object, the eight sections. Imports nothing | +| `ocda.load_ssp` | text -> model | +| `ocda.load_db` | sqlite -> model | +| `ocda.load` | what a path is, and loading it | +| `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries | +| `ocda.dump` | the model written out canonically, as an oracle | +| `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon | +| `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) | + +`ocda.model` importing nothing is the property that matters: a writer +depends on the model and on neither loader. + +## Use + +```json +"dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } } +``` + +```d +import ocda.load; +auto doc = load("document.ocda.db", "de"); // or "document.ssp" +foreach (obj; doc.body) { ... } +``` + +Link `sqlite3`; the library declares the dozen functions it needs rather +than taking a binding from the registry, so there is nothing to fetch. + +## The reader + +```sh +nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl| + --images|--verify --image-dir=<d>|--languages|--files|--report-unknown|--validate] + <file>.ssp|<file>.ocda.db|<file>.sisupod [lang] +``` + +The library's reader, one program where `sisudoc-ssp-reader` and +`sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp` +reader's, moved onto the library's model, so every mode reads every form +and **says the same thing of a document's `.ssp`, `.ocda.db` and +`.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries +files; `--validate` checks a `.ssp` as text and refuses anything else. + +It lives in `src/read/` beside the library, and dub keeps it out of the +library: the configuration `library` excludes `src/read/*`, the +configuration `ocda-read` builds it. A program using the library by dub, or +by naming `src/ocda/*.d` as the writer's flake does, never compiles it. + +## The two checks + +```sh +nix build .#ocda-agree && ./result/bin/ocda-agree <doc>.ssp <doc>.ocda.db [lang] +nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] <file> [lang] +``` + +`agree` loads both and compares ten fields of each object, naming the field +where they part company. `agree` found the blank-first-text-line defect and +was not enough for anything subtler. + +**`ocda-dump` is the stronger check**: the whole loaded model, every field, +written out in a form fully determined by the model, so that two +implementations can be compared with `diff`. It is an oracle, not an output +format; anything reading it to reconstruct a document has misunderstood it. +It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is +the point of having it. + +It found four disagreements `agree` could not see. They are in the +repository README and in the history. + +## Build + +```sh +nix build # all three programs, in one result/bin +nix build .#ocda-read # the reader +nix build .#ocda-agree # the agreement check +nix build .#ocda-dump # the canonical dump +nix develop # ldc, dub, dtools, sqlite, zstd +dub build # the library +dub build --config=ocda-read # the reader +``` +#+END_SRC + +** dub.json + +#+HEADER: :tangle ../dlang/dub.json +#+HEADER: :noweb yes +#+BEGIN_SRC json +{ + "authors": [ "Ralph Amissah" ], + "copyright": "Copyright (C) 2026 Ralph Amissah", + "name": "ocda-lib", + "version": "0.1.0", + "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations", + "homepage": "https://sisudoc.org", + "license": "AGPL-3.0+", + "sourcePaths": [ "./src" ], + "importPaths": [ "./src" ], + "libs": [ "sqlite3", "zstd" ], + "configurations": [ + { + "name": "library", + "targetType": "library", + "excludedSourceFiles": [ "src/read/*" ] + }, + { + "name": "ocda-read", + "targetType": "executable", + "targetName": "ocda-read", + "mainSourceFile": "src/read/main.d" + } + ] +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* nixpkgs source + +#+NAME: nixpkgs_unstable +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> +#+END_SRC + +* __END__ diff --git a/org/gleam-read.org b/org/gleam-read.org deleted file mode 100644 index 05fa2d5..0000000 --- a/org/gleam-read.org +++ /dev/null @@ -1,520 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: show2levels hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** main -*** tangle - -#+HEADER: :tangle ../gleam/src/read/main.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<read>> -#+END_SRC - -*** source - -#+NAME: read -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// ocda-read, in Gleam: reads the abstraction, in any of the forms the -//// library reads, and says what is in it. -//// -//// ocda-read [--tree|--outline|--languages|--files|--report-unknown] -//// <file>.ssp | <file>.ocda.db | <file>.zip [lang] -//// -//// The modes the Gleam readers had (the summary, the outline, the unknown -//// report, and the database's languages and files), over the library, so -//// that each reads every form. Their output is the D ocda-read's, byte for -//// byte: two readers in unrelated languages, neither written from the other -//// line by line, agreeing is what says each has read the model and not its -//// own idea of it. A `.sisupod` is the one form the Gleam library cannot -//// open (its zstd frame; see `ocda/load_pod`), and it says so. - -import gleam/dict -import gleam/int -import gleam/io -import gleam/list -import gleam/result -import gleam/string -import ocda/inventory.{type Language} -import ocda/load -import ocda/model.{type Document, type Object} - -/// The format major this reader knows. -const major_read = "2" - -type Options { - Options( - tree: Bool, - languages: Bool, - files: Bool, - report_unknown: Bool, - any_version: Bool, - lang: String, - positional: List(String), - ) -} - -/// Run with the arguments after the program name; returns the exit status. -pub fn run(args: List(String)) -> Int { - case parse(args, Options(False, False, False, False, False, "", [])) { - Error(message) -> { - io.println_error("ocda-read: " <> message <> "; --help lists the options") - 2 - } - Ok(Options(positional: [], ..)) -> { - help() - 0 - } - Ok(options) -> read(options) - } -} - -fn parse(args: List(String), o: Options) -> Result(Options, String) { - case args { - [] -> Ok(Options(..o, positional: list.reverse(o.positional))) - ["--help", ..] | ["-h", ..] -> Ok(Options(..o, positional: [])) - ["--tree", ..rest] | ["--outline", ..rest] -> - parse(rest, Options(..o, tree: True)) - ["--languages", ..rest] -> parse(rest, Options(..o, languages: True)) - ["--files", ..rest] -> parse(rest, Options(..o, files: True)) - ["--report-unknown", ..rest] -> - parse(rest, Options(..o, report_unknown: True)) - ["--any-version", ..rest] -> parse(rest, Options(..o, any_version: True)) - ["--lang", code, ..rest] -> parse(rest, Options(..o, lang: code)) - [flag, ..rest] -> - case string.starts_with(flag, "--lang=") { - True -> parse(rest, Options(..o, lang: string.drop_start(flag, 7))) - False -> - case string.starts_with(flag, "-") { - True -> - case - list.contains( - [ - "--html", "--llm-text", "--llm-jsonl", "--images", "--verify", - "--validate", - ], - flag, - ) - { - True -> - Error(flag <> " is a mode of the D ocda-read, not of this one") - False -> Error("Unrecognized option " <> flag) - } - False -> - parse(rest, Options(..o, positional: [flag, ..o.positional])) - } - } - } -} - -fn help() -> Nil { - io.println("ocda-read - reads the document abstraction, in any of its forms") - io.println("") - io.println( - "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.zip [lang]", - ) - io.println("") - io.println(" --tree the headings, as a tree") - io.println(" --outline the same as --tree") - io.println(" --languages the languages the file holds") - io.println(" --files what a .ocda.db carries besides the abstraction") - io.println(" --report-unknown what was read and not understood") - io.println(" --lang=<code> the language to read (default: the first)") - io.println(" --any-version read a format major this reader does not know") -} - -fn read(o: Options) -> Int { - let assert [path, ..more] = o.positional - let lang = case o.lang, more { - "", [given, ..] -> given - l, _ -> l - } - let lang_named = lang != "" - let loaded = { - use held <- result.try(inventory.languages(path)) - let lang = case lang, held { - "", [first, ..] -> first.lang - l, _ -> l - } - use _ <- result.try(case - held != [] && !list.any(held, fn(l) { l.lang == lang }) - { - True -> - Error( - "no language '" - <> lang - <> "' in " - <> path - <> "; it holds: " - <> string.join(list.map(held, fn(l) { l.lang }), " "), - ) - False -> Ok(Nil) - }) - use doc <- result.try(load.load(path, lang, o.any_version)) - Ok(#(held, lang, doc)) - } - case loaded { - Error(message) -> { - io.println_error("Error reading " <> path <> ": " <> message) - 2 - } - Ok(#(held, lang, doc)) -> - case gate(path, doc, o.any_version) { - False -> 3 - True -> { - summary(path, doc, held, lang, lang_named) - case o.languages { - True -> print_languages(held) - False -> Nil - } - case o.files { - True -> print_files(path) - False -> Nil - } - case o.tree { - True -> tree(doc) - False -> Nil - } - case o.report_unknown { - True -> report_unknown(doc) - False -> Nil - } - case o.tree || o.languages || o.files || o.report_unknown { - True -> Nil - False -> - io.println( - "\n(this is the summary: --help lists the modes, each adding its output)", - ) - } - 0 - } - } - } -} - -/// The major part, and refuse what is not known. -fn gate(path: String, doc: Document, any_version: Bool) -> Bool { - case doc.format_version, model.version_major(doc) == major_read, any_version { - "", _, _ -> { - io.println_error("warning: " <> path <> " declares no format version") - True - } - _, True, _ -> True - v, False, True -> { - io.println_error( - "warning: " - <> path - <> " is format v" - <> v - <> ", this reader knows v" - <> major_read - <> ".x; reading anyway", - ) - True - } - v, False, False -> { - io.println_error( - "error: " - <> path - <> " is format v" - <> v - <> ", this reader knows v" - <> major_read - <> ".x", - ) - io.println_error( - " a major version differs: a property may have changed meaning or gone away.", - ) - io.println_error(" --any-version reads it regardless.") - False - } - } -} - -const sections = [ - "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", - "blurb", "tail", -] - -/// The sections whose images are counted: every one but the tail, as the D -/// reader counts them. -const image_sections = [ - "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", - "blurb", -] - -fn summary( - path: String, - doc: Document, - held: List(Language), - lang: String, - lang_named: Bool, -) -> Nil { - io.println("Read: " <> path) - case doc.format_version { - "" -> Nil - v -> io.println(" Format: v" <> v) - } - case dict.get(doc.source, "language") { - Error(_) -> Nil - Ok(language) -> { - let of = case dict.get(doc.source, "languages") { - Ok(ls) -> " of " <> ls - Error(_) -> "" - } - let digest = case dict.get(doc.source, "digest") { - Ok(d) -> ", markup digest " <> string.slice(d, 0, 16) <> "..." - Error(_) -> "" - } - io.println(" Source: " <> language <> of <> digest) - } - } - case list.length(held) > 1 && !lang_named { - True -> io.println(" (reading " <> lang <> "; --lang=<code> for another)") - False -> Nil - } - io.println(" Title: " <> model.meta(doc, "title.main")) - io.println(" Author: " <> model.meta(doc, "creator.author")) - io.println( - " Sections: " - <> string.join( - list.map(sections, fn(s) { - s <> "=" <> int.to_string(list.length(model.section(doc, s))) - }), - " ", - ), - ) - let records = - list.fold(image_sections, 0, fn(n, s) { - list.fold(model.section(doc, s), n, fn(n, o) { - n + list.length(o.images) - }) - }) - case records { - 0 -> Nil - 1 -> io.println(" Images: 1 record") - n -> io.println(" Images: " <> int.to_string(n) <> " records") - } -} - -fn print_languages(held: List(Language)) -> Nil { - io.println("") - io.println(" lang source ssp digest") - list.each(held, fn(l) { - let digest = case string.length(l.ssp_digest) > 16 { - True -> string.slice(l.ssp_digest, 0, 16) <> "..." - False -> l.ssp_digest - } - io.println( - " " - <> string.pad_end(l.lang, 7, " ") - <> " " - <> string.pad_end(l.source_filename, 31, " ") - <> " " - <> digest, - ) - }) -} - -fn print_files(path: String) -> Nil { - io.println("") - case inventory.is_carrier(path) { - False -> - io.println( - " carried files: only a .ocda.db carries files; this is " - <> load.source_name(load.source_of(path)), - ) - True -> - case inventory.carried(path) { - Error(message) -> io.println_error("error: " <> message) - Ok([]) -> io.println(" carries no files") - Ok(rows) -> { - io.println(" carried files:") - io.println(" role stored n original in the file") - list.each(rows, fn(r) { - io.println( - " " - <> string.pad_end(r.role, 9, " ") - <> " " - <> string.pad_end(r.compression, 7, " ") - <> " " - <> string.pad_end(r.count, 5, " ") - <> " " - <> string.pad_end(r.bytes, 13, " ") - <> " " - <> r.stored, - ) - }) - } - } - } -} - -/// The headings, as a tree: from the document's root heading, down the -/// children each heading lists, as the D reader walks it. -fn tree(doc: Document) -> Nil { - io.println("Document: " <> model.meta(doc, "title.main")) - case model.meta(doc, "title.subtitle") { - "" -> Nil - sub -> io.println(" " <> sub) - } - io.println("Author: " <> model.meta(doc, "creator.author")) - io.println("") - let headings = - model.section(doc, "body") - |> list.filter(fn(o) { o.is_a == "heading" && o.ocn > 0 }) - |> list.map(fn(o) { #(o.ocn, o) }) - |> dict.from_list - case model.section(doc, "head") { - [root, ..] if root.is_a == "heading" -> { - io.println(heading_line(root, 0)) - walk(root, headings, 1) - } - _ -> Nil - } -} - -fn walk(parent: Object, headings: dict.Dict(Int, Object), depth: Int) -> Nil { - list.each(parent.children, fn(ocn) { - case dict.get(headings, ocn) { - Ok(child) -> { - io.println(heading_line(child, depth)) - walk(child, headings, depth + 1) - } - Error(_) -> Nil - } - }) -} - -fn heading_line(o: Object, depth: Int) -> String { - let level = case o.heading_level { - "" -> "?" - l -> l - } - let ocn = case o.ocn > 0 { - True -> " [" <> int.to_string(o.ocn) <> "]" - False -> "" - } - string.repeat(" ", depth) <> level <> "~" <> ocn <> " " <> strip_markup(o.text) -} - -/// The internal markers a heading's text may carry, removed for display, as -/// the D reader removes them. -fn strip_markup(text: String) -> String { - text - |> string.replace("⑆*", "") - |> string.replace("⑆!", "") - |> string.replace("⑆/", "") - |> string.replace("⑆_", "") - |> string.replace("⑆^", "") - |> string.replace("⑆,", "") - |> string.replace("┨", "") - |> string.replace("┣", "") - |> string.replace("┙", " ") - |> string.replace("┘", "\n") -} - -/// Say what was read and not understood. -fn report_unknown(doc: Document) -> Nil { - let u = doc.unknown - case model.any_unknown(u) { - False -> - io.println( - "\nread and understood: every property, block and line was known", - ) - True -> { - io.println("\nread and NOT understood:") - section_of_unknown(u.property, " properties ignored (", fn(name, n) { - " ." - <> string.pad_end(name, 26, " ") - <> " " - <> string.pad_start(int.to_string(n), 8, " ") - }) - section_of_unknown(u.block, " blocks ignored (", fn(name, n) { - " @" - <> string.pad_end(name, 26, " ") - <> " " - <> string.pad_start(int.to_string(n), 8, " ") - }) - case dict.size(u.line) { - 0 -> Nil - _ -> { - io.println(" lines of no known shape, by first character:") - dict.to_list(u.line) - |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) - |> list.each(fn(e) { - io.println( - " " - <> string.pad_end("\"" <> e.0 <> "\"", 27, " ") - <> " " - <> string.pad_start(int.to_string(e.1), 8, " "), - ) - }) - } - } - io.println( - " (ignoring these is conforming within a major version; the count is so that it is not silent)", - ) - } - } -} - -fn section_of_unknown( - counts: dict.Dict(String, Int), - heading: String, - line: fn(String, Int) -> String, -) -> Nil { - case dict.size(counts) { - 0 -> Nil - n -> { - io.println(heading <> int.to_string(n) <> " distinct):") - dict.to_list(counts) - |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) - |> list.each(fn(e) { io.println(line(e.0, e.1)) }) - } - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-test.org b/org/gleam-test.org deleted file mode 100644 index 402c6e0..0000000 --- a/org/gleam-test.org +++ /dev/null @@ -1,209 +0,0 @@ --*- mode: org -*- -#+TITLE: gleam lib -#+DESCRIPTION: gleam lib -#+FILETAGS: :spine:build:tools: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -* gleam :gleam: -** gleam -*** tangle - -#+HEADER: :tangle ../gleam/test/ocda_lib_test.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<test>> -#+END_SRC - -*** gleam source - -#+NAME: test -#+HEADER: :no-expand -#+BEGIN_SRC gleam -import gleam/dict -import gleam/list -import simplifile -import gleam/string -import gleeunit -import gleeunit/should -import ocda/dump -import ocda/load -import ocda/load_ssp -import ocda/model - -pub fn main() { - gleeunit.main() -} - -const minimal = "% SiSU Document Abstraction v2.0 -% Source: example.sst - -@source { - language: en - languages: en - digest: ABC -} - -@meta { - title.main: An Example -} - -@body { -[1] heading :A the_title -.part: frontmatter -.sha256: DEAD -| An Example - -[2] para -.part: body -.parent: 1 -| First line -| -| Third line, after a blank one - -} -" - -pub fn version_test() { - let doc = load_ssp.parse(minimal) - doc.format_version |> should.equal("2.0") - model.version_major(doc) |> should.equal("2") - doc.source_filename |> should.equal("example.sst") -} - -pub fn preamble_test() { - let doc = load_ssp.parse(minimal) - model.meta(doc, "title.main") |> should.equal("An Example") -} - -pub fn objects_test() { - let doc = load_ssp.parse(minimal) - let body = model.section(doc, "body") - body |> list.length |> should.equal(2) -} - -/// The blank line in the middle of an object's text is kept, and so is the -/// count: three lines, not two. -pub fn blank_text_line_test() { - let doc = load_ssp.parse(minimal) - case model.section(doc, "body") { - [_, second] -> - second.text - |> should.equal("First line\n\nThird line, after a blank one") - _ -> should.fail() - } -} - -/// A heading's identifier comes from its declaration line; an object whose -/// identifier is simply its ocn has it derived, because the text form does -/// not write what it can work out. -pub fn derived_identifier_test() { - let doc = load_ssp.parse(minimal) - case model.section(doc, "body") { - [first, second] -> { - first.identifier |> should.equal("the_title") - second.identifier |> should.equal("2") - } - _ -> should.fail() - } -} - -pub fn source_of_test() { - load.source_of("a/b.ssp") |> should.equal(load.Ssp) - load.source_of("a/b.ocda.db") |> should.equal(load.OcdaDb) - load.source_of("a/b.sst") |> should.equal(load.Unknown) -} - -/// The dump omits a field at its default, so a minimal document gives a -/// short dump and every line in it is a fact. -pub fn dump_test() { - let text = dump.document(load_ssp.parse(minimal)) - text |> string.contains(contain: "version 2.0") |> should.be_true - text |> string.contains(contain: "meta title.main An Example") |> should.be_true - text |> string.contains(contain: "object body 0") |> should.be_true - text |> string.contains(contain: "level A") |> should.be_true - // a false boolean is not written at all - text |> string.contains(contain: "bullet") |> should.be_false -} - - -/// A property is read, which the fixture above had to be corrected to show: -/// a property line is `.name: value` at the start of the line, and an -/// indented one is a line of no known shape. The fixture indented them, so -/// every property in it was being counted as malformed and the tests around -/// it passed for the wrong reason. -pub fn property_test() { - let doc = load_ssp.parse(minimal) - case model.section(doc, "body") { - [first, second] -> { - first.part |> should.equal("frontmatter") - first.sha256 |> should.equal("DEAD") - second.parent_ocn |> should.equal(1) - } - _ -> should.fail() - } - // and nothing in the fixture is malformed any more - dict_size(doc.unknown.line) |> should.equal(0) -} - -@external(erlang, "maps", "size") -fn dict_size(d: dict.Dict(a, b)) -> Int - -/// What a path is, including the two archive forms. -pub fn pod_source_test() { - load.source_of("a/b.sisupod") |> should.equal(load.Pod) - load.source_of("a/b.zip") |> should.equal(load.Pod) -} - -/// A zstd-wrapped archive is refused, and the refusal says why rather than -/// claiming the file is malformed. This is the one place the D and Gleam -/// libraries differ in what they accept, and it is a packaging limit: the -/// only zstd on the BEAM is a NIF that fetches zstd in its own build. -pub fn zstd_refusal_test() { - let path = "test/fixture-zstd-frame.sisupod" - let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>) - let assert Error(message) = load.load(path, "", False) - message |> string.contains(contain: "zstd") |> should.be_true - message |> string.contains(contain: "zstd -d") |> should.be_true - let _ = simplifile.delete(path) - Nil -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ diff --git a/org/gleam-ocda.org b/org/gleam.org index c10554e..56d8f0d 100644 --- a/org/gleam-ocda.org +++ b/org/gleam.org @@ -15,7 +15,7 @@ - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX -* gleam :gleam: +* ocda ** ocda lib *** tangle @@ -177,7 +177,7 @@ fn fail(message: String) -> Nil { fn halt(code: Int) -> Nil #+END_SRC -* gleam :gleam: +* ocda ** model *** tangle @@ -2703,6 +2703,908 @@ fn ints(xs: List(Int)) -> String { sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). #+END_SRC +* read +** main +*** tangle + +#+HEADER: :tangle ../gleam/src/read/main.gleam +#+HEADER: :noweb yes +#+BEGIN_SRC org +////<<doc_header_including_copyright_and_license>> +<<read>> +#+END_SRC + +*** source + +#+NAME: read +#+HEADER: :no-expand +#+BEGIN_SRC gleam +//// ocda-read, in Gleam: reads the abstraction, in any of the forms the +//// library reads, and says what is in it. +//// +//// ocda-read [--tree|--outline|--languages|--files|--report-unknown] +//// <file>.ssp | <file>.ocda.db | <file>.zip [lang] +//// +//// The modes the Gleam readers had (the summary, the outline, the unknown +//// report, and the database's languages and files), over the library, so +//// that each reads every form. Their output is the D ocda-read's, byte for +//// byte: two readers in unrelated languages, neither written from the other +//// line by line, agreeing is what says each has read the model and not its +//// own idea of it. A `.sisupod` is the one form the Gleam library cannot +//// open (its zstd frame; see `ocda/load_pod`), and it says so. + +import gleam/dict +import gleam/int +import gleam/io +import gleam/list +import gleam/result +import gleam/string +import ocda/inventory.{type Language} +import ocda/load +import ocda/model.{type Document, type Object} + +/// The format major this reader knows. +const major_read = "2" + +type Options { + Options( + tree: Bool, + languages: Bool, + files: Bool, + report_unknown: Bool, + any_version: Bool, + lang: String, + positional: List(String), + ) +} + +/// Run with the arguments after the program name; returns the exit status. +pub fn run(args: List(String)) -> Int { + case parse(args, Options(False, False, False, False, False, "", [])) { + Error(message) -> { + io.println_error("ocda-read: " <> message <> "; --help lists the options") + 2 + } + Ok(Options(positional: [], ..)) -> { + help() + 0 + } + Ok(options) -> read(options) + } +} + +fn parse(args: List(String), o: Options) -> Result(Options, String) { + case args { + [] -> Ok(Options(..o, positional: list.reverse(o.positional))) + ["--help", ..] | ["-h", ..] -> Ok(Options(..o, positional: [])) + ["--tree", ..rest] | ["--outline", ..rest] -> + parse(rest, Options(..o, tree: True)) + ["--languages", ..rest] -> parse(rest, Options(..o, languages: True)) + ["--files", ..rest] -> parse(rest, Options(..o, files: True)) + ["--report-unknown", ..rest] -> + parse(rest, Options(..o, report_unknown: True)) + ["--any-version", ..rest] -> parse(rest, Options(..o, any_version: True)) + ["--lang", code, ..rest] -> parse(rest, Options(..o, lang: code)) + [flag, ..rest] -> + case string.starts_with(flag, "--lang=") { + True -> parse(rest, Options(..o, lang: string.drop_start(flag, 7))) + False -> + case string.starts_with(flag, "-") { + True -> + case + list.contains( + [ + "--html", "--llm-text", "--llm-jsonl", "--images", "--verify", + "--validate", + ], + flag, + ) + { + True -> + Error(flag <> " is a mode of the D ocda-read, not of this one") + False -> Error("Unrecognized option " <> flag) + } + False -> + parse(rest, Options(..o, positional: [flag, ..o.positional])) + } + } + } +} + +fn help() -> Nil { + io.println("ocda-read - reads the document abstraction, in any of its forms") + io.println("") + io.println( + "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.zip [lang]", + ) + io.println("") + io.println(" --tree the headings, as a tree") + io.println(" --outline the same as --tree") + io.println(" --languages the languages the file holds") + io.println(" --files what a .ocda.db carries besides the abstraction") + io.println(" --report-unknown what was read and not understood") + io.println(" --lang=<code> the language to read (default: the first)") + io.println(" --any-version read a format major this reader does not know") +} + +fn read(o: Options) -> Int { + let assert [path, ..more] = o.positional + let lang = case o.lang, more { + "", [given, ..] -> given + l, _ -> l + } + let lang_named = lang != "" + let loaded = { + use held <- result.try(inventory.languages(path)) + let lang = case lang, held { + "", [first, ..] -> first.lang + l, _ -> l + } + use _ <- result.try(case + held != [] && !list.any(held, fn(l) { l.lang == lang }) + { + True -> + Error( + "no language '" + <> lang + <> "' in " + <> path + <> "; it holds: " + <> string.join(list.map(held, fn(l) { l.lang }), " "), + ) + False -> Ok(Nil) + }) + use doc <- result.try(load.load(path, lang, o.any_version)) + Ok(#(held, lang, doc)) + } + case loaded { + Error(message) -> { + io.println_error("Error reading " <> path <> ": " <> message) + 2 + } + Ok(#(held, lang, doc)) -> + case gate(path, doc, o.any_version) { + False -> 3 + True -> { + summary(path, doc, held, lang, lang_named) + case o.languages { + True -> print_languages(held) + False -> Nil + } + case o.files { + True -> print_files(path) + False -> Nil + } + case o.tree { + True -> tree(doc) + False -> Nil + } + case o.report_unknown { + True -> report_unknown(doc) + False -> Nil + } + case o.tree || o.languages || o.files || o.report_unknown { + True -> Nil + False -> + io.println( + "\n(this is the summary: --help lists the modes, each adding its output)", + ) + } + 0 + } + } + } +} + +/// The major part, and refuse what is not known. +fn gate(path: String, doc: Document, any_version: Bool) -> Bool { + case doc.format_version, model.version_major(doc) == major_read, any_version { + "", _, _ -> { + io.println_error("warning: " <> path <> " declares no format version") + True + } + _, True, _ -> True + v, False, True -> { + io.println_error( + "warning: " + <> path + <> " is format v" + <> v + <> ", this reader knows v" + <> major_read + <> ".x; reading anyway", + ) + True + } + v, False, False -> { + io.println_error( + "error: " + <> path + <> " is format v" + <> v + <> ", this reader knows v" + <> major_read + <> ".x", + ) + io.println_error( + " a major version differs: a property may have changed meaning or gone away.", + ) + io.println_error(" --any-version reads it regardless.") + False + } + } +} + +const sections = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", + "blurb", "tail", +] + +/// The sections whose images are counted: every one but the tail, as the D +/// reader counts them. +const image_sections = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", + "blurb", +] + +fn summary( + path: String, + doc: Document, + held: List(Language), + lang: String, + lang_named: Bool, +) -> Nil { + io.println("Read: " <> path) + case doc.format_version { + "" -> Nil + v -> io.println(" Format: v" <> v) + } + case dict.get(doc.source, "language") { + Error(_) -> Nil + Ok(language) -> { + let of = case dict.get(doc.source, "languages") { + Ok(ls) -> " of " <> ls + Error(_) -> "" + } + let digest = case dict.get(doc.source, "digest") { + Ok(d) -> ", markup digest " <> string.slice(d, 0, 16) <> "..." + Error(_) -> "" + } + io.println(" Source: " <> language <> of <> digest) + } + } + case list.length(held) > 1 && !lang_named { + True -> io.println(" (reading " <> lang <> "; --lang=<code> for another)") + False -> Nil + } + io.println(" Title: " <> model.meta(doc, "title.main")) + io.println(" Author: " <> model.meta(doc, "creator.author")) + io.println( + " Sections: " + <> string.join( + list.map(sections, fn(s) { + s <> "=" <> int.to_string(list.length(model.section(doc, s))) + }), + " ", + ), + ) + let records = + list.fold(image_sections, 0, fn(n, s) { + list.fold(model.section(doc, s), n, fn(n, o) { + n + list.length(o.images) + }) + }) + case records { + 0 -> Nil + 1 -> io.println(" Images: 1 record") + n -> io.println(" Images: " <> int.to_string(n) <> " records") + } +} + +fn print_languages(held: List(Language)) -> Nil { + io.println("") + io.println(" lang source ssp digest") + list.each(held, fn(l) { + let digest = case string.length(l.ssp_digest) > 16 { + True -> string.slice(l.ssp_digest, 0, 16) <> "..." + False -> l.ssp_digest + } + io.println( + " " + <> string.pad_end(l.lang, 7, " ") + <> " " + <> string.pad_end(l.source_filename, 31, " ") + <> " " + <> digest, + ) + }) +} + +fn print_files(path: String) -> Nil { + io.println("") + case inventory.is_carrier(path) { + False -> + io.println( + " carried files: only a .ocda.db carries files; this is " + <> load.source_name(load.source_of(path)), + ) + True -> + case inventory.carried(path) { + Error(message) -> io.println_error("error: " <> message) + Ok([]) -> io.println(" carries no files") + Ok(rows) -> { + io.println(" carried files:") + io.println(" role stored n original in the file") + list.each(rows, fn(r) { + io.println( + " " + <> string.pad_end(r.role, 9, " ") + <> " " + <> string.pad_end(r.compression, 7, " ") + <> " " + <> string.pad_end(r.count, 5, " ") + <> " " + <> string.pad_end(r.bytes, 13, " ") + <> " " + <> r.stored, + ) + }) + } + } + } +} + +/// The headings, as a tree: from the document's root heading, down the +/// children each heading lists, as the D reader walks it. +fn tree(doc: Document) -> Nil { + io.println("Document: " <> model.meta(doc, "title.main")) + case model.meta(doc, "title.subtitle") { + "" -> Nil + sub -> io.println(" " <> sub) + } + io.println("Author: " <> model.meta(doc, "creator.author")) + io.println("") + let headings = + model.section(doc, "body") + |> list.filter(fn(o) { o.is_a == "heading" && o.ocn > 0 }) + |> list.map(fn(o) { #(o.ocn, o) }) + |> dict.from_list + case model.section(doc, "head") { + [root, ..] if root.is_a == "heading" -> { + io.println(heading_line(root, 0)) + walk(root, headings, 1) + } + _ -> Nil + } +} + +fn walk(parent: Object, headings: dict.Dict(Int, Object), depth: Int) -> Nil { + list.each(parent.children, fn(ocn) { + case dict.get(headings, ocn) { + Ok(child) -> { + io.println(heading_line(child, depth)) + walk(child, headings, depth + 1) + } + Error(_) -> Nil + } + }) +} + +fn heading_line(o: Object, depth: Int) -> String { + let level = case o.heading_level { + "" -> "?" + l -> l + } + let ocn = case o.ocn > 0 { + True -> " [" <> int.to_string(o.ocn) <> "]" + False -> "" + } + string.repeat(" ", depth) <> level <> "~" <> ocn <> " " <> strip_markup(o.text) +} + +/// The internal markers a heading's text may carry, removed for display, as +/// the D reader removes them. +fn strip_markup(text: String) -> String { + text + |> string.replace("⑆*", "") + |> string.replace("⑆!", "") + |> string.replace("⑆/", "") + |> string.replace("⑆_", "") + |> string.replace("⑆^", "") + |> string.replace("⑆,", "") + |> string.replace("┨", "") + |> string.replace("┣", "") + |> string.replace("┙", " ") + |> string.replace("┘", "\n") +} + +/// Say what was read and not understood. +fn report_unknown(doc: Document) -> Nil { + let u = doc.unknown + case model.any_unknown(u) { + False -> + io.println( + "\nread and understood: every property, block and line was known", + ) + True -> { + io.println("\nread and NOT understood:") + section_of_unknown(u.property, " properties ignored (", fn(name, n) { + " ." + <> string.pad_end(name, 26, " ") + <> " " + <> string.pad_start(int.to_string(n), 8, " ") + }) + section_of_unknown(u.block, " blocks ignored (", fn(name, n) { + " @" + <> string.pad_end(name, 26, " ") + <> " " + <> string.pad_start(int.to_string(n), 8, " ") + }) + case dict.size(u.line) { + 0 -> Nil + _ -> { + io.println(" lines of no known shape, by first character:") + dict.to_list(u.line) + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.each(fn(e) { + io.println( + " " + <> string.pad_end("\"" <> e.0 <> "\"", 27, " ") + <> " " + <> string.pad_start(int.to_string(e.1), 8, " "), + ) + }) + } + } + io.println( + " (ignoring these is conforming within a major version; the count is so that it is not silent)", + ) + } + } +} + +fn section_of_unknown( + counts: dict.Dict(String, Int), + heading: String, + line: fn(String, Int) -> String, +) -> Nil { + case dict.size(counts) { + 0 -> Nil + n -> { + io.println(heading <> int.to_string(n) <> " distinct):") + dict.to_list(counts) + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.each(fn(e) { io.println(line(e.0, e.1)) }) + } + } +} +#+END_SRC + +* test +** gleam +*** tangle + +#+HEADER: :tangle ../gleam/test/ocda_lib_test.gleam +#+HEADER: :noweb yes +#+BEGIN_SRC org +////<<doc_header_including_copyright_and_license>> +<<test>> +#+END_SRC + +*** gleam source + +#+NAME: test +#+HEADER: :no-expand +#+BEGIN_SRC gleam +import gleam/dict +import gleam/list +import simplifile +import gleam/string +import gleeunit +import gleeunit/should +import ocda/dump +import ocda/load +import ocda/load_ssp +import ocda/model + +pub fn main() { + gleeunit.main() +} + +const minimal = "% SiSU Document Abstraction v2.0 +% Source: example.sst + +@source { + language: en + languages: en + digest: ABC +} + +@meta { + title.main: An Example +} + +@body { +[1] heading :A the_title +.part: frontmatter +.sha256: DEAD +| An Example + +[2] para +.part: body +.parent: 1 +| First line +| +| Third line, after a blank one + +} +" + +pub fn version_test() { + let doc = load_ssp.parse(minimal) + doc.format_version |> should.equal("2.0") + model.version_major(doc) |> should.equal("2") + doc.source_filename |> should.equal("example.sst") +} + +pub fn preamble_test() { + let doc = load_ssp.parse(minimal) + model.meta(doc, "title.main") |> should.equal("An Example") +} + +pub fn objects_test() { + let doc = load_ssp.parse(minimal) + let body = model.section(doc, "body") + body |> list.length |> should.equal(2) +} + +/// The blank line in the middle of an object's text is kept, and so is the +/// count: three lines, not two. +pub fn blank_text_line_test() { + let doc = load_ssp.parse(minimal) + case model.section(doc, "body") { + [_, second] -> + second.text + |> should.equal("First line\n\nThird line, after a blank one") + _ -> should.fail() + } +} + +/// A heading's identifier comes from its declaration line; an object whose +/// identifier is simply its ocn has it derived, because the text form does +/// not write what it can work out. +pub fn derived_identifier_test() { + let doc = load_ssp.parse(minimal) + case model.section(doc, "body") { + [first, second] -> { + first.identifier |> should.equal("the_title") + second.identifier |> should.equal("2") + } + _ -> should.fail() + } +} + +pub fn source_of_test() { + load.source_of("a/b.ssp") |> should.equal(load.Ssp) + load.source_of("a/b.ocda.db") |> should.equal(load.OcdaDb) + load.source_of("a/b.sst") |> should.equal(load.Unknown) +} + +/// The dump omits a field at its default, so a minimal document gives a +/// short dump and every line in it is a fact. +pub fn dump_test() { + let text = dump.document(load_ssp.parse(minimal)) + text |> string.contains(contain: "version 2.0") |> should.be_true + text |> string.contains(contain: "meta title.main An Example") |> should.be_true + text |> string.contains(contain: "object body 0") |> should.be_true + text |> string.contains(contain: "level A") |> should.be_true + // a false boolean is not written at all + text |> string.contains(contain: "bullet") |> should.be_false +} + + +/// A property is read, which the fixture above had to be corrected to show: +/// a property line is `.name: value` at the start of the line, and an +/// indented one is a line of no known shape. The fixture indented them, so +/// every property in it was being counted as malformed and the tests around +/// it passed for the wrong reason. +pub fn property_test() { + let doc = load_ssp.parse(minimal) + case model.section(doc, "body") { + [first, second] -> { + first.part |> should.equal("frontmatter") + first.sha256 |> should.equal("DEAD") + second.parent_ocn |> should.equal(1) + } + _ -> should.fail() + } + // and nothing in the fixture is malformed any more + dict_size(doc.unknown.line) |> should.equal(0) +} + +@external(erlang, "maps", "size") +fn dict_size(d: dict.Dict(a, b)) -> Int + +/// What a path is, including the two archive forms. +pub fn pod_source_test() { + load.source_of("a/b.sisupod") |> should.equal(load.Pod) + load.source_of("a/b.zip") |> should.equal(load.Pod) +} + +/// A zstd-wrapped archive is refused, and the refusal says why rather than +/// claiming the file is malformed. This is the one place the D and Gleam +/// libraries differ in what they accept, and it is a packaging limit: the +/// only zstd on the BEAM is a NIF that fetches zstd in its own build. +pub fn zstd_refusal_test() { + let path = "test/fixture-zstd-frame.sisupod" + let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>) + let assert Error(message) = load.load(path, "", False) + message |> string.contains(contain: "zstd") |> should.be_true + message |> string.contains(contain: "zstd -d") |> should.be_true + let _ = simplifile.delete(path) + Nil +} +#+END_SRC + +* config +** nix flake.nix :flake: + +#+HEADER: :tangle ../gleam/flake.nix +#+HEADER: :noweb yes +#+BEGIN_SRC nix +{ + description = "ocda-lib: the object-centric document abstraction as a Gleam library"; + inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + + name = "ocda_lib"; + + /* rebar3 with the port compiler already in it. + esqlite builds a NIF, and its rebar3 configuration asks for the + `pc` plugin, which rebar3 would otherwise fetch from Hex at build + time: impossible in a sandbox, and the build fails with + "missing_package pc". nixpkgs carries the plugin, so the answer is + a rebar3 that has it before it starts. + */ + rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { + plugins = [ pkgs.beamPackages.pc ]; + }; + + /* The Hex dependencies, as manifest.toml pins them. + . + There is no Gleam builder in nixpkgs and `gleam build` fetches from + Hex, so a sandboxed build cannot reach them. A fixed output + derivation may, because its result is pinned by hash. + . + When a dependency changes, manifest.toml changes and this hash must + change with it: build, take the hash nix reports, put it here. + */ + deps = pkgs.stdenv.mkDerivation { + pname = "${name}-deps"; + version = "0.1.0"; + src = ./.; + nativeBuildInputs = [ pkgs.gleam pkgs.cacert ]; + buildPhase = '' + export HOME=$TMPDIR + gleam deps download + ''; + installPhase = '' + mkdir -p $out + cp -r build/packages/* $out/ + ''; + dontFixup = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c="; + }; + in { + packages.${system} = { + default = self.packages.${system}.${name}; + /* Shipped as an Erlang shipment rather than as an escript, because + sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into + a shared object, and a NIF cannot be loaded from inside an + escript archive, the runtime needing a real file on disk. The + shipment keeps priv/ on disk, so it loads. + */ + ${name} = pkgs.stdenv.mkDerivation { + pname = "ocda-lib"; + version = "0.1.0"; + src = ./.; + nativeBuildInputs = [ + pkgs.gleam + pkgs.erlang + rebar3-pc + pkgs.makeWrapper + ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + # the dependencies, in place of the fetch gleam would do + mkdir -p build/packages + cp -r ${deps}/* build/packages/ + chmod -R u+w build/packages + gleam export erlang-shipment + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib $out/bin + cp -r build/erlang-shipment $out/lib/${name} + makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \ + --add-flags run \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} + # the reader under its own name, as the D flake installs it + makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \ + --add-flags "run read" \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} + runHook postInstall + ''; + meta = { + description = "the abstraction as a Gleam library, with its reader, dump and agreement check"; + mainProgram = "ocda-lib"; + }; + }; + }; + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ]; + }; + }; +} +#+END_SRC + +** README.md + +#+HEADER: :tangle ../gleam/README.md +#+HEADER: :noweb yes +#+BEGIN_SRC md +# ocda-lib, in Gleam + +The abstraction's model and both its loaders, as a Gleam library, with its +reader and the two programs that put it to the question. + +| | | +|------------------|-----------------------------------------------------------------------------------------| +| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | +| `ocda/load_ssp` | text -> model | +| `ocda/load_db` | sqlite -> model | +| `ocda/load` | what a path is, and loading it | +| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | +| `ocda/dump` | the model written out canonically, as an oracle | +| `read/main` | `ocda-read`, the reader | +| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | + +## Use + +```gleam +import ocda/load +import ocda/model + +case load.load(path, "en", False) { + Error(why) -> io.println_error(why) + Ok(doc) -> { + // and from here a writer neither knows nor cares which of the two + // files it was handed + list.each(model.section(doc, "body"), fn(obj) { ... }) + } +} +``` + +## The reader + +```sh +nix build +./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] + <file>.ssp|<file>.ocda.db|<file>.zip [lang] +``` + +`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers +had, over the library, so each reads every form this library reads; the +others are the D reader's. **Its output is the D `ocda-read`'s, byte for +byte**, in every mode both have. In `src/read/`, because Gleam compiles +nothing outside `src/`; a program depending on the library compiles it too, +and does not use it. + +A `.sisupod` it refuses, saying why: the zstd frame, below. + +## The two programs + +```sh +nix build +./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] +./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] +``` + +`agree` asks whether the two loaders give the same model. `dump` writes +the whole model out canonically, and is the stronger check of the two, +because it is **byte for byte what the D library's `ocda-dump` writes**. +So the four questions can all be asked with `diff`: + +| | | +|-----------------------------|-----------------------------| +| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | +| Gleam: `.ssp` == `.ocda.db` | 36 of 36 | +| `.ssp`: D == Gleam | 36 of 36 | +| `.ocda.db`: D == Gleam | 36 of 36 | + +## Deliberately a mirror of the D library + +Field for field, name for name, including where that is not the most +Gleamish choice: + +- `text` is one `String` with the lines joined by newline, not a + `List(String)`. So an object whose only text line is blank is + indistinguishable from an object with no text, *in both languages, in the + same place*. A difference that existed in one and not the other would be + worse than a loss both share, because it would show up in every dump and + mean nothing. +- the properties are typed fields rather than a list of pairs. A writer + wants `obj.parent_ocn`, not a lookup that might miss. + +The point is that the two agree. Making them agree is much cheaper when +they are shaped alike, and the shape is the specification's, not either +language's. + +## What the agreement found + +Four disagreements, none of which a comparison of section counts or of a +handful of fields would have shown. All four are recorded in the D +library's history; the short version: + +1. `.identifier` and `.anchor` are omitted by the text form when they can + be worked out (the identifier is the ocn, the anchor is the segment). + Both loaders now derive them. Over one document that is 1,314 + identifiers. +2. `make` and `doc_has` are one metadata table in the database, + distinguished by a key prefix. Both loaders unprefix them. +3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` + as `\/` and every heading with a slash in it came back wrong. +4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, + which is one language. It is a fact about the file, so neither loader + puts it in the document. + +## Build + +```sh +nix build # sandboxed, no network +nix develop -c gleam test +``` + +Shipped as an Erlang shipment rather than an escript, because sqlite on the +BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. +The dependency hash in `flake.nix` pins what `manifest.toml` pins; change +one and the other must change with it. + +## Format + +Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, +`ssp-format-specification.org` and `ocda-db-format-specification.org` for the +two serialisations. At format **2.x**. +#+END_SRC + +** gleam.toml + +#+HEADER: :tangle ../gleam/gleam.toml +#+HEADER: :noweb yes +#+BEGIN_SRC toml +name = "ocda_lib" +version = "0.1.0" +description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" +licences = ["AGPL-3.0-or-later"] + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +argv = ">= 1.1.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +sqlight = ">= 1.0.0 and < 2.0.0" +gleam_json = ">= 3.0.0 and < 4.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +#+END_SRC + * org includes ** project version @@ -2728,4 +3630,12 @@ sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC +* nixpkgs source + +#+NAME: nixpkgs_unstable +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> +#+END_SRC + * __END__ |
