summaryrefslogtreecommitdiffhomepage
path: root/org
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-10-02 13:57:43 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-10-02 21:39:21 -0400
commitc80a06efd9a5b6f709d6d4eddb6d6ef78f6502f2 (patch)
tree1c6036b58c7e3d1697f9a5c7f4982170d648a50c /org
parenttrack (diff)
track
Diffstat (limited to 'org')
-rw-r--r--org/d-ocda.org1163
-rw-r--r--org/d-read.org574
-rw-r--r--org/gleam-ocda.org2876
3 files changed, 4592 insertions, 21 deletions
diff --git a/org/d-ocda.org b/org/d-ocda.org
new file mode 100644
index 0000000..218a8cf
--- /dev/null
+++ b/org/d-ocda.org
@@ -0,0 +1,1163 @@
+-*- mode: org -*-
+#+TITLE: dlang load
+#+DESCRIPTION: dlang lib
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* dlang
+** load
+*** load dlang
+
+#+HEADER: :tangle "../dlang/src/ocda/load.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load;
+
+/+ ↓ what a path is, and loading it.
+ .
+ One place that decides, so that every reader and every writer accepts
+ the same inputs and a new input type is added once. This mirrors
+ spine's own ocda/abstraction/load.d, which names the sources it can
+ enter the pipeline from and tells them apart by suffix.
+ .
+ Deliberately *not* here: .sst and .ssm markup. Reading markup means
+ implementing the parser, which is spine's work and a far larger thing
+ than a reader or a sample writer needs. This library reads
+ abstractions.
++/
+
+import std.algorithm : endsWith;
+import ocda.model;
+import ocda.load_ssp;
+import ocda.load_db;
+import ocda.load_pod;
+
+enum Source {
+ ssp, // .ssp, the abstraction as text
+ ocda_db, // .ocda.db, the abstraction as sqlite
+ pod, // .sisupod or .zip, an archive carrying a .ssp
+ unknown,
+}
+
+Source sourceOf(string path) {
+ if (path.endsWith(".ocda.db")) { return Source.ocda_db; }
+ if (path.endsWith(".ssp")) { return Source.ssp; }
+ /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote
+ before, and every pod already published carries that name, so both
+ are read. What is inside decides how it is opened, not the name.
+ +/
+ if (path.endsWith(".sisupod")) { return Source.pod; }
+ if (path.endsWith(".zip")) { return Source.pod; }
+ if (path.endsWith(".db")) { return Source.ocda_db; }
+ return Source.unknown;
+}
+
+string sourceName(Source s) {
+ final switch (s) {
+ case Source.ssp: return ".ssp (the abstraction as text)";
+ case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)";
+ case Source.pod: return "a pod archive carrying a .ssp";
+ case Source.unknown: return "not an abstraction this library reads";
+ }
+}
+
+class OcdaLoadError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ load whatever the path is into the model.
+ lang names which language to take from a .ocda.db, which holds
+ every language of its document, and from a pod, which may carry more
+ than one .ssp. A .ssp is one language by construction and ignores it.
++/
+OcdaDocument load(string path, string lang = "", bool any_version = false) {
+ final switch (sourceOf(path)) {
+ case Source.ssp: return parseSspFile(path);
+ case Source.ocda_db: return loadDbFile(path, lang, any_version);
+ case Source.pod: return loadPodFile(path, lang);
+ case Source.unknown:
+ throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)
+ ~ " (expected .ssp, .ocda.db, .sisupod or .zip)");
+ }
+}
+#+END_SRC
+
+*** load db dlang
+
+#+HEADER: :tangle "../dlang/src/ocda/load_db.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_db;
+
+/+ ↓ build the model from a .ocda.db.
+ .
+ The point of this module is that it returns *the same value*
+ ocda.load_ssp returns from the same document's .ssp. A writer is then
+ a function from the model to bytes, and cannot tell which
+ serialisation it was given, exactly as spine's own writers cannot.
+ .
+ Written from the sisudoc-abstraction-format specification,
+ ocda-db-format-specification.md, whose section 4 gives the column by
+ column correspondence this module implements.
++/
+
+import std.array : join, split;
+import std.json : JSONType, parseJSON;
+import std.conv : to;
+import std.algorithm : startsWith;
+import std.string : strip;
+import ocda.model;
+import ocda.sqlite;
+
+/+ ↓ the schema major this loader knows (specification section 2) +/
+enum schema_major_read = "2";
+
+class OcdaDbError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ every language the database holds, in manifest order +/
+struct DbLanguage {
+ int doc_id;
+ string lang;
+ string source_filename;
+ string ssp_digest;
+}
+
+DbLanguage[] dbLanguages(ref Db db) {
+ DbLanguage[] out_;
+ foreach (r; db.rows(
+ "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
+ FROM documents ORDER BY id")) {
+ out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]);
+ }
+ return out_;
+}
+
+/+ ↓ load one language of a database as the model.
+ lang == "" takes the first language the manifest gave.
++/
+OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+
+ if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) {
+ throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;"
+ ~ " it does not look like a .ocda.db");
+ }
+
+ /+ ↓ specification sections 2 and 3.2: the rows describing the file rather
+ than a document in it are the only ones with doc_id null +/
+ string schema_version = db.one(
+ "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'");
+ if (schema_version.length > 0 && !any_version) {
+ auto major = schema_version.split(".")[0];
+ if (major != schema_major_read) {
+ throw new OcdaDbError(path ~ " is schema v" ~ schema_version
+ ~ ", this loader knows v" ~ schema_major_read ~ ".x");
+ }
+ }
+
+ auto languages = dbLanguages(db);
+ if (languages.length == 0) {
+ throw new OcdaDbError(path ~ ": no documents rows");
+ }
+ DbLanguage chosen = languages[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } }
+ if (!found) {
+ string have;
+ foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; }
+ throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have);
+ }
+ }
+
+ OcdaDocument doc;
+ doc.format_version = schema_version;
+ doc.source_filename = chosen.source_filename;
+
+ /+ ↓ @source of the .ssp corresponds to the documents row and the
+ source.digest metadata (specification section 5.1 of the .ssp, and
+ section 7 here) +/
+ doc.source["language"] = chosen.lang;
+ string languages_joined;
+ foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; }
+ doc.source["languages"] = languages_joined;
+ auto digest = db.one("SELECT value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'");
+ if (digest.length > 0) { doc.source["digest"] = digest; }
+
+ /+ ↓ @meta: every metadata row of this language that is not a source.*
+ property. source.* describes the artefact rather than the document
+ and lives in @source in the text form.
+ +/
+ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " ORDER BY key")) {
+ string key = r[0];
+ if (key.startsWith("source.")) { continue; }
+ /+ ↓ translation.ocn_aligned says whether the languages of this
+ document share one numbering. A .ocda.db can know it because it
+ holds every language; a .ssp is one language and cannot. So it is
+ a fact about the *file* rather than about the document, and putting
+ it in the document's metadata would let a writer tell which
+ serialisation it was handed, which is the one thing this library
+ exists to prevent. Left out until the model has a place for
+ collection-wide facts.
+ +/
+ if (key.startsWith("translation.")) { continue; }
+ /+ ↓ the database has one metadata table where the text form has three
+ blocks, and tells them apart by a key prefix. Unprefixing them
+ here is what makes the two loaders give one model: a writer asks
+ doc.make["footer"] whichever serialisation it was handed, and
+ never doc.metadata["make.footer"].
+ +/
+ if (key.startsWith("make.")) {
+ doc.make[key["make.".length .. $]] = r[1].strip;
+ } else if (key.startsWith("doc_has.")) {
+ doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
+ } else {
+ doc.metadata[key] = r[1].strip;
+ }
+ }
+
+ /+ ↓ 2.1 added the container range as two columns. A 2.0 database has
+ neither, and reads as having no range, which is what 2.0 meant. +/
+ bool has_range = false;
+ foreach (c; db.rows("PRAGMA table_info(objects)")) {
+ if (c[1] == "ocn_range_first") { has_range = true; }
+ }
+ string range_cols = has_range
+ ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
+ : "0, 0";
+
+ /+ ↓ the objects, in document order within each section +/
+ foreach (r; db.rows(
+ "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''),
+ coalesce(is_of_section,''), coalesce(is_of_type,''),
+ coalesce(heading_level,-1), coalesce(identifier,''),
+ coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
+ coalesce(children,''), coalesce(ancestors,''),
+ coalesce(dummy_heading,0), coalesce(object_number_off,0),
+ coalesce(indent_base,0), coalesce(indent_hang,0),
+ coalesce(bullet,0), coalesce(lang,''),
+ coalesce(has_links,0), coalesce(has_notes_reg,0),
+ coalesce(has_notes_star,0), coalesce(has_images,0),
+ coalesce(segment,''), coalesce(segment_prev,''),
+ coalesce(segment_next,''), coalesce(anchor,''),
+ coalesce(segment_html_is,''),
+ coalesce(table_cols,0), coalesce(table_widths,''),
+ coalesce(table_aligns,''), coalesce(table_header,0),
+ coalesce(code_linenumbers,0),
+ coalesce(heading_ancestors_text,''), coalesce(sha256,''),
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ "
+ FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
+ ~ " ORDER BY CASE section
+ WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
+ WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5
+ WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7
+ WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) {
+ OcdaObject obj;
+ obj.section = r[0];
+ obj.ocn = r[2].to!int;
+ obj.is_a = r[3];
+ obj.part = r[4];
+ obj.own_section = (r[5] == r[0]) ? "" : r[5];
+ obj.is_of_type = r[6];
+ /+ ↓ the database holds the level as a number, the text form as the
+ letter or digit the markup wrote. Model section 4 is the mapping. +/
+ obj.heading_level = levelName(r[7].to!int);
+ obj.identifier = r[8];
+ obj.parent_ocn = r[9].to!int;
+ obj.last_descendant_ocn = r[10].to!int;
+ obj.children = jsonInts(r[11]);
+ obj.ancestors = jsonInts(r[12]);
+ obj.dummy_heading = (r[13] != "0");
+ obj.ocn_off = (r[14] != "0");
+ obj.indent_base = r[15].to!int;
+ obj.indent_hang = r[16].to!int;
+ obj.bullet = (r[17] != "0");
+ obj.lang = r[18];
+ obj.has_links = (r[19] != "0");
+ obj.has_notes_reg = (r[20] != "0");
+ obj.has_notes_star = (r[21] != "0");
+ obj.has_images = (r[22] != "0");
+ obj.segment = r[23];
+ obj.segment_prev = r[24];
+ obj.segment_next = r[25];
+ obj.anchor = r[26];
+ obj.segment_html_is = r[27];
+ obj.heading_lev_anchor = r[37];
+ obj.segment_epub = r[38];
+ obj.segment_epub_is = r[39];
+ obj.ocn_range_first = r[40].to!int;
+ obj.ocn_range_last = r[41].to!int;
+ obj.table_cols = r[28].to!int;
+ obj.table_widths = jsonWords(r[29]);
+ obj.table_aligns = jsonWords(r[30]);
+ obj.table_header = (r[31] != "0");
+ obj.code_linenumbers = (r[32] != "0");
+ obj.heading_ancestors_text = jsonJoined(r[33]);
+ obj.sha256 = r[34];
+ obj.text = r[35];
+
+ /+ ↓ the four repeatable properties of the text form are child tables
+ here (specification section 4) +/
+ string object_id = r[36];
+ foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0),
+ coalesce(width,0), coalesce(height,0),
+ coalesce(missing,0)
+ FROM object_images WHERE object_id = " ~ object_id
+ ~ " ORDER BY seq")) {
+ OcdaImage img;
+ img.name = i[0];
+ img.sha256 = i[1];
+ img.bytes = i[2].to!ulong;
+ img.width = i[3].to!int;
+ img.height = i[4].to!int;
+ img.missing = (i[5] != "0");
+ obj.images ~= img;
+ }
+ foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.stow_link ~= l[0];
+ }
+ foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.anchor_tags ~= a[0];
+ }
+
+ auto section = doc.sectionByName(r[0]);
+ if (section !is null) { *section ~= obj; }
+ }
+ countUnread(db, chosen.doc_id, doc);
+ return doc;
+}
+
+/+ ↓ what the database holds and the model does not.
+ .
+ The text loader meets every property line and counts those it does not
+ keep, so a reader can say what was read and not understood. This
+ loader asks for columns by name, and would never see the ones it did
+ not ask for: it would report that it had understood everything, which
+ is the silence the count exists to prevent. So it counts them, on the
+ conditions under which the .ssp writes the same property (specification
+ section 6.3), and the report is the same from either form: checked,
+ 36 of 36.
++/
+private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) {
+ string id = doc_id.to!string;
+ void count(string property, string where) {
+ auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id
+ ~ " AND " ~ where).to!size_t;
+ if (n > 0) { doc.unknown.property[property] = n; }
+ }
+ count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''");
+ count("dom_status", "coalesce(dom_status,'') != ''");
+ count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''");
+ count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL");
+ count("parent_lev", "coalesce(parent_lev,0) != 0");
+ count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''");
+ count("attrib", "coalesce(attrib,'') != ''");
+ if (db.has("object_subtoc")) {
+ auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o"
+ ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t;
+ if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; }
+ }
+}
+
+/+ ↓ model section 4: the eight levels and their indices +/
+private string levelName(int index) {
+ switch (index) {
+ case 0: return "A";
+ case 1: return "B";
+ case 2: return "C";
+ case 3: return "D";
+ case 4: return "1";
+ case 5: return "2";
+ case 6: return "3";
+ case 7: return "4";
+ default: return "";
+ }
+}
+
+/+ ↓ the JSON columns, read as JSON.
+ .
+ The fixed width arrays are JSON in the database and space separated
+ (or "|" separated) in the text form. This was first written as a
+ character scan that threw away brackets and quotes, which is wrong the
+ moment a value holds an escape: Phobos' JSONValue.toString escapes
+ "/" as "\/", so every heading containing a slash came back with a
+ backslash in it. 36 of 36 documents differed on
+ heading_ancestors_text and nothing else had shown it.
+ .
+ So: parse. The values are small, there are few of them, and a real
+ parser is shorter than the scan it replaces.
++/
+private string[] jsonStrings(string v) {
+ if (v.length == 0) { return []; }
+ string[] out_;
+ try {
+ auto j = parseJSON(v);
+ if (j.type != JSONType.array) { return []; }
+ foreach (e; j.array) {
+ switch (e.type) {
+ case JSONType.string: out_ ~= e.str; break;
+ case JSONType.integer: out_ ~= e.integer.to!string; break;
+ case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
+ case JSONType.float_: out_ ~= e.floating.to!string; break;
+ case JSONType.true_: out_ ~= "true"; break;
+ case JSONType.false_: out_ ~= "false"; break;
+ default: out_ ~= ""; break;
+ }
+ }
+ } catch (Exception) {
+ return [];
+ }
+ return out_;
+}
+
+private int[] jsonInts(string v) {
+ int[] out_;
+ foreach (s; jsonStrings(v)) {
+ if (s.length == 0) { continue; }
+ try { out_ ~= s.to!int; } catch (Exception) {}
+ }
+ return out_;
+}
+
+/+ ↓ space separated in the text form: table_widths, table_aligns +/
+private string jsonWords(string v) {
+ return jsonStrings(v).join(" ");
+}
+
+/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
+ form and a JSON array here. The empty slots are significant: they say
+ which levels the heading sits under, so the separators carry as much
+ as the text does.
++/
+private string jsonJoined(string v) {
+ auto parts = jsonStrings(v);
+ if (parts.length == 0) { return ""; }
+ return parts.join("|");
+}
+#+END_SRC
+
+*** load pod dlang
+
+#+HEADER: :tangle "../dlang/src/ocda/load_pod.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_pod;
+
+/+ ↓ the abstraction out of a pod archive.
+ .
+ A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same
+ archive unwrapped, which is what spine published before it wrapped them.
+ Either way, a pod built with --pod2 carries the abstraction inside it,
+ at media/abstraction/<uid>.<lang>.ssp, and that is what is read.
+ .
+ So this is a third input form and *not* a third model: the .ssp inside
+ is parsed by ocda.load_ssp and comes back as the same value. A writer
+ over the library gains .sisupod without being touched, which is the
+ whole reason the dispatch is in one place.
+ .
+ A pod built without --pod2 carries markup and no abstraction. That is
+ refused, and the refusal says which flag makes a pod this reader can
+ read, because "no abstraction in it" is a fact about the pod rather
+ than about the reader.
++/
+
+import std.algorithm : endsWith, sort;
+import std.array : array;
+import std.file : read;
+import std.string : indexOf;
+import ocda.model;
+import ocda.load_ssp;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ a limit on what a frame may expand to.
+ A reader of an artefact it did not make declines to be told how much
+ memory to allocate. 512 MiB is far above the largest pod in the
+ reference collection (4 MiB) and far below anything that matters.
++/
+enum size_t max_pod_size = 512 * 1024 * 1024;
+
+class OcdaPodError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ which languages of the abstraction the pod carries, in sorted order +/
+string[] podLanguages(string path) {
+ auto bytes = archiveBytes(path);
+ string[] out_;
+ foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) {
+ string lang = langOfSspName(e.name);
+ if (lang.length > 0) { out_ ~= lang; }
+ }
+ return out_.sort.array;
+}
+
+/+ ↓ load one language of the abstraction a pod carries.
+ lang == "" takes the only one there is, and where there is a choice it
+ says so rather than choosing, as the .ocda.db loader does.
++/
+OcdaDocument loadPodFile(string path, string lang = "") {
+ auto bytes = archiveBytes(path);
+ auto entries = zipEntries(bytes);
+ auto ssps = zipFindSuffix(entries, ".ssp");
+ if (ssps.length == 0) {
+ throw new OcdaPodError(path ~ ": carries no .ssp."
+ ~ " A pod carries the abstraction only when spine was asked for it:"
+ ~ " spine --pod2 puts it in, spine --source does not.");
+ }
+ ZipEntry chosen = ssps[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (e; ssps) {
+ if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; }
+ }
+ if (!found) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": no language '" ~ lang
+ ~ "'; it holds: " ~ have);
+ }
+ } else if (ssps.length > 1) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": holds more than one language ("
+ ~ have ~ "); name the one wanted");
+ }
+ auto ssp = zipRead(bytes, chosen);
+ return parseSsp(cast(string) ssp.idup);
+}
+
+/+ ↓ the archive, unwrapped if it was wrapped.
+ The bytes decide, not the name: a .sisupod is a zstd frame and a .zip
+ is not, and a file misnamed either way still reads.
++/
+package ubyte[] archiveBytes(string path) {
+ auto bytes = cast(ubyte[]) read(path);
+ return zstdIsFrame(bytes)
+ ? zstdDecompress(bytes, max_pod_size)
+ : bytes;
+}
+
+/+ ↓ <uid>.<lang>.ssp -> <lang> +/
+package string langOfSspName(string name) {
+ if (!name.endsWith(".ssp")) { return ""; }
+ string stem = name[0 .. $ - 4];
+ auto dot = stem.lastIndexOf('.');
+ return (dot > 0) ? stem[dot + 1 .. $] : "";
+}
+
+private ptrdiff_t lastIndexOf(string s, char c) {
+ for (ptrdiff_t i = s.length - 1; i >= 0; i--) {
+ if (s[i] == c) { return i; }
+ }
+ return -1;
+}
+#+END_SRC
+
+* load ssp dlang
+
+#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.load_ssp;
+
+import std.algorithm : splitter, startsWith;
+import std.array : array, split;
+import std.conv : to, ConvException;
+import std.file : readText;
+import std.stdio : writeln;
+import std.string : strip, stripLeft, indexOf;
+import ocda.model;
+
+OcdaDocument parseSspFile(string filename) {
+ string content = readText(filename);
+ return parseSsp(content);
+}
+
+OcdaDocument parseSsp(string content) {
+ OcdaDocument doc;
+ OcdaObject current_obj;
+ bool in_object = false;
+ /+ ↓ whether a text line has been seen for the object being read.
+ Not "is the text non-empty": an object whose first text line is blank
+ (a verse opening on an empty line, which the sample set has seven of)
+ would then lose it, the next line overwriting rather than appending.
+ +/
+ bool seen_text = false;
+ string current_section = "";
+ string current_block = ""; // meta, make, doc_has, or section name
+
+ foreach (line; content.splitter("\n")) {
+ /+ ↓ comment. The first line of a .ssp declares the format version, and
+ it is the one comment that carries meaning: a reader checks the
+ major part and refuses what it does not know rather than loading
+ an unfamiliar artefact half populated in silence.
+ +/
+ if (line.length > 0 && line[0] == '%') {
+ enum _banner = "% SiSU Document Abstraction v";
+ enum _source = "% Source: ";
+ if (doc.format_version.length == 0 && line.startsWith(_banner)) {
+ doc.format_version = line[_banner.length .. $].strip;
+ } else if (doc.source_filename.length == 0 && line.startsWith(_source)) {
+ doc.source_filename = line[_source.length .. $].strip;
+ }
+ continue;
+ }
+
+ /+ ↓ blank line - finalize current object +/
+ if (line.length == 0 || line.strip.length == 0) {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = OcdaObject.init;
+ in_object = false;
+ seen_text = false;
+ }
+ continue;
+ }
+
+ /+ ↓ section/block open +/
+ if (line[0] == '@') {
+ auto brace_pos = line.indexOf('{');
+ if (brace_pos > 0) {
+ string name = line[1 .. brace_pos].strip;
+ current_block = name;
+ if (!isPreambleBlock(name)) {
+ current_section = name;
+ /+ ↓ a block that is neither preamble nor a section this reader
+ knows: counted, so that --report-unknown can say so +/
+ if (doc.sectionByName(name) is null) {
+ doc.unknown.block[name]++;
+ }
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ section/block close +/
+ if (line[0] == '}') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = OcdaObject.init;
+ in_object = false;
+ seen_text = false;
+ }
+ current_block = "";
+ continue;
+ }
+
+ /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/
+ if (isPreambleBlock(current_block)) {
+ string stripped = line.stripLeft;
+ auto colon_pos = stripped.indexOf(':');
+ if (colon_pos > 0) {
+ string key = stripped[0 .. colon_pos].strip;
+ string value = (colon_pos + 1 < stripped.length)
+ ? stripped[colon_pos + 1 .. $].strip : "";
+ if (current_block == "meta") { doc.metadata[key] = value; }
+ else if (current_block == "make") { doc.make[key] = value; }
+ else if (current_block == "source") { doc.source[key] = value; }
+ else { doc.doc_has[key] = value; }
+ }
+ continue;
+ }
+
+ /+ ↓ object declaration: [N] type ... +/
+ if (line[0] == '[') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+ current_obj = OcdaObject.init;
+ current_obj.section = current_section;
+ in_object = true;
+ seen_text = false;
+ parseObjectDeclaration(line, current_obj);
+ continue;
+ }
+
+ /+ ↓ property: .name: value +/
+ if (line[0] == '.') {
+ if (in_object) {
+ parseProperty(line, current_obj, doc.unknown);
+ }
+ continue;
+ }
+
+ /+ ↓ text content: | content +/
+ if (line.length >= 1 && line[0] == '|') {
+ if (in_object) {
+ string text_content = (line.length > 2) ? line[2 .. $] : "";
+ if (seen_text) {
+ current_obj.text ~= "\n" ~ text_content;
+ } else {
+ current_obj.text = text_content;
+ seen_text = true;
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ no known shape. Counted by its first character so that a reader
+ can be told the file held something this one did not expect
+ +/
+ doc.unknown.line[line[0 .. 1].idup]++;
+ }
+
+ /+ ↓ finalize last object +/
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+
+ return doc;
+}
+
+private void parseObjectDeclaration(string line, ref OcdaObject obj) {
+ /+ parse: [N] type :level identifier +/
+ auto bracket_close = line.indexOf(']');
+ if (bracket_close < 0) return;
+ try {
+ obj.ocn = line[1 .. bracket_close].strip.to!int;
+ } catch (ConvException) {
+ obj.ocn = 0;
+ }
+ string rest = (bracket_close + 1 < line.length)
+ ? line[bracket_close + 1 .. $].strip : "";
+ if (rest.length == 0) return;
+
+ auto parts = rest.split(" ");
+ if (parts.length == 0) return;
+ obj.is_a = parts[0];
+
+ if (obj.is_a == "heading" && parts.length >= 2) {
+ /+ parse :A, :B, :1, etc. +/
+ string lev = parts[1];
+ if (lev.length >= 2 && lev[0] == ':') {
+ obj.heading_level = lev[1 .. $];
+ }
+ /+ optional identifier after level +/
+ if (parts.length >= 3) {
+ obj.identifier = parts[2];
+ }
+ }
+}
+
+/+ ↓ the blocks that are document preamble rather than sections of objects +/
+private bool isPreambleBlock(string name) {
+ return name == "source" || name == "meta" || name == "make" || name == "doc_has";
+}
+
+private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) {
+ /+ parse: .name: value +/
+ auto colon_pos = line.indexOf(':');
+ if (colon_pos < 1) return;
+ string name = line[1 .. colon_pos].strip;
+ string value = (colon_pos + 1 < line.length)
+ ? line[colon_pos + 1 .. $].strip : "";
+
+ switch (name) {
+ case "part": obj.part = value; break;
+ case "is_of_type": obj.is_of_type = value; break;
+ case "section": obj.own_section = value; break;
+ case "identifier": obj.identifier = value; break;
+ case "segment_html_is": obj.segment_html_is = value; break;
+ /+ ↓ the epub segments at heading levels 1 to 4 where the html
+ segments at level 4 alone, so the two are different files and
+ the abstraction records both +/
+ case "segment_epub": obj.segment_epub = value; break;
+ case "segment_epub_is": obj.segment_epub_is = value; break;
+ case "table_aligns": obj.table_aligns = value; break;
+ case "stow_link": obj.stow_link ~= value; break;
+ case "anchor_tag": obj.anchor_tags ~= value; break;
+ case "parent": obj.parent_ocn = toIntSafe(value); break;
+ case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break;
+ case "ocn_range": {
+ auto r = parseIntArray(value);
+ if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; }
+ break;
+ }
+ case "children": obj.children = parseIntArray(value); break;
+ case "ancestors": obj.ancestors = parseIntArray(value); break;
+ case "dummy": obj.dummy_heading = (value == "true"); break;
+ case "ocn_off": obj.ocn_off = (value == "true"); break;
+ case "indent":
+ auto parts = value.split(" ");
+ if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]);
+ if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]);
+ break;
+ case "bullet": obj.bullet = (value == "true"); break;
+ case "lang": obj.lang = value; break;
+ case "has":
+ foreach (flag; value.split(" ")) {
+ switch (flag) {
+ case "links": obj.has_links = true; break;
+ case "notes_reg": obj.has_notes_reg = true; break;
+ case "notes_star": obj.has_notes_star = true; break;
+ case "images": obj.has_images = true; break;
+ default: break;
+ }
+ }
+ break;
+ case "segment": obj.segment = value; break;
+ case "segment_prev": obj.segment_prev = value; break;
+ case "segment_next": obj.segment_next = value; break;
+ case "anchor": obj.anchor = value; break;
+ case "heading_lev_anchor": obj.heading_lev_anchor = value; break;
+ case "table_cols": obj.table_cols = toIntSafe(value); break;
+ case "table_widths": obj.table_widths = value; break;
+ case "table_header": obj.table_header = (value == "true"); break;
+ case "code_syntax": obj.code_syntax = value; break;
+ case "code_linenumbers": obj.code_linenumbers = (value == "true"); break;
+ case "heading_ancestors_text": obj.heading_ancestors_text = value; break;
+ case "sha256": obj.sha256 = value; break;
+ case "image": obj.images ~= parseImageRecord(value); break;
+ /+ ↓ a property this reader has no use for, or has never heard of.
+ Ignored, which is the conforming behaviour within a major
+ version, but counted rather than passed over in silence
+ +/
+ default: unknown.property[name]++; break;
+ }
+}
+
+private OcdaImage parseImageRecord(string value) {
+ /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...]
+ filename first, remaining fields order-independent; unrecognised
+ fields are kept rather than discarded +/
+ OcdaImage img;
+ bool first = true;
+ foreach (field; value.split(" ")) {
+ if (field.length == 0) continue;
+ if (first) {
+ img.name = field;
+ first = false;
+ continue;
+ }
+ auto colon_pos = field.indexOf(':');
+ if (colon_pos < 1) continue;
+ string key = field[0 .. colon_pos];
+ string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : "";
+ switch (key) {
+ case "sha256": img.sha256 = val; break;
+ case "bytes": img.bytes = toUlongSafe(val); break;
+ case "px":
+ auto dims = val.split("x");
+ if (dims.length == 2) {
+ img.width = toIntSafe(dims[0]);
+ img.height = toIntSafe(dims[1]);
+ }
+ break;
+ case "missing": img.missing = (val == "true"); break;
+ default: img.fields[key] = val; break;
+ }
+ }
+ return img;
+}
+
+/+ ↓ two values the text form leaves out because they can be worked out.
+ .
+ The .ssp emits a property only when it carries information, so
+ .identifier is absent when it is simply the ocn, and .anchor is absent
+ when it is the segment name. The .ocda.db, being columns rather than
+ lines, stores both unconditionally. So a loader that takes the text
+ form literally gives a *different model* from the one the database
+ gives, and the difference is not small: in the reference collection
+ the identifier is omitted on all but a handful of objects.
+ .
+ The model is the thing both serialisations describe, so the loader
+ fills them in. Found by dumping both models and diffing them, which no
+ amount of comparing section counts would have shown.
++/
+private void deriveOmitted(ref OcdaObject obj) {
+ import std.conv : to;
+ if (obj.identifier.length == 0 && obj.ocn != 0) {
+ obj.identifier = obj.ocn.to!string;
+ }
+ if (obj.anchor.length == 0 && obj.segment.length > 0) {
+ obj.anchor = obj.segment;
+ }
+}
+
+private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) {
+ deriveOmitted(obj);
+ auto sec = doc.sectionByName(section);
+ if (sec !is null) {
+ ,*sec ~= obj;
+ }
+}
+
+private int toIntSafe(string s) {
+ try { return s.strip.to!int; }
+ catch (ConvException) { return 0; }
+}
+
+private ulong toUlongSafe(string s) {
+ try { return s.strip.to!ulong; }
+ catch (ConvException) { return 0; }
+}
+
+private int[] parseIntArray(string s) {
+ int[] result;
+ foreach (part; s.split(" ")) {
+ auto v = toIntSafe(part);
+ result ~= v;
+ }
+ return result;
+}
+#+END_SRC
+
+** compression
+*** dlang read zip
+
+#+HEADER: :tangle "../dlang/src/ocda/zip.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.zip;
+
+/+ ↓ enough of the zip format to find one file in a pod and read it.
+ .
+ Not a zip library. A pod archive holds a handful of entries, spine
+ writes them **stored** (uncompressed) so that the outer zstd frame does
+ the compressing once over the whole thing, and what is wanted from it
+ here is one named entry. So: read the central directory, find the
+ entry, return its bytes.
+ .
+ Deflate is *refused rather than mishandled*. Phobos can inflate, but a
+ library that says it read an entry it in fact skipped is worse than one
+ that says it cannot, and spine's own archives do not need it.
++/
+
+import std.algorithm : endsWith;
+import std.conv : to;
+
+class OcdaZipError : Exception {
+ this(string msg) { super(msg); }
+}
+
+struct ZipEntry {
+ string name;
+ uint method; // 0 stored, 8 deflate
+ uint compressed;
+ uint uncompressed;
+ uint local_offset;
+}
+
+/+ ↓ the entries, from the central directory at the end of the file +/
+ZipEntry[] zipEntries(const(ubyte)[] bytes) {
+ enum uint eocd_sig = 0x06054b50;
+ enum uint cd_sig = 0x02014b50;
+ if (bytes.length < 22) {
+ throw new OcdaZipError("too short to be a zip archive");
+ }
+ /+ ↓ the end of central directory record is last, and may be followed by a
+ comment of up to 65535 bytes, so it is searched for backwards +/
+ ptrdiff_t eocd = -1;
+ ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0;
+ for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) {
+ if (u32(bytes, i) == eocd_sig) { eocd = i; break; }
+ }
+ if (eocd < 0) {
+ throw new OcdaZipError("no end of central directory record;"
+ ~ " it does not look like a zip archive");
+ }
+ uint count = u16(bytes, eocd + 10);
+ uint offset = u32(bytes, eocd + 16);
+ ZipEntry[] out_;
+ size_t p = offset;
+ foreach (_; 0 .. count) {
+ if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) {
+ throw new OcdaZipError("central directory entry "
+ ~ out_.length.to!string ~ " is malformed");
+ }
+ ZipEntry e;
+ e.method = u16(bytes, p + 10);
+ e.compressed = u32(bytes, p + 20);
+ e.uncompressed = u32(bytes, p + 24);
+ uint name_len = u16(bytes, p + 28);
+ uint extra_len = u16(bytes, p + 30);
+ uint cmnt_len = u16(bytes, p + 32);
+ e.local_offset = u32(bytes, p + 42);
+ e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup;
+ out_ ~= e;
+ p += 46 + name_len + extra_len + cmnt_len;
+ }
+ return out_;
+}
+
+/+ ↓ one entry's bytes, from its local header +/
+ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) {
+ enum uint local_sig = 0x04034b50;
+ size_t p = e.local_offset;
+ if (p + 30 > bytes.length || u32(bytes, p) != local_sig) {
+ throw new OcdaZipError(e.name ~ ": local header is malformed");
+ }
+ uint name_len = u16(bytes, p + 26);
+ uint extra_len = u16(bytes, p + 28);
+ size_t start = p + 30 + name_len + extra_len;
+ size_t end = start + e.compressed;
+ if (end > bytes.length) {
+ throw new OcdaZipError(e.name ~ ": runs past the end of the archive");
+ }
+ if (e.method != 0) {
+ throw new OcdaZipError(e.name ~ ": compression method "
+ ~ e.method.to!string ~ "; this reader takes only stored entries,"
+ ~ " which is what spine writes");
+ }
+ return bytes[start .. end].dup;
+}
+
+/+ ↓ the first entry whose name ends this way, or none.
+ Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid.
++/
+ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) {
+ ZipEntry[] out_;
+ foreach (e; entries) {
+ if (e.name.endsWith(suffix)) { out_ ~= e; }
+ }
+ return out_;
+}
+
+private uint u16(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i] | (cast(uint) b[i + 1] << 8);
+}
+
+private uint u32(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i]
+ | (cast(uint) b[i + 1] << 8)
+ | (cast(uint) b[i + 2] << 16)
+ | (cast(uint) b[i + 3] << 24);
+}
+#+END_SRC
+
+*** dlang read zstd
+
+#+HEADER: :tangle "../dlang/src/ocda/zstd.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module ocda.zstd;
+
+/+ ↓ libzstd, declared rather than bound.
+ .
+ A .sisupod is one zstd frame wrapping a zip archive, so reading one
+ needs exactly two things from the library: is this a frame, and what is
+ in it. Both are declared here and the C library is linked, which is the
+ same arrangement ocda.sqlite uses: no vendored tree, no generated
+ header, and the prototypes say what is being called.
++/
+
+extern (C) @nogc nothrow private {
+ size_t ZSTD_decompress(void* dst, size_t dstCapacity,
+ const(void)* src, size_t compressedSize);
+ ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
+ uint ZSTD_isError(size_t code);
+ const(char)* ZSTD_getErrorName(size_t code);
+}
+
+/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/
+private enum ulong contentSizeUnknown = ulong.max;
+private enum ulong contentSizeError = ulong.max - 1;
+
+/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/
+private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD];
+
+class OcdaZstdError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ does this begin with a zstd frame?
+ Asked of the bytes rather than of the filename, because what is inside
+ decides how a thing is opened. A .sisupod is a frame; a .zip published
+ before spine wrapped them is not, and both are read.
++/
+bool zstdIsFrame(const(ubyte)[] bytes) {
+ if (bytes.length < zstd_magic.length) { return false; }
+ foreach (i, b; zstd_magic) {
+ if (bytes[i] != b) { return false; }
+ }
+ return true;
+}
+
+/+ ↓ one frame in, its contents out.
+ max_size is a refusal rather than a buffer size: a frame declares how
+ much it expands to, and a frame that claims more than the caller is
+ willing to hold is not decompressed at all. A reader of an artefact it
+ did not make wants that limit.
++/
+ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) {
+ if (!zstdIsFrame(bytes)) {
+ throw new OcdaZstdError("not a zstd frame");
+ }
+ ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length);
+ if (declared == contentSizeError) {
+ throw new OcdaZstdError("zstd frame: could not read its content size");
+ }
+ if (declared == contentSizeUnknown) {
+ throw new OcdaZstdError("zstd frame declares no content size;"
+ ~ " this reader takes only frames that do");
+ }
+ if (declared > max_size) {
+ import std.conv : to;
+ throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string
+ ~ " bytes, over the " ~ max_size.to!string ~ " byte limit");
+ }
+ auto out_ = new ubyte[](cast(size_t) declared);
+ size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length);
+ if (ZSTD_isError(got)) {
+ import std.string : fromStringz;
+ throw new OcdaZstdError("zstd: "
+ ~ ZSTD_getErrorName(got).fromStringz.idup);
+ }
+ if (got != declared) {
+ throw new OcdaZstdError("zstd frame expanded to a different size"
+ ~ " than it declared");
+ }
+ return out_;
+}
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* __END__
diff --git a/org/d-read.org b/org/d-read.org
index 9cf2416..c9f0334 100644
--- a/org/d-read.org
+++ b/org/d-read.org
@@ -16,7 +16,8 @@
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
* dlang :dlang:
-** dlang read main
+** main & tree
+*** dlang read main
#+HEADER: :tangle "../dlang/src/read/main.d"
#+BEGIN_SRC d
@@ -251,7 +252,7 @@ void printFiles(string path) {
}
#+END_SRC
-** dlang read tree
+*** dlang read tree
#+HEADER: :tangle "../dlang/src/read/tree.d"
#+BEGIN_SRC d
@@ -327,7 +328,8 @@ private string stripInternalMarkup(string text) {
}
#+END_SRC
-** dlang read verify
+** check
+*** dlang read verify
#+HEADER: :tangle "../dlang/src/read/verify.d"
#+BEGIN_SRC d
@@ -546,7 +548,7 @@ private int beInt(const(ubyte)[] b) {
}
#+END_SRC
-** dlang read validate
+*** dlang read validate
#+HEADER: :tangle "../dlang/src/read/validate.d"
#+BEGIN_SRC d
@@ -818,42 +820,573 @@ void reportValidation(string filename, SspFinding[] found) {
}
#+END_SRC
-** dlang read report unknown
+*** dlang read report unknown
#+HEADER: :tangle "../dlang/src/read/report_unknown.d"
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
-#+END_SRC
+module read.report_unknown;
-** dlang read zip
+import std.algorithm : sort;
+import std.array : array;
+import std.stdio : writeln, writefln;
+import ocda.model;
-#+HEADER: :tangle "../dlang/src/read/zip.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
+/+ ↓ say what was read and not understood.
+ This is what makes the reader usable as a check on the format
+ specification: a reader that ignores what it does not know is
+ conforming, but one that cannot say what it ignored cannot tell you
+ whether the specification it was written from was complete. Silence
+ and understanding look the same from outside.
++/
+void reportUnknown(ref OcdaDocument doc) {
+ if (!doc.unknown.any) {
+ writeln("\nread and understood: every property, block and line was known");
+ return;
+ }
+ writeln("\nread and NOT understood:");
+ if (doc.unknown.property.length > 0) {
+ writeln(" properties ignored (", doc.unknown.property.length, " distinct):");
+ foreach (name; doc.unknown.property.keys.sort) {
+ writefln(" .%-26s %8d", name, doc.unknown.property[name]);
+ }
+ }
+ if (doc.unknown.block.length > 0) {
+ writeln(" blocks ignored (", doc.unknown.block.length, " distinct):");
+ foreach (name; doc.unknown.block.keys.sort) {
+ writefln(" @%-26s %8d", name, doc.unknown.block[name]);
+ }
+ }
+ if (doc.unknown.line.length > 0) {
+ writeln(" lines of no known shape, by first character:");
+ foreach (c; doc.unknown.line.keys.sort) {
+ writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]);
+ }
+ }
+ writeln(" (ignoring these is conforming within a major version;",
+ " the count is so that it is not silent)");
+}
#+END_SRC
-** dlang read zstd
+** misc
+*** dlang read html
-#+HEADER: :tangle "../dlang/src/read/zstd.d"
+#+HEADER: :tangle "../dlang/src/read/html.d"
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
-#+END_SRC
+module read.html;
-** dlang read html
+import std.array : replace, split, join, appender;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, startsWith;
+import ocda.model;
-#+HEADER: :tangle "../dlang/src/read/html.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
+void outputHtmlScroll(OcdaDocument doc, string output_file) {
+ auto buf = appender!(string[]);
+
+ buf ~= htmlHead(doc);
+
+ /+ ↓ title +/
+ if (doc.head.length > 0) {
+ auto title_obj = doc.head[0];
+ buf ~= `<div class="title">`;
+ buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`;
+ if (doc.meta("title.subtitle").length > 0)
+ buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`;
+ buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`;
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ table of contents +/
+ if (doc.toc.length > 0) {
+ buf ~= `<div class="toc">`;
+ buf ~= `<h2>Table of Contents</h2>`;
+ foreach (obj; doc.toc) {
+ if (obj.is_a == "toc") {
+ string text = convertInlineMarkup(obj.text);
+ int indent = obj.indent_base;
+ string cls = (indent > 4) ? "toc-sub" : "toc-entry";
+ buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ buf ~= `<hr/>`;
+ }
+
+ /+ ↓ body +/
+ buf ~= `<div class="body">`;
+ foreach (obj; doc.body) {
+ buf ~= renderObject(obj);
+ }
+ buf ~= `</div>`;
+
+ /+ ↓ endnotes +/
+ bool has_endnotes = false;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") { has_endnotes = true; break; }
+ }
+ if (has_endnotes) {
+ buf ~= `<div class="endnotes">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Endnotes</h2>`;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") {
+ string text = convertInlineMarkup(obj.text);
+ string parent_ref = (obj.parent_ocn > 0)
+ ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>`
+ : "";
+ buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ glossary +/
+ if (doc.glossary.length > 0) {
+ buf ~= `<div class="glossary">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Glossary</h2>`;
+ foreach (obj; doc.glossary) {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`;
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ bookindex +/
+ bool has_bookindex = false;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") { has_bookindex = true; break; }
+ }
+ if (has_bookindex) {
+ buf ~= `<div class="bookindex">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Book Index</h2>`;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="index-entry">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ buf ~= htmlTail();
+
+ /+ ↓ write output +/
+ string[] output = buf.data;
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; output) f.writeln(line);
+ } else {
+ foreach (line; output) writeln(line);
+ }
+}
+
+private string[] renderObject(OcdaObject obj) {
+ /+ ↓ markup commentary is carried by the abstraction and is not output.
+ Rendering it would put the author notes into the document, which
+ spine itself does not do
+ +/
+ if (!obj.renders) { return []; }
+ string[] lines;
+ string text = convertInlineMarkup(obj.text);
+ string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off)
+ ? ` id="o` ~ obj.ocn.to!string ~ `"`
+ : "";
+ string ocn_display = (obj.ocn > 0 && !obj.ocn_off)
+ ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>`
+ : "";
+
+ switch (obj.is_a) {
+ case "heading":
+ string tag = headingTag(obj.heading_level);
+ string seg_anchor = (obj.segment.length > 0)
+ ? ` id="` ~ htmlEscape(obj.segment) ~ `"`
+ : ocn_anchor;
+ lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`;
+ break;
+ case "para":
+ string indent_cls = "";
+ if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`;
+ else if (obj.bullet) indent_cls = ` class="bullet"`;
+ lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ case "code":
+ lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`;
+ break;
+ case "table":
+ lines ~= renderTable(obj, ocn_anchor);
+ break;
+ case "group":
+ lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`;
+ break;
+ case "block":
+ lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`;
+ break;
+ case "verse":
+ lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`;
+ break;
+ case "quote":
+ lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`;
+ break;
+ default:
+ if (text.length > 0)
+ lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ }
+ return lines;
+}
+
+private string headingTag(string level) {
+ switch (level) {
+ case "A": return "h1";
+ case "B": return "h2";
+ case "C": return "h3";
+ case "D": return "h4";
+ case "1": return "h3";
+ case "2": return "h4";
+ case "3": return "h5";
+ case "4": return "h6";
+ default: return "h4";
+ }
+}
+
+private string renderTable(OcdaObject obj, string ocn_anchor) {
+ string[] lines;
+ lines ~= `<table` ~ ocn_anchor ~ `>`;
+ /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/
+ auto rows = obj.text.split("\n");
+ bool first_row = obj.table_header;
+ foreach (row; rows) {
+ if (row.strip.length == 0) continue;
+ string tag = first_row ? "th" : "td";
+ lines ~= `<tr>`;
+ foreach (cell; row.split("┊")) {
+ lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`;
+ }
+ lines ~= `</tr>`;
+ first_row = false;
+ }
+ lines ~= `</table>`;
+ return lines.join("\n");
+}
+
+/+ ↓ convert spine internal Unicode markup to HTML +/
+string convertInlineMarkup(string text) {
+ if (text.length == 0) return "";
+ string result = htmlEscape(text);
+
+ /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc.
+ Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X
+ The ┨┣ form is used within heading content +/
+
+ /+ ↓ bold +/
+ result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>");
+ result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>");
+
+ /+ ↓ emphasis +/
+ result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>");
+ result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>");
+
+ /+ ↓ italic +/
+ result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>");
+ result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>");
+
+ /+ ↓ underscore +/
+ result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>");
+ result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>");
+
+ /+ ↓ superscript +/
+ result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>");
+ result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>");
+
+ /+ ↓ subscript +/
+ result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>");
+ result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>");
+
+ /+ ↓ monospace +/
+ result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>");
+
+ /+ ↓ links: ┥link text┝┤url├ +/
+ result = replaceLinks(result);
+
+ /+ ↓ footnote markers: 【N text】 +/
+ result = replaceFootnotes(result);
+
+ /+ ↓ line breaks +/
+ result = result.replace("┘", "<br/>");
+ result = result.replace("┙", "<br/>");
+
+ /+ ↓ non-breaking spaces +/
+ result = result.replace("░", "&nbsp;");
+
+ /+ ↓ bullet markers +/
+ result = result.replace("● ", "&#8226; ");
+
+ /+ ↓ page break markers +/
+ result = result.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ return result;
+}
+
+private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) {
+ /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/
+ string result = text;
+ while (true) {
+ auto open_pos = result.indexOf(open_mark);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + open_mark.length;
+ /+ ↓ look for closing ⑆X +/
+ string close_mark = "⑆" ~ close_suffix;
+ auto close_pos = indexOf(result[after_open .. $], close_mark);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + close_mark.length .. $];
+ }
+ return result;
+}
+
+private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) {
+ /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/
+ string result = text;
+ string full_open = open_mark ~ bracket_open;
+ while (true) {
+ auto open_pos = result.indexOf(full_open);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + full_open.length;
+ auto close_pos = indexOf(result[after_open .. $], bracket_close);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + bracket_close.length .. $];
+ }
+ return result;
+}
+
+private string replaceLinks(string text) {
+ /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/
+ string result = text;
+ while (true) {
+ auto link_open = result.indexOf("┥");
+ if (link_open < 0) break;
+ auto link_mid = indexOf(result[link_open .. $], "┝┤");
+ if (link_mid < 0) break;
+ link_mid += link_open;
+ auto link_close = indexOf(result[link_mid .. $], "├");
+ if (link_close < 0) break;
+ link_close += link_mid;
+ string link_text = result[link_open + "┥".length .. link_mid];
+ string url = result[link_mid + "┝┤".length .. link_close];
+ /+ ↓ handle internal links (¤ prefix) +/
+ if (url.length > 0 && url[0] == '¤') {
+ url = url[("¤").length .. $];
+ }
+ /+ ↓ handle OCN-only links (#N) +/
+ if (url.length > 0 && url[0] == '#') {
+ url = "#o" ~ url[1 .. $];
+ }
+ result = result[0 .. link_open]
+ ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>`
+ ~ result[link_close + "├".length .. $];
+ }
+ return result;
+}
+
+private string replaceFootnotes(string text) {
+ /+ ↓ replace 【N content】 with superscript note number +/
+ string result = text;
+ while (true) {
+ auto fn_open = result.indexOf("【");
+ if (fn_open < 0) break;
+ auto fn_close = indexOf(result[fn_open .. $], "】");
+ if (fn_close < 0) break;
+ fn_close += fn_open;
+ string fn_content = result[fn_open + "【".length .. fn_close];
+ /+ ↓ extract note number (first token before space) +/
+ auto space_pos = fn_content.indexOf(' ');
+ string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content;
+ result = result[0 .. fn_open]
+ ~ `<sup class="fn">` ~ note_num ~ `</sup>`
+ ~ result[fn_close + "】".length .. $];
+ }
+ return result;
+}
+
+string htmlEscape(string text) {
+ return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+ .replace("\"", "&quot;");
+}
+
+private string[] htmlHead(OcdaDocument doc) {
+ string[] lines;
+ lines ~= `<!DOCTYPE html>`;
+ lines ~= `<html>`;
+ lines ~= `<head>`;
+ lines ~= `<meta charset="utf-8"/>`;
+ lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`;
+ lines ~= `<style>`;
+ lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`;
+ lines ~= ` .title { text-align: center; margin-bottom: 2em; }`;
+ lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`;
+ lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`;
+ lines ~= ` .toc { margin-bottom: 2em; }`;
+ lines ~= ` .toc-entry { margin: 0.2em 0; }`;
+ lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`;
+ lines ~= ` .indent-1 { margin-left: 2em; }`;
+ lines ~= ` .indent-2 { margin-left: 4em; }`;
+ lines ~= ` .indent-3 { margin-left: 6em; }`;
+ lines ~= ` .bullet { margin-left: 2em; }`;
+ lines ~= ` .bullet:before { content: "\2022 "; }`;
+ lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`;
+ lines ~= ` .group { margin: 1em 2em; }`;
+ lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`;
+ lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`;
+ lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`;
+ lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`;
+ lines ~= ` th { background: #f0f0f0; }`;
+ lines ~= ` .endnote { font-size: 0.9em; }`;
+ lines ~= ` .fn { color: #666; }`;
+ lines ~= ` .index-entry { font-size: 0.9em; }`;
+ lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`;
+ lines ~= `</style>`;
+ lines ~= `</head>`;
+ lines ~= `<body>`;
+ return lines;
+}
+
+private string[] htmlTail() {
+ return [`</body>`, `</html>`];
+}
#+END_SRC
-** dlang read images
+*** dlang read images
#+HEADER: :tangle "../dlang/src/read/images.d"
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
+module read.images;
+
+import std.array : join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import ocda.model;
+
+/+ ↓ the .image records of a document, gathered per distinct file, with the
+ objects referencing each file, in order of first appearance +/
+struct ImageEntry {
+ string name;
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+ bool missing;
+ string[] refs;
+ string[] other_sha256; // same filename, differing digest
+}
+
+struct ImageRecords {
+ int records; // .image lines, an image referenced twice counts twice
+ ImageEntry[] entries; // distinct files
+}
+
+ImageRecords gatherImageRecords(OcdaDocument doc) {
+ ImageRecords ret;
+ size_t[string] seen;
+ foreach (section; sectionNames) {
+ auto objs = doc.sectionByName(section);
+ if (objs is null) continue;
+ foreach (obj; *objs) {
+ foreach (img; obj.images) {
+ ret.records += 1;
+ string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]";
+ if (auto at = img.name in seen) {
+ if (ret.entries[*at].sha256 != img.sha256
+ && img.sha256.length > 0
+ && ret.entries[*at].sha256.length > 0) {
+ ret.entries[*at].other_sha256 ~= img.sha256;
+ }
+ ret.entries[*at].refs ~= _ref;
+ } else {
+ ImageEntry e_;
+ e_.name = img.name;
+ e_.sha256 = img.sha256;
+ e_.bytes = img.bytes;
+ e_.width = img.width;
+ e_.height = img.height;
+ e_.missing = img.missing;
+ e_.refs = [_ref];
+ seen[img.name] = ret.entries.length;
+ ret.entries ~= e_;
+ }
+ }
+ }
+ }
+ return ret;
+}
+
+/+ ↓ image manifest, line oriented and greppable:
+
+ <sha256> <bytes> <px> <filename> refs: body[97] body[412]
+ MISSING <filename> refs: body[840]
+
+ The point of the record in the .ssp is verification (is the image
+ shipped in this pod the image the abstraction was built from), so this
+ output is the shape a verifier wants: one line per distinct file.
++/
+void outputImages(OcdaDocument doc, string output_file) {
+ auto gathered = gatherImageRecords(doc);
+ string[] lines;
+ int missing = 0;
+ foreach (e; gathered.entries) {
+ if (e.missing) missing += 1;
+ }
+ lines ~= "Images: " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ ", " ~ missing.to!string ~ " missing";
+ lines ~= "";
+ foreach (e; gathered.entries) {
+ string px = (e.width > 0 && e.height > 0)
+ ? e.width.to!string ~ "x" ~ e.height.to!string : "-";
+ lines ~= (e.missing)
+ ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ")
+ : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px
+ ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ");
+ /+ ↓ one filename, more than one digest: the .ssp is inconsistent,
+ or two distinct files share a name +/
+ foreach (other; e.other_sha256) {
+ lines ~= " WARNING differing digest for same filename: " ~ other;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+/+ ↓ count of .image records in a document, for the parse summary +/
+int imageRecordCount(OcdaDocument doc) {
+ return gatherImageRecords(doc).records;
+}
+
+package immutable string[] sectionNames = [
+ "head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb",
+];
#+END_SRC
-** dlang read llm text
+** llm
+*** dlang read llm text
#+HEADER: :tangle "../dlang/src/read/llm_text.d"
#+BEGIN_SRC d
@@ -1042,7 +1575,7 @@ string stripMarkup(string text) {
}
#+END_SRC
-** dlang read llm jsonl
+*** dlang read llm jsonl
#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d"
#+BEGIN_SRC d
@@ -1196,4 +1729,3 @@ private string jsonStr(string s) {
#+END_SRC
* __END__
-
diff --git a/org/gleam-ocda.org b/org/gleam-ocda.org
new file mode 100644
index 0000000..a7cac9a
--- /dev/null
+++ b/org/gleam-ocda.org
@@ -0,0 +1,2876 @@
+-*- mode: org -*-
+#+TITLE: gleam lib
+#+DESCRIPTION: gleam lib
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :noweb yes :padline no
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* gleam :gleam:
+** README.md
+*** tangle
+
+#+HEADER: :tangle ../gleam/README.md
+#+HEADER: :noweb yes
+#+BEGIN_SRC md
+# ocda-lib, in Gleam
+
+The abstraction's model and both its loaders, as a Gleam library, with its
+reader and the two programs that put it to the question.
+
+| | |
+|------------------|-----------------------------------------------------------------------------------------|
+| `ocda/model` | the document, the object, the image. Depends on nothing but the standard library |
+| `ocda/load_ssp` | text -> model |
+| `ocda/load_db` | sqlite -> model |
+| `ocda/load` | what a path is, and loading it |
+| `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries |
+| `ocda/dump` | the model written out canonically, as an oracle |
+| `read/main` | `ocda-read`, the reader |
+| `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks |
+
+## Use
+
+```gleam
+import ocda/load
+import ocda/model
+
+case load.load(path, "en", False) {
+ Error(why) -> io.println_error(why)
+ Ok(doc) -> {
+ // and from here a writer neither knows nor cares which of the two
+ // files it was handed
+ list.each(model.section(doc, "body"), fn(obj) { ... })
+ }
+}
+```
+
+## The reader
+
+```sh
+nix build
+./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown]
+ <file>.ssp|<file>.ocda.db|<file>.zip [lang]
+```
+
+`ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers
+had, over the library, so each reads every form this library reads; the
+others are the D reader's. **Its output is the D `ocda-read`'s, byte for
+byte**, in every mode both have. In `src/read/`, because Gleam compiles
+nothing outside `src/`; a program depending on the library compiles it too,
+and does not use it.
+
+A `.sisupod` it refuses, saying why: the zstd frame, below.
+
+## The two programs
+
+```sh
+nix build
+./result/bin/ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+./result/bin/ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+```
+
+`agree` asks whether the two loaders give the same model. `dump` writes
+the whole model out canonically, and is the stronger check of the two,
+because it is **byte for byte what the D library's `ocda-dump` writes**.
+So the four questions can all be asked with `diff`:
+
+| | |
+|-----------------------------|-----------------------------|
+| D: `.ssp` == `.ocda.db` | 36 of 36 document-languages |
+| Gleam: `.ssp` == `.ocda.db` | 36 of 36 |
+| `.ssp`: D == Gleam | 36 of 36 |
+| `.ocda.db`: D == Gleam | 36 of 36 |
+
+## Deliberately a mirror of the D library
+
+Field for field, name for name, including where that is not the most
+Gleamish choice:
+
+- `text` is one `String` with the lines joined by newline, not a
+ `List(String)`. So an object whose only text line is blank is
+ indistinguishable from an object with no text, *in both languages, in the
+ same place*. A difference that existed in one and not the other would be
+ worse than a loss both share, because it would show up in every dump and
+ mean nothing.
+- the properties are typed fields rather than a list of pairs. A writer
+ wants `obj.parent_ocn`, not a lookup that might miss.
+
+The point is that the two agree. Making them agree is much cheaper when
+they are shaped alike, and the shape is the specification's, not either
+language's.
+
+## What the agreement found
+
+Four disagreements, none of which a comparison of section counts or of a
+handful of fields would have shown. All four are recorded in the D
+library's history; the short version:
+
+1. `.identifier` and `.anchor` are omitted by the text form when they can
+ be worked out (the identifier is the ocn, the anchor is the segment).
+ Both loaders now derive them. Over one document that is 1,314
+ identifiers.
+2. `make` and `doc_has` are one metadata table in the database,
+ distinguished by a key prefix. Both loaders unprefix them.
+3. The JSON columns must be *parsed*, not scanned: the encoder writes `/`
+ as `\/` and every heading with a slash in it came back wrong.
+4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`,
+ which is one language. It is a fact about the file, so neither loader
+ puts it in the document.
+
+## Build
+
+```sh
+nix build # sandboxed, no network
+nix develop -c gleam test
+```
+
+Shipped as an Erlang shipment rather than an escript, because sqlite on the
+BEAM is a NIF and a NIF cannot be loaded from inside an escript archive.
+The dependency hash in `flake.nix` pins what `manifest.toml` pins; change
+one and the other must change with it.
+
+## Format
+
+Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model,
+`ssp-format-specification.org` and `ocda-db-format-specification.org` for the
+two serialisations. At format **2.x**.
+#+END_SRC
+
+** gleam.toml
+*** tangle
+
+#+HEADER: :tangle ../gleam/gleam.toml
+#+HEADER: :noweb yes
+#+BEGIN_SRC toml
+name = "ocda_lib"
+version = "0.1.0"
+description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations"
+licences = ["AGPL-3.0-or-later"]
+
+[dependencies]
+gleam_stdlib = ">= 1.0.0 and < 2.0.0"
+argv = ">= 1.1.0 and < 2.0.0"
+simplifile = ">= 2.7.0 and < 3.0.0"
+sqlight = ">= 1.0.0 and < 2.0.0"
+gleam_json = ">= 3.0.0 and < 4.0.0"
+
+[dev-dependencies]
+gleeunit = ">= 1.0.0 and < 2.0.0"
+#+END_SRC
+
+** gleam ocda_lib
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda_lib.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<ocda_lib>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: ocda_lib
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// ocda-lib, in Gleam: the abstraction as a library, put to the question.
+////
+//// A library has no binary, so this is the programs over it: the reader,
+//// and the two that test its claims.
+////
+//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+//// ocda-lib read [options] <file> [lang] (also installed as ocda-read)
+////
+//// `dump` writes the whole loaded model out canonically, byte for byte as
+//// the D library's `ocda-dump` writes it, so that the two implementations
+//// can be diffed. `agree` asks the same question the D library's
+//// `ocda-agree` asks, within this language: does loading a document's
+//// `.ssp` give what loading its `.ocda.db` gives?
+
+import argv
+import gleam/int
+import gleam/io
+import gleam/list
+import gleam/string
+import ocda/dump
+import ocda/load
+import ocda/model.{type Document}
+import read/main as reader
+
+pub fn main() {
+ case argv.load().arguments {
+ ["dump", "--unknown", path] -> dump_file(path, "", True)
+ ["dump", "--unknown", path, lang] -> dump_file(path, lang, True)
+ ["dump", path] -> dump_file(path, "", False)
+ ["dump", path, lang] -> dump_file(path, lang, False)
+ ["agree", ssp, db] -> agree(ssp, db, "")
+ ["agree", ssp, db, lang] -> agree(ssp, db, lang)
+ ["read", ..rest] ->
+ case reader.run(rest) {
+ 0 -> Nil
+ code -> halt(code)
+ }
+ _ -> usage()
+ }
+}
+
+fn usage() -> Nil {
+ io.println_error(
+ "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+ ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+ ocda-lib read [--help] [options] <file> [lang]",
+ )
+ halt(2)
+}
+
+fn dump_file(path: String, lang: String, unknown: Bool) -> Nil {
+ case load.load(path, lang, False) {
+ Error(message) -> fail(message)
+ Ok(doc) ->
+ case unknown {
+ True -> io.print(dump.unknown(doc))
+ False -> io.print(dump.document(doc))
+ }
+ }
+}
+
+/// The library's whole purpose is that a writer cannot tell which
+/// serialisation it was handed. That is only true if the two loaders give
+/// the same model, so this asks, and says where they part company rather
+/// than only that they did.
+fn agree(ssp_path: String, db_path: String, lang: String) -> Nil {
+ case load.load(ssp_path, "", False), load.load(db_path, lang, False) {
+ Error(message), _ -> fail(message)
+ _, Error(message) -> fail(message)
+ Ok(from_ssp), Ok(from_db) -> {
+ let differences = compare(from_ssp, from_db)
+ case differences {
+ [] -> io.println("agree: the two loaders give the same model")
+ _ -> {
+ list.each(list.take(differences, 12), io.println)
+ case list.length(differences) > 12 {
+ True -> io.println(" ... stopping after 12")
+ False -> Nil
+ }
+ io.println(int.to_string(list.length(differences)) <> " difference(s)")
+ halt(1)
+ }
+ }
+ }
+ }
+}
+
+/// Compared through the dump rather than field by field, which is both
+/// shorter and stricter: the dump carries every field, so nothing can
+/// differ without showing here.
+fn compare(from_ssp: Document, from_db: Document) -> List(String) {
+ let a = dump.document(from_ssp)
+ let b = dump.document(from_db)
+ case a == b {
+ True -> []
+ False -> diff_lines(a, b)
+ }
+}
+
+fn diff_lines(a: String, b: String) -> List(String) {
+ let la = split_lines(a)
+ let lb = split_lines(b)
+ case list.length(la) == list.length(lb) {
+ False -> [
+ " differ: line count",
+ " .ssp: " <> int.to_string(list.length(la)),
+ " .ocda.db: " <> int.to_string(list.length(lb)),
+ ..pairwise(la, lb)
+ ]
+ True -> pairwise(la, lb)
+ }
+}
+
+fn pairwise(la: List(String), lb: List(String)) -> List(String) {
+ list.zip(la, lb)
+ |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) })
+ |> list.filter(fn(t) { t.1 != t.2 })
+ |> list.map(fn(t) {
+ " differ: line "
+ <> int.to_string(t.0 + 1)
+ <> "\n .ssp: "
+ <> t.1
+ <> "\n .ocda.db: "
+ <> t.2
+ })
+}
+
+fn split_lines(text: String) -> List(String) {
+ case text {
+ "" -> []
+ _ -> string.split(text, on: "\n")
+ }
+}
+
+fn fail(message: String) -> Nil {
+ io.println_error("error: " <> message)
+ halt(2)
+}
+
+/// Gleam has no standard exit, and the exit status is what a sweep script
+/// reads.
+@external(erlang, "erlang", "halt")
+fn halt(code: Int) -> Nil
+#+END_SRC
+
+** erlang
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda_ffi.erl
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+%%<<doc_header_including_copyright_and_license>>
+<<erlang_ffi>>
+#+END_SRC
+
+*** erlang source
+
+#+NAME: erlang_ffi
+#+HEADER: :no-expand
+#+BEGIN_SRC erlang
+%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's
+%% crypto has and the Gleam standard library does not. Upper case hex, as
+%% the database stores it and the D library writes it.
+-module(ocda_ffi).
+-export([sha256_hex/1]).
+
+sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).
+#+END_SRC
+
+** model gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/model.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<model>>
+#+END_SRC
+
+#+NAME: model
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// The object-centric document abstraction, as Gleam types.
+////
+//// A deliberate mirror of the D library's `ocda.model`, field for field.
+//// The two are meant to agree, and the cheapest way to make that testable
+//// is for them to hold the same things under the same names: the dump of a
+//// loaded document is then identical by construction rather than by
+//// negotiation.
+////
+//// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for
+//// the model, the two format specifications for what each serialisation
+//// says about it.
+
+import gleam/dict.{type Dict}
+import gleam/list
+import gleam/string
+
+/// The format major version this library knows.
+///
+/// The `.ssp` and the `.ocda.db` carry one version between them, declared
+/// in band. The minor part rises when a property is added and a reader of
+/// the newer format still reads the older, so only the major is checked.
+pub const format_major_read = "2"
+
+/// The nine sections, in the order the format gives them, which is not the
+/// order a sorted key list or a `group by` would give.
+pub const section_names = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
+ "blurb", "tail",
+]
+
+/// The four blocks that are document preamble rather than sections of
+/// objects.
+pub const preamble_blocks = ["source", "meta", "make", "doc_has"]
+
+/// A repeatable `.image` record: the file as the abstraction found it.
+pub type Image {
+ Image(
+ /// filename as the object refers to it
+ name: String,
+ /// hex, uppercase
+ sha256: String,
+ bytes: Int,
+ /// the file's own pixel dimensions, not any display width
+ width: Int,
+ height: Int,
+ /// absent or unreadable, so no digest and no size
+ missing: Bool,
+ /// any further key:value fields, kept rather than dropped
+ fields: Dict(String, String),
+ )
+}
+
+pub fn new_image() -> Image {
+ Image(
+ name: "",
+ sha256: "",
+ bytes: 0,
+ width: 0,
+ height: 0,
+ missing: False,
+ fields: dict.new(),
+ )
+}
+
+/// One object of the abstraction.
+///
+/// `text` is one string with the lines joined by newline, as in the D
+/// library and for the same reason: an object whose only text line is
+/// blank is then indistinguishable from an object with no text, in both
+/// libraries, in the same place. A difference that exists in one language
+/// and not the other would be worse than a loss both share.
+pub type Object {
+ Object(
+ ocn: Int,
+ /// one of the sixteen kinds
+ is_a: String,
+ /// the section this object was found in
+ section: String,
+ /// headings: A B C D 1 2 3 4
+ heading_level: String,
+ identifier: String,
+ part: String,
+ /// how the text behaves: para, block, comment
+ is_of_type: String,
+ /// the object's own section, when it differs from the block it sits in
+ own_section: String,
+ segment_html_is: String,
+ segment_epub: String,
+ segment_epub_is: String,
+ table_aligns: String,
+ stow_link: List(String),
+ anchor_tags: List(String),
+ parent_ocn: Int,
+ last_descendant_ocn: Int,
+ ocn_range_first: Int,
+ ocn_range_last: Int,
+ children: List(Int),
+ ancestors: List(Int),
+ dummy_heading: Bool,
+ ocn_off: Bool,
+ indent_base: Int,
+ indent_hang: Int,
+ bullet: Bool,
+ lang: String,
+ has_links: Bool,
+ has_notes_reg: Bool,
+ has_notes_star: Bool,
+ has_images: Bool,
+ segment: String,
+ segment_prev: String,
+ segment_next: String,
+ anchor: String,
+ heading_lev_anchor: String,
+ table_cols: Int,
+ table_widths: String,
+ table_header: Bool,
+ code_syntax: String,
+ code_linenumbers: Bool,
+ heading_ancestors_text: String,
+ sha256: String,
+ images: List(Image),
+ text: String,
+ )
+}
+
+pub fn new_object() -> Object {
+ Object(
+ ocn: 0,
+ is_a: "",
+ section: "",
+ heading_level: "",
+ identifier: "",
+ part: "",
+ is_of_type: "",
+ own_section: "",
+ segment_html_is: "",
+ segment_epub: "",
+ segment_epub_is: "",
+ table_aligns: "",
+ stow_link: [],
+ anchor_tags: [],
+ parent_ocn: 0,
+ last_descendant_ocn: 0,
+ ocn_range_first: 0,
+ ocn_range_last: 0,
+ children: [],
+ ancestors: [],
+ dummy_heading: False,
+ ocn_off: False,
+ indent_base: 0,
+ indent_hang: 0,
+ bullet: False,
+ lang: "",
+ has_links: False,
+ has_notes_reg: False,
+ has_notes_star: False,
+ has_images: False,
+ segment: "",
+ segment_prev: "",
+ segment_next: "",
+ anchor: "",
+ heading_lev_anchor: "",
+ table_cols: 0,
+ table_widths: "",
+ table_header: False,
+ code_syntax: "",
+ code_linenumbers: False,
+ heading_ancestors_text: "",
+ sha256: "",
+ images: [],
+ text: "",
+ )
+}
+
+/// Whether this object is rendered at all.
+///
+/// `is_of_type` "comment" is markup commentary: carried by the abstraction
+/// so that nothing is lost, and not put into output. An output that
+/// renders it leaks the author's notes into the document, which is what
+/// spine itself does not do.
+pub fn renders(object: Object) -> Bool {
+ object.is_of_type != "comment"
+}
+
+/// The section this object belongs to: the block it sits in, unless it
+/// says otherwise.
+pub fn section_of(object: Object) -> String {
+ case object.own_section {
+ "" -> object.section
+ own -> own
+ }
+}
+
+/// What the loader met and did not understand.
+///
+/// A loader that silently drops what it does not know cannot say whether
+/// it understood the document, and so cannot be used to check a
+/// specification against an implementation: silence and understanding look
+/// alike.
+pub type Unknown {
+ Unknown(
+ /// `.name` -> times seen
+ property: Dict(String, Int),
+ /// `@name` -> times seen
+ block: Dict(String, Int),
+ /// first character -> times seen, for lines of no known shape
+ line: Dict(String, Int),
+ )
+}
+
+pub fn new_unknown() -> Unknown {
+ Unknown(property: dict.new(), block: dict.new(), line: dict.new())
+}
+
+pub fn any_unknown(unknown: Unknown) -> Bool {
+ dict.size(unknown.property) > 0
+ || dict.size(unknown.block) > 0
+ || dict.size(unknown.line) > 0
+}
+
+pub type Document {
+ Document(
+ /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when
+ /// absent
+ format_version: String,
+ source_filename: String,
+ /// language, languages, digest
+ source: Dict(String, String),
+ unknown: Unknown,
+ metadata: Dict(String, String),
+ make: Dict(String, String),
+ doc_has: Dict(String, String),
+ /// section name -> objects, in document order
+ sections: Dict(String, List(Object)),
+ )
+}
+
+pub fn new_document() -> Document {
+ Document(
+ format_version: "",
+ source_filename: "",
+ source: dict.new(),
+ unknown: new_unknown(),
+ metadata: dict.new(),
+ make: dict.new(),
+ doc_has: dict.new(),
+ sections: dict.new(),
+ )
+}
+
+/// The major part of the declared version, "" when none was declared.
+pub fn version_major(document: Document) -> String {
+ case string.split_once(document.format_version, ".") {
+ Ok(#(major, _)) -> major
+ Error(_) -> document.format_version
+ }
+}
+
+pub fn meta(document: Document, key: String) -> String {
+ case dict.get(document.metadata, key) {
+ Ok(value) -> value
+ Error(_) -> ""
+ }
+}
+
+/// The objects of a named section, empty when the document has none.
+pub fn section(document: Document, name: String) -> List(Object) {
+ case dict.get(document.sections, name) {
+ Ok(objects) -> objects
+ Error(_) -> []
+ }
+}
+
+/// Is this one of the nine sections of the format?
+pub fn is_section(name: String) -> Bool {
+ list.contains(section_names, name)
+}
+
+/// Is this one of the four preamble blocks?
+pub fn is_preamble_block(name: String) -> Bool {
+ list.contains(preamble_blocks, name)
+}
+#+END_SRC
+
+** inventory gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/inventory.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<inventory>>
+#+END_SRC
+
+#+NAME: inventory
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// What a source holds, beyond the one document the model is loaded from.
+////
+//// The model is one language of one document. Two questions a reader asks
+//// are about the file instead: which languages it holds, and, for a
+//// database, what else it carries. They are here rather than in a reader
+//// because the answer depends on the input form, which is the library's to
+//// know, and so that a reader asking them does not learn which form it was
+//// handed. A mirror of the D library's `ocda.inventory`.
+////
+//// `languages` answers for every form, and answers the same: the languages
+//// in the order the manifest gave them (which every `.ssp` records, in
+//// `@source languages`), each with the markup file it was made from and the
+//// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a
+//// pod it is taken from the bytes, which are the same bytes.
+////
+//// `carried` is the database's: a `.ocda.db` holds the markup, images,
+//// configuration and catalogues it was built from, a `.ssp` holds none of
+//// them, and a pod's are its archive entries rather than anything the
+//// abstraction says.
+
+import gleam/bit_array
+import gleam/dict
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/load_db
+import ocda/load_pod
+import ocda/load_ssp
+import ocda/zip
+import simplifile
+
+pub type Language {
+ Language(lang: String, source_filename: String, ssp_digest: String)
+}
+
+pub type Carried {
+ Carried(
+ role: String,
+ compression: String,
+ count: String,
+ bytes: String,
+ stored: String,
+ )
+}
+
+pub fn languages(path: String) -> Result(List(Language), String) {
+ case load.source_of(path) {
+ load.OcdaDb -> {
+ use held <- result.try(load_db.languages_of_file(path))
+ Ok(
+ list.map(held, fn(l) {
+ Language(l.lang, l.source_filename, l.ssp_digest)
+ }),
+ )
+ }
+ load.Ssp -> {
+ use bytes <- result.try(
+ simplifile.read_bits(path)
+ |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }),
+ )
+ use text <- result.try(
+ bit_array.to_string(bytes)
+ |> result.replace_error(path <> ": not valid UTF-8"),
+ )
+ let doc = load_ssp.parse(text)
+ Ok([
+ Language(
+ source_value(doc.source, "language"),
+ doc.source_filename,
+ sha256_hex(bytes),
+ ),
+ ])
+ }
+ load.Pod -> {
+ use archive <- result.try(load_pod.archive_bytes(path))
+ use entries <- result.try(zip.entries(archive))
+ use found <- result.try(
+ zip.find_suffix(entries, ".ssp")
+ |> list.try_map(fn(e) {
+ use bytes <- result.try(zip.read(archive, e))
+ use text <- result.try(
+ bit_array.to_string(bytes)
+ |> result.replace_error(e.name <> ": not valid UTF-8"),
+ )
+ let doc = load_ssp.parse(text)
+ let lang = result.unwrap(load_pod.lang_of(e.name), "")
+ Ok(#(
+ Language(lang, doc.source_filename, sha256_hex(bytes)),
+ string.split(source_value(doc.source, "languages"), " "),
+ ))
+ }),
+ )
+ let order = case found {
+ [#(_, o), ..] -> list.filter(o, fn(l) { l != "" })
+ [] -> []
+ }
+ let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list
+ // the manifest's order, and anything it did not name after it
+ let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) })
+ let rest =
+ list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) })
+ |> list.map(fn(f) { f.0 })
+ Ok(list.append(named, rest))
+ }
+ load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown))
+ }
+}
+
+/// Only a `.ocda.db` carries files.
+pub fn is_carrier(path: String) -> Bool {
+ load.source_of(path) == load.OcdaDb
+}
+
+pub fn carried(path: String) -> Result(List(Carried), String) {
+ case is_carrier(path) {
+ False -> Ok([])
+ True -> {
+ use rows <- result.try(load_db.carried_of_file(path))
+ Ok(
+ list.filter_map(rows, fn(r) {
+ case r {
+ [role, compression, n, bytes, stored] ->
+ Ok(Carried(role, compression, n, bytes, stored))
+ _ -> Error(Nil)
+ }
+ }),
+ )
+ }
+ }
+}
+
+fn source_value(source: dict.Dict(String, String), key: String) -> String {
+ result.unwrap(dict.get(source, key), "")
+}
+
+@external(erlang, "ocda_ffi", "sha256_hex")
+fn sha256_hex(bytes: BitArray) -> String
+#+END_SRC
+
+** load gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/load.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<load>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: load
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// What a path is, and loading it.
+////
+//// One place that decides, so that every reader and every writer accepts
+//// the same inputs and a new input form is added once. This mirrors
+//// spine's own `ocda/abstraction/load.d`, which names the sources it can
+//// enter the pipeline from and tells them apart by suffix.
+////
+//// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means
+//// implementing the parser, which is spine's work and a far larger thing
+//// than a reader or a sample writer needs. This library reads
+//// abstractions.
+
+import gleam/list
+import gleam/string
+import ocda/load_db
+import ocda/load_pod
+import ocda/load_ssp
+import ocda/model.{type Document}
+
+pub type Source {
+ /// `.ssp`, the abstraction as text
+ Ssp
+ /// `.ocda.db`, the abstraction as sqlite
+ OcdaDb
+ /// `.sisupod` or `.zip`, an archive carrying a `.ssp`
+ Pod
+ Unknown
+}
+
+/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote
+/// before, and every pod already published carries that name, so both are
+/// recognised. What is inside decides how it is opened, not the name.
+const suffixes = [
+ #(".ocda.db", OcdaDb),
+ #(".ssp", Ssp),
+ #(".sisupod", Pod),
+ #(".zip", Pod),
+ #(".db", OcdaDb),
+]
+
+pub fn source_of(path: String) -> Source {
+ case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) {
+ Ok(#(_, source)) -> source
+ Error(_) -> Unknown
+ }
+}
+
+pub fn source_name(source: Source) -> String {
+ case source {
+ Ssp -> ".ssp (the abstraction as text)"
+ OcdaDb -> ".ocda.db (the abstraction as sqlite)"
+ Pod -> "a pod archive carrying a .ssp"
+ Unknown -> "not an abstraction this library reads"
+ }
+}
+
+/// Load whatever the path is into the model.
+///
+/// `lang` names which language to take from a `.ocda.db`, which holds
+/// every language of its document, and from a pod, which may carry more than
+/// one `.ssp`. A `.ssp` is one language by construction and ignores it.
+pub fn load(
+ path: String,
+ lang: String,
+ any_version: Bool,
+) -> Result(Document, String) {
+ case source_of(path) {
+ Ssp -> load_ssp.load_file(path)
+ OcdaDb -> load_db.load_file(path, lang, any_version)
+ Pod -> load_pod.load_file(path, lang)
+ Unknown ->
+ Error(
+ path
+ <> ": "
+ <> source_name(Unknown)
+ <> " (expected .ssp, .ocda.db, .sisupod or .zip)",
+ )
+ }
+}
+#+END_SRC
+
+** load pod
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/load_pod.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<load_pod>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: load_pod
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// The abstraction out of a pod archive.
+////
+//// A pod built with `--pod2` carries the abstraction inside it, at
+//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is
+//// a third input form and *not* a third model: the `.ssp` inside is parsed by
+//// `ocda/load_ssp` and comes back as the same value. A writer over the
+//// library gains pods without being touched, which is the whole reason the
+//// dispatch is in one place.
+////
+//// ## The one thing this cannot do
+////
+//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip
+//// and **not the frame**, because there is no zstd on the BEAM without a NIF,
+//// and the one that exists, `ezstd`, clones zstd from github in its build, so
+//// it cannot be built in a sandbox without patching it. The D library has
+//// both, through a declared `libzstd`.
+////
+//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says
+//// why rather than pretending the file is malformed. It is the only place the
+//// two libraries differ in what they accept, and it is a packaging limit
+//// rather than anything about the format.
+
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load_ssp
+import ocda/model.{type Document}
+import ocda/zip
+import simplifile
+
+/// The magic number of a zstd frame, little endian 0xFD2FB528.
+const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>>
+
+/// Which languages of the abstraction the pod carries, sorted.
+pub fn languages(path: String) -> Result(List(String), String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ zip.find_suffix(entries, ".ssp")
+ |> list.filter_map(fn(e) { lang_of(e.name) })
+ |> list.sort(string.compare)
+ |> Ok
+}
+
+/// Load one language of the abstraction a pod carries.
+///
+/// `lang` == "" takes the only one there is, and where there is a choice it
+/// says so rather than choosing, as the `.ocda.db` loader does.
+pub fn load_file(path: String, lang: String) -> Result(Document, String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ let ssps = zip.find_suffix(entries, ".ssp")
+ use chosen <- result.try(case ssps, lang {
+ [], _ ->
+ Error(
+ path
+ <> ": carries no .ssp. A pod carries the abstraction only when spine"
+ <> " was asked for it: spine --pod2 puts it in, spine --source does not.",
+ )
+ [only], "" -> Ok(only)
+ _, "" ->
+ Error(
+ path
+ <> ": holds more than one language ("
+ <> string.join(langs(ssps), " ")
+ <> "); name the one wanted",
+ )
+ _, wanted ->
+ case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) {
+ Ok(found) -> Ok(found)
+ Error(_) ->
+ Error(
+ path
+ <> ": no language '"
+ <> wanted
+ <> "'; it holds: "
+ <> string.join(langs(ssps), " "),
+ )
+ }
+ })
+ use content <- result.try(zip.read(bytes, chosen))
+ case bit_array_to_string(content) {
+ Ok(text) -> Ok(load_ssp.parse(text))
+ Error(_) -> Error(chosen.name <> ": not valid UTF-8")
+ }
+}
+
+fn langs(entries: List(zip.Entry)) -> List(String) {
+ list.filter_map(entries, fn(e) { lang_of(e.name) })
+}
+
+/// `<uid>.<lang>.ssp` -> `<lang>`
+pub fn lang_of(name: String) -> Result(String, Nil) {
+ case string.ends_with(name, ".ssp") {
+ False -> Error(Nil)
+ True -> {
+ let stem = string.drop_end(name, 4)
+ case string.split(stem, on: ".") |> list.last {
+ Ok(lang) if lang != stem -> Ok(lang)
+ _ -> Error(Nil)
+ }
+ }
+ }
+}
+
+/// The bytes decide, not the name: a file misnamed either way is treated by
+/// what it holds.
+pub fn archive_bytes(path: String) -> Result(BitArray, String) {
+ case simplifile.read_bits(path) {
+ Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
+ Ok(bytes) ->
+ case is_zstd_frame(bytes) {
+ False -> Ok(bytes)
+ True ->
+ Error(
+ path
+ <> " is a zstd frame wrapping the archive, and this library has no"
+ <> " zstd: the only one on the BEAM is a NIF that fetches zstd in"
+ <> " its own build, so it cannot be built sandboxed. Unwrap it"
+ <> " first (zstd -d) and read the archive, or use the D library,"
+ <> " which reads both.",
+ )
+ }
+ }
+}
+
+fn is_zstd_frame(bytes: BitArray) -> Bool {
+ case bytes {
+ <<head:bytes-size(4), _:bits>> -> head == zstd_magic
+ _ -> False
+ }
+}
+
+@external(erlang, "unicode", "characters_to_binary")
+fn bit_array_to_string_ffi(b: BitArray) -> String
+
+pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
+ Ok(bit_array_to_string_ffi(b))
+}
+#+END_SRC
+
+** load ssp
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/load_ssp.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<load_ssp>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: load_ssp
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// Loading a `.ssp` into the model.
+////
+//// The format is line oriented and every line is one of eight shapes told
+//// apart by its first character, so the loader is a fold over the lines
+//// carrying the state the shapes need.
+////
+//// Written against `ssp-format-specification.org`. Where it and the D
+//// library differ the D library is the bug, or this is: they are meant to
+//// give the same model, and `ocda-dump` says whether they do.
+
+import gleam/dict.{type Dict}
+import gleam/int
+import gleam/list
+import gleam/option
+import gleam/result
+import gleam/string
+import ocda/model.{type Document, type Image, type Object, Image, Object}
+import simplifile
+
+pub fn load_file(path: String) -> Result(Document, String) {
+ case simplifile.read(path) {
+ Ok(content) -> Ok(parse(content))
+ Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
+ }
+}
+
+type State {
+ State(
+ document: Document,
+ /// the block currently open, "" when none
+ block: String,
+ /// the section currently open, "" when in the preamble
+ section: String,
+ /// objects of the current section, newest first
+ objects: List(Object),
+ /// the object being read, if any
+ current: Result(Object, Nil),
+ /// text lines of the object being read, newest first
+ text: List(String),
+ /// whether a text line has been seen for the object being read.
+ /// Not "is the text non-empty": an object whose first text line is
+ /// blank, which the reference collection has seven of, would then lose
+ /// it.
+ seen_text: Bool,
+ )
+}
+
+pub fn parse(content: String) -> Document {
+ content
+ |> string.split(on: "\n")
+ |> list.fold(from: initial(), with: line)
+ |> finish
+}
+
+fn initial() -> State {
+ State(
+ document: model.new_document(),
+ block: "",
+ section: "",
+ objects: [],
+ current: Error(Nil),
+ text: [],
+ seen_text: False,
+ )
+}
+
+fn line(state: State, raw: String) -> State {
+ case raw {
+ "% SiSU Document Abstraction v" <> version ->
+ case state.document.format_version {
+ "" ->
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ format_version: string.trim(version),
+ ),
+ )
+ _ -> state
+ }
+ "% Source: " <> filename ->
+ case state.document.source_filename {
+ "" ->
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ source_filename: string.trim(filename),
+ ),
+ )
+ _ -> state
+ }
+ "%" <> _ -> state
+ "@" <> rest -> open_block(state, rest)
+ "}" <> _ -> close_block(state)
+ "[" <> rest -> declaration(state, rest)
+ "." <> rest -> property(state, rest)
+ "|" <> rest -> text_line(state, rest)
+ _ ->
+ case string.trim(raw) {
+ // a blank line ends the object being read
+ "" -> commit(state)
+ _ ->
+ case model.is_preamble_block(state.block) {
+ True -> preamble_pair(state, raw)
+ False -> unknown_line(state, raw)
+ }
+ }
+ }
+}
+
+fn open_block(state: State, rest: String) -> State {
+ let name = case string.split_once(rest, "{") {
+ Ok(#(name, _)) -> string.trim(name)
+ Error(_) -> string.trim(rest)
+ }
+ case model.is_preamble_block(name) {
+ True -> State(..flush(state), block: name)
+ False -> {
+ let state = flush(state)
+ let document = case model.is_section(name) {
+ True -> state.document
+ // a block that is neither preamble nor one of the nine sections:
+ // counted, so that a reader can say the file held something it did
+ // not expect
+ False ->
+ model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ block: bump(state.document.unknown.block, name),
+ ),
+ )
+ }
+ State(..state, block: name, section: name, document: document)
+ }
+ }
+}
+
+fn close_block(state: State) -> State {
+ State(..flush(state), block: "")
+}
+
+/// Commit the object being read, then put the section's objects into the
+/// document and close the section.
+fn flush(state: State) -> State {
+ let state = commit(state)
+ case state.section, state.objects {
+ "", _ -> state
+ _, [] -> State(..state, section: "", objects: [])
+ name, objects ->
+ State(
+ ..state,
+ section: "",
+ objects: [],
+ document: model.Document(
+ ..state.document,
+ sections: add_section(
+ state.document.sections,
+ name,
+ list.reverse(objects),
+ ),
+ ),
+ )
+ }
+}
+
+fn preamble_pair(state: State, raw: String) -> State {
+ case string.split_once(string.trim_start(raw), ":") {
+ Error(_) -> state
+ Ok(#(key, value)) -> {
+ let key = string.trim(key)
+ let value = string.trim(value)
+ let document = state.document
+ let document = case state.block {
+ "meta" ->
+ model.Document(
+ ..document,
+ metadata: dict.insert(document.metadata, key, value),
+ )
+ "make" ->
+ model.Document(..document, make: dict.insert(document.make, key, value))
+ "source" ->
+ model.Document(
+ ..document,
+ source: dict.insert(document.source, key, value),
+ )
+ _ ->
+ model.Document(
+ ..document,
+ doc_has: dict.insert(document.doc_has, key, value),
+ )
+ }
+ State(..state, document: document)
+ }
+ }
+}
+
+/// `[<ocn>] <kind>`, and for a heading `[<ocn>] heading :<level>
+/// [<identifier>]`.
+fn declaration(state: State, rest: String) -> State {
+ let state = commit(state)
+ case string.split_once(rest, "]") {
+ Error(_) -> state
+ Ok(#(ocn_text, tail)) -> {
+ let ocn = parse_int(string.trim(ocn_text))
+ let object =
+ Object(..model.new_object(), ocn: ocn, section: state.section)
+ let object = case string.split(string.trim(tail), on: " ") {
+ ["heading", level, ..identifier] ->
+ Object(
+ ..object,
+ is_a: "heading",
+ heading_level: string.drop_start(level, 1),
+ identifier: string.join(identifier, " "),
+ )
+ [""] -> object
+ [kind, ..] -> Object(..object, is_a: kind)
+ [] -> object
+ }
+ State(..state, current: Ok(object), text: [], seen_text: False)
+ }
+ }
+}
+
+fn property(state: State, rest: String) -> State {
+ case state.current, string.split_once(rest, ":") {
+ Ok(object), Ok(#(name, raw_value)) -> {
+ let name = string.trim(name)
+ let value = string.trim(raw_value)
+ let #(object, known) = apply_property(object, name, value)
+ let document = case known {
+ True -> state.document
+ False ->
+ model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ property: bump(state.document.unknown.property, name),
+ ),
+ )
+ }
+ State(..state, current: Ok(object), document: document)
+ }
+ _, _ -> state
+ }
+}
+
+/// The properties this library reads, and what each sets.
+///
+/// A property it has no use for is counted rather than passed over:
+/// ignoring one is conforming within a major version, doing so in silence
+/// is what stops a reader being a check on the specification.
+fn apply_property(
+ object: Object,
+ name: String,
+ value: String,
+) -> #(Object, Bool) {
+ case name {
+ "part" -> #(Object(..object, part: value), True)
+ "is_of_type" -> #(Object(..object, is_of_type: value), True)
+ "section" -> #(Object(..object, own_section: value), True)
+ "identifier" -> #(Object(..object, identifier: value), True)
+ "segment_html_is" -> #(Object(..object, segment_html_is: value), True)
+ "segment_epub" -> #(Object(..object, segment_epub: value), True)
+ "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True)
+ "table_aligns" -> #(Object(..object, table_aligns: value), True)
+ "stow_link" -> #(
+ Object(..object, stow_link: list.append(object.stow_link, [value])),
+ True,
+ )
+ "anchor_tag" -> #(
+ Object(..object, anchor_tags: list.append(object.anchor_tags, [value])),
+ True,
+ )
+ "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True)
+ "last_descendant" -> #(
+ Object(..object, last_descendant_ocn: parse_int(value)),
+ True,
+ )
+ "ocn_range" ->
+ case numbers(value) {
+ [first, last] -> #(
+ Object(..object, ocn_range_first: first, ocn_range_last: last),
+ True,
+ )
+ _ -> #(object, True)
+ }
+ "children" -> #(Object(..object, children: numbers(value)), True)
+ "ancestors" -> #(Object(..object, ancestors: numbers(value)), True)
+ "dummy" -> #(Object(..object, dummy_heading: value == "true"), True)
+ "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True)
+ "indent" -> {
+ let parts = string.split(value, on: " ")
+ let base = case parts {
+ [first, ..] -> parse_int(first)
+ [] -> 0
+ }
+ let hang = case parts {
+ [_, second, ..] -> parse_int(second)
+ _ -> 0
+ }
+ #(Object(..object, indent_base: base, indent_hang: hang), True)
+ }
+ "bullet" -> #(Object(..object, bullet: value == "true"), True)
+ "lang" -> #(Object(..object, lang: value), True)
+ "has" -> #(has_flags(object, value), True)
+ "segment" -> #(Object(..object, segment: value), True)
+ "segment_prev" -> #(Object(..object, segment_prev: value), True)
+ "segment_next" -> #(Object(..object, segment_next: value), True)
+ "anchor" -> #(Object(..object, anchor: value), True)
+ "heading_lev_anchor" -> #(
+ Object(..object, heading_lev_anchor: value),
+ True,
+ )
+ "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True)
+ "table_widths" -> #(Object(..object, table_widths: value), True)
+ "table_header" -> #(Object(..object, table_header: value == "true"), True)
+ "code_syntax" -> #(Object(..object, code_syntax: value), True)
+ "code_linenumbers" -> #(
+ Object(..object, code_linenumbers: value == "true"),
+ True,
+ )
+ "heading_ancestors_text" -> #(
+ Object(..object, heading_ancestors_text: value),
+ True,
+ )
+ "sha256" -> #(Object(..object, sha256: value), True)
+ "image" -> #(
+ Object(..object, images: list.append(object.images, [image(value)])),
+ True,
+ )
+ _ -> #(object, False)
+ }
+}
+
+fn has_flags(object: Object, value: String) -> Object {
+ value
+ |> string.split(on: " ")
+ |> list.fold(from: object, with: fn(obj, flag) {
+ case flag {
+ "links" -> Object(..obj, has_links: True)
+ "notes_reg" -> Object(..obj, has_notes_reg: True)
+ "notes_star" -> Object(..obj, has_notes_star: True)
+ "images" -> Object(..obj, has_images: True)
+ _ -> obj
+ }
+ })
+}
+
+/// `<filename> sha256:<HEX> bytes:<n> [px:<w>x<h>] [missing:true] [...]`
+///
+/// The filename comes first, the rest are order-independent, and a field
+/// this library does not recognise is kept rather than discarded.
+fn image(value: String) -> Image {
+ case string.split(value, on: " ") {
+ [] -> model.new_image()
+ [name, ..fields] ->
+ list.fold(fields, model.new_image(), fn(img, field) {
+ case string.split_once(field, ":") {
+ Error(_) -> img
+ Ok(#(key, val)) ->
+ case key {
+ "sha256" -> Image(..img, sha256: val)
+ "bytes" -> Image(..img, bytes: parse_int(val))
+ "px" ->
+ case string.split_once(val, "x") {
+ Ok(#(w, h)) ->
+ Image(..img, width: parse_int(w), height: parse_int(h))
+ Error(_) -> img
+ }
+ "missing" -> Image(..img, missing: val == "true")
+ _ -> Image(..img, fields: dict.insert(img.fields, key, val))
+ }
+ }
+ })
+ |> fn(img) { Image(..img, name: name) }
+ }
+}
+
+/// `| <text>`, and `| ` alone for a blank line. Never a bare `|`: the
+/// emitter writes the space.
+fn text_line(state: State, rest: String) -> State {
+ case state.current {
+ Error(_) -> state
+ Ok(_) -> {
+ let content = string.drop_start(rest, 1)
+ State(..state, text: [content, ..state.text], seen_text: True)
+ }
+ }
+}
+
+fn unknown_line(state: State, raw: String) -> State {
+ let first = string.slice(raw, 0, 1)
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ line: bump(state.document.unknown.line, first),
+ ),
+ ),
+ )
+}
+
+fn commit(state: State) -> State {
+ case state.current {
+ Error(_) -> state
+ Ok(object) -> {
+ let text = case state.seen_text {
+ True -> string.join(list.reverse(state.text), "\n")
+ False -> ""
+ }
+ let object = derive_omitted(Object(..object, text: text))
+ let objects = case model.is_section(state.section) {
+ True -> [object, ..state.objects]
+ False -> state.objects
+ }
+ State(
+ ..state,
+ current: Error(Nil),
+ text: [],
+ seen_text: False,
+ objects: objects,
+ )
+ }
+ }
+}
+
+/// Two values the text form leaves out because they can be worked out.
+///
+/// `.identifier` is absent when it is simply the ocn, `.anchor` when it is
+/// the segment name. The `.ocda.db`, being columns rather than lines,
+/// stores both unconditionally. So a loader that takes the text form
+/// literally gives a different model from the one the database gives, and
+/// over one document that is 1,314 identifiers and 69 anchors.
+fn derive_omitted(object: Object) -> Object {
+ let object = case object.identifier, object.ocn {
+ "", 0 -> object
+ "", ocn -> Object(..object, identifier: int.to_string(ocn))
+ _, _ -> object
+ }
+ case object.anchor, object.segment {
+ "", "" -> object
+ "", segment -> Object(..object, anchor: segment)
+ _, _ -> object
+ }
+}
+
+fn finish(state: State) -> Document {
+ flush(state).document
+}
+
+fn add_section(
+ sections: Dict(String, List(Object)),
+ name: String,
+ objects: List(Object),
+) -> Dict(String, List(Object)) {
+ dict.upsert(sections, name, fn(existing) {
+ case existing {
+ option.Some(previous) -> list.append(previous, objects)
+ option.None -> objects
+ }
+ })
+}
+
+fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) {
+ dict.upsert(counts, key, fn(existing) {
+ case existing {
+ option.Some(n) -> n + 1
+ option.None -> 1
+ }
+ })
+}
+
+fn parse_int(value: String) -> Int {
+ value |> string.trim |> int.parse |> result.unwrap(0)
+}
+
+fn numbers(value: String) -> List(Int) {
+ value
+ |> string.split(on: " ")
+ |> list.filter(fn(s) { s != "" })
+ |> list.map(parse_int)
+}
+#+END_SRC
+
+** load db
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/load_db.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<load_db>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: load_db
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// Loading a `.ocda.db` into the model.
+////
+//// The point of this module is that it returns *the same value*
+//// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is
+//// then a function from the model to bytes and cannot tell which
+//// serialisation it was given, exactly as spine's own writers cannot.
+////
+//// Written against `ocda-db-format-specification.org`, whose section 4
+//// gives the column by column correspondence this module implements.
+
+import gleam/dict
+import gleam/dynamic/decode
+import gleam/float
+import gleam/int
+import gleam/json
+import gleam/list
+import gleam/option
+import gleam/result
+import gleam/string
+import ocda/model.{type Document, Image, Object}
+import sqlight
+
+/// The schema major version this loader knows.
+pub const schema_major_read = "2"
+
+/// One language of the database, as the manifest gives it.
+pub type Language {
+ Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String)
+}
+
+/// Load one language of a database as the model. `lang` == "" takes the
+/// first language the manifest gives.
+pub fn load_file(
+ path: String,
+ lang: String,
+ any_version: Bool,
+) -> Result(Document, String) {
+ // read only: a loader has no business writing, and a read-write open
+ // *creates* an empty database where a path is mistyped, which then reads
+ // as a document with nothing in it, a failure that looks like an answer
+ use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+
+ use _ <- result.try(case
+ has(conn, "documents") && has(conn, "objects") && has(conn, "metadata")
+ {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ path
+ <> ": no documents/objects/metadata tables; it does not look like a .ocda.db",
+ )
+ })
+
+ let schema_version =
+ one(conn, "select value from metadata where doc_id is null
+ and key = 'schema.version'")
+ use _ <- result.try(case schema_version, any_version {
+ "", _ -> Ok(Nil)
+ _, True -> Ok(Nil)
+ v, False ->
+ case major_of(v) == schema_major_read {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ path
+ <> " is schema v"
+ <> v
+ <> ", this loader knows v"
+ <> schema_major_read
+ <> ".x",
+ )
+ }
+ })
+
+ let languages = languages(conn)
+ use chosen <- result.try(case languages, lang {
+ [], _ -> Error(path <> ": no documents rows")
+ [first, ..], "" -> Ok(first)
+ _, wanted ->
+ case list.find(languages, fn(l) { l.lang == wanted }) {
+ Ok(found) -> Ok(found)
+ Error(_) ->
+ Error(
+ path
+ <> ": no language '"
+ <> wanted
+ <> "'; it holds: "
+ <> string.join(list.map(languages, fn(l) { l.lang }), " "),
+ )
+ }
+ })
+
+ let source =
+ dict.new()
+ |> dict.insert("language", chosen.lang)
+ |> dict.insert(
+ "languages",
+ string.join(list.map(languages, fn(l) { l.lang }), " "),
+ )
+ let digest = meta_value(conn, chosen.doc_id, "source.digest")
+ let source = case digest {
+ "" -> source
+ d -> dict.insert(source, "digest", d)
+ }
+
+ let document =
+ model.Document(
+ ..model.new_document(),
+ format_version: schema_version,
+ source_filename: chosen.source_filename,
+ source: source,
+ )
+ let document = metadata(conn, chosen.doc_id, document)
+ let document = objects(conn, chosen.doc_id, document)
+ let document = count_unread(conn, chosen.doc_id, document)
+ Ok(document)
+}
+
+fn with_connection(
+ uri: String,
+ path: String,
+ then: fn(sqlight.Connection) -> Result(a, String),
+) -> Result(a, String) {
+ case sqlight.open(uri) {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let outcome = then(conn)
+ let _ = sqlight.close(conn)
+ outcome
+ }
+ }
+}
+
+/// Every language the database holds, in manifest order.
+pub fn languages(conn: sqlight.Connection) -> List(Language) {
+ let d = {
+ use id <- decode.field(0, decode.int)
+ use lang <- decode.field(1, decode.string)
+ use source <- decode.field(2, decode.string)
+ use digest <- decode.field(3, decode.string)
+ decode.success(Language(id, lang, source, digest))
+ }
+ sqlight.query(
+ "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
+ from documents order by id",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+/// The three preamble blocks of the text form, out of the one metadata
+/// table the database keeps them in.
+fn metadata(
+ conn: sqlight.Connection,
+ doc_id: Int,
+ document: Document,
+) -> Document {
+ let d = {
+ use key <- decode.field(0, decode.string)
+ use value <- decode.field(1, decode.string)
+ decode.success(#(key, value))
+ }
+ let rows =
+ sqlight.query(
+ "select key, value from metadata where doc_id = "
+ <> int.to_string(doc_id)
+ <> " order by key",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+
+ list.fold(rows, document, fn(doc, row) {
+ let #(key, raw) = row
+ let value = string.trim(raw)
+ case key {
+ // source.* describes the artefact rather than the document, and is
+ // @source in the text form
+ "source." <> _ -> doc
+ // translation.ocn_aligned says whether the languages of this document
+ // share one numbering. A .ocda.db can know it because it holds every
+ // language; a .ssp is one language and cannot. It is a fact about the
+ // file, not the document, and carrying it here would let a writer
+ // tell which serialisation it was handed.
+ "translation." <> _ -> doc
+ // the database tells the three blocks apart by a key prefix.
+ // Unprefixing them is what makes the two loaders give one model: a
+ // writer asks make["footer"], never metadata["make.footer"].
+ "make." <> rest ->
+ model.Document(..doc, make: dict.insert(doc.make, rest, value))
+ "doc_has." <> rest ->
+ model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value))
+ _ ->
+ model.Document(..doc, metadata: dict.insert(doc.metadata, key, value))
+ }
+ })
+}
+
+/// The objects, in document order within each section, the sections in the
+/// order the format gives rather than the alphabetical one sqlite would.
+fn objects(
+ conn: sqlight.Connection,
+ doc_id: Int,
+ document: Document,
+) -> Document {
+ let d = {
+ use section <- decode.field(0, decode.string)
+ use ocn <- decode.field(2, decode.int)
+ use is_a <- decode.field(3, decode.string)
+ use part <- decode.field(4, decode.string)
+ use is_of_section <- decode.field(5, decode.string)
+ use is_of_type <- decode.field(6, decode.string)
+ use heading_level <- decode.field(7, decode.int)
+ use identifier <- decode.field(8, decode.string)
+ use parent <- decode.field(9, decode.int)
+ use last_descendant <- decode.field(10, decode.int)
+ use children <- decode.field(11, decode.string)
+ use ancestors <- decode.field(12, decode.string)
+ use dummy <- decode.field(13, decode.int)
+ use ocn_off <- decode.field(14, decode.int)
+ use indent_base <- decode.field(15, decode.int)
+ use indent_hang <- decode.field(16, decode.int)
+ use bullet <- decode.field(17, decode.int)
+ use lang <- decode.field(18, decode.string)
+ use has_links <- decode.field(19, decode.int)
+ use has_notes_reg <- decode.field(20, decode.int)
+ use has_notes_star <- decode.field(21, decode.int)
+ use has_images <- decode.field(22, decode.int)
+ use segment <- decode.field(23, decode.string)
+ use segment_prev <- decode.field(24, decode.string)
+ use segment_next <- decode.field(25, decode.string)
+ use anchor <- decode.field(26, decode.string)
+ use segment_html_is <- decode.field(27, decode.string)
+ use table_cols <- decode.field(28, decode.int)
+ use table_widths <- decode.field(29, decode.string)
+ use table_aligns <- decode.field(30, decode.string)
+ use table_header <- decode.field(31, decode.int)
+ use code_linenumbers <- decode.field(32, decode.int)
+ use hat <- decode.field(33, decode.string)
+ use sha256 <- decode.field(34, decode.string)
+ use text <- decode.field(35, decode.string)
+ use id <- decode.field(36, decode.int)
+ use hla <- decode.field(37, decode.string)
+ use seg_epub <- decode.field(38, decode.string)
+ use seg_epub_is <- decode.field(39, decode.string)
+ use range_first <- decode.field(40, decode.int)
+ use range_last <- decode.field(41, decode.int)
+ decode.success(#(
+ id,
+ Object(
+ ocn: ocn,
+ is_a: is_a,
+ section: section,
+ heading_level: level_name(heading_level),
+ identifier: identifier,
+ part: part,
+ is_of_type: is_of_type,
+ own_section: case is_of_section == section {
+ True -> ""
+ False -> is_of_section
+ },
+ segment_html_is: segment_html_is,
+ table_aligns: json_words(table_aligns),
+ stow_link: [],
+ anchor_tags: [],
+ parent_ocn: parent,
+ last_descendant_ocn: last_descendant,
+ ocn_range_first: range_first,
+ ocn_range_last: range_last,
+ children: json_ints(children),
+ ancestors: json_ints(ancestors),
+ dummy_heading: dummy != 0,
+ ocn_off: ocn_off != 0,
+ indent_base: indent_base,
+ indent_hang: indent_hang,
+ bullet: bullet != 0,
+ lang: lang,
+ has_links: has_links != 0,
+ has_notes_reg: has_notes_reg != 0,
+ has_notes_star: has_notes_star != 0,
+ has_images: has_images != 0,
+ segment: segment,
+ segment_prev: segment_prev,
+ segment_next: segment_next,
+ anchor: anchor,
+ heading_lev_anchor: hla,
+ segment_epub: seg_epub,
+ segment_epub_is: seg_epub_is,
+ table_cols: table_cols,
+ table_widths: json_words(table_widths),
+ table_header: table_header != 0,
+ code_syntax: "",
+ code_linenumbers: code_linenumbers != 0,
+ heading_ancestors_text: json_joined(hat),
+ sha256: sha256,
+ images: [],
+ text: text,
+ ),
+ ))
+ }
+ let rows =
+ sqlight.query(
+ object_sql(doc_id, has_range_columns(conn)),
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+
+ // Built newest-first per section and reversed once at the end. Appending
+ // to the tail of each section instead would be quadratic, and the largest
+ // document in the reference collection has some thousands of objects in
+ // one section.
+ let sections =
+ list.fold(rows, dict.new(), fn(acc, row) {
+ let #(id, object) = row
+ // the four repeatable properties of the text form are child tables
+ // here, one query each
+ let object =
+ Object(
+ ..object,
+ images: images(conn, id),
+ stow_link: links(conn, id),
+ anchor_tags: anchors(conn, id),
+ )
+ case model.is_section(object.section) {
+ False -> acc
+ True ->
+ dict.upsert(acc, object.section, fn(existing) {
+ case existing {
+ option.Some(previous) -> [object, ..previous]
+ option.None -> [object]
+ }
+ })
+ }
+ })
+ |> dict.map_values(fn(_, objects) { list.reverse(objects) })
+ model.Document(..document, sections: sections)
+}
+
+/// 2.1 added the container range as two columns. A 2.0 database has
+/// neither, and reads as having no range, which is what 2.0 meant.
+fn has_range_columns(conn: sqlight.Connection) -> Bool {
+ let d = decode.field(0, decode.int, decode.success)
+ case
+ sqlight.query(
+ "select count(*) from pragma_table_info('objects')
+ where name = 'ocn_range_first'",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ {
+ Ok([n]) -> n > 0
+ _ -> False
+ }
+}
+
+fn object_sql(doc_id: Int, has_range: Bool) -> String {
+ let range_cols = case has_range {
+ True -> "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)"
+ False -> "0, 0"
+ }
+ "select section, seq, ocn, is_a, coalesce(is_of_part,''),
+ coalesce(is_of_section,''), coalesce(is_of_type,''),
+ coalesce(heading_level,-1), coalesce(identifier,''),
+ coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
+ coalesce(children,''), coalesce(ancestors,''),
+ coalesce(dummy_heading,0), coalesce(object_number_off,0),
+ coalesce(indent_base,0), coalesce(indent_hang,0),
+ coalesce(bullet,0), coalesce(lang,''),
+ coalesce(has_links,0), coalesce(has_notes_reg,0),
+ coalesce(has_notes_star,0), coalesce(has_images,0),
+ coalesce(segment,''), coalesce(segment_prev,''),
+ coalesce(segment_next,''), coalesce(anchor,''),
+ coalesce(segment_html_is,''),
+ coalesce(table_cols,0), coalesce(table_widths,''),
+ coalesce(table_aligns,''), coalesce(table_header,0),
+ coalesce(code_linenumbers,0),
+ coalesce(heading_ancestors_text,''), coalesce(sha256,''),
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,''), " <> range_cols <> "
+ from objects where doc_id = " <> int.to_string(doc_id) <> "
+ order by case section
+ when 'head' then 1 when 'toc' then 2 when 'body' then 3
+ when 'endnotes' then 4 when 'glossary' then 5
+ when 'bibliography' then 6 when 'bookindex' then 7
+ when 'blurb' then 8 when 'tail' then 9 else 10 end, seq"
+}
+
+fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) {
+ let d = {
+ use name <- decode.field(0, decode.string)
+ use sha256 <- decode.field(1, decode.string)
+ use bytes <- decode.field(2, decode.int)
+ use width <- decode.field(3, decode.int)
+ use height <- decode.field(4, decode.int)
+ use missing <- decode.field(5, decode.int)
+ decode.success(Image(
+ name: name,
+ sha256: sha256,
+ bytes: bytes,
+ width: width,
+ height: height,
+ missing: missing != 0,
+ fields: dict.new(),
+ ))
+ }
+ sqlight.query(
+ "select name, coalesce(sha256,''), coalesce(bytes,0),
+ coalesce(width,0), coalesce(height,0), coalesce(missing,0)
+ from object_images where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn links(conn: sqlight.Connection, object_id: Int) -> List(String) {
+ let d = {
+ use url <- decode.field(0, decode.string)
+ decode.success(url)
+ }
+ sqlight.query(
+ "select url from object_links where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn anchors(conn: sqlight.Connection, object_id: Int) -> List(String) {
+ let d = {
+ use anchor <- decode.field(0, decode.string)
+ decode.success(anchor)
+ }
+ sqlight.query(
+ "select anchor from object_anchors where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn one(conn: sqlight.Connection, sql: String) -> String {
+ let d = {
+ use v <- decode.field(0, decode.string)
+ decode.success(v)
+ }
+ case sqlight.query(sql, on: conn, with: [], expecting: d) {
+ Ok([v, ..]) -> v
+ _ -> ""
+ }
+}
+
+fn has(conn: sqlight.Connection, name: String) -> Bool {
+ one(
+ conn,
+ "select cast(count(*) as text) from sqlite_master where name = '"
+ <> name
+ <> "'",
+ )
+ != "0"
+}
+
+fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String {
+ one(
+ conn,
+ "select value from metadata where doc_id = "
+ <> int.to_string(doc_id)
+ <> " and key = '"
+ <> key
+ <> "'",
+ )
+}
+
+fn major_of(version: String) -> String {
+ case string.split_once(version, ".") {
+ Ok(#(major, _)) -> major
+ Error(_) -> version
+ }
+}
+
+/// The eight heading levels and their indices.
+fn level_name(index: Int) -> String {
+ case index {
+ 0 -> "A"
+ 1 -> "B"
+ 2 -> "C"
+ 3 -> "D"
+ 4 -> "1"
+ 5 -> "2"
+ 6 -> "3"
+ 7 -> "4"
+ _ -> ""
+ }
+}
+
+/// The JSON columns, read as JSON.
+///
+/// Not as a character scan that strips brackets and quotes: that is wrong
+/// the moment a value holds an escape, and spine's encoder writes "/" as
+/// "\/", so every heading containing a slash comes back with a backslash in
+/// it. The D library had exactly that bug and all 36 document-languages
+/// differed on heading_ancestors_text because of it.
+fn json_strings(value: String) -> List(String) {
+ case value {
+ "" -> []
+ v ->
+ case json.parse(v, decode.list(of: loose_string())) {
+ Ok(items) -> items
+ Error(_) -> []
+ }
+ }
+}
+
+/// An element that may be a string or a number, which is what these columns
+/// hold: children and ancestors are int arrays, table_aligns and
+/// heading_ancestors_text string arrays, and table_widths an int array
+/// except where the columns divide evenly, when it is [33.3333,...]. One
+/// table in the reference collection does that, and without the float
+/// branch the widths came back as three empty strings: the fallback turned
+/// a value it could not read into a value that looked read.
+fn loose_string() -> decode.Decoder(String) {
+ decode.one_of(decode.string, [
+ decode.int |> decode.map(int.to_string),
+ decode.float |> decode.map(float.to_string),
+ decode.success(""),
+ ])
+}
+
+fn json_ints(value: String) -> List(Int) {
+ value
+ |> json_strings
+ |> list.filter(fn(s) { s != "" })
+ |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) })
+}
+
+/// Space separated in the text form: table_widths, table_aligns.
+fn json_words(value: String) -> String {
+ value |> json_strings |> string.join(" ")
+}
+
+/// heading_ancestors_text is eight slots, joined with "|" in the text form
+/// and a JSON array here. The empty slots are significant: they say which
+/// levels the heading sits under.
+fn json_joined(value: String) -> String {
+ case json_strings(value) {
+ [] -> ""
+ parts -> string.join(parts, "|")
+ }
+}
+
+/// What the database holds and the model does not.
+///
+/// The text loader meets every property line and counts those it does not
+/// keep, so a reader can say what was read and not understood. This loader
+/// asks for columns by name, and would never see the ones it did not ask
+/// for: it would report that it had understood everything, which is the
+/// silence the count exists to prevent. So it counts them, on the
+/// conditions under which the `.ssp` writes the same property
+/// (specification section 6.3), and the report is the same from either
+/// form, as the D loader's is.
+fn count_unread(
+ conn: sqlight.Connection,
+ doc_id: Int,
+ document: Document,
+) -> Document {
+ let id = int.to_string(doc_id)
+ let counts = [
+ #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"),
+ #("dom_status", "coalesce(dom_status,'') != ''"),
+ #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"),
+ #("heading_lev_collapsed", "heading_lev_collapsed is not null"),
+ #("parent_lev", "coalesce(parent_lev,0) != 0"),
+ #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"),
+ #("attrib", "coalesce(attrib,'') != ''"),
+ ]
+ let property =
+ list.fold(counts, document.unknown.property, fn(acc, c) {
+ let n =
+ count(
+ conn,
+ "select count(*) from objects where doc_id = " <> id <> " and " <> c.1,
+ )
+ case n > 0 {
+ True -> dict.insert(acc, c.0, n)
+ False -> acc
+ }
+ })
+ let property = case has(conn, "object_subtoc") {
+ False -> property
+ True -> {
+ let n =
+ count(
+ conn,
+ "select count(*) from object_subtoc s join objects o
+ on o.id = s.object_id where o.doc_id = " <> id,
+ )
+ case n > 0 {
+ True -> dict.insert(property, "lev4_subtoc", n)
+ False -> property
+ }
+ }
+ }
+ model.Document(
+ ..document,
+ unknown: model.Unknown(..document.unknown, property: property),
+ )
+}
+
+fn count(conn: sqlight.Connection, sql: String) -> Int {
+ let d = decode.field(0, decode.int, decode.success)
+ case sqlight.query(sql, on: conn, with: [], expecting: d) {
+ Ok([n, ..]) -> n
+ _ -> 0
+ }
+}
+
+/// Every language the database at `path` holds, in manifest order.
+pub fn languages_of_file(path: String) -> Result(List(Language), String) {
+ use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+ Ok(languages(conn))
+}
+
+/// What the database at `path` carries besides the abstraction, by role
+/// and storage: role, compression, count, original bytes, stored bytes.
+pub fn carried_of_file(path: String) -> Result(List(List(String)), String) {
+ use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+ case has(conn, "files") {
+ False -> Ok([])
+ True -> {
+ let d = {
+ use role <- decode.field(0, decode.string)
+ use compression <- decode.field(1, decode.string)
+ use n <- decode.field(2, decode.int)
+ use bytes <- decode.field(3, decode.int)
+ use stored <- decode.field(4, decode.int)
+ decode.success([
+ role,
+ compression,
+ int.to_string(n),
+ int.to_string(bytes),
+ int.to_string(stored),
+ ])
+ }
+ sqlight.query(
+ "select role, compression, count(*), sum(bytes), sum(length(data))
+ from files group by role, compression order by role",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.map_error(fn(e) { path <> ": " <> e.message })
+ }
+ }
+}
+#+END_SRC
+
+** zip gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/zip.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<zip>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: zip
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// Enough of the zip format to find one file in a pod and read it.
+////
+//// Not a zip library. A pod archive holds a handful of entries, spine writes
+//// them **stored** (uncompressed) so that the outer zstd frame does the
+//// compressing once over the whole thing, and what is wanted from it here is
+//// one named entry.
+////
+//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could
+//// inflate it, but a library that says it read an entry it in fact skipped is
+//// worse than one that says it cannot, and spine's own archives do not need
+//// it.
+
+import gleam/bit_array
+import gleam/int
+import gleam/list
+import gleam/result
+import gleam/string
+
+pub type Entry {
+ Entry(
+ name: String,
+ /// 0 stored, 8 deflate
+ method: Int,
+ compressed: Int,
+ uncompressed: Int,
+ local_offset: Int,
+ )
+}
+
+const eocd_sig = 0x06054b50
+
+const cd_sig = 0x02014b50
+
+const local_sig = 0x04034b50
+
+/// The entries, from the central directory at the end of the file.
+pub fn entries(bytes: BitArray) -> Result(List(Entry), String) {
+ let size = bit_array.byte_size(bytes)
+ use _ <- result.try(case size >= 22 {
+ True -> Ok(Nil)
+ False -> Error("too short to be a zip archive")
+ })
+ use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size)))
+ use count <- result.try(u16(bytes, eocd + 10))
+ use offset <- result.try(u32(bytes, eocd + 16))
+ read_central(bytes, offset, count, [])
+}
+
+/// The end of central directory record is last, and may be followed by a
+/// comment of up to 65535 bytes, so it is searched for backwards.
+fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) {
+ case at < lowest {
+ True ->
+ Error(
+ "no end of central directory record; it does not look like a zip archive",
+ )
+ False ->
+ case u32(bytes, at) {
+ Ok(sig) if sig == eocd_sig -> Ok(at)
+ _ -> find_eocd(bytes, at - 1, lowest)
+ }
+ }
+}
+
+fn lowest(size: Int) -> Int {
+ case size > 22 + 65_535 {
+ True -> size - 22 - 65_535
+ False -> 0
+ }
+}
+
+fn read_central(
+ bytes: BitArray,
+ at: Int,
+ remaining: Int,
+ acc: List(Entry),
+) -> Result(List(Entry), String) {
+ case remaining {
+ 0 -> Ok(list.reverse(acc))
+ _ -> {
+ use sig <- result.try(u32(bytes, at))
+ use _ <- result.try(case sig == cd_sig {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ "central directory entry "
+ <> int.to_string(list.length(acc))
+ <> " is malformed",
+ )
+ })
+ use method <- result.try(u16(bytes, at + 10))
+ use compressed <- result.try(u32(bytes, at + 20))
+ use uncompressed <- result.try(u32(bytes, at + 24))
+ use name_len <- result.try(u16(bytes, at + 28))
+ use extra_len <- result.try(u16(bytes, at + 30))
+ use cmnt_len <- result.try(u16(bytes, at + 32))
+ use local_offset <- result.try(u32(bytes, at + 42))
+ use name <- result.try(text(bytes, at + 46, name_len))
+ let entry =
+ Entry(
+ name: name,
+ method: method,
+ compressed: compressed,
+ uncompressed: uncompressed,
+ local_offset: local_offset,
+ )
+ read_central(
+ bytes,
+ at + 46 + name_len + extra_len + cmnt_len,
+ remaining - 1,
+ [entry, ..acc],
+ )
+ }
+ }
+}
+
+/// One entry's bytes, from its local header.
+pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) {
+ use sig <- result.try(u32(bytes, entry.local_offset))
+ use _ <- result.try(case sig == local_sig {
+ True -> Ok(Nil)
+ False -> Error(entry.name <> ": local header is malformed")
+ })
+ use name_len <- result.try(u16(bytes, entry.local_offset + 26))
+ use extra_len <- result.try(u16(bytes, entry.local_offset + 28))
+ use _ <- result.try(case entry.method {
+ 0 -> Ok(Nil)
+ m ->
+ Error(
+ entry.name
+ <> ": compression method "
+ <> int.to_string(m)
+ <> "; this reader takes only stored entries, which is what spine writes",
+ )
+ })
+ let start = entry.local_offset + 30 + name_len + extra_len
+ case bit_array.slice(bytes, start, entry.compressed) {
+ Ok(slice) -> Ok(slice)
+ Error(_) -> Error(entry.name <> ": runs past the end of the archive")
+ }
+}
+
+/// The entries whose names end this way, in archive order. Used to find
+/// media/abstraction/<uid>.<lang>.ssp without knowing the uid.
+pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) {
+ list.filter(entries, fn(e) { string.ends_with(e.name, suffix) })
+}
+
+fn u16(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 2) {
+ Ok(<<v:little-size(16)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn u32(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 4) {
+ Ok(<<v:little-size(32)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
+ case bit_array.slice(bytes, at, len) {
+ Ok(slice) ->
+ bit_array.to_string(slice)
+ |> result.replace_error("an entry name is not valid UTF-8")
+ Error(_) -> Error("an entry name runs past the end of the archive")
+ }
+}
+#+END_SRC
+
+** imapges gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/images.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<images>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: images
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// An image's bytes, from whichever source the document came from.
+////
+//// The model *names* images, with digest, byte size and pixel size; it does
+//// not hold them. That is right - an abstraction of a document is not a
+//// bundle of its assets - but a writer of any format that shows images needs
+//// the bytes, so something has to know where they are. This is that
+//// something, and it is in the library rather than in a writer because the
+//// answer depends on the *input form* and not on the output.
+////
+//// | `.ocda.db` | the `files` table, role 'image', by name |
+//// | `.zip` | the archive entry whose path ends in `/image/<name>` |
+//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot |
+//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` |
+
+import gleam/bit_array
+import gleam/dynamic/decode
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/zip
+import simplifile
+import sqlight
+
+/// The bytes, or an error saying why not.
+pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) {
+ case load.source_of(source) {
+ load.OcdaDb -> from_db(source, name)
+ load.Pod -> from_archive(source, name)
+ load.Ssp -> from_beside(source, name)
+ load.Unknown -> Error(source <> ": not a source this library reads")
+ }
+}
+
+/// Can this source give images at all?
+///
+/// A writer asks before it decides whether to emit them, because an `<img>`
+/// pointing at a file it did not include is worse than no image.
+pub fn can_give_images(source: String) -> Bool {
+ case load.source_of(source) {
+ load.OcdaDb -> True
+ load.Pod -> zip_readable(source)
+ load.Ssp -> image_dir_beside(source) != ""
+ load.Unknown -> False
+ }
+}
+
+fn zip_readable(path: String) -> Bool {
+ case simplifile.read_bits(path) {
+ Error(_) -> False
+ Ok(bytes) ->
+ case zip.entries(bytes) {
+ Ok(_) -> True
+ Error(_) -> False
+ }
+ }
+}
+
+fn from_db(path: String, name: String) -> Result(BitArray, String) {
+ case sqlight.open("file:" <> path <> "?mode=ro") {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let out =
+ sqlight.query(
+ "select data from files where role = 'image' and name = ?1",
+ on: conn,
+ with: [sqlight.text(name)],
+ expecting: blob(),
+ )
+ |> result.replace_error(path <> ": could not read " <> name)
+ let _ = sqlight.close(conn)
+ case out {
+ Ok([data, ..]) -> Ok(data)
+ Ok([]) -> Error(name <> ": not in this database")
+ Error(e) -> Error(e)
+ }
+ }
+ }
+}
+
+/// A blob column, read as bytes rather than as text: an image read as text
+/// stops at the first zero byte, and a png's ninth byte is zero.
+fn blob() -> decode.Decoder(BitArray) {
+ use v <- decode.field(0, decode.bit_array)
+ decode.success(v)
+}
+
+fn from_archive(path: String, name: String) -> Result(BitArray, String) {
+ use bytes <- result.try(
+ simplifile.read_bits(path)
+ |> result.replace_error(path <> ": cannot read"),
+ )
+ use entries <- result.try(zip.entries(bytes))
+ case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) {
+ Error(_) -> Error(name <> ": not in this archive")
+ Ok(entry) -> zip.read(bytes, entry)
+ }
+}
+
+fn from_beside(path: String, name: String) -> Result(BitArray, String) {
+ case image_dir_beside(path) {
+ "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it")
+ dir ->
+ simplifile.read_bits(dir <> "/" <> name)
+ |> result.replace_error(dir <> "/" <> name <> ": cannot read")
+ }
+}
+
+/// `media/abstraction/<x>.ssp` -> `media/image/`
+fn image_dir_beside(path: String) -> String {
+ let parts = string.split(path, on: "/")
+ let dir = case list.reverse(parts) {
+ [_file, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ case string.ends_with(dir, "abstraction") {
+ False -> ""
+ True -> {
+ let parent = case list.reverse(string.split(dir, on: "/")) {
+ [_, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ let image_dir = parent <> "/image"
+ case simplifile.is_directory(image_dir) {
+ Ok(True) -> image_dir
+ _ -> ""
+ }
+ }
+ }
+}
+
+/// Unused here, kept so that a caller can size a read before making it.
+pub fn byte_size(b: BitArray) -> Int {
+ bit_array.byte_size(b)
+}
+#+END_SRC
+
+** dump gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/dump.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<dump>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: dump
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// The model, written out canonically.
+////
+//// Not an output format: an *oracle*. Two libraries in two languages
+//// implement the same model from the same specification, and the cheap way
+//// to ask whether they agree is to have each write the whole loaded
+//// document as text and diff the two. Byte for byte identical to the D
+//// library's `ocda/dump.d`, which is the whole point of it.
+////
+//// The rules, which both implementations follow:
+////
+//// - every line is `<key> <value>`, the value running to end of line,
+//// and only one free-form value per line;
+//// - a field at its default is *omitted*, so the output says what the
+//// document has rather than what the type declares;
+//// - dictionaries come in sorted key order, because neither language's
+//// map has an order worth relying on;
+//// - sections come in the fixed order of the format.
+////
+//// Anything that reads this to reconstruct a document has misunderstood
+//// it. Load the `.ssp` or the `.ocda.db`.
+
+import gleam/dict.{type Dict}
+import gleam/int
+import gleam/list
+import gleam/string
+import ocda/model.{type Document, type Object}
+
+pub fn document(doc: Document) -> String {
+ let lines =
+ []
+ |> kv("version", doc.format_version)
+ |> kv("source_filename", doc.source_filename)
+ |> entries("source", doc.source)
+ |> entries("meta", doc.metadata)
+ |> entries("make", doc.make)
+ |> entries("doc_has", doc.doc_has)
+ |> counts(doc)
+ |> objects(doc)
+ lines |> list.reverse |> join
+}
+
+/// What the loader met and did not understand, sorted.
+///
+/// Separate from the document dump: it describes the loader, not the
+/// document, and the `.ocda.db` loader has nothing to report where the
+/// `.ssp` loader does.
+pub fn unknown(doc: Document) -> String {
+ []
+ |> numbered("property", doc.unknown.property)
+ |> numbered("block", doc.unknown.block)
+ |> numbered("line", doc.unknown.line)
+ |> list.reverse
+ |> join
+}
+
+fn join(lines: List(String)) -> String {
+ case lines {
+ [] -> ""
+ _ -> string.join(lines, "\n") <> "\n"
+ }
+}
+
+/// Accumulated newest-first and reversed once, so that nothing appends to
+/// the tail of a list that grows to a few hundred thousand lines.
+fn kv(acc: List(String), key: String, value: String) -> List(String) {
+ case value {
+ "" -> acc
+ v -> [key <> " " <> v, ..acc]
+ }
+}
+
+fn kn(acc: List(String), key: String, value: Int) -> List(String) {
+ case value {
+ 0 -> acc
+ n -> [key <> " " <> int.to_string(n), ..acc]
+ }
+}
+
+fn kb(acc: List(String), key: String, value: Bool) -> List(String) {
+ case value {
+ False -> acc
+ True -> [key <> " 1", ..acc]
+ }
+}
+
+fn entries(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, String),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines]
+ })
+}
+
+fn numbered(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, Int),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines]
+ })
+}
+
+fn counts(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ case model.section(doc, name) {
+ [] -> lines
+ objects ->
+ [
+ "section " <> name <> " " <> int.to_string(list.length(objects)),
+ ..lines
+ ]
+ }
+ })
+}
+
+fn objects(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ model.section(doc, name)
+ |> list.index_fold(lines, fn(l, obj, i) {
+ object(l, name, i, obj)
+ })
+ })
+}
+
+fn object(
+ acc: List(String),
+ section: String,
+ index: Int,
+ obj: Object,
+) -> List(String) {
+ let acc = ["object " <> section <> " " <> int.to_string(index), ..acc]
+ acc
+ |> kn("ocn", obj.ocn)
+ |> kv("is_a", obj.is_a)
+ |> kv("own_section", obj.own_section)
+ |> kv("level", obj.heading_level)
+ |> kv("identifier", obj.identifier)
+ |> kv("part", obj.part)
+ |> kv("is_of_type", obj.is_of_type)
+ |> kv("segment_html_is", obj.segment_html_is)
+ |> kv("segment_epub", obj.segment_epub)
+ |> kv("segment_epub_is", obj.segment_epub_is)
+ |> kn("parent", obj.parent_ocn)
+ |> kn("last_descendant", obj.last_descendant_ocn)
+ |> kn("ocn_range_first", obj.ocn_range_first)
+ |> kn("ocn_range_last", obj.ocn_range_last)
+ |> kv("children", ints(obj.children))
+ |> kv("ancestors", ints(obj.ancestors))
+ |> kb("dummy", obj.dummy_heading)
+ |> kb("ocn_off", obj.ocn_off)
+ |> kn("indent_base", obj.indent_base)
+ |> kn("indent_hang", obj.indent_hang)
+ |> kb("bullet", obj.bullet)
+ |> kv("lang", obj.lang)
+ |> kb("has_links", obj.has_links)
+ |> kb("has_notes_reg", obj.has_notes_reg)
+ |> kb("has_notes_star", obj.has_notes_star)
+ |> kb("has_images", obj.has_images)
+ |> kv("segment", obj.segment)
+ |> kv("segment_prev", obj.segment_prev)
+ |> kv("segment_next", obj.segment_next)
+ |> kv("anchor", obj.anchor)
+ |> kv("heading_lev_anchor", obj.heading_lev_anchor)
+ |> kn("table_cols", obj.table_cols)
+ |> kv("table_widths", obj.table_widths)
+ |> kv("table_aligns", obj.table_aligns)
+ |> kb("table_header", obj.table_header)
+ |> kv("code_syntax", obj.code_syntax)
+ |> kb("code_linenumbers", obj.code_linenumbers)
+ |> kv("heading_ancestors_text", obj.heading_ancestors_text)
+ |> kv("sha256", obj.sha256)
+ |> links(obj)
+ |> anchor_tags(obj)
+ |> images(obj)
+ |> text(obj)
+}
+
+fn links(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) })
+}
+
+fn anchor_tags(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) })
+}
+
+fn images(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.images, acc, fn(lines, img) {
+ // the name last, because a filename may hold a space where none of the
+ // other fields can
+ let lines =
+ kv(
+ lines,
+ "image",
+ img.sha256
+ <> " "
+ <> int.to_string(img.bytes)
+ <> " "
+ <> int.to_string(img.width)
+ <> "x"
+ <> int.to_string(img.height)
+ <> " "
+ <> case img.missing {
+ True -> "1"
+ False -> "0"
+ }
+ <> " "
+ <> img.name,
+ )
+ img.fields
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(lines, fn(l, pair) {
+ kv(l, "image_field", pair.0 <> " " <> pair.1)
+ })
+ })
+}
+
+/// Text, when there is any.
+///
+/// An object whose only text line is blank holds "" here, indistinguishable
+/// from no text at all, because the model joins the lines into one string.
+/// Both libraries lose the same distinction in the same place, which is
+/// what makes them comparable; it is recorded rather than hidden.
+fn text(acc: List(String), obj: Object) -> List(String) {
+ case obj.text {
+ "" -> acc
+ t -> {
+ let lines = string.split(t, on: "\n")
+ let acc = kv(acc, "text", int.to_string(list.length(lines)))
+ list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] })
+ }
+ }
+}
+
+fn ints(xs: List(Int)) -> String {
+ case xs {
+ [] -> ""
+ _ -> xs |> list.map(int.to_string) |> string.join(",")
+ }
+}
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* __END__