summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/ocda/load_db.d
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-26 09:47:07 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-26 09:47:07 -0400
commitfda48e417435e31a3122bfa7ca563333628aed99 (patch)
treec91f50006de299a9db42247bc21df4f59f9c7a82 /dlang/src/ocda/load_db.d
format 2.0: abstraction as library, in D and Gleam
The object-centric document abstraction that sisudoc-spine builds from SiSU markup, read as a library: the model, and a loader for each form it is written in - the .ssp (text), the .ocda.db (sqlite), and a pod archive carrying a .ssp (.sisupod, .zip). Begun 2026-09-25; this is the library as it stood at format 2.0. The readers (sisudoc-ssp-reader, sisudoc-ocda-db-reader) each read one form and print. A writer needs neither: it needs the document. So the model and the loaders live here, ocda.model imports nothing, and a program built on the library cannot tell which form it was handed. ocda.load is the one place that decides what a path is, so a new input form is added once and every program gains it. The model and the .ssp loader began as the .ssp reader's code, moved and renamed to the level they belong to; the .ocda.db loader is new. The Gleam library is a deliberate field-for-field mirror of the D one, including where that is not the most idiomatic Gleam: a difference in one language and not the other would show in every comparison and mean nothing. The claim holds only if the loaders agree, so the library asks: ocda-agree <doc>.ssp <doc>.ocda.db [lang] ten fields, per object ocda-dump <file> [lang] the whole model, canonically The dump is an oracle, not an output format, and the D and Gleam dumps are byte identical, so four questions are a diff: D and Gleam, each from the .ssp and from the .ocda.db. Over the markup document samples all four hold, every document-language; a .sisupod gives the same model as the .ssp beside it. None of that held at first, and what the checks found is the record of how the specification became enough: - an object whose first text line is blank lost it (the loader asked whether the text was empty, not whether a line had been seen); seven sample documents open verse that way. Found by agree; fixed here and in the .ssp reader, which carried the same code. - found by the dump: the identifier and the anchor are omitted by the text form when they can be derived and stored by the database always, so the loaders derive them, and the rule went into the specification (derived values); make and doc_has share one table in the database under a key prefix, which the loaders strip; the JSON columns were scanned, not parsed, and Phobos writes "/" as "\/"; and translation.ocn_aligned is a fact about the file, not the document, so neither loader puts it in the model. - in Gleam, a JSON element decoder fell through to a fallback giving "", so a float column read as empty strings: a value that could not be read turned into one that looked read. - found by the writers: anchor_tags, heading_lev_anchor, segment_epub and segment_epub_is were in the specification and read by nothing. ocda.images gives an image's bytes from whichever form the document came from (the database's files table, the archive entry, or beside a .ssp at media/image/), and says beforehand whether it can. The Gleam library reads a .zip pod but not a .sisupod: the zstd frame around it has no implementation on the BEAM that builds sandboxed. It says so, and says zstd -d, rather than calling the file malformed. gleam/ builds with nix-gleam's builder, kept in gleam/nix-gleam/ (MIT) Use: dlang/ nix build ocda-dump, ocda-agree; or a dub path dependency on dlang/ gleam/ nix build ocda-lib dump | agree See README.md, and dlang/README.md and gleam/README.md. (assisted by Claude-Code)
Diffstat (limited to 'dlang/src/ocda/load_db.d')
-rw-r--r--dlang/src/ocda/load_db.d310
1 files changed, 310 insertions, 0 deletions
diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d
new file mode 100644
index 0000000..dec0626
--- /dev/null
+++ b/dlang/src/ocda/load_db.d
@@ -0,0 +1,310 @@
+module ocda.load_db;
+
+/+ ↓ build the model from a .ocda.db.
+ .
+ The point of this module is that it returns *the same value*
+ ocda.load_ssp returns from the same document's .ssp. A writer is then
+ a function from the model to bytes, and cannot tell which
+ serialisation it was given, exactly as spine's own writers cannot.
+ .
+ Written from the sisudoc-abstraction-format specification,
+ ocda-db-format-specification.md, whose section 4 gives the column by
+ column correspondence this module implements.
++/
+
+import std.array : join, split;
+import std.json : JSONType, parseJSON;
+import std.conv : to;
+import std.algorithm : startsWith;
+import std.string : strip;
+import ocda.model;
+import ocda.sqlite;
+
+/+ ↓ the schema major this loader knows (specification section 2) +/
+enum schema_major_read = "2";
+
+class OcdaDbError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ every language the database holds, in manifest order +/
+struct DbLanguage {
+ int doc_id;
+ string lang;
+ string source_filename;
+ string ssp_digest;
+}
+
+DbLanguage[] dbLanguages(ref Db db) {
+ DbLanguage[] out_;
+ foreach (r; db.rows(
+ "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
+ FROM documents ORDER BY id")) {
+ out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]);
+ }
+ return out_;
+}
+
+/+ ↓ load one language of a database as the model.
+ lang == "" takes the first language the manifest gave.
++/
+OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+
+ if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) {
+ throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;"
+ ~ " it does not look like a .ocda.db");
+ }
+
+ /+ ↓ specification sections 2 and 3.2: the rows describing the file rather
+ than a document in it are the only ones with doc_id null +/
+ string schema_version = db.one(
+ "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'");
+ if (schema_version.length > 0 && !any_version) {
+ auto major = schema_version.split(".")[0];
+ if (major != schema_major_read) {
+ throw new OcdaDbError(path ~ " is schema v" ~ schema_version
+ ~ ", this loader knows v" ~ schema_major_read ~ ".x");
+ }
+ }
+
+ auto languages = dbLanguages(db);
+ if (languages.length == 0) {
+ throw new OcdaDbError(path ~ ": no documents rows");
+ }
+ DbLanguage chosen = languages[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } }
+ if (!found) {
+ string have;
+ foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; }
+ throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have);
+ }
+ }
+
+ OcdaDocument doc;
+ doc.format_version = schema_version;
+ doc.source_filename = chosen.source_filename;
+
+ /+ ↓ @source of the .ssp corresponds to the documents row and the
+ source.digest metadata (specification section 5.1 of the .ssp, and
+ section 7 here) +/
+ doc.source["language"] = chosen.lang;
+ string languages_joined;
+ foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; }
+ doc.source["languages"] = languages_joined;
+ auto digest = db.one("SELECT value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'");
+ if (digest.length > 0) { doc.source["digest"] = digest; }
+
+ /+ ↓ @meta: every metadata row of this language that is not a source.*
+ property. source.* describes the artefact rather than the document
+ and lives in @source in the text form.
+ +/
+ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
+ ~ chosen.doc_id.to!string ~ " ORDER BY key")) {
+ string key = r[0];
+ if (key.startsWith("source.")) { continue; }
+ /+ ↓ translation.ocn_aligned says whether the languages of this
+ document share one numbering. A .ocda.db can know it because it
+ holds every language; a .ssp is one language and cannot. So it is
+ a fact about the *file* rather than about the document, and putting
+ it in the document's metadata would let a writer tell which
+ serialisation it was handed, which is the one thing this library
+ exists to prevent. Left out until the model has a place for
+ collection-wide facts.
+ +/
+ if (key.startsWith("translation.")) { continue; }
+ /+ ↓ the database has one metadata table where the text form has three
+ blocks, and tells them apart by a key prefix. Unprefixing them
+ here is what makes the two loaders give one model: a writer asks
+ doc.make["footer"] whichever serialisation it was handed, and
+ never doc.metadata["make.footer"].
+ +/
+ if (key.startsWith("make.")) {
+ doc.make[key["make.".length .. $]] = r[1].strip;
+ } else if (key.startsWith("doc_has.")) {
+ doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
+ } else {
+ doc.metadata[key] = r[1].strip;
+ }
+ }
+
+ /+ ↓ the objects, in document order within each section +/
+ foreach (r; db.rows(
+ "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''),
+ coalesce(is_of_section,''), coalesce(is_of_type,''),
+ coalesce(heading_level,-1), coalesce(identifier,''),
+ coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
+ coalesce(children,''), coalesce(ancestors,''),
+ coalesce(dummy_heading,0), coalesce(object_number_off,0),
+ coalesce(indent_base,0), coalesce(indent_hang,0),
+ coalesce(bullet,0), coalesce(lang,''),
+ coalesce(has_links,0), coalesce(has_notes_reg,0),
+ coalesce(has_notes_star,0), coalesce(has_images,0),
+ coalesce(segment,''), coalesce(segment_prev,''),
+ coalesce(segment_next,''), coalesce(anchor,''),
+ coalesce(segment_html_is,''),
+ coalesce(table_cols,0), coalesce(table_widths,''),
+ coalesce(table_aligns,''), coalesce(table_header,0),
+ coalesce(code_linenumbers,0),
+ coalesce(heading_ancestors_text,''), coalesce(sha256,''),
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,'')
+ FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
+ ~ " ORDER BY CASE section
+ WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
+ WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5
+ WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7
+ WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) {
+ OcdaObject obj;
+ obj.section = r[0];
+ obj.ocn = r[2].to!int;
+ obj.is_a = r[3];
+ obj.part = r[4];
+ obj.own_section = (r[5] == r[0]) ? "" : r[5];
+ obj.is_of_type = r[6];
+ /+ ↓ the database holds the level as a number, the text form as the
+ letter or digit the markup wrote. Model section 4 is the mapping. +/
+ obj.heading_level = levelName(r[7].to!int);
+ obj.identifier = r[8];
+ obj.parent_ocn = r[9].to!int;
+ obj.last_descendant_ocn = r[10].to!int;
+ obj.children = jsonInts(r[11]);
+ obj.ancestors = jsonInts(r[12]);
+ obj.dummy_heading = (r[13] != "0");
+ obj.ocn_off = (r[14] != "0");
+ obj.indent_base = r[15].to!int;
+ obj.indent_hang = r[16].to!int;
+ obj.bullet = (r[17] != "0");
+ obj.lang = r[18];
+ obj.has_links = (r[19] != "0");
+ obj.has_notes_reg = (r[20] != "0");
+ obj.has_notes_star = (r[21] != "0");
+ obj.has_images = (r[22] != "0");
+ obj.segment = r[23];
+ obj.segment_prev = r[24];
+ obj.segment_next = r[25];
+ obj.anchor = r[26];
+ obj.segment_html_is = r[27];
+ obj.heading_lev_anchor = r[37];
+ obj.segment_epub = r[38];
+ obj.segment_epub_is = r[39];
+ obj.table_cols = r[28].to!int;
+ obj.table_widths = jsonWords(r[29]);
+ obj.table_aligns = jsonWords(r[30]);
+ obj.table_header = (r[31] != "0");
+ obj.code_linenumbers = (r[32] != "0");
+ obj.heading_ancestors_text = jsonJoined(r[33]);
+ obj.sha256 = r[34];
+ obj.text = r[35];
+
+ /+ ↓ the four repeatable properties of the text form are child tables
+ here (specification section 4) +/
+ string object_id = r[36];
+ foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0),
+ coalesce(width,0), coalesce(height,0),
+ coalesce(missing,0)
+ FROM object_images WHERE object_id = " ~ object_id
+ ~ " ORDER BY seq")) {
+ OcdaImage img;
+ img.name = i[0];
+ img.sha256 = i[1];
+ img.bytes = i[2].to!ulong;
+ img.width = i[3].to!int;
+ img.height = i[4].to!int;
+ img.missing = (i[5] != "0");
+ obj.images ~= img;
+ }
+ foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.stow_link ~= l[0];
+ }
+ foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = "
+ ~ object_id ~ " ORDER BY seq")) {
+ obj.anchor_tags ~= a[0];
+ }
+
+ auto section = doc.sectionByName(r[0]);
+ if (section !is null) { *section ~= obj; }
+ }
+ return doc;
+}
+
+/+ ↓ model section 4: the eight levels and their indices +/
+private string levelName(int index) {
+ switch (index) {
+ case 0: return "A";
+ case 1: return "B";
+ case 2: return "C";
+ case 3: return "D";
+ case 4: return "1";
+ case 5: return "2";
+ case 6: return "3";
+ case 7: return "4";
+ default: return "";
+ }
+}
+
+/+ ↓ the JSON columns, read as JSON.
+ .
+ The fixed width arrays are JSON in the database and space separated
+ (or "|" separated) in the text form. This was first written as a
+ character scan that threw away brackets and quotes, which is wrong the
+ moment a value holds an escape: Phobos' JSONValue.toString escapes
+ "/" as "\/", so every heading containing a slash came back with a
+ backslash in it. 36 of 36 documents differed on
+ heading_ancestors_text and nothing else had shown it.
+ .
+ So: parse. The values are small, there are few of them, and a real
+ parser is shorter than the scan it replaces.
++/
+private string[] jsonStrings(string v) {
+ if (v.length == 0) { return []; }
+ string[] out_;
+ try {
+ auto j = parseJSON(v);
+ if (j.type != JSONType.array) { return []; }
+ foreach (e; j.array) {
+ switch (e.type) {
+ case JSONType.string: out_ ~= e.str; break;
+ case JSONType.integer: out_ ~= e.integer.to!string; break;
+ case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
+ case JSONType.float_: out_ ~= e.floating.to!string; break;
+ case JSONType.true_: out_ ~= "true"; break;
+ case JSONType.false_: out_ ~= "false"; break;
+ default: out_ ~= ""; break;
+ }
+ }
+ } catch (Exception) {
+ return [];
+ }
+ return out_;
+}
+
+private int[] jsonInts(string v) {
+ int[] out_;
+ foreach (s; jsonStrings(v)) {
+ if (s.length == 0) { continue; }
+ try { out_ ~= s.to!int; } catch (Exception) {}
+ }
+ return out_;
+}
+
+/+ ↓ space separated in the text form: table_widths, table_aligns +/
+private string jsonWords(string v) {
+ return jsonStrings(v).join(" ");
+}
+
+/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
+ form and a JSON array here. The empty slots are significant: they say
+ which levels the heading sits under, so the separators carry as much
+ as the text does.
++/
+private string jsonJoined(string v) {
+ auto parts = jsonStrings(v);
+ if (parts.length == 0) { return ""; }
+ return parts.join("|");
+}