-*- mode: org -*- #+TITLE: dlang load #+DESCRIPTION: dlang lib #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code #+PROPERTY: header-args+ :noweb yes :padline no #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * ocda ** model #+HEADER: :tangle "../dlang/src/ocda/model.d" #+BEGIN_SRC d <> module ocda.model; struct OcdaImage { string name; // image filename as referenced by the object string sha256; // hex-encoded (uppercase) digest of the file ulong bytes; // file size int width; // file's own pixel width (not display width) int height; // file's own pixel height bool missing; // file absent or unreadable, no sha256/bytes string[string] fields; // any further/unrecognised key:value fields } struct OcdaObject { int ocn; string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) string identifier; // heading: from the declaration line; otherwise .identifier string part; // frontmatter, body, backmatter string is_of_type; // para, block, comment: how the text behaves string own_section; // .section, when it differs from the block it sits in string segment_html_is;// the html segment this object sits in string segment_epub; // the epub segment this heading opens string segment_epub_is;// the epub segment this object sits in string table_aligns; // l c r per column string[] stow_link; // links extracted from the object string[] anchor_tags; // named targets the markup placed on this object int parent_ocn; int last_descendant_ocn; int ocn_range_first;// a container (a poem): the first and last int ocn_range_last; // ocn of what it holds; 0 otherwise (2.1) int[] children; // direct child heading OCNs int[] ancestors; // markedup_ancestors[8] bool dummy_heading; bool ocn_off; int indent_base; int indent_hang; bool bullet; string lang; bool has_links; bool has_notes_reg; bool has_notes_star; bool has_images; string segment; string segment_prev; string segment_next; string anchor; string heading_lev_anchor; // a heading's own named target int table_cols; string table_widths; bool table_header; string code_syntax; bool code_linenumbers; string heading_ancestors_text; string sha256; OcdaImage[] images; // repeatable .image records, in order string text; // content (lines joined with newline) /+ ↓ whether this object is rendered at all. is_of_type "comment" is markup commentary: carried by the abstraction so that nothing is lost, and not put into output. An output that renders it leaks the author notes into the document, which is what spine itself does not do. +/ bool renders() const { return is_of_type != "comment"; } /+ ↓ the section this object belongs to, which is the block it sits in unless it says otherwise +/ string sectionOf() const { return (own_section.length > 0) ? own_section : section; } } /+ ↓ what the reader met and did not understand. A reader that silently drops what it does not know cannot say whether it understood the document, and cannot be used to check a specification against an implementation: silence and understanding look alike. So the parser counts what it ignored, and --report-unknown prints it. +/ struct OcdaUnknown { size_t[string] property; // .name -> times seen size_t[string] block; // @name -> times seen size_t[string] line; // first char -> times seen, lines of no known shape bool any() const { return property.length > 0 || block.length > 0 || line.length > 0; } } struct OcdaDocument { string format_version; // as declared on line 1, "" when absent string source_filename; // from the "% Source:" line string[string] source; // @source: language, languages, digest OcdaUnknown unknown; // what was met and not understood string[string] metadata; // key -> value string[string] make; // key -> value string[string] doc_has; // key -> value OcdaObject[] head; OcdaObject[] toc; OcdaObject[] body; OcdaObject[] endnotes; OcdaObject[] glossary; OcdaObject[] bibliography; OcdaObject[] bookindex; OcdaObject[] blurb; OcdaObject[] tail; /+ ↓ the major part of the declared version, "" when none was declared +/ string formatVersionMajor() const { import std.string : indexOf; if (format_version.length == 0) { return ""; } auto dot = format_version.indexOf('.'); return (dot > 0) ? format_version[0 .. dot] : format_version; } string meta(string key) { if (auto p = key in metadata) return *p; return ""; } OcdaObject[]* sectionByName(string name) { switch (name) { case "head": return &head; case "toc": return &toc; case "body": return &body; case "endnotes": return &endnotes; case "glossary": return &glossary; case "bibliography": return &bibliography; case "bookindex": return &bookindex; case "blurb": return &blurb; case "tail": return &tail; default: return null; } } } /+ ↓ the format major version this reader knows. The .ssp and the .ocda.db carry one version between them, declared in band: the .ssp on its first line, the database in its schema.version row. The minor part is raised when a property is added, and a reader of the newer format still reads the older one, so only the major part is checked here. Raise this when the reader has been brought to a new major, not before: the point of the gate is to refuse what has not been looked at. +/ enum ssp_format_major_read = "2"; #+END_SRC ** load ** what to load *** inventory #+HEADER: :tangle "../dlang/src/ocda/inventory.d" #+BEGIN_SRC d <> module ocda.inventory; /+ ↓ what a source holds, beyond the one document the model is loaded from. . The model is one language of one document. Two questions a reader asks are about the file instead: which languages it holds, and, for a database, what else it carries. They are here rather than in a reader because the answer depends on the input form, which is the library's to know, and so that a reader asking them does not learn which form it was handed. . languages() answers for every form, and answers the same: the languages in the order the manifest gave them (which every .ssp records, in @source languages), each with the markup file it was made from and the sha256 of its .ssp. The database stores that digest; for a .ssp and a pod it is taken from the bytes, which are the same bytes. . carried() is the database's: a .ocda.db holds the markup, images, configuration and catalogues it was built from, a .ssp holds none of them, and a pod's are its archive entries rather than anything the abstraction says. So it answers for a database and says so otherwise. +/ import std.algorithm : endsWith, countUntil; import std.array : array, split; import std.conv : to; import std.digest : toHexString, LetterCase; import std.digest.sha : sha256Of; import std.file : read; import std.string : strip; import ocda.model; import ocda.load; import ocda.load_ssp; import ocda.load_db; import ocda.load_pod; import ocda.sqlite; import ocda.zip; struct OcdaLanguage { string lang; string source_filename; // the markup file the abstraction was made from string ssp_digest; // sha256 of that language's .ssp, upper case hex } struct OcdaCarried { string role; // source, image, conf, manifest, tools string compression; // how it is stored: none, zstd string count; string bytes; // the files' own size, whatever the storage string stored; // what they take in the database } OcdaLanguage[] languages(string path) { final switch (sourceOf(path)) { case Source.ocda_db: auto db = Db.open(path); scope(exit) { db.close(); } OcdaLanguage[] out_; foreach (l; dbLanguages(db)) { out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); } return out_; case Source.ssp: auto bytes = cast(ubyte[]) read(path); auto doc = parseSsp(cast(string) bytes.idup); return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; case Source.pod: auto archive = archiveBytes(path); OcdaLanguage[string] found; string[] order; foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { auto bytes = zipRead(archive, e); auto doc = parseSsp(cast(string) bytes.idup); string lang = langOfSspName(e.name); found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); if (order.length == 0) { order = sourceValue(doc, "languages").split; } } /+ ↓ the manifest's order, and anything it did not name after it +/ OcdaLanguage[] out_; foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } return out_; case Source.unknown: throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); } } /+ ↓ what a database carries, by role and storage; empty for any other form, which isCarrier() says in advance +/ bool isCarrier(string path) { return sourceOf(path) == Source.ocda_db; } OcdaCarried[] carried(string path) { if (!isCarrier(path)) { return []; } auto db = Db.open(path); scope(exit) { db.close(); } if (!db.has("files")) { return []; } OcdaCarried[] out_; foreach (r; db.rows( "SELECT role, compression, count(*), sum(bytes), sum(length(data)) FROM files GROUP BY role, compression ORDER BY role")) { out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); } return out_; } private string sourceValue(ref OcdaDocument doc, string key) { if (auto p = key in doc.source) { return *p; } return ""; } private string hexOf(const(ubyte)[] bytes) { return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; } #+END_SRC *** load **** load #+HEADER: :tangle "../dlang/src/ocda/load.d" #+BEGIN_SRC d <> module ocda.load; /+ ↓ what a path is, and loading it. . One place that decides, so that every reader and every writer accepts the same inputs and a new input type is added once. This mirrors spine's own ocda/abstraction/load.d, which names the sources it can enter the pipeline from and tells them apart by suffix. . Deliberately *not* here: .sst and .ssm markup. Reading markup means implementing the parser, which is spine's work and a far larger thing than a reader or a sample writer needs. This library reads abstractions. +/ import std.algorithm : endsWith; import ocda.model; import ocda.load_ssp; import ocda.load_db; import ocda.load_pod; enum Source { ssp, // .ssp, the abstraction as text ocda_db, // .ocda.db, the abstraction as sqlite pod, // .sisupod or .zip, an archive carrying a .ssp unknown, } Source sourceOf(string path) { if (path.endsWith(".ocda.db")) { return Source.ocda_db; } if (path.endsWith(".ssp")) { return Source.ssp; } /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote before, and every pod already published carries that name, so both are read. What is inside decides how it is opened, not the name. +/ if (path.endsWith(".sisupod")) { return Source.pod; } if (path.endsWith(".zip")) { return Source.pod; } if (path.endsWith(".db")) { return Source.ocda_db; } return Source.unknown; } string sourceName(Source s) { final switch (s) { case Source.ssp: return ".ssp (the abstraction as text)"; case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; case Source.pod: return "a pod archive carrying a .ssp"; case Source.unknown: return "not an abstraction this library reads"; } } class OcdaLoadError : Exception { this(string msg) { super(msg); } } /+ ↓ load whatever the path is into the model. lang names which language to take from a .ocda.db, which holds every language of its document, and from a pod, which may carry more than one .ssp. A .ssp is one language by construction and ignores it. +/ OcdaDocument load(string path, string lang = "", bool any_version = false) { final switch (sourceOf(path)) { case Source.ssp: return parseSspFile(path); case Source.ocda_db: return loadDbFile(path, lang, any_version); case Source.pod: return loadPodFile(path, lang); case Source.unknown: throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); } } #+END_SRC **** load db #+HEADER: :tangle "../dlang/src/ocda/load_db.d" #+BEGIN_SRC d <> module ocda.load_db; /+ ↓ build the model from a .ocda.db. . The point of this module is that it returns *the same value* ocda.load_ssp returns from the same document's .ssp. A writer is then a function from the model to bytes, and cannot tell which serialisation it was given, exactly as spine's own writers cannot. . Written from the sisudoc-abstraction-format specification, ocda-db-format-specification.md, whose section 4 gives the column by column correspondence this module implements. +/ import std.array : join, split; import std.json : JSONType, parseJSON; import std.conv : to; import std.algorithm : startsWith; import std.string : strip; import ocda.model; import ocda.sqlite; /+ ↓ the schema major this loader knows (specification section 2) +/ enum schema_major_read = "2"; class OcdaDbError : Exception { this(string msg) { super(msg); } } /+ ↓ every language the database holds, in manifest order +/ struct DbLanguage { int doc_id; string lang; string source_filename; string ssp_digest; } DbLanguage[] dbLanguages(ref Db db) { DbLanguage[] out_; foreach (r; db.rows( "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') FROM documents ORDER BY id")) { out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); } return out_; } /+ ↓ load one language of a database as the model. lang == "" takes the first language the manifest gave. +/ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { auto db = Db.open(path); scope(exit) { db.close(); } if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" ~ " it does not look like a .ocda.db"); } /+ ↓ specification sections 2 and 3.2: the rows describing the file rather than a document in it are the only ones with doc_id null +/ string schema_version = db.one( "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); if (schema_version.length > 0 && !any_version) { auto major = schema_version.split(".")[0]; if (major != schema_major_read) { throw new OcdaDbError(path ~ " is schema v" ~ schema_version ~ ", this loader knows v" ~ schema_major_read ~ ".x"); } } auto languages = dbLanguages(db); if (languages.length == 0) { throw new OcdaDbError(path ~ ": no documents rows"); } DbLanguage chosen = languages[0]; if (lang.length > 0) { bool found = false; foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } if (!found) { string have; foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); } } OcdaDocument doc; doc.format_version = schema_version; doc.source_filename = chosen.source_filename; /+ ↓ @source of the .ssp corresponds to the documents row and the source.digest metadata (specification section 5.1 of the .ssp, and section 7 here) +/ doc.source["language"] = chosen.lang; string languages_joined; foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } doc.source["languages"] = languages_joined; auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); if (digest.length > 0) { doc.source["digest"] = digest; } /+ ↓ @meta: every metadata row of this language that is not a source.* property. source.* describes the artefact rather than the document and lives in @source in the text form. +/ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY key")) { string key = r[0]; if (key.startsWith("source.")) { continue; } /+ ↓ translation.ocn_aligned says whether the languages of this document share one numbering. A .ocda.db can know it because it holds every language; a .ssp is one language and cannot. So it is a fact about the *file* rather than about the document, and putting it in the document's metadata would let a writer tell which serialisation it was handed, which is the one thing this library exists to prevent. Left out until the model has a place for collection-wide facts. +/ if (key.startsWith("translation.")) { continue; } /+ ↓ the database has one metadata table where the text form has three blocks, and tells them apart by a key prefix. Unprefixing them here is what makes the two loaders give one model: a writer asks doc.make["footer"] whichever serialisation it was handed, and never doc.metadata["make.footer"]. +/ if (key.startsWith("make.")) { doc.make[key["make.".length .. $]] = r[1].strip; } else if (key.startsWith("doc_has.")) { doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; } else { doc.metadata[key] = r[1].strip; } } /+ ↓ 2.1 added the container range as two columns. A 2.0 database has neither, and reads as having no range, which is what 2.0 meant. +/ bool has_range = false; foreach (c; db.rows("PRAGMA table_info(objects)")) { if (c[1] == "ocn_range_first") { has_range = true; } } string range_cols = has_range ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" : "0, 0"; /+ ↓ the objects, in document order within each section +/ foreach (r; db.rows( "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), coalesce(is_of_section,''), coalesce(is_of_type,''), coalesce(heading_level,-1), coalesce(identifier,''), coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), coalesce(children,''), coalesce(ancestors,''), coalesce(dummy_heading,0), coalesce(object_number_off,0), coalesce(indent_base,0), coalesce(indent_hang,0), coalesce(bullet,0), coalesce(lang,''), coalesce(has_links,0), coalesce(has_notes_reg,0), coalesce(has_notes_star,0), coalesce(has_images,0), coalesce(segment,''), coalesce(segment_prev,''), coalesce(segment_next,''), coalesce(anchor,''), coalesce(segment_html_is,''), coalesce(table_cols,0), coalesce(table_widths,''), coalesce(table_aligns,''), coalesce(table_header,0), coalesce(code_linenumbers,0), coalesce(heading_ancestors_text,''), coalesce(sha256,''), coalesce(text,''), id, coalesce(heading_lev_anchor,''), coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ " FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY CASE section WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { OcdaObject obj; obj.section = r[0]; obj.ocn = r[2].to!int; obj.is_a = r[3]; obj.part = r[4]; obj.own_section = (r[5] == r[0]) ? "" : r[5]; obj.is_of_type = r[6]; /+ ↓ the database holds the level as a number, the text form as the letter or digit the markup wrote. Model section 4 is the mapping. +/ obj.heading_level = levelName(r[7].to!int); obj.identifier = r[8]; obj.parent_ocn = r[9].to!int; obj.last_descendant_ocn = r[10].to!int; obj.children = jsonInts(r[11]); obj.ancestors = jsonInts(r[12]); obj.dummy_heading = (r[13] != "0"); obj.ocn_off = (r[14] != "0"); obj.indent_base = r[15].to!int; obj.indent_hang = r[16].to!int; obj.bullet = (r[17] != "0"); obj.lang = r[18]; obj.has_links = (r[19] != "0"); obj.has_notes_reg = (r[20] != "0"); obj.has_notes_star = (r[21] != "0"); obj.has_images = (r[22] != "0"); obj.segment = r[23]; obj.segment_prev = r[24]; obj.segment_next = r[25]; obj.anchor = r[26]; obj.segment_html_is = r[27]; obj.heading_lev_anchor = r[37]; obj.segment_epub = r[38]; obj.segment_epub_is = r[39]; obj.ocn_range_first = r[40].to!int; obj.ocn_range_last = r[41].to!int; obj.table_cols = r[28].to!int; obj.table_widths = jsonWords(r[29]); obj.table_aligns = jsonWords(r[30]); obj.table_header = (r[31] != "0"); obj.code_linenumbers = (r[32] != "0"); obj.heading_ancestors_text = jsonJoined(r[33]); obj.sha256 = r[34]; obj.text = r[35]; /+ ↓ the four repeatable properties of the text form are child tables here (specification section 4) +/ string object_id = r[36]; foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), coalesce(width,0), coalesce(height,0), coalesce(missing,0) FROM object_images WHERE object_id = " ~ object_id ~ " ORDER BY seq")) { OcdaImage img; img.name = i[0]; img.sha256 = i[1]; img.bytes = i[2].to!ulong; img.width = i[3].to!int; img.height = i[4].to!int; img.missing = (i[5] != "0"); obj.images ~= img; } foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " ~ object_id ~ " ORDER BY seq")) { obj.stow_link ~= l[0]; } foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " ~ object_id ~ " ORDER BY seq")) { obj.anchor_tags ~= a[0]; } auto section = doc.sectionByName(r[0]); if (section !is null) { *section ~= obj; } } countUnread(db, chosen.doc_id, doc); return doc; } /+ ↓ what the database holds and the model does not. . The text loader meets every property line and counts those it does not keep, so a reader can say what was read and not understood. This loader asks for columns by name, and would never see the ones it did not ask for: it would report that it had understood everything, which is the silence the count exists to prevent. So it counts them, on the conditions under which the .ssp writes the same property (specification section 6.3), and the report is the same from either form: checked, 36 of 36. +/ private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { string id = doc_id.to!string; void count(string property, string where) { auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id ~ " AND " ~ where).to!size_t; if (n > 0) { doc.unknown.property[property] = n; } } count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); count("dom_status", "coalesce(dom_status,'') != ''"); count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); count("parent_lev", "coalesce(parent_lev,0) != 0"); count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); count("attrib", "coalesce(attrib,'') != ''"); if (db.has("object_subtoc")) { auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } } } /+ ↓ model section 4: the eight levels and their indices +/ private string levelName(int index) { switch (index) { case 0: return "A"; case 1: return "B"; case 2: return "C"; case 3: return "D"; case 4: return "1"; case 5: return "2"; case 6: return "3"; case 7: return "4"; default: return ""; } } /+ ↓ the JSON columns, read as JSON. . The fixed width arrays are JSON in the database and space separated (or "|" separated) in the text form. This was first written as a character scan that threw away brackets and quotes, which is wrong the moment a value holds an escape: Phobos' JSONValue.toString escapes "/" as "\/", so every heading containing a slash came back with a backslash in it. 36 of 36 documents differed on heading_ancestors_text and nothing else had shown it. . So: parse. The values are small, there are few of them, and a real parser is shorter than the scan it replaces. +/ private string[] jsonStrings(string v) { if (v.length == 0) { return []; } string[] out_; try { auto j = parseJSON(v); if (j.type != JSONType.array) { return []; } foreach (e; j.array) { switch (e.type) { case JSONType.string: out_ ~= e.str; break; case JSONType.integer: out_ ~= e.integer.to!string; break; case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; case JSONType.float_: out_ ~= e.floating.to!string; break; case JSONType.true_: out_ ~= "true"; break; case JSONType.false_: out_ ~= "false"; break; default: out_ ~= ""; break; } } } catch (Exception) { return []; } return out_; } private int[] jsonInts(string v) { int[] out_; foreach (s; jsonStrings(v)) { if (s.length == 0) { continue; } try { out_ ~= s.to!int; } catch (Exception) {} } return out_; } /+ ↓ space separated in the text form: table_widths, table_aligns +/ private string jsonWords(string v) { return jsonStrings(v).join(" "); } /+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text form and a JSON array here. The empty slots are significant: they say which levels the heading sits under, so the separators carry as much as the text does. +/ private string jsonJoined(string v) { auto parts = jsonStrings(v); if (parts.length == 0) { return ""; } return parts.join("|"); } #+END_SRC **** load pod #+HEADER: :tangle "../dlang/src/ocda/load_pod.d" #+BEGIN_SRC d <> module ocda.load_pod; /+ ↓ the abstraction out of a pod archive. . A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same archive unwrapped, which is what spine published before it wrapped them. Either way, a pod built with --pod2 carries the abstraction inside it, at media/abstraction/..ssp, and that is what is read. . So this is a third input form and *not* a third model: the .ssp inside is parsed by ocda.load_ssp and comes back as the same value. A writer over the library gains .sisupod without being touched, which is the whole reason the dispatch is in one place. . A pod built without --pod2 carries markup and no abstraction. That is refused, and the refusal says which flag makes a pod this reader can read, because "no abstraction in it" is a fact about the pod rather than about the reader. +/ import std.algorithm : endsWith, sort; import std.array : array; import std.file : read; import std.string : indexOf; import ocda.model; import ocda.load_ssp; import ocda.zip; import ocda.zstd; /+ ↓ a limit on what a frame may expand to. A reader of an artefact it did not make declines to be told how much memory to allocate. 512 MiB is far above the largest pod in the reference collection (4 MiB) and far below anything that matters. +/ enum size_t max_pod_size = 512 * 1024 * 1024; class OcdaPodError : Exception { this(string msg) { super(msg); } } /+ ↓ which languages of the abstraction the pod carries, in sorted order +/ string[] podLanguages(string path) { auto bytes = archiveBytes(path); string[] out_; foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { string lang = langOfSspName(e.name); if (lang.length > 0) { out_ ~= lang; } } return out_.sort.array; } /+ ↓ load one language of the abstraction a pod carries. lang == "" takes the only one there is, and where there is a choice it says so rather than choosing, as the .ocda.db loader does. +/ OcdaDocument loadPodFile(string path, string lang = "") { auto bytes = archiveBytes(path); auto entries = zipEntries(bytes); auto ssps = zipFindSuffix(entries, ".ssp"); if (ssps.length == 0) { throw new OcdaPodError(path ~ ": carries no .ssp." ~ " A pod carries the abstraction only when spine was asked for it:" ~ " spine --pod2 puts it in, spine --source does not."); } ZipEntry chosen = ssps[0]; if (lang.length > 0) { bool found = false; foreach (e; ssps) { if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } } if (!found) { string have; foreach (i, e; ssps) { have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); } throw new OcdaPodError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); } } else if (ssps.length > 1) { string have; foreach (i, e; ssps) { have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); } throw new OcdaPodError(path ~ ": holds more than one language (" ~ have ~ "); name the one wanted"); } auto ssp = zipRead(bytes, chosen); return parseSsp(cast(string) ssp.idup); } /+ ↓ the archive, unwrapped if it was wrapped. The bytes decide, not the name: a .sisupod is a zstd frame and a .zip is not, and a file misnamed either way still reads. +/ package ubyte[] archiveBytes(string path) { auto bytes = cast(ubyte[]) read(path); return zstdIsFrame(bytes) ? zstdDecompress(bytes, max_pod_size) : bytes; } /+ ↓ ..ssp -> +/ package string langOfSspName(string name) { if (!name.endsWith(".ssp")) { return ""; } string stem = name[0 .. $ - 4]; auto dot = stem.lastIndexOf('.'); return (dot > 0) ? stem[dot + 1 .. $] : ""; } private ptrdiff_t lastIndexOf(string s, char c) { for (ptrdiff_t i = s.length - 1; i >= 0; i--) { if (s[i] == c) { return i; } } return -1; } #+END_SRC **** load ssp #+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" #+BEGIN_SRC d <> module ocda.load_ssp; import std.algorithm : splitter, startsWith; import std.array : array, split; import std.conv : to, ConvException; import std.file : readText; import std.stdio : writeln; import std.string : strip, stripLeft, indexOf; import ocda.model; OcdaDocument parseSspFile(string filename) { string content = readText(filename); return parseSsp(content); } OcdaDocument parseSsp(string content) { OcdaDocument doc; OcdaObject current_obj; bool in_object = false; /+ ↓ whether a text line has been seen for the object being read. Not "is the text non-empty": an object whose first text line is blank (a verse opening on an empty line, which the sample set has seven of) would then lose it, the next line overwriting rather than appending. +/ bool seen_text = false; string current_section = ""; string current_block = ""; // meta, make, doc_has, or section name foreach (line; content.splitter("\n")) { /+ ↓ comment. The first line of a .ssp declares the format version, and it is the one comment that carries meaning: a reader checks the major part and refuses what it does not know rather than loading an unfamiliar artefact half populated in silence. +/ if (line.length > 0 && line[0] == '%') { enum _banner = "% SiSU Document Abstraction v"; enum _source = "% Source: "; if (doc.format_version.length == 0 && line.startsWith(_banner)) { doc.format_version = line[_banner.length .. $].strip; } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { doc.source_filename = line[_source.length .. $].strip; } continue; } /+ ↓ blank line - finalize current object +/ if (line.length == 0 || line.strip.length == 0) { if (in_object) { commitObject(doc, current_section, current_obj); current_obj = OcdaObject.init; in_object = false; seen_text = false; } continue; } /+ ↓ section/block open +/ if (line[0] == '@') { auto brace_pos = line.indexOf('{'); if (brace_pos > 0) { string name = line[1 .. brace_pos].strip; current_block = name; if (!isPreambleBlock(name)) { current_section = name; /+ ↓ a block that is neither preamble nor a section this reader knows: counted, so that --report-unknown can say so +/ if (doc.sectionByName(name) is null) { doc.unknown.block[name]++; } } } continue; } /+ ↓ section/block close +/ if (line[0] == '}') { if (in_object) { commitObject(doc, current_section, current_obj); current_obj = OcdaObject.init; in_object = false; seen_text = false; } current_block = ""; continue; } /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ if (isPreambleBlock(current_block)) { string stripped = line.stripLeft; auto colon_pos = stripped.indexOf(':'); if (colon_pos > 0) { string key = stripped[0 .. colon_pos].strip; string value = (colon_pos + 1 < stripped.length) ? stripped[colon_pos + 1 .. $].strip : ""; if (current_block == "meta") { doc.metadata[key] = value; } else if (current_block == "make") { doc.make[key] = value; } else if (current_block == "source") { doc.source[key] = value; } else { doc.doc_has[key] = value; } } continue; } /+ ↓ object declaration: [N] type ... +/ if (line[0] == '[') { if (in_object) { commitObject(doc, current_section, current_obj); } current_obj = OcdaObject.init; current_obj.section = current_section; in_object = true; seen_text = false; parseObjectDeclaration(line, current_obj); continue; } /+ ↓ property: .name: value +/ if (line[0] == '.') { if (in_object) { parseProperty(line, current_obj, doc.unknown); } continue; } /+ ↓ text content: | content +/ if (line.length >= 1 && line[0] == '|') { if (in_object) { string text_content = (line.length > 2) ? line[2 .. $] : ""; if (seen_text) { current_obj.text ~= "\n" ~ text_content; } else { current_obj.text = text_content; seen_text = true; } } continue; } /+ ↓ no known shape. Counted by its first character so that a reader can be told the file held something this one did not expect +/ doc.unknown.line[line[0 .. 1].idup]++; } /+ ↓ finalize last object +/ if (in_object) { commitObject(doc, current_section, current_obj); } return doc; } private void parseObjectDeclaration(string line, ref OcdaObject obj) { /+ parse: [N] type :level identifier +/ auto bracket_close = line.indexOf(']'); if (bracket_close < 0) return; try { obj.ocn = line[1 .. bracket_close].strip.to!int; } catch (ConvException) { obj.ocn = 0; } string rest = (bracket_close + 1 < line.length) ? line[bracket_close + 1 .. $].strip : ""; if (rest.length == 0) return; auto parts = rest.split(" "); if (parts.length == 0) return; obj.is_a = parts[0]; if (obj.is_a == "heading" && parts.length >= 2) { /+ parse :A, :B, :1, etc. +/ string lev = parts[1]; if (lev.length >= 2 && lev[0] == ':') { obj.heading_level = lev[1 .. $]; } /+ optional identifier after level +/ if (parts.length >= 3) { obj.identifier = parts[2]; } } } /+ ↓ the blocks that are document preamble rather than sections of objects +/ private bool isPreambleBlock(string name) { return name == "source" || name == "meta" || name == "make" || name == "doc_has"; } private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { /+ parse: .name: value +/ auto colon_pos = line.indexOf(':'); if (colon_pos < 1) return; string name = line[1 .. colon_pos].strip; string value = (colon_pos + 1 < line.length) ? line[colon_pos + 1 .. $].strip : ""; switch (name) { case "part": obj.part = value; break; case "is_of_type": obj.is_of_type = value; break; case "section": obj.own_section = value; break; case "identifier": obj.identifier = value; break; case "segment_html_is": obj.segment_html_is = value; break; /+ ↓ the epub segments at heading levels 1 to 4 where the html segments at level 4 alone, so the two are different files and the abstraction records both +/ case "segment_epub": obj.segment_epub = value; break; case "segment_epub_is": obj.segment_epub_is = value; break; case "table_aligns": obj.table_aligns = value; break; case "stow_link": obj.stow_link ~= value; break; case "anchor_tag": obj.anchor_tags ~= value; break; case "parent": obj.parent_ocn = toIntSafe(value); break; case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; case "ocn_range": { auto r = parseIntArray(value); if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; } break; } case "children": obj.children = parseIntArray(value); break; case "ancestors": obj.ancestors = parseIntArray(value); break; case "dummy": obj.dummy_heading = (value == "true"); break; case "ocn_off": obj.ocn_off = (value == "true"); break; case "indent": auto parts = value.split(" "); if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); break; case "bullet": obj.bullet = (value == "true"); break; case "lang": obj.lang = value; break; case "has": foreach (flag; value.split(" ")) { switch (flag) { case "links": obj.has_links = true; break; case "notes_reg": obj.has_notes_reg = true; break; case "notes_star": obj.has_notes_star = true; break; case "images": obj.has_images = true; break; default: break; } } break; case "segment": obj.segment = value; break; case "segment_prev": obj.segment_prev = value; break; case "segment_next": obj.segment_next = value; break; case "anchor": obj.anchor = value; break; case "heading_lev_anchor": obj.heading_lev_anchor = value; break; case "table_cols": obj.table_cols = toIntSafe(value); break; case "table_widths": obj.table_widths = value; break; case "table_header": obj.table_header = (value == "true"); break; case "code_syntax": obj.code_syntax = value; break; case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; case "heading_ancestors_text": obj.heading_ancestors_text = value; break; case "sha256": obj.sha256 = value; break; case "image": obj.images ~= parseImageRecord(value); break; /+ ↓ a property this reader has no use for, or has never heard of. Ignored, which is the conforming behaviour within a major version, but counted rather than passed over in silence +/ default: unknown.property[name]++; break; } } private OcdaImage parseImageRecord(string value) { /+ parse: sha256: bytes: [key:value ...] filename first, remaining fields order-independent; unrecognised fields are kept rather than discarded +/ OcdaImage img; bool first = true; foreach (field; value.split(" ")) { if (field.length == 0) continue; if (first) { img.name = field; first = false; continue; } auto colon_pos = field.indexOf(':'); if (colon_pos < 1) continue; string key = field[0 .. colon_pos]; string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; switch (key) { case "sha256": img.sha256 = val; break; case "bytes": img.bytes = toUlongSafe(val); break; case "px": auto dims = val.split("x"); if (dims.length == 2) { img.width = toIntSafe(dims[0]); img.height = toIntSafe(dims[1]); } break; case "missing": img.missing = (val == "true"); break; default: img.fields[key] = val; break; } } return img; } /+ ↓ two values the text form leaves out because they can be worked out. . The .ssp emits a property only when it carries information, so .identifier is absent when it is simply the ocn, and .anchor is absent when it is the segment name. The .ocda.db, being columns rather than lines, stores both unconditionally. So a loader that takes the text form literally gives a *different model* from the one the database gives, and the difference is not small: in the reference collection the identifier is omitted on all but a handful of objects. . The model is the thing both serialisations describe, so the loader fills them in. Found by dumping both models and diffing them, which no amount of comparing section counts would have shown. +/ private void deriveOmitted(ref OcdaObject obj) { import std.conv : to; if (obj.identifier.length == 0 && obj.ocn != 0) { obj.identifier = obj.ocn.to!string; } if (obj.anchor.length == 0 && obj.segment.length > 0) { obj.anchor = obj.segment; } } private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { deriveOmitted(obj); auto sec = doc.sectionByName(section); if (sec !is null) { ,*sec ~= obj; } } private int toIntSafe(string s) { try { return s.strip.to!int; } catch (ConvException) { return 0; } } private ulong toUlongSafe(string s) { try { return s.strip.to!ulong; } catch (ConvException) { return 0; } } private int[] parseIntArray(string s) { int[] result; foreach (part; s.split(" ")) { auto v = toIntSafe(part); result ~= v; } return result; } #+END_SRC *** images #+HEADER: :tangle "../dlang/src/ocda/images.d" #+BEGIN_SRC d <> module ocda.images; /+ ↓ an image's bytes, from whichever source the document came from. . The model *names* images, with digest, byte size and pixel size; it does not hold them. That is right - an abstraction of a document is not a bundle of its assets - but a writer of any format that shows images needs the bytes, so something has to know where they are. This is that something, and it is in the library rather than in a writer because the answer depends on the *input form* and not on the output. . Three answers, and one refusal: . .ocda.db the files table, role 'image', by name .sisupod the archive entry whose path ends in /image/ .zip the same .ssp beside it: a pod puts the abstraction in media/abstraction/ and its images in media/image/, so the path is documented rather than guessed . A .ssp that is not in a pod has nowhere to look, and that is reported as absent rather than as an error: a reader that cannot find an image should say so and carry on, because the document is still readable. +/ import std.algorithm : endsWith; import std.array : array; import std.file : exists, isFile, read; import std.path : buildPath, dirName; import ocda.load; import ocda.sqlite; import ocda.zip; import ocda.zstd; /+ ↓ the bytes, or an empty slice when the source has no such image +/ ubyte[] imageBytes(string source, string name) { final switch (sourceOf(source)) { case Source.ocda_db: return fromDb(source, name); case Source.pod: return fromArchive(source, name); case Source.ssp: return fromBeside(source, name); case Source.unknown: return []; } } /+ ↓ can this source give images at all? A writer asks before it decides whether to emit them, because an pointing at a file it did not include is worse than no image. +/ bool canGiveImages(string source) { final switch (sourceOf(source)) { case Source.ocda_db: return true; case Source.pod: return true; case Source.ssp: return imageDirBeside(source).length > 0; case Source.unknown: return false; } } private ubyte[] fromDb(string path, string name) { auto db = Db.open(path); scope(exit) { db.close(); } if (!db.has("files")) { return []; } /+ ↓ the name is from the abstraction, not from a caller, and the quote doubling is what keeps it a value rather than syntax +/ return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '" ~ sqlQuote(name) ~ "'"); } private ubyte[] fromArchive(string path, string name) { auto bytes = cast(ubyte[]) read(path); if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); } auto entries = zipEntries(bytes); foreach (e; entries) { if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); } } return []; } private ubyte[] fromBeside(string path, string name) { string dir = imageDirBeside(path); if (dir.length == 0) { return []; } string file = buildPath(dir, name); return file.exists && file.isFile ? cast(ubyte[]) read(file) : []; } /+ ↓ media/abstraction/.ssp -> media/image/ +/ private string imageDirBeside(string path) { string dir = path.dirName; if (!dir.endsWith("abstraction")) { return ""; } string image_dir = buildPath(dir.dirName, "image"); return image_dir.exists ? image_dir : ""; } /+ ↓ a single quote doubled is how sql says a single quote +/ private string sqlQuote(string s) { string out_; foreach (c; s) { out_ ~= (c == '\'') ? "''" : [c]; } return out_; } #+END_SRC ** sqlite #+HEADER: :tangle "../dlang/src/ocda/sqlite.d" #+BEGIN_SRC d <> module ocda.sqlite; /+ ↓ the sqlite3 C API, declared rather than depended upon. . A binding from the dub registry would be a dependency to fetch, and the point of this reader is that it builds in a sandbox with no network. The dozen functions a reader needs are declared here and the library is linked directly, which costs a page of extern(C) and saves a dependency. +/ import std.conv : to; import std.string : fromStringz, toStringz; extern (C) { struct sqlite3; struct sqlite3_stmt; int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs); int sqlite3_close(sqlite3* db); int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte, sqlite3_stmt** stmt, const(char)** tail); int sqlite3_step(sqlite3_stmt* stmt); int sqlite3_finalize(sqlite3_stmt* stmt); int sqlite3_column_count(sqlite3_stmt* stmt); const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col); const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col); int sqlite3_column_int(sqlite3_stmt* stmt, int col); int sqlite3_column_type(sqlite3_stmt* stmt, int col); const(char)* sqlite3_errmsg(sqlite3* db); const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col); int sqlite3_column_bytes(sqlite3_stmt* stmt, int col); } enum SQLITE_OK = 0; enum SQLITE_ROW = 100; enum SQLITE_DONE = 101; enum SQLITE_NULL = 5; enum SQLITE_OPEN_READONLY = 0x00000001; class SqliteError : Exception { this(string msg) { super(msg); } } /+ ↓ a database, opened read only. Read only is deliberate: a reader has no business writing, and a request that opens a database read-write can create an empty one where a path is mistyped. +/ struct Db { private sqlite3* _handle; static Db open(string path) { sqlite3* handle; auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null); if (rc != SQLITE_OK) { string msg = (handle is null) ? "cannot open " ~ path : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup; if (handle !is null) { sqlite3_close(handle); } throw new SqliteError(msg); } Db db; db._handle = handle; return db; } void close() { if (_handle !is null) { sqlite3_close(_handle); _handle = null; } } /+ ↓ every row of a query, each row as its columns in order +/ string[][] rows(string sql) { sqlite3_stmt* stmt; auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null); if (rc != SQLITE_OK) { throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql); } scope(exit) { sqlite3_finalize(stmt); } string[][] out_rows; int cols = sqlite3_column_count(stmt); while (sqlite3_step(stmt) == SQLITE_ROW) { string[] row; foreach (c; 0 .. cols) { row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL) ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup; } out_rows ~= row; } return out_rows; } /+ ↓ the first column of the first row, "" when there is none +/ string one(string sql) { auto r = rows(sql); return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; } /+ ↓ the first column of the first row as bytes, for a blob. Separate from one(), because a blob is not text: an image read as text stops at the first zero byte, and a png's ninth byte is zero. +/ ubyte[] blob(string sql) { sqlite3_stmt* stmt; if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) { throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup ~ " [" ~ sql ~ "]"); } scope(exit) { sqlite3_finalize(stmt); } if (sqlite3_step(stmt) != SQLITE_ROW) { return []; } int n = sqlite3_column_bytes(stmt, 0); if (n <= 0) { return []; } auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0); if (p is null) { return []; } return p[0 .. n].dup; } /+ ↓ does a table or view of this name exist +/ bool has(string name) { return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; } } #+END_SRC ** compression *** zip #+HEADER: :tangle "../dlang/src/ocda/zip.d" #+BEGIN_SRC d <> module ocda.zip; /+ ↓ enough of the zip format to find one file in a pod and read it. . Not a zip library. A pod archive holds a handful of entries, spine writes them **stored** (uncompressed) so that the outer zstd frame does the compressing once over the whole thing, and what is wanted from it here is one named entry. So: read the central directory, find the entry, return its bytes. . Deflate is *refused rather than mishandled*. Phobos can inflate, but a library that says it read an entry it in fact skipped is worse than one that says it cannot, and spine's own archives do not need it. +/ import std.algorithm : endsWith; import std.conv : to; class OcdaZipError : Exception { this(string msg) { super(msg); } } struct ZipEntry { string name; uint method; // 0 stored, 8 deflate uint compressed; uint uncompressed; uint local_offset; } /+ ↓ the entries, from the central directory at the end of the file +/ ZipEntry[] zipEntries(const(ubyte)[] bytes) { enum uint eocd_sig = 0x06054b50; enum uint cd_sig = 0x02014b50; if (bytes.length < 22) { throw new OcdaZipError("too short to be a zip archive"); } /+ ↓ the end of central directory record is last, and may be followed by a comment of up to 65535 bytes, so it is searched for backwards +/ ptrdiff_t eocd = -1; ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0; for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) { if (u32(bytes, i) == eocd_sig) { eocd = i; break; } } if (eocd < 0) { throw new OcdaZipError("no end of central directory record;" ~ " it does not look like a zip archive"); } uint count = u16(bytes, eocd + 10); uint offset = u32(bytes, eocd + 16); ZipEntry[] out_; size_t p = offset; foreach (_; 0 .. count) { if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) { throw new OcdaZipError("central directory entry " ~ out_.length.to!string ~ " is malformed"); } ZipEntry e; e.method = u16(bytes, p + 10); e.compressed = u32(bytes, p + 20); e.uncompressed = u32(bytes, p + 24); uint name_len = u16(bytes, p + 28); uint extra_len = u16(bytes, p + 30); uint cmnt_len = u16(bytes, p + 32); e.local_offset = u32(bytes, p + 42); e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup; out_ ~= e; p += 46 + name_len + extra_len + cmnt_len; } return out_; } /+ ↓ one entry's bytes, from its local header +/ ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) { enum uint local_sig = 0x04034b50; size_t p = e.local_offset; if (p + 30 > bytes.length || u32(bytes, p) != local_sig) { throw new OcdaZipError(e.name ~ ": local header is malformed"); } uint name_len = u16(bytes, p + 26); uint extra_len = u16(bytes, p + 28); size_t start = p + 30 + name_len + extra_len; size_t end = start + e.compressed; if (end > bytes.length) { throw new OcdaZipError(e.name ~ ": runs past the end of the archive"); } if (e.method != 0) { throw new OcdaZipError(e.name ~ ": compression method " ~ e.method.to!string ~ "; this reader takes only stored entries," ~ " which is what spine writes"); } return bytes[start .. end].dup; } /+ ↓ the first entry whose name ends this way, or none. Used to find media/abstraction/..ssp without knowing the uid. +/ ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) { ZipEntry[] out_; foreach (e; entries) { if (e.name.endsWith(suffix)) { out_ ~= e; } } return out_; } private uint u16(const(ubyte)[] b, size_t i) { return cast(uint) b[i] | (cast(uint) b[i + 1] << 8); } private uint u32(const(ubyte)[] b, size_t i) { return cast(uint) b[i] | (cast(uint) b[i + 1] << 8) | (cast(uint) b[i + 2] << 16) | (cast(uint) b[i + 3] << 24); } #+END_SRC *** zstd #+HEADER: :tangle "../dlang/src/ocda/zstd.d" #+BEGIN_SRC d <> module ocda.zstd; /+ ↓ libzstd, declared rather than bound. . A .sisupod is one zstd frame wrapping a zip archive, so reading one needs exactly two things from the library: is this a frame, and what is in it. Both are declared here and the C library is linked, which is the same arrangement ocda.sqlite uses: no vendored tree, no generated header, and the prototypes say what is being called. +/ extern (C) @nogc nothrow private { size_t ZSTD_decompress(void* dst, size_t dstCapacity, const(void)* src, size_t compressedSize); ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); uint ZSTD_isError(size_t code); const(char)* ZSTD_getErrorName(size_t code); } /+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/ private enum ulong contentSizeUnknown = ulong.max; private enum ulong contentSizeError = ulong.max - 1; /+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/ private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD]; class OcdaZstdError : Exception { this(string msg) { super(msg); } } /+ ↓ does this begin with a zstd frame? Asked of the bytes rather than of the filename, because what is inside decides how a thing is opened. A .sisupod is a frame; a .zip published before spine wrapped them is not, and both are read. +/ bool zstdIsFrame(const(ubyte)[] bytes) { if (bytes.length < zstd_magic.length) { return false; } foreach (i, b; zstd_magic) { if (bytes[i] != b) { return false; } } return true; } /+ ↓ one frame in, its contents out. max_size is a refusal rather than a buffer size: a frame declares how much it expands to, and a frame that claims more than the caller is willing to hold is not decompressed at all. A reader of an artefact it did not make wants that limit. +/ ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) { if (!zstdIsFrame(bytes)) { throw new OcdaZstdError("not a zstd frame"); } ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length); if (declared == contentSizeError) { throw new OcdaZstdError("zstd frame: could not read its content size"); } if (declared == contentSizeUnknown) { throw new OcdaZstdError("zstd frame declares no content size;" ~ " this reader takes only frames that do"); } if (declared > max_size) { import std.conv : to; throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string ~ " bytes, over the " ~ max_size.to!string ~ " byte limit"); } auto out_ = new ubyte[](cast(size_t) declared); size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length); if (ZSTD_isError(got)) { import std.string : fromStringz; throw new OcdaZstdError("zstd: " ~ ZSTD_getErrorName(got).fromStringz.idup); } if (got != declared) { throw new OcdaZstdError("zstd frame expanded to a different size" ~ " than it declared"); } return out_; } #+END_SRC ** dump #+HEADER: :tangle "../dlang/src/ocda/dump.d" #+BEGIN_SRC d <> module ocda.dump; /+ ↓ the model, written out canonically. . Not an output format: an *oracle*. Two libraries in two languages implement the same model from the same specification, and the cheap way to ask whether they agree is to have each write the whole loaded document as text and diff the two. That is only useful if the text is fully determined by the model, so: . - every line is " ", the value running to end of line, and only one free-form value per line; - a field at its default is *omitted*, so the output says what the document has rather than what the struct declares; - dictionaries are emitted in sorted key order, because neither language's hash map has an order worth relying on; - sections come in the fixed order of the format, not alphabetically. . Anything that reads this to reconstruct a document has misunderstood it. Load the .ssp or the .ocda.db. +/ import std.algorithm : sort, splitter; import std.array : appender, array, join; import std.conv : to; import ocda.model; /+ ↓ the nine sections, in format order +/ static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", "tail"]; string dumpDocument(ref OcdaDocument doc) { auto out_ = appender!string; void kv(string key, string value) { if (value.length == 0) { return; } out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; } void kn(string key, long value) { if (value == 0) { return; } kv(key, value.to!string); } void kb(string key, bool value) { if (!value) { return; } kv(key, "1"); } void dict(string prefix, string[string] d) { foreach (key; d.keys.dup.sort) { out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; out_ ~= d[key]; out_ ~= "\n"; } } kv("version", doc.format_version); kv("source_filename", doc.source_filename); dict("source", doc.source); dict("meta", doc.metadata); dict("make", doc.make); dict("doc_has", doc.doc_has); foreach (name; dump_sections) { auto sec = doc.sectionByName(name); if (sec is null || (*sec).length == 0) { continue; } kv("section", name ~ " " ~ (*sec).length.to!string); } foreach (name; dump_sections) { auto sec = doc.sectionByName(name); if (sec is null) { continue; } foreach (i, obj; *sec) { out_ ~= "object "; out_ ~= name; out_ ~= " "; out_ ~= i.to!string; out_ ~= "\n"; kn("ocn", obj.ocn); kv("is_a", obj.is_a); kv("own_section", obj.own_section); kv("level", obj.heading_level); kv("identifier", obj.identifier); kv("part", obj.part); kv("is_of_type", obj.is_of_type); kv("segment_html_is", obj.segment_html_is); kv("segment_epub", obj.segment_epub); kv("segment_epub_is", obj.segment_epub_is); kn("parent", obj.parent_ocn); kn("last_descendant", obj.last_descendant_ocn); kn("ocn_range_first", obj.ocn_range_first); kn("ocn_range_last", obj.ocn_range_last); kv("children", ints(obj.children)); kv("ancestors", ints(obj.ancestors)); kb("dummy", obj.dummy_heading); kb("ocn_off", obj.ocn_off); kn("indent_base", obj.indent_base); kn("indent_hang", obj.indent_hang); kb("bullet", obj.bullet); kv("lang", obj.lang); kb("has_links", obj.has_links); kb("has_notes_reg", obj.has_notes_reg); kb("has_notes_star", obj.has_notes_star); kb("has_images", obj.has_images); kv("segment", obj.segment); kv("segment_prev", obj.segment_prev); kv("segment_next", obj.segment_next); kv("anchor", obj.anchor); kv("heading_lev_anchor", obj.heading_lev_anchor); kn("table_cols", obj.table_cols); kv("table_widths", obj.table_widths); kv("table_aligns", obj.table_aligns); kb("table_header", obj.table_header); kv("code_syntax", obj.code_syntax); kb("code_linenumbers", obj.code_linenumbers); kv("heading_ancestors_text", obj.heading_ancestors_text); kv("sha256", obj.sha256); foreach (url; obj.stow_link) { kv("stow_link", url); } foreach (a; obj.anchor_tags) { kv("anchor_tag", a); } foreach (img; obj.images) { /+ ↓ the name last, because a filename may hold a space where none of the other fields can +/ kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " ~ (img.missing ? "1" : "0") ~ " " ~ img.name); foreach (key; img.fields.keys.dup.sort) { kv("image_field", key ~ " " ~ img.fields[key]); } } /+ ↓ text, when there is any. An object whose only text line is blank holds "" here, indistinguishable from no text at all, because the model joins the lines into one string. Both libraries lose the same distinction in the same place, which is what makes them comparable; it is recorded rather than hidden. +/ if (obj.text.length > 0) { auto lines = obj.text.splitter("\n").array; kv("text", lines.length.to!string); foreach (line; lines) { out_ ~= "| "; out_ ~= line; out_ ~= "\n"; } } } } return out_.data; } /+ ↓ what the reader met and did not understand, sorted. Separate from the document dump: it describes the reader, not the document, and the .ocda.db loader has nothing to report where the .ssp loader does. +/ string dumpUnknown(ref OcdaDocument doc) { auto out_ = appender!string; void counts(string prefix, size_t[string] d) { foreach (key; d.keys.dup.sort) { out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; out_ ~= d[key].to!string; out_ ~= "\n"; } } counts("property", doc.unknown.property); counts("block", doc.unknown.block); counts("line", doc.unknown.line); return out_.data; } private string ints(int[] xs) { if (xs.length == 0) { return ""; } string[] parts; foreach (x; xs) { parts ~= x.to!string; } return parts.join(","); } #+END_SRC * read ** main #+HEADER: :tangle "../dlang/src/read/main.d" #+BEGIN_SRC d <> /++ ocda-read - reads the object-centric document abstraction, in any of the forms it is written in, and says what is in it. ocda-read [options] .ssp | .ocda.db | .sisupod [lang] One reader where there were two: sisudoc-ssp-reader read a .ssp and sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is built on the library, so every mode reads every form, and a mode given the .ssp, the .ocda.db and the .sisupod of a document says the same thing of each: it is told what the document is, not which file it came from. Two things are about the file rather than the document, and say so: --files, what a .ocda.db carries besides the abstraction, and --validate, which checks a .ssp as text. Everything else is the model's. The output modules are the .ssp reader's, moved onto the library's model. +/ module read.main; import std.algorithm : canFind, map; import std.array : array, join; import std.getopt; import std.stdio : writeln, writefln, stderr; import ocda.model; import ocda.load; import ocda.inventory; import read.tree; import read.html; import read.llm_text; import read.llm_jsonl; import read.images; import read.verify; import read.report_unknown; import read.validate; int main(string[] args) { bool do_tree = false; bool do_html = false; bool do_llm_text = false; bool do_llm_jsonl = false; bool do_images = false; bool do_verify = false; bool do_report_unknown = false; bool do_validate = false; bool do_languages = false; bool do_files = false; bool any_version = false; string image_dir = ""; string output_file = ""; string lang = ""; GetoptResult helpInfo; try { helpInfo = getopt(args, "tree", "the headings, as a tree", &do_tree, "outline", "the same as --tree", &do_tree, "html", "the document as one html page", &do_html, "llm-text", "plain text with heading breadcrumbs", &do_llm_text, "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, "images", "the image records: digest, size, pixels", &do_images, "verify", "check the image records against --image-dir", &do_verify, "image-dir", "the images to verify against", &image_dir, "languages", "the languages the file holds", &do_languages, "files", "what a .ocda.db carries besides the abstraction", &do_files, "report-unknown", "what was read and not understood", &do_report_unknown, "validate", "check a .ssp against its specification, as text", &do_validate, "lang", "= the language to read (default: the first)", &lang, "output", "write the mode's output to a file", &output_file, "any-version", "read a format major this reader does not know", &any_version, ); } catch (GetOptException e) { stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); return 2; } if (helpInfo.helpWanted || args.length < 2) { defaultGetoptPrinter( "ocda-read - reads the document abstraction, in any of its forms\n\n" ~ "Usage: ocda-read [options] .ssp|.ocda.db|.sisupod [lang]\n", helpInfo.options, ); return 0; } string input_file = args[1]; if (lang.length == 0 && args.length > 2) { lang = args[2]; } bool lang_named = (lang.length > 0); /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not at all: there is no text form of a database to check, and a pod's .ssp is checked by taking it out +/ if (do_validate && sourceOf(input_file) != Source.ssp) { stderr.writeln("--validate checks a .ssp as text, against the .ssp", " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); return 2; } /+ ↓ which language. A .ocda.db and a pod may hold several; where none is named, every form reads the first in the manifest's order and says so, rather than the database taking one and the pod refusing +/ OcdaLanguage[] held; OcdaDocument doc; try { held = languages(input_file); if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { stderr.writeln("error: no language '", lang, "' in ", input_file, "; it holds: ", held.map!(l => l.lang).array.join(" ")); return 2; } doc = load(input_file, lang, any_version); } catch (Exception e) { stderr.writeln("Error reading ", input_file, ": ", e.msg); return 2; } /+ ↓ the version gate: the major part, and refuse what is not known +/ if (doc.format_version.length == 0) { stderr.writeln("warning: ", input_file, " declares no format version"); } else if (doc.formatVersionMajor != ssp_format_major_read) { if (any_version) { stderr.writeln("warning: ", input_file, " is format v", doc.format_version, ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); } else { stderr.writeln("error: ", input_file, " is format v", doc.format_version, ", this reader knows v", ssp_format_major_read, ".x"); stderr.writeln(" a major version differs: a property may have", " changed meaning or gone away."); stderr.writeln(" --any-version reads it regardless."); return 3; } } writeln("Read: ", input_file); if (doc.format_version.length > 0) { writeln(" Format: v", doc.format_version); } if (auto _lang = "language" in doc.source) { writeln(" Source: ", *_lang, ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); } if (held.length > 1 && !lang_named) { writeln(" (reading ", lang, "; --lang= for another)"); } writeln(" Title: ", doc.meta("title.main")); writeln(" Author: ", doc.meta("creator.author")); writeln(" Sections: head=", doc.head.length, " toc=", doc.toc.length, " body=", doc.body.length, " endnotes=", doc.endnotes.length, " glossary=", doc.glossary.length, " bibliography=", doc.bibliography.length, " bookindex=", doc.bookindex.length, " blurb=", doc.blurb.length, " tail=", doc.tail.length); auto image_records = imageRecordCount(doc); if (image_records > 0) { writeln(" Images: ", image_records, " record", (image_records == 1) ? "" : "s"); } if (do_languages) { printLanguages(held); } if (do_files) { printFiles(input_file); } if (do_tree) { outputTree(doc, output_file); } if (do_html) { outputHtmlScroll(doc, output_file); } if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } if (do_images) { outputImages(doc, output_file); } int _verify_failures = 0; if (do_verify) { if (image_dir.length == 0) { stderr.writeln("--verify needs --image-dir="); return 2; } _verify_failures = verifyImages(doc, image_dir, output_file); } if (do_report_unknown) { reportUnknown(doc); } int _validation_findings = 0; if (do_validate) { auto _found = validateSspFile(input_file, ssp_format_major_read); reportValidation(input_file, _found); _validation_findings = cast(int) _found.length; } if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images && !do_verify && !do_report_unknown && !do_validate && !do_languages && !do_files) { writeln("\n(this is the summary: --help lists the modes, each adding its output)"); } return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; } /+ ↓ the languages, each with the markup it was made from and its .ssp's digest: the same from every form, the database storing the digest and the others taking it from the bytes +/ void printLanguages(OcdaLanguage[] held) { writeln(); writeln(" lang source ssp digest"); foreach (l; held) { writefln(" %-7s %-31s %s", l.lang, l.source_filename, (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); } } /+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries nothing, and a pod's files are its archive entries rather than anything the abstraction records, so for those it says so +/ void printFiles(string path) { writeln(); if (!isCarrier(path)) { writeln(" carried files: only a .ocda.db carries files; this is ", sourceName(sourceOf(path))); return; } auto rows = carried(path); if (rows.length == 0) { writeln(" carries no files"); return; } writeln(" carried files:"); writeln(" role stored n original in the file"); foreach (r; rows) { writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); } } #+END_SRC ** check *** validate =.ssp= #+HEADER: :tangle "../dlang/src/read/validate.d" #+BEGIN_SRC d <> module read.validate; /+ ↓ check a .ssp against the format specification. . Written from specs/sisudoc-abstraction-format/ssp-format-specification.md and from nothing else: the point of it is to be what the specification says, executed, so that a file can be held against the document rather than against an implementation. Each check names the section it comes from, so a disagreement can be taken back to the text. +/ import std.algorithm : canFind, startsWith; import std.array : array, split; import std.conv : to; import std.file : readText; import std.stdio : writeln, writefln; import std.string : indexOf, strip; struct SspFinding { size_t line; // 1-based string section; // the specification section the rule is from string detail; } /+ ↓ specification section 6.3: the properties, in the order they are emitted. A reader may rely on the order, so a validator can check it +/ private immutable string[] property_order = [ "identifier", "part", "section", "parent", "last_descendant", "ocn_range", "children", "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", "table_header", "code_linenumbers", "stow_link", "segment", "anchor", "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", "segment_html_is", "segment_epub_is", "segment_lv4_is", "heading_ancestors_text", "lev4_subtoc", "anchor_tag", ]; /+ ↓ section 6.1: the sixteen kinds +/ private immutable string[] kinds = [ "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", "comment", "verse", "glossary", "quote", "poem", "block", "table", "blurb", "group", ]; /+ ↓ section 4: the nine sections, in the order they appear +/ private immutable string[] section_order = [ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", "tail", ]; /+ ↓ section 4: the four preamble blocks, in the order they appear +/ private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; /+ ↓ section 6.1: the heading levels +/ private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; private int propertyRank(string name) { foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } return -1; } SspFinding[] validateSspFile(string filename, string major_known) { SspFinding[] found; auto lines = readText(filename).split("\n"); bool seen_banner = false; bool seen_source_line = false; string block = ""; // the block currently open, "" when none int preamble_at = -1; // how far through the preamble order int section_at = -1; // how far through the section order bool in_object = false; int prev_rank = -1; size_t doc_has_keys = 0; bool in_text = false; // text lines come after properties foreach (i, line; lines) { size_t no = i + 1; /+ ↓ section 2: the version, on the first line +/ if (no == 1) { enum banner = "% SiSU Document Abstraction v"; if (!line.startsWith(banner)) { found ~= SspFinding(no, "2", "first line is not the version banner"); } else { seen_banner = true; string v = line[banner.length .. $].strip; auto dot = v.indexOf('.'); string major = (dot > 0) ? v[0 .. dot] : v; if (major != major_known) { found ~= SspFinding(no, "2.1", "format major v" ~ major ~ ", this validator knows v" ~ major_known); } } continue; } if (no == 2) { if (!line.startsWith("% Source: ")) { found ~= SspFinding(no, "4", "second line is not \"% Source: \""); } else { seen_source_line = true; } continue; } if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } /+ ↓ section 3: a comment is the file header only +/ if (line[0] == '%') { found ~= SspFinding(no, "3", "a comment outside the two header lines"); continue; } /+ ↓ block open +/ if (line[0] == '@') { auto brace = line.indexOf(" {"); if (brace < 1 || line[$ - 1] != '{') { found ~= SspFinding(no, "3", "block open is not \"@name {\""); continue; } string name = line[1 .. brace]; if (preamble_order.canFind(name)) { int at = 0; foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } if (at <= preamble_at) { found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); } preamble_at = at; if (section_at >= 0) { found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); } } else if (section_order.canFind(name)) { int at = 0; foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } if (at <= section_at) { found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); } section_at = at; } else { found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); } block = name; doc_has_keys = 0; continue; } if (line[0] == '}') { if (line.length != 1) { found ~= SspFinding(no, "3", "block close carries more than \"}\""); } /+ ↓ section 5.4: ten keys, always +/ if (block == "doc_has" && doc_has_keys != 10) { found ~= SspFinding(no, "5.4", "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); } block = ""; in_object = false; prev_rank = -1; continue; } /+ ↓ section 5: inside a preamble block, " key: value" +/ if (preamble_order.canFind(block)) { if (!line.startsWith(" ") || line.indexOf(": ") < 3) { found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); } else if (block == "doc_has") { doc_has_keys++; } continue; } /+ ↓ section 6.1: the object declaration line +/ if (line[0] == '[') { auto close = line.indexOf("] "); if (close < 2) { found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); continue; } string ocn = line[1 .. close]; foreach (c; ocn) { if (c < '0' || c > '9') { found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); break; } } auto rest = line[close + 2 .. $].split(" "); if (rest.length == 0 || !kinds.canFind(rest[0])) { found ~= SspFinding(no, "6.1", "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); } else if (rest[0] == "heading") { if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); } else if (!levels.canFind(rest[1][1 .. $])) { found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); } } in_object = true; in_text = false; prev_rank = -1; continue; } /+ ↓ section 6.3: properties, known and in order +/ if (line[0] == '.') { if (!in_object) { found ~= SspFinding(no, "6", "a property outside an object record"); continue; } if (in_text) { found ~= SspFinding(no, "6", "a property after the object text began"); } auto colon = line.indexOf(": "); if (colon < 2) { found ~= SspFinding(no, "3", "property is not \".name: value\""); continue; } string name = line[1 .. colon]; int rank = propertyRank(name); if (rank < 0) { found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); } else { /+ ↓ the four repeatable properties may recur at the same rank +/ if (rank < prev_rank) { found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); } prev_rank = rank; } continue; } /+ ↓ section 6.2: text lines +/ if (line[0] == '|') { if (!in_object) { found ~= SspFinding(no, "6.2", "a text line outside an object record"); } if (line.length < 2 || line[1] != ' ') { found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); } in_text = true; continue; } found ~= SspFinding(no, "3", "line of no shape the specification defines"); } if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } return found; } void reportValidation(string filename, SspFinding[] found) { if (found.length == 0) { writeln("\nvalid: ", filename, " conforms to the .ssp specification as this validator reads it"); return; } writefln("\n%d finding%s against the specification:", found.length, (found.length == 1) ? "" : "s"); foreach (f; found) { if (f.line > 0) { writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); } else { writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); } } } #+END_SRC *** verify images #+HEADER: :tangle "../dlang/src/read/verify.d" #+BEGIN_SRC d <> module read.verify; import std.algorithm : sort; import std.array : array, join; import std.conv : to; import std.digest : toHexString; import std.digest.sha : SHA256, digest; import std.file : dirEntries, exists, isDir, SpanMode, getSize; import std.path : baseName, buildPath, extension; import std.stdio : writeln, File; import std.string : toLower; import ocda.model; import read.images : gatherImageRecords; /+ ↓ verification: recompute what the .image records claim, against the images as they stand in a directory (a pod's media/image). ssp-reader --verify --image-dir=/media/image doc.ssp Reports one line per distinct file: OK won_benkler_2_1.png MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) PRESENT ffa.png (recorded missing, but the file is there) EXTRA unused.png (in directory, no record in this .ssp) Returns the number of failures. EXTRA is reported but is not a failure: a pod carries the images of every language version of a document, while one .ssp covers one language. +/ int verifyImages(OcdaDocument doc, string image_dir, string output_file) { string[] lines; int failures = 0; int ok = 0; int mismatch = 0; int absent = 0; int extra = 0; if (!(image_dir.exists) || !(image_dir.isDir)) { writeln("ERROR: image directory not found: ", image_dir); return 1; } auto gathered = gatherImageRecords(doc); bool[string] recorded; foreach (e; gathered.entries) { recorded[e.name] = true; string pth = buildPath(image_dir, e.name); if (!(pth.exists)) { if (e.missing) { /+ recorded missing, and missing here too: consistent +/ ok += 1; lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; } else { absent += 1; failures += 1; lines ~= "ABSENT " ~ e.name ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; } continue; } if (e.missing) { mismatch += 1; failures += 1; lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; continue; } auto file_ = readFileStat(pth); string[] differs; if (file_.sha256 != e.sha256) { differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; } if (file_.bytes != e.bytes) { differs ~= "bytes recorded " ~ e.bytes.to!string ~ " file " ~ file_.bytes.to!string; } /+ px is written only where the dimensions could be read, and read here only for the formats below, so compare only when both are known +/ if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 && (file_.width != e.width || file_.height != e.height)) { differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; } if (differs.length > 0) { mismatch += 1; failures += 1; lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); } else { ok += 1; lines ~= "OK " ~ e.name; } } /+ ↓ files in the directory that no record names +/ string[] in_dir; foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { if (!(entry.isFile)) continue; string nm = entry.name.baseName; switch (nm.extension.toLower) { case ".png": case ".jpg": case ".jpeg": case ".gif": in_dir ~= nm; break; default: break; } } in_dir.sort; foreach (nm; in_dir) { if (nm !in recorded) { extra += 1; lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; } } string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" ~ ((gathered.entries.length == 1) ? "" : "s") ~ " in " ~ gathered.records.to!string ~ " record" ~ ((gathered.records == 1) ? "" : "s") ~ ", against " ~ image_dir; lines = [head, ""] ~ lines; lines ~= ""; lines ~= "verify: " ~ ok.to!string ~ " ok, " ~ mismatch.to!string ~ " mismatch, " ~ absent.to!string ~ " absent, " ~ extra.to!string ~ " extra" ~ ((failures == 0) ? " PASS" : " FAIL"); /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } return failures; } private struct FileStat_ { string sha256; ulong bytes; int width; int height; } private FileStat_ readFileStat(string pth) { FileStat_ ret; auto f = File(pth, "rb"); ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); ret.sha256 = sha.toHexString.to!string; ret.bytes = pth.getSize; auto dim = readImageDimensions(pth); ret.width = dim[0]; ret.height = dim[1]; return ret; } /+ ↓ pixel dimensions from the file header, for the formats spine's markup admits (png, gif, jpg). Header read only, no decoding, and no dependency beyond phobos. Returns [0, 0] where the header cannot be read +/ private int[2] readImageDimensions(string pth) { int[2] ret = [0, 0]; ubyte[] head; try { auto f = File(pth, "rb"); head = new ubyte[](64 * 1024); head = f.rawRead(head); } catch (Exception ex) { return ret; } if (head.length >= 24 && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { /+ png: IHDR width & height, big endian, at 16 and 20 +/ ret[0] = beInt(head[16 .. 20]); ret[1] = beInt(head[20 .. 24]); return ret; } if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { /+ gif: logical screen width & height, little endian, at 6 and 8 +/ ret[0] = head[6] | (head[7] << 8); ret[1] = head[8] | (head[9] << 8); return ret; } if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { /+ jpeg: walk the segments to a start of frame, height then width, big endian, at offsets 5 and 7 of the segment +/ size_t i = 2; while (i + 9 < head.length) { if (head[i] != 0xff) { i += 1; continue; } ubyte marker = head[i + 1]; if (marker == 0xd8 || marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } if (marker == 0xff) { i += 1; continue; } size_t seg_len = (head[i + 2] << 8) | head[i + 3]; bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) || (marker >= 0xc5 && marker <= 0xc7) || (marker >= 0xc9 && marker <= 0xcb) || (marker >= 0xcd && marker <= 0xcf)); if (is_sof) { ret[1] = (head[i + 5] << 8) | head[i + 6]; ret[0] = (head[i + 7] << 8) | head[i + 8]; return ret; } if (seg_len < 2) break; i += 2 + seg_len; } } return ret; } private int beInt(const(ubyte)[] b) { return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); } #+END_SRC *** report unknown #+HEADER: :tangle "../dlang/src/read/report_unknown.d" #+BEGIN_SRC d <> module read.report_unknown; import std.algorithm : sort; import std.array : array; import std.stdio : writeln, writefln; import ocda.model; /+ ↓ say what was read and not understood. This is what makes the reader usable as a check on the format specification: a reader that ignores what it does not know is conforming, but one that cannot say what it ignored cannot tell you whether the specification it was written from was complete. Silence and understanding look the same from outside. +/ void reportUnknown(ref OcdaDocument doc) { if (!doc.unknown.any) { writeln("\nread and understood: every property, block and line was known"); return; } writeln("\nread and NOT understood:"); if (doc.unknown.property.length > 0) { writeln(" properties ignored (", doc.unknown.property.length, " distinct):"); foreach (name; doc.unknown.property.keys.sort) { writefln(" .%-26s %8d", name, doc.unknown.property[name]); } } if (doc.unknown.block.length > 0) { writeln(" blocks ignored (", doc.unknown.block.length, " distinct):"); foreach (name; doc.unknown.block.keys.sort) { writefln(" @%-26s %8d", name, doc.unknown.block[name]); } } if (doc.unknown.line.length > 0) { writeln(" lines of no known shape, by first character:"); foreach (c; doc.unknown.line.keys.sort) { writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]); } } writeln(" (ignoring these is conforming within a major version;", " the count is so that it is not silent)"); } #+END_SRC ** misc *** tree #+HEADER: :tangle "../dlang/src/read/tree.d" #+BEGIN_SRC d <> module read.tree; import std.conv : to; import std.stdio : writeln, writef, writefln, File; import std.array : replicate; import ocda.model; void outputTree(OcdaDocument doc, string output_file) { string[] lines; lines ~= "Document: " ~ doc.meta("title.main"); if (doc.meta("title.subtitle").length > 0) lines ~= " " ~ doc.meta("title.subtitle"); lines ~= "Author: " ~ doc.meta("creator.author"); lines ~= ""; /+ ↓ build OCN -> object lookup for body headings +/ OcdaObject[int] headings; foreach (obj; doc.body) { if (obj.is_a == "heading" && obj.ocn > 0) { headings[obj.ocn] = obj; } } /+ ↓ find root heading (from head section, or first body heading) +/ if (doc.head.length > 0 && doc.head[0].is_a == "heading") { auto root = doc.head[0]; lines ~= formatHeading(root, 0); walkChildren(root, headings, lines, 1); } /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } } private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { foreach (child_ocn; parent.children) { if (auto p = child_ocn in headings) { lines ~= formatHeading(*p, depth); walkChildren(*p, headings, lines, depth + 1); } } } private string formatHeading(OcdaObject obj, int depth) { string indent = replicate(" ", depth); string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; /+ ↓ strip internal markup from heading text for clean display +/ string text = stripInternalMarkup(obj.text); return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; } private string stripInternalMarkup(string text) { /+ ↓ simple removal of common internal markers for display +/ import std.regex : replaceAll, regex; import std.array : replace; string result = text; result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") .replace("┨", "").replace("┣", "") .replace("┙", " ").replace("┘", "\n"); return result; } #+END_SRC *** html #+HEADER: :tangle "../dlang/src/read/html.d" #+BEGIN_SRC d <> module read.html; import std.array : replace, split, join, appender; import std.conv : to; import std.stdio : writeln, File; import std.string : strip, indexOf, startsWith; import ocda.model; void outputHtmlScroll(OcdaDocument doc, string output_file) { auto buf = appender!(string[]); buf ~= htmlHead(doc); /+ ↓ title +/ if (doc.head.length > 0) { auto title_obj = doc.head[0]; buf ~= `
`; buf ~= `

` ~ htmlEscape(doc.meta("title.main")) ~ `

`; if (doc.meta("title.subtitle").length > 0) buf ~= `

` ~ htmlEscape(doc.meta("title.subtitle")) ~ `

`; buf ~= `

` ~ htmlEscape(doc.meta("creator.author")) ~ `

`; buf ~= `
`; } /+ ↓ table of contents +/ if (doc.toc.length > 0) { buf ~= `
`; buf ~= `

Table of Contents

`; foreach (obj; doc.toc) { if (obj.is_a == "toc") { string text = convertInlineMarkup(obj.text); int indent = obj.indent_base; string cls = (indent > 4) ? "toc-sub" : "toc-entry"; buf ~= `

` ~ text ~ `

`; } } buf ~= `
`; buf ~= `
`; } /+ ↓ body +/ buf ~= `
`; foreach (obj; doc.body) { buf ~= renderObject(obj); } buf ~= `
`; /+ ↓ endnotes +/ bool has_endnotes = false; foreach (obj; doc.endnotes) { if (obj.is_a == "endnote") { has_endnotes = true; break; } } if (has_endnotes) { buf ~= `
`; buf ~= `
`; buf ~= `

Endnotes

`; foreach (obj; doc.endnotes) { if (obj.is_a == "endnote") { string text = convertInlineMarkup(obj.text); string parent_ref = (obj.parent_ocn > 0) ? ` [` ~ obj.parent_ocn.to!string ~ `]` : ""; buf ~= `

` ~ text ~ parent_ref ~ `

`; } } buf ~= `
`; } /+ ↓ glossary +/ if (doc.glossary.length > 0) { buf ~= `
`; buf ~= `
`; buf ~= `

Glossary

`; foreach (obj; doc.glossary) { string text = convertInlineMarkup(obj.text); buf ~= `

` ~ text ~ `

`; } buf ~= `
`; } /+ ↓ bookindex +/ bool has_bookindex = false; foreach (obj; doc.bookindex) { if (obj.is_a == "bookindex") { has_bookindex = true; break; } } if (has_bookindex) { buf ~= `
`; buf ~= `
`; buf ~= `

Book Index

`; foreach (obj; doc.bookindex) { if (obj.is_a == "bookindex") { string text = convertInlineMarkup(obj.text); buf ~= `

` ~ text ~ `

`; } } buf ~= `
`; } buf ~= htmlTail(); /+ ↓ write output +/ string[] output = buf.data; if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; output) f.writeln(line); } else { foreach (line; output) writeln(line); } } private string[] renderObject(OcdaObject obj) { /+ ↓ markup commentary is carried by the abstraction and is not output. Rendering it would put the author notes into the document, which spine itself does not do +/ if (!obj.renders) { return []; } string[] lines; string text = convertInlineMarkup(obj.text); string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) ? ` id="o` ~ obj.ocn.to!string ~ `"` : ""; string ocn_display = (obj.ocn > 0 && !obj.ocn_off) ? `` ~ obj.ocn.to!string ~ `` : ""; switch (obj.is_a) { case "heading": string tag = headingTag(obj.heading_level); string seg_anchor = (obj.segment.length > 0) ? ` id="` ~ htmlEscape(obj.segment) ~ `"` : ocn_anchor; lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ ``; break; case "para": string indent_cls = ""; if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`; else if (obj.bullet) indent_cls = ` class="bullet"`; lines ~= `` ~ text ~ ocn_display ~ `

`; break; case "code": lines ~= `` ~ htmlEscape(obj.text) ~ ``; break; case "table": lines ~= renderTable(obj, ocn_anchor); break; case "group": lines ~= `` ~ text ~ ``; break; case "block": lines ~= `` ~ htmlEscape(obj.text) ~ ``; break; case "verse": lines ~= `` ~ text.replace("\n", "
\n") ~ ocn_display ~ `

`; break; case "quote": lines ~= `` ~ text ~ ocn_display ~ ``; break; default: if (text.length > 0) lines ~= `` ~ text ~ ocn_display ~ `

`; break; } return lines; } private string headingTag(string level) { switch (level) { case "A": return "h1"; case "B": return "h2"; case "C": return "h3"; case "D": return "h4"; case "1": return "h3"; case "2": return "h4"; case "3": return "h5"; case "4": return "h6"; default: return "h4"; } } private string renderTable(OcdaObject obj, string ocn_anchor) { string[] lines; lines ~= ``; /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/ auto rows = obj.text.split("\n"); bool first_row = obj.table_header; foreach (row; rows) { if (row.strip.length == 0) continue; string tag = first_row ? "th" : "td"; lines ~= ``; foreach (cell; row.split("┊")) { lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ ``; } lines ~= ``; first_row = false; } lines ~= ``; return lines.join("\n"); } /+ ↓ convert spine internal Unicode markup to HTML +/ string convertInlineMarkup(string text) { if (text.length == 0) return ""; string result = htmlEscape(text); /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc. Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X The ┨┣ form is used within heading content +/ /+ ↓ bold +/ result = replaceInlineMarker(result, "⑆!", "!", "", ""); result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "", ""); /+ ↓ emphasis +/ result = replaceInlineMarker(result, "⑆*", "*", "", ""); result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "", ""); /+ ↓ italic +/ result = replaceInlineMarker(result, "⑆/", "/", "", ""); result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "", ""); /+ ↓ underscore +/ result = replaceInlineMarker(result, "⑆_", "_", "", ""); result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "", ""); /+ ↓ superscript +/ result = replaceInlineMarker(result, "⑆^", "^", "", ""); result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "", ""); /+ ↓ subscript +/ result = replaceInlineMarker(result, "⑆,", ",", "", ""); result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "", ""); /+ ↓ monospace +/ result = replaceInlineMarker(result, "⑆■", "■", "", ""); /+ ↓ links: ┥link text┝┤url├ +/ result = replaceLinks(result); /+ ↓ footnote markers: 【N text】 +/ result = replaceFootnotes(result); /+ ↓ line breaks +/ result = result.replace("┘", "
"); result = result.replace("┙", "
"); /+ ↓ non-breaking spaces +/ result = result.replace("░", " "); /+ ↓ bullet markers +/ result = result.replace("● ", "• "); /+ ↓ page break markers +/ result = result.replace("┼", "").replace("┿", "").replace("╂", ""); return result; } private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) { /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/ string result = text; while (true) { auto open_pos = result.indexOf(open_mark); if (open_pos < 0) break; auto after_open = open_pos + open_mark.length; /+ ↓ look for closing ⑆X +/ string close_mark = "⑆" ~ close_suffix; auto close_pos = indexOf(result[after_open .. $], close_mark); if (close_pos < 0) break; close_pos += after_open; string content = result[after_open .. close_pos]; result = result[0 .. open_pos] ~ html_open ~ content ~ html_close ~ result[close_pos + close_mark.length .. $]; } return result; } private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) { /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/ string result = text; string full_open = open_mark ~ bracket_open; while (true) { auto open_pos = result.indexOf(full_open); if (open_pos < 0) break; auto after_open = open_pos + full_open.length; auto close_pos = indexOf(result[after_open .. $], bracket_close); if (close_pos < 0) break; close_pos += after_open; string content = result[after_open .. close_pos]; result = result[0 .. open_pos] ~ html_open ~ content ~ html_close ~ result[close_pos + bracket_close.length .. $]; } return result; } private string replaceLinks(string text) { /+ ↓ replace ┥link text┝┤url├ with link text +/ string result = text; while (true) { auto link_open = result.indexOf("┥"); if (link_open < 0) break; auto link_mid = indexOf(result[link_open .. $], "┝┤"); if (link_mid < 0) break; link_mid += link_open; auto link_close = indexOf(result[link_mid .. $], "├"); if (link_close < 0) break; link_close += link_mid; string link_text = result[link_open + "┥".length .. link_mid]; string url = result[link_mid + "┝┤".length .. link_close]; /+ ↓ handle internal links (¤ prefix) +/ if (url.length > 0 && url[0] == '¤') { url = url[("¤").length .. $]; } /+ ↓ handle OCN-only links (#N) +/ if (url.length > 0 && url[0] == '#') { url = "#o" ~ url[1 .. $]; } result = result[0 .. link_open] ~ `` ~ link_text ~ `` ~ result[link_close + "├".length .. $]; } return result; } private string replaceFootnotes(string text) { /+ ↓ replace 【N content】 with superscript note number +/ string result = text; while (true) { auto fn_open = result.indexOf("【"); if (fn_open < 0) break; auto fn_close = indexOf(result[fn_open .. $], "】"); if (fn_close < 0) break; fn_close += fn_open; string fn_content = result[fn_open + "【".length .. fn_close]; /+ ↓ extract note number (first token before space) +/ auto space_pos = fn_content.indexOf(' '); string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content; result = result[0 .. fn_open] ~ `` ~ note_num ~ `` ~ result[fn_close + "】".length .. $]; } return result; } string htmlEscape(string text) { return text.replace("&", "&").replace("<", "<").replace(">", ">") .replace("\"", """); } private string[] htmlHead(OcdaDocument doc) { string[] lines; lines ~= ``; lines ~= ``; lines ~= ``; lines ~= ``; lines ~= `` ~ htmlEscape(doc.meta("title.main")) ~ ``; lines ~= ``; lines ~= ``; lines ~= ``; return lines; } private string[] htmlTail() { return [``, ``]; } #+END_SRC *** images #+HEADER: :tangle "../dlang/src/read/images.d" #+BEGIN_SRC d <> module read.images; import std.array : join; import std.conv : to; import std.stdio : writeln, File; import ocda.model; /+ ↓ the .image records of a document, gathered per distinct file, with the objects referencing each file, in order of first appearance +/ struct ImageEntry { string name; string sha256; ulong bytes; int width; int height; bool missing; string[] refs; string[] other_sha256; // same filename, differing digest } struct ImageRecords { int records; // .image lines, an image referenced twice counts twice ImageEntry[] entries; // distinct files } ImageRecords gatherImageRecords(OcdaDocument doc) { ImageRecords ret; size_t[string] seen; foreach (section; sectionNames) { auto objs = doc.sectionByName(section); if (objs is null) continue; foreach (obj; *objs) { foreach (img; obj.images) { ret.records += 1; string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]"; if (auto at = img.name in seen) { if (ret.entries[*at].sha256 != img.sha256 && img.sha256.length > 0 && ret.entries[*at].sha256.length > 0) { ret.entries[*at].other_sha256 ~= img.sha256; } ret.entries[*at].refs ~= _ref; } else { ImageEntry e_; e_.name = img.name; e_.sha256 = img.sha256; e_.bytes = img.bytes; e_.width = img.width; e_.height = img.height; e_.missing = img.missing; e_.refs = [_ref]; seen[img.name] = ret.entries.length; ret.entries ~= e_; } } } } return ret; } /+ ↓ image manifest, line oriented and greppable: refs: body[97] body[412] MISSING refs: body[840] The point of the record in the .ssp is verification (is the image shipped in this pod the image the abstraction was built from), so this output is the shape a verifier wants: one line per distinct file. +/ void outputImages(OcdaDocument doc, string output_file) { auto gathered = gatherImageRecords(doc); string[] lines; int missing = 0; foreach (e; gathered.entries) { if (e.missing) missing += 1; } lines ~= "Images: " ~ gathered.records.to!string ~ " record" ~ ((gathered.records == 1) ? "" : "s") ~ ", " ~ gathered.entries.length.to!string ~ " distinct file" ~ ((gathered.entries.length == 1) ? "" : "s") ~ ", " ~ missing.to!string ~ " missing"; lines ~= ""; foreach (e; gathered.entries) { string px = (e.width > 0 && e.height > 0) ? e.width.to!string ~ "x" ~ e.height.to!string : "-"; lines ~= (e.missing) ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ") : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" "); /+ ↓ one filename, more than one digest: the .ssp is inconsistent, or two distinct files share a name +/ foreach (other; e.other_sha256) { lines ~= " WARNING differing digest for same filename: " ~ other; } } /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } } /+ ↓ count of .image records in a document, for the parse summary +/ int imageRecordCount(OcdaDocument doc) { return gatherImageRecords(doc).records; } package immutable string[] sectionNames = [ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", ]; #+END_SRC ** llm *** llm text #+HEADER: :tangle "../dlang/src/read/llm_text.d" #+BEGIN_SRC d <> module read.llm_text; import std.array : replace, split, join; import std.conv : to; import std.stdio : writeln, File; import std.string : strip, indexOf, stripRight; import ocda.model; /+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption format: ## Document Title > Chapter > Section > [OCN] Clean paragraph text with inline markup stripped. +/ void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { string[] lines; string doc_title = doc.meta("title.main"); /+ ↓ document header +/ lines ~= "# " ~ doc_title; if (doc.meta("title.subtitle").length > 0) lines ~= "# " ~ doc.meta("title.subtitle"); lines ~= "# " ~ doc.meta("creator.author"); lines ~= ""; /+ ↓ build heading text lookup from ancestors +/ string[int] heading_text; foreach (obj; doc.head) { if (obj.is_a == "heading" && obj.ocn > 0) heading_text[obj.ocn] = stripMarkup(obj.text); } foreach (obj; doc.body) { if (obj.is_a == "heading") heading_text[obj.ocn] = stripMarkup(obj.text); } /+ ↓ body objects +/ string current_breadcrumb = ""; foreach (obj; doc.body) { if (obj.is_a == "heading") { current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); lines ~= ""; lines ~= "## " ~ current_breadcrumb; lines ~= ""; continue; } /+ ↓ markup commentary is not content +/ if (!obj.renders) { continue; } string text = stripMarkup(obj.text); if (text.strip.length == 0) continue; string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) ? " [" ~ obj.ocn.to!string ~ "]" : ""; switch (obj.is_a) { case "para": lines ~= text ~ ocn_ref; lines ~= ""; break; case "code": lines ~= "```"; lines ~= obj.text; /+ preserve raw text for code +/ lines ~= "```"; lines ~= ""; break; case "table": lines ~= text ~ ocn_ref; lines ~= ""; break; case "verse": lines ~= text ~ ocn_ref; lines ~= ""; break; case "quote": lines ~= "> " ~ text ~ ocn_ref; lines ~= ""; break; case "group": case "block": lines ~= text ~ ocn_ref; lines ~= ""; break; default: if (text.length > 0) { lines ~= text ~ ocn_ref; lines ~= ""; } break; } } /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } } private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ string[] parts; if (ancestors_text.length > 0) { foreach (part; ancestors_text.split("|")) { string clean = stripMarkup(part).strip; if (clean.length > 0 && clean != doc_title) { /+ ↓ truncate very long heading text for breadcrumb +/ if (clean.length > 60) clean = clean[0 .. 57] ~ "..."; parts ~= clean; } } } /+ ↓ add the current heading itself +/ string heading_text = stripMarkup(obj.text).strip; if (heading_text.length > 60) heading_text = heading_text[0 .. 57] ~ "..."; string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; if (parts.length > 0) return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; else return doc_title ~ " > " ~ heading_text ~ ocn_ref; } /+ ↓ strip all internal Unicode markup to produce clean plain text +/ string stripMarkup(string text) { if (text.length == 0) return ""; string r = text; /+ ↓ links: ┥link text┝┤url├ -> link text +/ while (true) { auto lo = r.indexOf("┥"); if (lo < 0) break; auto lm = indexOf(r[lo .. $], "┝┤"); if (lm < 0) break; lm += lo; auto lc = indexOf(r[lm .. $], "├"); if (lc < 0) break; lc += lm; string link_text = r[lo + "┥".length .. lm]; r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; } /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ while (true) { auto fo = r.indexOf("【"); if (fo < 0) break; auto fc = indexOf(r[fo .. $], "】"); if (fc < 0) break; fc += fo; r = r[0 .. fo] ~ r[fc + "】".length .. $]; } /+ ↓ font face markers +/ r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") .replace("⑆■", "") .replace("┨", "").replace("┣*", "").replace("┣!", "") .replace("┣/", "").replace("┣_", "").replace("┣^", "") .replace("┣,", "").replace("┣", ""); /+ ↓ line breaks and special chars +/ r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); r = r.replace("░", " "); r = r.replace("● ", "- "); r = r.replace("┼", "").replace("┿", "").replace("╂", ""); /+ ↓ table cell separators +/ r = r.replace("┊", " | "); /+ ↓ clean up multiple spaces +/ while (r.indexOf(" ") >= 0) r = r.replace(" ", " "); return r.strip; } #+END_SRC *** llm jsonl #+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" #+BEGIN_SRC d <> module read.llm_jsonl; import std.array : split, join, replace; import std.conv : to; import std.stdio : writeln, File; import std.string : strip, indexOf, stripRight; import ocda.model; import read.llm_text : stripMarkup; /+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines one JSON object per line, each containing: doc, ocn, path (breadcrumb), type, text (clean), images (only where the object embeds one, so that a chunk whose text reads around a figure carries the figure's identity), index_terms (from bookindex entries associated with this object) +/ void outputLlmJsonl(OcdaDocument doc, string output_file) { string[] lines; string doc_id = doc.meta("title.main"); string doc_author = doc.meta("creator.author"); /+ ↓ build bookindex -> OCN mapping for index_terms bookindex objects in the .ssp have parent_ocn linking to body objects but the index terms are in the text of bookindex objects, not directly attached to body objects. We use a simpler approach: headings carry heading_ancestors_text which gives context. +/ /+ ↓ track current heading path +/ string current_path = ""; foreach (obj; doc.body) { if (obj.is_a == "heading") { current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); /+ ↓ emit heading as a chunk too +/ string heading_text = stripMarkup(obj.text); if (heading_text.strip.length > 0) { lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); } continue; } /+ ↓ markup commentary is not content, and least of all is it a chunk to embed: it would be indexed as though the author had written it into the document +/ if (!obj.renders) { continue; } string text = stripMarkup(obj.text); if (text.strip.length == 0) continue; lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); } /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } } private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { return `{"doc":` ~ jsonStr(doc_id) ~ `,"author":` ~ jsonStr(doc_author) ~ `,"ocn":` ~ ocn.to!string ~ `,"path":` ~ jsonStr(path) ~ `,"type":` ~ jsonStr(type) ~ `,"text":` ~ jsonStr(text) ~ jsonImages(images) ~ `}`; } private string jsonImages(OcdaImage[] images) { /+ ↓ omitted entirely where the object embeds no image +/ if (images.length == 0) return ""; string[] items; foreach (img; images) { string item = `{"name":` ~ jsonStr(img.name); if (img.missing) { item ~= `,"missing":true`; } else { item ~= `,"sha256":` ~ jsonStr(img.sha256) ~ `,"bytes":` ~ img.bytes.to!string; if (img.width > 0 && img.height > 0) { item ~= `,"width":` ~ img.width.to!string ~ `,"height":` ~ img.height.to!string; } } items ~= item ~ `}`; } return `,"images":[` ~ items.join(",") ~ `]`; } private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { string[] parts; if (ancestors_text.length > 0) { foreach (part; ancestors_text.split("|")) { string clean = stripMarkup(part).strip; if (clean.length > 0 && clean != doc_title) { parts ~= clean; } } } string heading_text = stripMarkup(obj.text).strip; if (parts.length > 0) return parts.join(" > ") ~ " > " ~ heading_text; else return heading_text; } private string jsonStr(string s) { /+ ↓ JSON string escaping +/ string r = s; r = r.replace(`\`, `\\`); r = r.replace(`"`, `\"`); r = r.replace("\n", `\n`); r = r.replace("\r", `\r`); r = r.replace("\t", `\t`); return `"` ~ r ~ `"`; } #+END_SRC * test ** test agree #+HEADER: :tangle "../dlang/test/agree.d" #+BEGIN_SRC d <> /++ Do the two loaders agree? The library's whole purpose is that a writer cannot tell which serialisation it was handed. That is only true if loading a document's .ssp and loading its .ocda.db give the same model, so this asks. Usage: agree .ssp .ocda.db [lang] +/ import std.conv : to; import std.stdio; import ocda.model; import ocda.load; int main(string[] args) { if (args.length < 3) { stderr.writeln("usage: agree .ssp .ocda.db [lang]"); return 2; } string lang = (args.length > 3) ? args[3] : ""; OcdaDocument a, b; try { a = load(args[1]); b = load(args[2], lang); } catch (Exception e) { stderr.writeln("error: ", e.msg); return 2; } int differences = 0; void differ(string what, string from_ssp, string from_db) { writeln(" differ: ", what); writeln(" .ssp: ", from_ssp); writeln(" .ocda.db: ", from_db); differences++; } if (a.format_version != b.format_version) { differ("format_version", a.format_version, b.format_version); } if (a.source_filename != b.source_filename) { differ("source_filename", a.source_filename, b.source_filename); } /+ ↓ the sections, object by object. Position within a section is the identity to compare on: an ocn is not unique and 0 is shared +/ static immutable sections = ["head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", "tail"]; foreach (name; sections) { auto sa = a.sectionByName(name); auto sb = b.sectionByName(name); if (sa is null || sb is null) { continue; } if ((*sa).length != (*sb).length) { differ(name ~ " object count", (*sa).length.to!string, (*sb).length.to!string); continue; } foreach (i, oa; *sa) { auto ob = (*sb)[i]; string at = name ~ "[" ~ i.to!string ~ "]"; if (oa.ocn != ob.ocn) { differ(at ~ " ocn", oa.ocn.to!string, ob.ocn.to!string); } if (oa.is_a != ob.is_a) { differ(at ~ " is_a", oa.is_a, ob.is_a); } if (oa.is_of_type != ob.is_of_type) { differ(at ~ " is_of_type", oa.is_of_type, ob.is_of_type); } if (oa.part != ob.part) { differ(at ~ " part", oa.part, ob.part); } if (oa.parent_ocn != ob.parent_ocn) { differ(at ~ " parent", oa.parent_ocn.to!string, ob.parent_ocn.to!string); } if (oa.heading_level != ob.heading_level) { differ(at ~ " level", oa.heading_level, ob.heading_level); } if (oa.sha256 != ob.sha256) { differ(at ~ " sha256", oa.sha256, ob.sha256); } if (oa.text != ob.text) { differ(at ~ " text", oa.text, ob.text); } if (oa.ancestors != ob.ancestors) { differ(at ~ " ancestors", oa.ancestors.to!string, ob.ancestors.to!string); } if (oa.images.length != ob.images.length) { differ(at ~ " image count", oa.images.length.to!string, ob.images.length.to!string); } if (differences > 12) { writeln(" ... stopping after 12"); return 1; } } } if (differences == 0) { writeln("agree: the two loaders give the same model"); return 0; } writeln(differences, " difference(s)"); return 1; } #+END_SRC ** test dump #+HEADER: :tangle "../dlang/test/dump.d" #+BEGIN_SRC d <> /++ Write a loaded abstraction out canonically. The oracle for everything that claims to implement this model: load a .ssp or a .ocda.db and print the model, deterministically, so that two implementations can be compared with diff. Usage: ocda-dump [--unknown] .ssp|.ocda.db [lang] +/ import std.stdio; import ocda.model; import ocda.load; import ocda.dump; int main(string[] args) { bool unknown = false; string[] rest; foreach (a; args[1 .. $]) { if (a == "--unknown") { unknown = true; } else { rest ~= a; } } if (rest.length < 1) { stderr.writeln("usage: ocda-dump [--unknown] .ssp|.ocda.db [lang]"); return 2; } string lang = (rest.length > 1) ? rest[1] : ""; OcdaDocument doc; try { doc = load(rest[0], lang); } catch (Exception e) { stderr.writeln("error: ", e.msg); return 2; } write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); return 0; } #+END_SRC * config ** nix flake.nix :flake: #+HEADER: :tangle ../dlang/flake.nix #+HEADER: :noweb yes #+BEGIN_SRC nix { description = "ocda-lib: the object-centric document abstraction as a D library"; inputs.nixpkgs.url = "<>"; outputs = { self, nixpkgs }: let system = "x86_64-linux"; # adjust for aarch64-linux / darwin pkgs = import nixpkgs { inherit system; }; /* A library, so the library itself is not a build product: a program that uses it compiles its modules alongside its own, as the writer's flake does, or takes it as a dub dependency. What is built here are programs over it: the reader, and the two that put the library to the question - the agreement check, which loads a document's .ssp and its .ocda.db and compares the two models, and the dump, which writes a loaded model out canonically so that an implementation in another language can be diffed against this one. . `mains` is what is compiled besides the library: the checks are one file each in test/, the reader is src/read/. */ mkTool = { tool, binary, what, mains, compiler, compilerBin }: pkgs.stdenv.mkDerivation { pname = "ocda-lib-${tool}"; version = "0.1.0"; src = self; nativeBuildInputs = [ compiler ]; buildInputs = [ pkgs.sqlite pkgs.zstd ]; buildPhase = '' runHook preBuild export HOME=$TMPDIR ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd runHook postBuild ''; installPhase = '' runHook preInstall mkdir -p $out/bin install -m755 ${tool} $out/bin/${binary} runHook postInstall ''; meta = { description = what; mainProgram = binary; }; }; ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; ocda-read = mkTool (ldc // { tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; what = "reads the abstraction in any of its forms and says what is in it"; }); ocda-agree = mkTool (ldc // { tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; what = "checks that the .ssp and .ocda.db loaders give one model"; }); ocda-dump = mkTool (ldc // { tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; what = "writes a loaded abstraction out canonically, as an oracle"; }); in { packages.${system} = { /* All three in one bin/, so that one build leaves every program in one result link. Built separately into the same link, each build replaced the one before, and a script looking for ocda-dump beside ocda-agree found nothing. */ default = pkgs.symlinkJoin { name = "ocda-lib-programs-0.1.0"; paths = [ ocda-read ocda-agree ocda-dump ]; }; inherit ocda-read ocda-agree ocda-dump; }; devShells.${system}.default = pkgs.mkShell { packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; }; }; } #+END_SRC ** README.md #+HEADER: :tangle ../dlang/README.md #+HEADER: :noweb yes #+BEGIN_SRC md # ocda-lib, in D The abstraction's model and both its loaders, as a dub library. ## Modules | | | |---|---| | `ocda.model` | the document, the object, the eight sections. Imports nothing | | `ocda.load_ssp` | text -> model | | `ocda.load_db` | sqlite -> model | | `ocda.load` | what a path is, and loading it | | `ocda.inventory` | what a file holds beyond the one document: its languages (every form alike), and what a `.ocda.db` carries | | `ocda.dump` | the model written out canonically, as an oracle | | `ocda.sqlite` | the sqlite3 C API, declared rather than depended upon | | `read.*` | `ocda-read`, the reader: in `src/read/`, and *not* part of the library (below) | `ocda.model` importing nothing is the property that matters: a writer depends on the model and on neither loader. ## Use ```json "dependencies": { "ocda-lib": { "path": "../../sisudoc-ocda-lib/dlang" } } ``` ```d import ocda.load; auto doc = load("document.ocda.db", "de"); // or "document.ssp" foreach (obj; doc.body) { ... } ``` Link `sqlite3`; the library declares the dozen functions it needs rather than taking a binding from the registry, so there is nothing to fetch. ## The reader ```sh nix build .#ocda-read && ./result/bin/ocda-read [--tree|--html|--llm-text|--llm-jsonl| --images|--verify --image-dir=|--languages|--files|--report-unknown|--validate] .ssp|.ocda.db|.sisupod [lang] ``` The library's reader, one program where `sisudoc-ssp-reader` and `sisudoc-ocda-db-reader` were two. Its output modules are the `.ssp` reader's, moved onto the library's model, so every mode reads every form and **says the same thing of a document's `.ssp`, `.ocda.db` and `.sisupod`**. `--files` answers for a `.ocda.db`, the one form that carries files; `--validate` checks a `.ssp` as text and refuses anything else. It lives in `src/read/` beside the library, and dub keeps it out of the library: the configuration `library` excludes `src/read/*`, the configuration `ocda-read` builds it. A program using the library by dub, or by naming `src/ocda/*.d` as the writer's flake does, never compiles it. ## The two checks ```sh nix build .#ocda-agree && ./result/bin/ocda-agree .ssp .ocda.db [lang] nix build .#ocda-dump && ./result/bin/ocda-dump [--unknown] [lang] ``` `agree` loads both and compares ten fields of each object, naming the field where they part company. `agree` found the blank-first-text-line defect and was not enough for anything subtler. **`ocda-dump` is the stronger check**: the whole loaded model, every field, written out in a form fully determined by the model, so that two implementations can be compared with `diff`. It is an oracle, not an output format; anything reading it to reconstruct a document has misunderstood it. It agrees byte for byte with the Gleam library's `ocda-lib dump`, which is the point of having it. It found four disagreements `agree` could not see. They are in the repository README and in the history. ## Build ```sh nix build # all three programs, in one result/bin nix build .#ocda-read # the reader nix build .#ocda-agree # the agreement check nix build .#ocda-dump # the canonical dump nix develop # ldc, dub, dtools, sqlite, zstd dub build # the library dub build --config=ocda-read # the reader ``` #+END_SRC ** dub.json #+HEADER: :tangle ../dlang/dub.json #+HEADER: :noweb yes #+BEGIN_SRC json { "authors": [ "Ralph Amissah" ], "copyright": "Copyright (C) 2026 Ralph Amissah", "name": "ocda-lib", "version": "0.1.0", "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations", "homepage": "https://sisudoc.org", "license": "AGPL-3.0+", "sourcePaths": [ "./src" ], "importPaths": [ "./src" ], "libs": [ "sqlite3", "zstd" ], "configurations": [ { "name": "library", "targetType": "library", "excludedSourceFiles": [ "src/read/*" ] }, { "name": "ocda-read", "targetType": "executable", "targetName": "ocda-read", "mainSourceFile": "src/read/main.d" } ] } #+END_SRC * org includes ** project version #+NAME: spine_version #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> #+END_SRC ** year #+NAME: year #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC ** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * nixpkgs source #+NAME: nixpkgs_unstable #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> #+END_SRC * __END__