From 4e4f2f8c2b2c043077390cb4ccf7ce97ffe4ac6b Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 12 Oct 2026 20:24:11 -0400 Subject: format 2.1: ocda-read, the readers folded in ocda-read, one reader where there were two. The .ssp reader and the .ocda.db reader each had their own parser; this one is built on the library, so every mode reads a .ssp, a .ocda.db and a pod, and says the same thing of each. Its output modules are the .ssp reader's, copied (the reader repositories are kept, frozen) and moved onto the library's model; --languages and --files come from the database reader. ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images| --verify --image-dir=|--languages|--files| --report-unknown|--validate] [lang] In D every mode of both readers; in Gleam the modes the Gleam readers had (the summary, --tree, --languages, --files, --report-unknown), whose output is the D reader's, byte for byte. In each language it lives in src/read/ and is not part of the library: dub configurations keep it out of a dependant's build in D, and Gleam compiles all of src/ but a dependant need not use it. Checked over the markup document samples: against the .ssp reader on a .ssp, byte for byte after the first line in every D mode but the summary, whose closing hint differs; against the database reader on a .ocda.db, --files and --languages (its id column left out); the same output from the .ssp, the .ocda.db and the .sisupod; and D == Gleam. The database reader's --outline is not reproduced: it ordered headings within their own section, so that other sections' headings fell among the body's. --outline here is --tree. What writing it found in the library: the database loader could not say what it had not understood. It asked for columns by name and so never saw the rest, and reported that it had understood everything. It now counts them on the conditions under which the .ssp writes the same property, and --report-unknown is the same from every form. ocda.inventory answers two questions about the file rather than the document: its languages, in the manifest's order, each with the markup it was made from and the sha256 of its .ssp, for every form alike; and what a .ocda.db carries besides the abstraction. In Gleam the sha256 comes from OTP's crypto through a four line Erlang FFI. The flake's default is all three D programs in one result: built separately into the same link, each replaced the one before. (assisted by Claude-Code) --- dlang/src/ocda/inventory.d | 117 +++++++++++++++++++++++++++++++++++++++++++++ dlang/src/ocda/load_db.d | 33 +++++++++++++ dlang/src/ocda/load_pod.d | 4 +- 3 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 dlang/src/ocda/inventory.d (limited to 'dlang/src/ocda') diff --git a/dlang/src/ocda/inventory.d b/dlang/src/ocda/inventory.d new file mode 100644 index 0000000..dba2156 --- /dev/null +++ b/dlang/src/ocda/inventory.d @@ -0,0 +1,117 @@ +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index eb3d71b..2f59437 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -242,9 +242,42 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) auto section = doc.sectionByName(r[0]); if (section !is null) { *section ~= obj; } } + countUnread(db, chosen.doc_id, doc); return doc; } +/+ ↓ what the database holds and the model does not. + . + The text loader meets every property line and counts those it does not + keep, so a reader can say what was read and not understood. This + loader asks for columns by name, and would never see the ones it did + not ask for: it would report that it had understood everything, which + is the silence the count exists to prevent. So it counts them, on the + conditions under which the .ssp writes the same property (specification + section 6.3), and the report is the same from either form: checked, + 36 of 36. ++/ +private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { + string id = doc_id.to!string; + void count(string property, string where) { + auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id + ~ " AND " ~ where).to!size_t; + if (n > 0) { doc.unknown.property[property] = n; } + } + count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); + count("dom_status", "coalesce(dom_status,'') != ''"); + count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); + count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); + count("parent_lev", "coalesce(parent_lev,0) != 0"); + count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); + count("attrib", "coalesce(attrib,'') != ''"); + if (db.has("object_subtoc")) { + auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" + ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; + if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } + } +} + /+ ↓ model section 4: the eight levels and their indices +/ private string levelName(int index) { switch (index) { diff --git a/dlang/src/ocda/load_pod.d b/dlang/src/ocda/load_pod.d index 75993be..8f59fe3 100644 --- a/dlang/src/ocda/load_pod.d +++ b/dlang/src/ocda/load_pod.d @@ -92,7 +92,7 @@ OcdaDocument loadPodFile(string path, string lang = "") { The bytes decide, not the name: a .sisupod is a zstd frame and a .zip is not, and a file misnamed either way still reads. +/ -private ubyte[] archiveBytes(string path) { +package ubyte[] archiveBytes(string path) { auto bytes = cast(ubyte[]) read(path); return zstdIsFrame(bytes) ? zstdDecompress(bytes, max_pod_size) @@ -100,7 +100,7 @@ private ubyte[] archiveBytes(string path) { } /+ ↓ ..ssp -> +/ -private string langOfSspName(string name) { +package string langOfSspName(string name) { if (!name.endsWith(".ssp")) { return ""; } string stem = name[0 .. $ - 4]; auto dot = stem.lastIndexOf('.'); -- cgit v1.2.3