diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-04 11:43:53 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-04 11:43:53 -0400 |
| commit | a489b74853a57c2f4bd12bbdf43f9f293e5f210a (patch) | |
| tree | 9cd67b3f8193e08001f7632860cadd92599a3f06 | |
| parent | track (diff) | |
track orgifyorgify
| -rw-r--r-- | dlang/src/read/main.d | 4 | ||||
| -rw-r--r-- | dlang/src/read/verify.d | 6 | ||||
| -rw-r--r-- | org/d-ocda.org | 258 | ||||
| -rw-r--r-- | org/d-read.org | 601 | ||||
| -rw-r--r-- | org/gleam-ocda.org | 340 |
5 files changed, 608 insertions, 601 deletions
diff --git a/dlang/src/read/main.d b/dlang/src/read/main.d index 6e81f29..84690c5 100644 --- a/dlang/src/read/main.d +++ b/dlang/src/read/main.d @@ -102,8 +102,8 @@ int main(string[] args) { GetoptResult helpInfo; try { helpInfo = getopt(args, - "tree", "the headings, as a tree", &do_tree, - "outline", "the same as --tree", &do_tree, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, "html", "the document as one html page", &do_html, "llm-text", "plain text with heading breadcrumbs", &do_llm_text, "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, diff --git a/dlang/src/read/verify.d b/dlang/src/read/verify.d index 0322242..21d4a87 100644 --- a/dlang/src/read/verify.d +++ b/dlang/src/read/verify.d @@ -205,7 +205,8 @@ private FileStat_ readFileStat(string pth) { /+ ↓ pixel dimensions from the file header, for the formats spine's markup admits (png, gif, jpg). Header read only, no decoding, and no dependency - beyond phobos. Returns [0, 0] where the header cannot be read +/ + beyond phobos. Returns [0, 0] where the header cannot be read ++/ private int[2] readImageDimensions(string pth) { int[2] ret = [0, 0]; ubyte[] head; @@ -231,7 +232,8 @@ private int[2] readImageDimensions(string pth) { } if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { /+ jpeg: walk the segments to a start of frame, height then width, - big endian, at offsets 5 and 7 of the segment +/ + big endian, at offsets 5 and 7 of the segment + +/ size_t i = 2; while (i + 9 < head.length) { if (head[i] != 0xff) { i += 1; continue; } diff --git a/org/d-ocda.org b/org/d-ocda.org index dd50bde..c4510af 100644 --- a/org/d-ocda.org +++ b/org/d-ocda.org @@ -173,7 +173,133 @@ enum ssp_format_major_read = "2"; #+END_SRC ** load +** what to load +*** inventory + +#+HEADER: :tangle "../dlang/src/ocda/inventory.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} +#+END_SRC + *** load +**** load #+HEADER: :tangle "../dlang/src/ocda/load.d" #+BEGIN_SRC d @@ -249,7 +375,7 @@ OcdaDocument load(string path, string lang = "", bool any_version = false) { } #+END_SRC -*** load db +**** load db #+HEADER: :tangle "../dlang/src/ocda/load_db.d" #+BEGIN_SRC d @@ -611,7 +737,7 @@ private string jsonJoined(string v) { } #+END_SRC -*** load pod +**** load pod #+HEADER: :tangle "../dlang/src/ocda/load_pod.d" #+BEGIN_SRC d @@ -733,7 +859,7 @@ private ptrdiff_t lastIndexOf(string s, char c) { } #+END_SRC -*** load ssp +**** load ssp #+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" #+BEGIN_SRC d @@ -1082,131 +1208,7 @@ private int[] parseIntArray(string s) { } #+END_SRC -** inventory - -#+HEADER: :tangle "../dlang/src/ocda/inventory.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.inventory; - -/+ ↓ what a source holds, beyond the one document the model is loaded from. - . - The model is one language of one document. Two questions a reader asks - are about the file instead: which languages it holds, and, for a - database, what else it carries. They are here rather than in a reader - because the answer depends on the input form, which is the library's - to know, and so that a reader asking them does not learn which form it - was handed. - . - languages() answers for every form, and answers the same: the - languages in the order the manifest gave them (which every .ssp - records, in @source languages), each with the markup file it was made - from and the sha256 of its .ssp. The database stores that digest; for a - .ssp and a pod it is taken from the bytes, which are the same bytes. - . - carried() is the database's: a .ocda.db holds the markup, images, - configuration and catalogues it was built from, a .ssp holds none of - them, and a pod's are its archive entries rather than anything the - abstraction says. So it answers for a database and says so otherwise. -+/ - -import std.algorithm : endsWith, countUntil; -import std.array : array, split; -import std.conv : to; -import std.digest : toHexString, LetterCase; -import std.digest.sha : sha256Of; -import std.file : read; -import std.string : strip; -import ocda.model; -import ocda.load; -import ocda.load_ssp; -import ocda.load_db; -import ocda.load_pod; -import ocda.sqlite; -import ocda.zip; - -struct OcdaLanguage { - string lang; - string source_filename; // the markup file the abstraction was made from - string ssp_digest; // sha256 of that language's .ssp, upper case hex -} - -struct OcdaCarried { - string role; // source, image, conf, manifest, tools - string compression; // how it is stored: none, zstd - string count; - string bytes; // the files' own size, whatever the storage - string stored; // what they take in the database -} - -OcdaLanguage[] languages(string path) { - final switch (sourceOf(path)) { - case Source.ocda_db: - auto db = Db.open(path); - scope(exit) { db.close(); } - OcdaLanguage[] out_; - foreach (l; dbLanguages(db)) { - out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); - } - return out_; - case Source.ssp: - auto bytes = cast(ubyte[]) read(path); - auto doc = parseSsp(cast(string) bytes.idup); - return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; - case Source.pod: - auto archive = archiveBytes(path); - OcdaLanguage[string] found; - string[] order; - foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { - auto bytes = zipRead(archive, e); - auto doc = parseSsp(cast(string) bytes.idup); - string lang = langOfSspName(e.name); - found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); - if (order.length == 0) { - order = sourceValue(doc, "languages").split; - } - } - /+ ↓ the manifest's order, and anything it did not name after it +/ - OcdaLanguage[] out_; - foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } - foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } - return out_; - case Source.unknown: - throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); - } -} - -/+ ↓ what a database carries, by role and storage; empty for any other form, - which isCarrier() says in advance +/ -bool isCarrier(string path) { - return sourceOf(path) == Source.ocda_db; -} - -OcdaCarried[] carried(string path) { - if (!isCarrier(path)) { return []; } - auto db = Db.open(path); - scope(exit) { db.close(); } - if (!db.has("files")) { return []; } - OcdaCarried[] out_; - foreach (r; db.rows( - "SELECT role, compression, count(*), sum(bytes), sum(length(data)) - FROM files GROUP BY role, compression ORDER BY role")) { - out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); - } - return out_; -} - -private string sourceValue(ref OcdaDocument doc, string key) { - if (auto p = key in doc.source) { return *p; } - return ""; -} - -private string hexOf(const(ubyte)[] bytes) { - return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; -} -#+END_SRC - -** images +*** images #+HEADER: :tangle "../dlang/src/ocda/images.d" #+BEGIN_SRC d diff --git a/org/d-read.org b/org/d-read.org index 44bb5a0..9652e21 100644 --- a/org/d-read.org +++ b/org/d-read.org @@ -16,8 +16,7 @@ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * dlang :dlang: -** main & tree -*** read main +** main #+HEADER: :tangle "../dlang/src/read/main.d" #+BEGIN_SRC d @@ -77,8 +76,8 @@ int main(string[] args) { GetoptResult helpInfo; try { helpInfo = getopt(args, - "tree", "the headings, as a tree", &do_tree, - "outline", "the same as --tree", &do_tree, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, "html", "the document as one html page", &do_html, "llm-text", "plain text with heading breadcrumbs", &do_llm_text, "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, @@ -252,303 +251,8 @@ void printFiles(string path) { } #+END_SRC -*** read tree - -#+HEADER: :tangle "../dlang/src/read/tree.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.tree; - -import std.conv : to; -import std.stdio : writeln, writef, writefln, File; -import std.array : replicate; -import ocda.model; - -void outputTree(OcdaDocument doc, string output_file) { - string[] lines; - - lines ~= "Document: " ~ doc.meta("title.main"); - if (doc.meta("title.subtitle").length > 0) - lines ~= " " ~ doc.meta("title.subtitle"); - lines ~= "Author: " ~ doc.meta("creator.author"); - lines ~= ""; - - /+ ↓ build OCN -> object lookup for body headings +/ - OcdaObject[int] headings; - foreach (obj; doc.body) { - if (obj.is_a == "heading" && obj.ocn > 0) { - headings[obj.ocn] = obj; - } - } - - /+ ↓ find root heading (from head section, or first body heading) +/ - if (doc.head.length > 0 && doc.head[0].is_a == "heading") { - auto root = doc.head[0]; - lines ~= formatHeading(root, 0); - walkChildren(root, headings, lines, 1); - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { - foreach (child_ocn; parent.children) { - if (auto p = child_ocn in headings) { - lines ~= formatHeading(*p, depth); - walkChildren(*p, headings, lines, depth + 1); - } - } -} - -private string formatHeading(OcdaObject obj, int depth) { - string indent = replicate(" ", depth); - string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; - string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; - /+ ↓ strip internal markup from heading text for clean display +/ - string text = stripInternalMarkup(obj.text); - return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; -} - -private string stripInternalMarkup(string text) { - /+ ↓ simple removal of common internal markers for display +/ - import std.regex : replaceAll, regex; - import std.array : replace; - string result = text; - result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") - .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") - .replace("┨", "").replace("┣", "") - .replace("┙", " ").replace("┘", "\n"); - return result; -} -#+END_SRC - ** check -*** read verify - -#+HEADER: :tangle "../dlang/src/read/verify.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module read.verify; - -import std.algorithm : sort; -import std.array : array, join; -import std.conv : to; -import std.digest : toHexString; -import std.digest.sha : SHA256, digest; -import std.file : dirEntries, exists, isDir, SpanMode, getSize; -import std.path : baseName, buildPath, extension; -import std.stdio : writeln, File; -import std.string : toLower; -import ocda.model; -import read.images : gatherImageRecords; - -/+ ↓ verification: recompute what the .image records claim, against the - images as they stand in a directory (a pod's media/image). - - ssp-reader --verify --image-dir=<pod>/media/image doc.ssp - - Reports one line per distinct file: - - OK won_benkler_2_1.png - MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... - ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) - PRESENT ffa.png (recorded missing, but the file is there) - EXTRA unused.png (in directory, no record in this .ssp) - - Returns the number of failures. EXTRA is reported but is not a failure: - a pod carries the images of every language version of a document, while - one .ssp covers one language. -+/ -int verifyImages(OcdaDocument doc, string image_dir, string output_file) { - string[] lines; - int failures = 0; - int ok = 0; - int mismatch = 0; - int absent = 0; - int extra = 0; - - if (!(image_dir.exists) || !(image_dir.isDir)) { - writeln("ERROR: image directory not found: ", image_dir); - return 1; - } - auto gathered = gatherImageRecords(doc); - bool[string] recorded; - - foreach (e; gathered.entries) { - recorded[e.name] = true; - string pth = buildPath(image_dir, e.name); - if (!(pth.exists)) { - if (e.missing) { - /+ recorded missing, and missing here too: consistent +/ - ok += 1; - lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; - } else { - absent += 1; - failures += 1; - lines ~= "ABSENT " ~ e.name - ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; - } - continue; - } - if (e.missing) { - mismatch += 1; - failures += 1; - lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; - continue; - } - auto file_ = readFileStat(pth); - string[] differs; - if (file_.sha256 != e.sha256) { - differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; - } - if (file_.bytes != e.bytes) { - differs ~= "bytes recorded " ~ e.bytes.to!string - ~ " file " ~ file_.bytes.to!string; - } - /+ px is written only where the dimensions could be read, and read here - only for the formats below, so compare only when both are known +/ - if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 - && (file_.width != e.width || file_.height != e.height)) { - differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string - ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; - } - if (differs.length > 0) { - mismatch += 1; - failures += 1; - lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); - } else { - ok += 1; - lines ~= "OK " ~ e.name; - } - } - - /+ ↓ files in the directory that no record names +/ - string[] in_dir; - foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { - if (!(entry.isFile)) continue; - string nm = entry.name.baseName; - switch (nm.extension.toLower) { - case ".png": case ".jpg": case ".jpeg": case ".gif": - in_dir ~= nm; - break; - default: break; - } - } - in_dir.sort; - foreach (nm; in_dir) { - if (nm !in recorded) { - extra += 1; - lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; - } - } - - string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" - ~ ((gathered.entries.length == 1) ? "" : "s") - ~ " in " ~ gathered.records.to!string ~ " record" - ~ ((gathered.records == 1) ? "" : "s") - ~ ", against " ~ image_dir; - lines = [head, ""] ~ lines; - lines ~= ""; - lines ~= "verify: " ~ ok.to!string ~ " ok, " - ~ mismatch.to!string ~ " mismatch, " - ~ absent.to!string ~ " absent, " - ~ extra.to!string ~ " extra" - ~ ((failures == 0) ? " PASS" : " FAIL"); - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } - return failures; -} - -private struct FileStat_ { - string sha256; - ulong bytes; - int width; - int height; -} - -private FileStat_ readFileStat(string pth) { - FileStat_ ret; - auto f = File(pth, "rb"); - ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); - ret.sha256 = sha.toHexString.to!string; - ret.bytes = pth.getSize; - auto dim = readImageDimensions(pth); - ret.width = dim[0]; - ret.height = dim[1]; - return ret; -} - -/+ ↓ pixel dimensions from the file header, for the formats spine's markup - admits (png, gif, jpg). Header read only, no decoding, and no dependency - beyond phobos. Returns [0, 0] where the header cannot be read +/ -private int[2] readImageDimensions(string pth) { - int[2] ret = [0, 0]; - ubyte[] head; - try { - auto f = File(pth, "rb"); - head = new ubyte[](64 * 1024); - head = f.rawRead(head); - } catch (Exception ex) { - return ret; - } - if (head.length >= 24 - && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { - /+ png: IHDR width & height, big endian, at 16 and 20 +/ - ret[0] = beInt(head[16 .. 20]); - ret[1] = beInt(head[20 .. 24]); - return ret; - } - if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { - /+ gif: logical screen width & height, little endian, at 6 and 8 +/ - ret[0] = head[6] | (head[7] << 8); - ret[1] = head[8] | (head[9] << 8); - return ret; - } - if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { - /+ jpeg: walk the segments to a start of frame, height then width, - big endian, at offsets 5 and 7 of the segment +/ - size_t i = 2; - while (i + 9 < head.length) { - if (head[i] != 0xff) { i += 1; continue; } - ubyte marker = head[i + 1]; - if (marker == 0xd8 || marker == 0x01 - || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } - if (marker == 0xff) { i += 1; continue; } - size_t seg_len = (head[i + 2] << 8) | head[i + 3]; - bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) - || (marker >= 0xc5 && marker <= 0xc7) - || (marker >= 0xc9 && marker <= 0xcb) - || (marker >= 0xcd && marker <= 0xcf)); - if (is_sof) { - ret[1] = (head[i + 5] << 8) | head[i + 6]; - ret[0] = (head[i + 7] << 8) | head[i + 8]; - return ret; - } - if (seg_len < 2) break; - i += 2 + seg_len; - } - } - return ret; -} - -private int beInt(const(ubyte)[] b) { - return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); -} -#+END_SRC - -*** read validate +*** read validate =.ssp= #+HEADER: :tangle "../dlang/src/read/validate.d" #+BEGIN_SRC d @@ -820,6 +524,227 @@ void reportValidation(string filename, SspFinding[] found) { } #+END_SRC +*** read verify images + +#+HEADER: :tangle "../dlang/src/read/verify.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.verify; + +import std.algorithm : sort; +import std.array : array, join; +import std.conv : to; +import std.digest : toHexString; +import std.digest.sha : SHA256, digest; +import std.file : dirEntries, exists, isDir, SpanMode, getSize; +import std.path : baseName, buildPath, extension; +import std.stdio : writeln, File; +import std.string : toLower; +import ocda.model; +import read.images : gatherImageRecords; + +/+ ↓ verification: recompute what the .image records claim, against the + images as they stand in a directory (a pod's media/image). + + ssp-reader --verify --image-dir=<pod>/media/image doc.ssp + + Reports one line per distinct file: + + OK won_benkler_2_1.png + MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... + ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) + PRESENT ffa.png (recorded missing, but the file is there) + EXTRA unused.png (in directory, no record in this .ssp) + + Returns the number of failures. EXTRA is reported but is not a failure: + a pod carries the images of every language version of a document, while + one .ssp covers one language. ++/ +int verifyImages(OcdaDocument doc, string image_dir, string output_file) { + string[] lines; + int failures = 0; + int ok = 0; + int mismatch = 0; + int absent = 0; + int extra = 0; + + if (!(image_dir.exists) || !(image_dir.isDir)) { + writeln("ERROR: image directory not found: ", image_dir); + return 1; + } + auto gathered = gatherImageRecords(doc); + bool[string] recorded; + + foreach (e; gathered.entries) { + recorded[e.name] = true; + string pth = buildPath(image_dir, e.name); + if (!(pth.exists)) { + if (e.missing) { + /+ recorded missing, and missing here too: consistent +/ + ok += 1; + lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; + } else { + absent += 1; + failures += 1; + lines ~= "ABSENT " ~ e.name + ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; + } + continue; + } + if (e.missing) { + mismatch += 1; + failures += 1; + lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; + continue; + } + auto file_ = readFileStat(pth); + string[] differs; + if (file_.sha256 != e.sha256) { + differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; + } + if (file_.bytes != e.bytes) { + differs ~= "bytes recorded " ~ e.bytes.to!string + ~ " file " ~ file_.bytes.to!string; + } + /+ px is written only where the dimensions could be read, and read here + only for the formats below, so compare only when both are known +/ + if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 + && (file_.width != e.width || file_.height != e.height)) { + differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string + ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; + } + if (differs.length > 0) { + mismatch += 1; + failures += 1; + lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); + } else { + ok += 1; + lines ~= "OK " ~ e.name; + } + } + + /+ ↓ files in the directory that no record names +/ + string[] in_dir; + foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { + if (!(entry.isFile)) continue; + string nm = entry.name.baseName; + switch (nm.extension.toLower) { + case ".png": case ".jpg": case ".jpeg": case ".gif": + in_dir ~= nm; + break; + default: break; + } + } + in_dir.sort; + foreach (nm; in_dir) { + if (nm !in recorded) { + extra += 1; + lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; + } + } + + string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ " in " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", against " ~ image_dir; + lines = [head, ""] ~ lines; + lines ~= ""; + lines ~= "verify: " ~ ok.to!string ~ " ok, " + ~ mismatch.to!string ~ " mismatch, " + ~ absent.to!string ~ " absent, " + ~ extra.to!string ~ " extra" + ~ ((failures == 0) ? " PASS" : " FAIL"); + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } + return failures; +} + +private struct FileStat_ { + string sha256; + ulong bytes; + int width; + int height; +} + +private FileStat_ readFileStat(string pth) { + FileStat_ ret; + auto f = File(pth, "rb"); + ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); + ret.sha256 = sha.toHexString.to!string; + ret.bytes = pth.getSize; + auto dim = readImageDimensions(pth); + ret.width = dim[0]; + ret.height = dim[1]; + return ret; +} + +/+ ↓ pixel dimensions from the file header, for the formats spine's markup + admits (png, gif, jpg). Header read only, no decoding, and no dependency + beyond phobos. Returns [0, 0] where the header cannot be read ++/ +private int[2] readImageDimensions(string pth) { + int[2] ret = [0, 0]; + ubyte[] head; + try { + auto f = File(pth, "rb"); + head = new ubyte[](64 * 1024); + head = f.rawRead(head); + } catch (Exception ex) { + return ret; + } + if (head.length >= 24 + && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { + /+ png: IHDR width & height, big endian, at 16 and 20 +/ + ret[0] = beInt(head[16 .. 20]); + ret[1] = beInt(head[20 .. 24]); + return ret; + } + if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { + /+ gif: logical screen width & height, little endian, at 6 and 8 +/ + ret[0] = head[6] | (head[7] << 8); + ret[1] = head[8] | (head[9] << 8); + return ret; + } + if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { + /+ jpeg: walk the segments to a start of frame, height then width, + big endian, at offsets 5 and 7 of the segment + +/ + size_t i = 2; + while (i + 9 < head.length) { + if (head[i] != 0xff) { i += 1; continue; } + ubyte marker = head[i + 1]; + if (marker == 0xd8 || marker == 0x01 + || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } + if (marker == 0xff) { i += 1; continue; } + size_t seg_len = (head[i + 2] << 8) | head[i + 3]; + bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf)); + if (is_sof) { + ret[1] = (head[i + 5] << 8) | head[i + 6]; + ret[0] = (head[i + 7] << 8) | head[i + 8]; + return ret; + } + if (seg_len < 2) break; + i += 2 + seg_len; + } + } + return ret; +} + +private int beInt(const(ubyte)[] b) { + return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); +} +#+END_SRC + *** read report unknown #+HEADER: :tangle "../dlang/src/read/report_unknown.d" @@ -869,6 +794,82 @@ void reportUnknown(ref OcdaDocument doc) { #+END_SRC ** misc +*** read tree + +#+HEADER: :tangle "../dlang/src/read/tree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.tree; + +import std.conv : to; +import std.stdio : writeln, writef, writefln, File; +import std.array : replicate; +import ocda.model; + +void outputTree(OcdaDocument doc, string output_file) { + string[] lines; + + lines ~= "Document: " ~ doc.meta("title.main"); + if (doc.meta("title.subtitle").length > 0) + lines ~= " " ~ doc.meta("title.subtitle"); + lines ~= "Author: " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build OCN -> object lookup for body headings +/ + OcdaObject[int] headings; + foreach (obj; doc.body) { + if (obj.is_a == "heading" && obj.ocn > 0) { + headings[obj.ocn] = obj; + } + } + + /+ ↓ find root heading (from head section, or first body heading) +/ + if (doc.head.length > 0 && doc.head[0].is_a == "heading") { + auto root = doc.head[0]; + lines ~= formatHeading(root, 0); + walkChildren(root, headings, lines, 1); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { + foreach (child_ocn; parent.children) { + if (auto p = child_ocn in headings) { + lines ~= formatHeading(*p, depth); + walkChildren(*p, headings, lines, depth + 1); + } + } +} + +private string formatHeading(OcdaObject obj, int depth) { + string indent = replicate(" ", depth); + string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; + string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + /+ ↓ strip internal markup from heading text for clean display +/ + string text = stripInternalMarkup(obj.text); + return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; +} + +private string stripInternalMarkup(string text) { + /+ ↓ simple removal of common internal markers for display +/ + import std.regex : replaceAll, regex; + import std.array : replace; + string result = text; + result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("┨", "").replace("┣", "") + .replace("┙", " ").replace("┘", "\n"); + return result; +} +#+END_SRC + *** read html #+HEADER: :tangle "../dlang/src/read/html.d" diff --git a/org/gleam-ocda.org b/org/gleam-ocda.org index 2cb2e20..c10554e 100644 --- a/org/gleam-ocda.org +++ b/org/gleam-ocda.org @@ -177,6 +177,7 @@ fn fail(message: String) -> Nil { fn halt(code: Int) -> Nil #+END_SRC +* gleam :gleam: ** model *** tangle @@ -476,8 +477,9 @@ pub fn is_preamble_block(name: String) -> Bool { } #+END_SRC -** inventory -*** tangle +** what to load +*** inventory +**** tangle #+HEADER: :tangle ../gleam/src/ocda/inventory.gleam #+HEADER: :noweb yes @@ -486,7 +488,7 @@ pub fn is_preamble_block(name: String) -> Bool { <<inventory>> #+END_SRC -*** source +**** source #+NAME: inventory #+HEADER: :no-expand @@ -631,8 +633,9 @@ fn source_value(source: dict.Dict(String, String), key: String) -> String { fn sha256_hex(bytes: BitArray) -> String #+END_SRC -** load -*** tangle +*** load +**** load +***** tangle #+HEADER: :tangle ../gleam/src/ocda/load.gleam #+HEADER: :noweb yes @@ -640,8 +643,7 @@ fn sha256_hex(bytes: BitArray) -> String ////<<doc_header_including_copyright_and_license>> <<load>> #+END_SRC - -*** source +***** source #+NAME: load #+HEADER: :no-expand @@ -727,8 +729,8 @@ pub fn load( } #+END_SRC -** load pod -*** tangle +**** load pod +***** tangle #+HEADER: :tangle ../gleam/src/ocda/load_pod.gleam #+HEADER: :noweb yes @@ -737,7 +739,7 @@ pub fn load( <<load_pod>> #+END_SRC -*** source +***** source #+NAME: load_pod #+HEADER: :no-expand @@ -882,8 +884,8 @@ pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { } #+END_SRC -** load ssp -*** tangle +**** load ssp +***** tangle #+HEADER: :tangle ../gleam/src/ocda/load_ssp.gleam #+HEADER: :noweb yes @@ -892,7 +894,7 @@ pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { <<load_ssp>> #+END_SRC -*** source +***** source #+NAME: load_ssp #+HEADER: :no-expand @@ -1385,8 +1387,8 @@ fn numbers(value: String) -> List(Int) { } #+END_SRC -** load db -*** tangle +**** load db +***** tangle #+HEADER: :tangle ../gleam/src/ocda/load_db.gleam #+HEADER: :noweb yes @@ -1395,7 +1397,7 @@ fn numbers(value: String) -> List(Int) { <<load_db>> #+END_SRC -*** source +***** source #+NAME: load_db #+HEADER: :no-expand @@ -2071,7 +2073,159 @@ pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { } #+END_SRC -** zip +*** images +**** tangle + +#+HEADER: :tangle ../gleam/src/ocda/images.gleam +#+HEADER: :noweb yes +#+BEGIN_SRC org +////<<doc_header_including_copyright_and_license>> +<<images>> +#+END_SRC + +**** source + +#+NAME: images +#+HEADER: :no-expand +#+BEGIN_SRC gleam +//// An image's bytes, from whichever source the document came from. +//// +//// The model *names* images, with digest, byte size and pixel size; it does +//// not hold them. That is right - an abstraction of a document is not a +//// bundle of its assets - but a writer of any format that shows images needs +//// the bytes, so something has to know where they are. This is that +//// something, and it is in the library rather than in a writer because the +//// answer depends on the *input form* and not on the output. +//// +//// | `.ocda.db` | the `files` table, role 'image', by name | +//// | `.zip` | the archive entry whose path ends in `/image/<name>` | +//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot | +//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` | + +import gleam/bit_array +import gleam/dynamic/decode +import gleam/list +import gleam/result +import gleam/string +import ocda/load +import ocda/zip +import simplifile +import sqlight + +/// The bytes, or an error saying why not. +pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) { + case load.source_of(source) { + load.OcdaDb -> from_db(source, name) + load.Pod -> from_archive(source, name) + load.Ssp -> from_beside(source, name) + load.Unknown -> Error(source <> ": not a source this library reads") + } +} + +/// Can this source give images at all? +/// +/// A writer asks before it decides whether to emit them, because an `<img>` +/// pointing at a file it did not include is worse than no image. +pub fn can_give_images(source: String) -> Bool { + case load.source_of(source) { + load.OcdaDb -> True + load.Pod -> zip_readable(source) + load.Ssp -> image_dir_beside(source) != "" + load.Unknown -> False + } +} + +fn zip_readable(path: String) -> Bool { + case simplifile.read_bits(path) { + Error(_) -> False + Ok(bytes) -> + case zip.entries(bytes) { + Ok(_) -> True + Error(_) -> False + } + } +} + +fn from_db(path: String, name: String) -> Result(BitArray, String) { + case sqlight.open("file:" <> path <> "?mode=ro") { + Error(e) -> Error(path <> ": " <> e.message) + Ok(conn) -> { + let out = + sqlight.query( + "select data from files where role = 'image' and name = ?1", + on: conn, + with: [sqlight.text(name)], + expecting: blob(), + ) + |> result.replace_error(path <> ": could not read " <> name) + let _ = sqlight.close(conn) + case out { + Ok([data, ..]) -> Ok(data) + Ok([]) -> Error(name <> ": not in this database") + Error(e) -> Error(e) + } + } + } +} + +/// A blob column, read as bytes rather than as text: an image read as text +/// stops at the first zero byte, and a png's ninth byte is zero. +fn blob() -> decode.Decoder(BitArray) { + use v <- decode.field(0, decode.bit_array) + decode.success(v) +} + +fn from_archive(path: String, name: String) -> Result(BitArray, String) { + use bytes <- result.try( + simplifile.read_bits(path) + |> result.replace_error(path <> ": cannot read"), + ) + use entries <- result.try(zip.entries(bytes)) + case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) { + Error(_) -> Error(name <> ": not in this archive") + Ok(entry) -> zip.read(bytes, entry) + } +} + +fn from_beside(path: String, name: String) -> Result(BitArray, String) { + case image_dir_beside(path) { + "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it") + dir -> + simplifile.read_bits(dir <> "/" <> name) + |> result.replace_error(dir <> "/" <> name <> ": cannot read") + } +} + +/// `media/abstraction/<x>.ssp` -> `media/image/` +fn image_dir_beside(path: String) -> String { + let parts = string.split(path, on: "/") + let dir = case list.reverse(parts) { + [_file, ..rest] -> string.join(list.reverse(rest), "/") + [] -> "" + } + case string.ends_with(dir, "abstraction") { + False -> "" + True -> { + let parent = case list.reverse(string.split(dir, on: "/")) { + [_, ..rest] -> string.join(list.reverse(rest), "/") + [] -> "" + } + let image_dir = parent <> "/image" + case simplifile.is_directory(image_dir) { + Ok(True) -> image_dir + _ -> "" + } + } + } +} + +/// Unused here, kept so that a caller can size a read before making it. +pub fn byte_size(b: BitArray) -> Int { + bit_array.byte_size(b) +} +#+END_SRC + +** compression - zip *** tangle #+HEADER: :tangle ../gleam/src/ocda/zip.gleam @@ -2258,158 +2412,6 @@ fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { } #+END_SRC -** imapges -*** tangle - -#+HEADER: :tangle ../gleam/src/ocda/images.gleam -#+HEADER: :noweb yes -#+BEGIN_SRC org -////<<doc_header_including_copyright_and_license>> -<<images>> -#+END_SRC - -*** source - -#+NAME: images -#+HEADER: :no-expand -#+BEGIN_SRC gleam -//// An image's bytes, from whichever source the document came from. -//// -//// The model *names* images, with digest, byte size and pixel size; it does -//// not hold them. That is right - an abstraction of a document is not a -//// bundle of its assets - but a writer of any format that shows images needs -//// the bytes, so something has to know where they are. This is that -//// something, and it is in the library rather than in a writer because the -//// answer depends on the *input form* and not on the output. -//// -//// | `.ocda.db` | the `files` table, role 'image', by name | -//// | `.zip` | the archive entry whose path ends in `/image/<name>` | -//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot | -//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` | - -import gleam/bit_array -import gleam/dynamic/decode -import gleam/list -import gleam/result -import gleam/string -import ocda/load -import ocda/zip -import simplifile -import sqlight - -/// The bytes, or an error saying why not. -pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) { - case load.source_of(source) { - load.OcdaDb -> from_db(source, name) - load.Pod -> from_archive(source, name) - load.Ssp -> from_beside(source, name) - load.Unknown -> Error(source <> ": not a source this library reads") - } -} - -/// Can this source give images at all? -/// -/// A writer asks before it decides whether to emit them, because an `<img>` -/// pointing at a file it did not include is worse than no image. -pub fn can_give_images(source: String) -> Bool { - case load.source_of(source) { - load.OcdaDb -> True - load.Pod -> zip_readable(source) - load.Ssp -> image_dir_beside(source) != "" - load.Unknown -> False - } -} - -fn zip_readable(path: String) -> Bool { - case simplifile.read_bits(path) { - Error(_) -> False - Ok(bytes) -> - case zip.entries(bytes) { - Ok(_) -> True - Error(_) -> False - } - } -} - -fn from_db(path: String, name: String) -> Result(BitArray, String) { - case sqlight.open("file:" <> path <> "?mode=ro") { - Error(e) -> Error(path <> ": " <> e.message) - Ok(conn) -> { - let out = - sqlight.query( - "select data from files where role = 'image' and name = ?1", - on: conn, - with: [sqlight.text(name)], - expecting: blob(), - ) - |> result.replace_error(path <> ": could not read " <> name) - let _ = sqlight.close(conn) - case out { - Ok([data, ..]) -> Ok(data) - Ok([]) -> Error(name <> ": not in this database") - Error(e) -> Error(e) - } - } - } -} - -/// A blob column, read as bytes rather than as text: an image read as text -/// stops at the first zero byte, and a png's ninth byte is zero. -fn blob() -> decode.Decoder(BitArray) { - use v <- decode.field(0, decode.bit_array) - decode.success(v) -} - -fn from_archive(path: String, name: String) -> Result(BitArray, String) { - use bytes <- result.try( - simplifile.read_bits(path) - |> result.replace_error(path <> ": cannot read"), - ) - use entries <- result.try(zip.entries(bytes)) - case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) { - Error(_) -> Error(name <> ": not in this archive") - Ok(entry) -> zip.read(bytes, entry) - } -} - -fn from_beside(path: String, name: String) -> Result(BitArray, String) { - case image_dir_beside(path) { - "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it") - dir -> - simplifile.read_bits(dir <> "/" <> name) - |> result.replace_error(dir <> "/" <> name <> ": cannot read") - } -} - -/// `media/abstraction/<x>.ssp` -> `media/image/` -fn image_dir_beside(path: String) -> String { - let parts = string.split(path, on: "/") - let dir = case list.reverse(parts) { - [_file, ..rest] -> string.join(list.reverse(rest), "/") - [] -> "" - } - case string.ends_with(dir, "abstraction") { - False -> "" - True -> { - let parent = case list.reverse(string.split(dir, on: "/")) { - [_, ..rest] -> string.join(list.reverse(rest), "/") - [] -> "" - } - let image_dir = parent <> "/image" - case simplifile.is_directory(image_dir) { - Ok(True) -> image_dir - _ -> "" - } - } - } -} - -/// Unused here, kept so that a caller can size a read before making it. -pub fn byte_size(b: BitArray) -> Int { - bit_array.byte_size(b) -} -#+END_SRC - ** dump *** tangle |
