summaryrefslogtreecommitdiffhomepage
path: root/org/d-read.org
diff options
context:
space:
mode:
Diffstat (limited to 'org/d-read.org')
-rw-r--r--org/d-read.org601
1 files changed, 301 insertions, 300 deletions
diff --git a/org/d-read.org b/org/d-read.org
index 44bb5a0..9652e21 100644
--- a/org/d-read.org
+++ b/org/d-read.org
@@ -16,8 +16,7 @@
after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
* dlang :dlang:
-** main & tree
-*** read main
+** main
#+HEADER: :tangle "../dlang/src/read/main.d"
#+BEGIN_SRC d
@@ -77,8 +76,8 @@ int main(string[] args) {
GetoptResult helpInfo;
try {
helpInfo = getopt(args,
- "tree", "the headings, as a tree", &do_tree,
- "outline", "the same as --tree", &do_tree,
+ "tree", "the headings, as a tree", &do_tree,
+ "outline", "the same as --tree", &do_tree,
"html", "the document as one html page", &do_html,
"llm-text", "plain text with heading breadcrumbs", &do_llm_text,
"llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl,
@@ -252,303 +251,8 @@ void printFiles(string path) {
}
#+END_SRC
-*** read tree
-
-#+HEADER: :tangle "../dlang/src/read/tree.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.tree;
-
-import std.conv : to;
-import std.stdio : writeln, writef, writefln, File;
-import std.array : replicate;
-import ocda.model;
-
-void outputTree(OcdaDocument doc, string output_file) {
- string[] lines;
-
- lines ~= "Document: " ~ doc.meta("title.main");
- if (doc.meta("title.subtitle").length > 0)
- lines ~= " " ~ doc.meta("title.subtitle");
- lines ~= "Author: " ~ doc.meta("creator.author");
- lines ~= "";
-
- /+ ↓ build OCN -> object lookup for body headings +/
- OcdaObject[int] headings;
- foreach (obj; doc.body) {
- if (obj.is_a == "heading" && obj.ocn > 0) {
- headings[obj.ocn] = obj;
- }
- }
-
- /+ ↓ find root heading (from head section, or first body heading) +/
- if (doc.head.length > 0 && doc.head[0].is_a == "heading") {
- auto root = doc.head[0];
- lines ~= formatHeading(root, 0);
- walkChildren(root, headings, lines, 1);
- }
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
-}
-
-private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) {
- foreach (child_ocn; parent.children) {
- if (auto p = child_ocn in headings) {
- lines ~= formatHeading(*p, depth);
- walkChildren(*p, headings, lines, depth + 1);
- }
- }
-}
-
-private string formatHeading(OcdaObject obj, int depth) {
- string indent = replicate(" ", depth);
- string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?";
- string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
- /+ ↓ strip internal markup from heading text for clean display +/
- string text = stripInternalMarkup(obj.text);
- return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text;
-}
-
-private string stripInternalMarkup(string text) {
- /+ ↓ simple removal of common internal markers for display +/
- import std.regex : replaceAll, regex;
- import std.array : replace;
- string result = text;
- result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
- .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
- .replace("┨", "").replace("┣", "")
- .replace("┙", " ").replace("┘", "\n");
- return result;
-}
-#+END_SRC
-
** check
-*** read verify
-
-#+HEADER: :tangle "../dlang/src/read/verify.d"
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module read.verify;
-
-import std.algorithm : sort;
-import std.array : array, join;
-import std.conv : to;
-import std.digest : toHexString;
-import std.digest.sha : SHA256, digest;
-import std.file : dirEntries, exists, isDir, SpanMode, getSize;
-import std.path : baseName, buildPath, extension;
-import std.stdio : writeln, File;
-import std.string : toLower;
-import ocda.model;
-import read.images : gatherImageRecords;
-
-/+ ↓ verification: recompute what the .image records claim, against the
- images as they stand in a directory (a pod's media/image).
-
- ssp-reader --verify --image-dir=<pod>/media/image doc.ssp
-
- Reports one line per distinct file:
-
- OK won_benkler_2_1.png
- MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D...
- ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory)
- PRESENT ffa.png (recorded missing, but the file is there)
- EXTRA unused.png (in directory, no record in this .ssp)
-
- Returns the number of failures. EXTRA is reported but is not a failure:
- a pod carries the images of every language version of a document, while
- one .ssp covers one language.
-+/
-int verifyImages(OcdaDocument doc, string image_dir, string output_file) {
- string[] lines;
- int failures = 0;
- int ok = 0;
- int mismatch = 0;
- int absent = 0;
- int extra = 0;
-
- if (!(image_dir.exists) || !(image_dir.isDir)) {
- writeln("ERROR: image directory not found: ", image_dir);
- return 1;
- }
- auto gathered = gatherImageRecords(doc);
- bool[string] recorded;
-
- foreach (e; gathered.entries) {
- recorded[e.name] = true;
- string pth = buildPath(image_dir, e.name);
- if (!(pth.exists)) {
- if (e.missing) {
- /+ recorded missing, and missing here too: consistent +/
- ok += 1;
- lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)";
- } else {
- absent += 1;
- failures += 1;
- lines ~= "ABSENT " ~ e.name
- ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)";
- }
- continue;
- }
- if (e.missing) {
- mismatch += 1;
- failures += 1;
- lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)";
- continue;
- }
- auto file_ = readFileStat(pth);
- string[] differs;
- if (file_.sha256 != e.sha256) {
- differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256;
- }
- if (file_.bytes != e.bytes) {
- differs ~= "bytes recorded " ~ e.bytes.to!string
- ~ " file " ~ file_.bytes.to!string;
- }
- /+ px is written only where the dimensions could be read, and read here
- only for the formats below, so compare only when both are known +/
- if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0
- && (file_.width != e.width || file_.height != e.height)) {
- differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string
- ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string;
- }
- if (differs.length > 0) {
- mismatch += 1;
- failures += 1;
- lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; ");
- } else {
- ok += 1;
- lines ~= "OK " ~ e.name;
- }
- }
-
- /+ ↓ files in the directory that no record names +/
- string[] in_dir;
- foreach (entry; dirEntries(image_dir, SpanMode.shallow)) {
- if (!(entry.isFile)) continue;
- string nm = entry.name.baseName;
- switch (nm.extension.toLower) {
- case ".png": case ".jpg": case ".jpeg": case ".gif":
- in_dir ~= nm;
- break;
- default: break;
- }
- }
- in_dir.sort;
- foreach (nm; in_dir) {
- if (nm !in recorded) {
- extra += 1;
- lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)";
- }
- }
-
- string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file"
- ~ ((gathered.entries.length == 1) ? "" : "s")
- ~ " in " ~ gathered.records.to!string ~ " record"
- ~ ((gathered.records == 1) ? "" : "s")
- ~ ", against " ~ image_dir;
- lines = [head, ""] ~ lines;
- lines ~= "";
- lines ~= "verify: " ~ ok.to!string ~ " ok, "
- ~ mismatch.to!string ~ " mismatch, "
- ~ absent.to!string ~ " absent, "
- ~ extra.to!string ~ " extra"
- ~ ((failures == 0) ? " PASS" : " FAIL");
-
- /+ ↓ write output +/
- if (output_file.length > 0) {
- auto f = File(output_file, "w");
- foreach (line; lines) f.writeln(line);
- } else {
- foreach (line; lines) writeln(line);
- }
- return failures;
-}
-
-private struct FileStat_ {
- string sha256;
- ulong bytes;
- int width;
- int height;
-}
-
-private FileStat_ readFileStat(string pth) {
- FileStat_ ret;
- auto f = File(pth, "rb");
- ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024));
- ret.sha256 = sha.toHexString.to!string;
- ret.bytes = pth.getSize;
- auto dim = readImageDimensions(pth);
- ret.width = dim[0];
- ret.height = dim[1];
- return ret;
-}
-
-/+ ↓ pixel dimensions from the file header, for the formats spine's markup
- admits (png, gif, jpg). Header read only, no decoding, and no dependency
- beyond phobos. Returns [0, 0] where the header cannot be read +/
-private int[2] readImageDimensions(string pth) {
- int[2] ret = [0, 0];
- ubyte[] head;
- try {
- auto f = File(pth, "rb");
- head = new ubyte[](64 * 1024);
- head = f.rawRead(head);
- } catch (Exception ex) {
- return ret;
- }
- if (head.length >= 24
- && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
- /+ png: IHDR width & height, big endian, at 16 and 20 +/
- ret[0] = beInt(head[16 .. 20]);
- ret[1] = beInt(head[20 .. 24]);
- return ret;
- }
- if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) {
- /+ gif: logical screen width & height, little endian, at 6 and 8 +/
- ret[0] = head[6] | (head[7] << 8);
- ret[1] = head[8] | (head[9] << 8);
- return ret;
- }
- if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) {
- /+ jpeg: walk the segments to a start of frame, height then width,
- big endian, at offsets 5 and 7 of the segment +/
- size_t i = 2;
- while (i + 9 < head.length) {
- if (head[i] != 0xff) { i += 1; continue; }
- ubyte marker = head[i + 1];
- if (marker == 0xd8 || marker == 0x01
- || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }
- if (marker == 0xff) { i += 1; continue; }
- size_t seg_len = (head[i + 2] << 8) | head[i + 3];
- bool is_sof = ((marker >= 0xc0 && marker <= 0xc3)
- || (marker >= 0xc5 && marker <= 0xc7)
- || (marker >= 0xc9 && marker <= 0xcb)
- || (marker >= 0xcd && marker <= 0xcf));
- if (is_sof) {
- ret[1] = (head[i + 5] << 8) | head[i + 6];
- ret[0] = (head[i + 7] << 8) | head[i + 8];
- return ret;
- }
- if (seg_len < 2) break;
- i += 2 + seg_len;
- }
- }
- return ret;
-}
-
-private int beInt(const(ubyte)[] b) {
- return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]);
-}
-#+END_SRC
-
-*** read validate
+*** read validate =.ssp=
#+HEADER: :tangle "../dlang/src/read/validate.d"
#+BEGIN_SRC d
@@ -820,6 +524,227 @@ void reportValidation(string filename, SspFinding[] found) {
}
#+END_SRC
+*** read verify images
+
+#+HEADER: :tangle "../dlang/src/read/verify.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.verify;
+
+import std.algorithm : sort;
+import std.array : array, join;
+import std.conv : to;
+import std.digest : toHexString;
+import std.digest.sha : SHA256, digest;
+import std.file : dirEntries, exists, isDir, SpanMode, getSize;
+import std.path : baseName, buildPath, extension;
+import std.stdio : writeln, File;
+import std.string : toLower;
+import ocda.model;
+import read.images : gatherImageRecords;
+
+/+ ↓ verification: recompute what the .image records claim, against the
+ images as they stand in a directory (a pod's media/image).
+
+ ssp-reader --verify --image-dir=<pod>/media/image doc.ssp
+
+ Reports one line per distinct file:
+
+ OK won_benkler_2_1.png
+ MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D...
+ ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory)
+ PRESENT ffa.png (recorded missing, but the file is there)
+ EXTRA unused.png (in directory, no record in this .ssp)
+
+ Returns the number of failures. EXTRA is reported but is not a failure:
+ a pod carries the images of every language version of a document, while
+ one .ssp covers one language.
++/
+int verifyImages(OcdaDocument doc, string image_dir, string output_file) {
+ string[] lines;
+ int failures = 0;
+ int ok = 0;
+ int mismatch = 0;
+ int absent = 0;
+ int extra = 0;
+
+ if (!(image_dir.exists) || !(image_dir.isDir)) {
+ writeln("ERROR: image directory not found: ", image_dir);
+ return 1;
+ }
+ auto gathered = gatherImageRecords(doc);
+ bool[string] recorded;
+
+ foreach (e; gathered.entries) {
+ recorded[e.name] = true;
+ string pth = buildPath(image_dir, e.name);
+ if (!(pth.exists)) {
+ if (e.missing) {
+ /+ recorded missing, and missing here too: consistent +/
+ ok += 1;
+ lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)";
+ } else {
+ absent += 1;
+ failures += 1;
+ lines ~= "ABSENT " ~ e.name
+ ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)";
+ }
+ continue;
+ }
+ if (e.missing) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)";
+ continue;
+ }
+ auto file_ = readFileStat(pth);
+ string[] differs;
+ if (file_.sha256 != e.sha256) {
+ differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256;
+ }
+ if (file_.bytes != e.bytes) {
+ differs ~= "bytes recorded " ~ e.bytes.to!string
+ ~ " file " ~ file_.bytes.to!string;
+ }
+ /+ px is written only where the dimensions could be read, and read here
+ only for the formats below, so compare only when both are known +/
+ if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0
+ && (file_.width != e.width || file_.height != e.height)) {
+ differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string
+ ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string;
+ }
+ if (differs.length > 0) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; ");
+ } else {
+ ok += 1;
+ lines ~= "OK " ~ e.name;
+ }
+ }
+
+ /+ ↓ files in the directory that no record names +/
+ string[] in_dir;
+ foreach (entry; dirEntries(image_dir, SpanMode.shallow)) {
+ if (!(entry.isFile)) continue;
+ string nm = entry.name.baseName;
+ switch (nm.extension.toLower) {
+ case ".png": case ".jpg": case ".jpeg": case ".gif":
+ in_dir ~= nm;
+ break;
+ default: break;
+ }
+ }
+ in_dir.sort;
+ foreach (nm; in_dir) {
+ if (nm !in recorded) {
+ extra += 1;
+ lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)";
+ }
+ }
+
+ string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ " in " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", against " ~ image_dir;
+ lines = [head, ""] ~ lines;
+ lines ~= "";
+ lines ~= "verify: " ~ ok.to!string ~ " ok, "
+ ~ mismatch.to!string ~ " mismatch, "
+ ~ absent.to!string ~ " absent, "
+ ~ extra.to!string ~ " extra"
+ ~ ((failures == 0) ? " PASS" : " FAIL");
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+ return failures;
+}
+
+private struct FileStat_ {
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+}
+
+private FileStat_ readFileStat(string pth) {
+ FileStat_ ret;
+ auto f = File(pth, "rb");
+ ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024));
+ ret.sha256 = sha.toHexString.to!string;
+ ret.bytes = pth.getSize;
+ auto dim = readImageDimensions(pth);
+ ret.width = dim[0];
+ ret.height = dim[1];
+ return ret;
+}
+
+/+ ↓ pixel dimensions from the file header, for the formats spine's markup
+ admits (png, gif, jpg). Header read only, no decoding, and no dependency
+ beyond phobos. Returns [0, 0] where the header cannot be read
++/
+private int[2] readImageDimensions(string pth) {
+ int[2] ret = [0, 0];
+ ubyte[] head;
+ try {
+ auto f = File(pth, "rb");
+ head = new ubyte[](64 * 1024);
+ head = f.rawRead(head);
+ } catch (Exception ex) {
+ return ret;
+ }
+ if (head.length >= 24
+ && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
+ /+ png: IHDR width & height, big endian, at 16 and 20 +/
+ ret[0] = beInt(head[16 .. 20]);
+ ret[1] = beInt(head[20 .. 24]);
+ return ret;
+ }
+ if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) {
+ /+ gif: logical screen width & height, little endian, at 6 and 8 +/
+ ret[0] = head[6] | (head[7] << 8);
+ ret[1] = head[8] | (head[9] << 8);
+ return ret;
+ }
+ if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) {
+ /+ jpeg: walk the segments to a start of frame, height then width,
+ big endian, at offsets 5 and 7 of the segment
+ +/
+ size_t i = 2;
+ while (i + 9 < head.length) {
+ if (head[i] != 0xff) { i += 1; continue; }
+ ubyte marker = head[i + 1];
+ if (marker == 0xd8 || marker == 0x01
+ || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }
+ if (marker == 0xff) { i += 1; continue; }
+ size_t seg_len = (head[i + 2] << 8) | head[i + 3];
+ bool is_sof = ((marker >= 0xc0 && marker <= 0xc3)
+ || (marker >= 0xc5 && marker <= 0xc7)
+ || (marker >= 0xc9 && marker <= 0xcb)
+ || (marker >= 0xcd && marker <= 0xcf));
+ if (is_sof) {
+ ret[1] = (head[i + 5] << 8) | head[i + 6];
+ ret[0] = (head[i + 7] << 8) | head[i + 8];
+ return ret;
+ }
+ if (seg_len < 2) break;
+ i += 2 + seg_len;
+ }
+ }
+ return ret;
+}
+
+private int beInt(const(ubyte)[] b) {
+ return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]);
+}
+#+END_SRC
+
*** read report unknown
#+HEADER: :tangle "../dlang/src/read/report_unknown.d"
@@ -869,6 +794,82 @@ void reportUnknown(ref OcdaDocument doc) {
#+END_SRC
** misc
+*** read tree
+
+#+HEADER: :tangle "../dlang/src/read/tree.d"
+#+BEGIN_SRC d
+<<doc_header_including_copyright_and_license>>
+module read.tree;
+
+import std.conv : to;
+import std.stdio : writeln, writef, writefln, File;
+import std.array : replicate;
+import ocda.model;
+
+void outputTree(OcdaDocument doc, string output_file) {
+ string[] lines;
+
+ lines ~= "Document: " ~ doc.meta("title.main");
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= " " ~ doc.meta("title.subtitle");
+ lines ~= "Author: " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build OCN -> object lookup for body headings +/
+ OcdaObject[int] headings;
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading" && obj.ocn > 0) {
+ headings[obj.ocn] = obj;
+ }
+ }
+
+ /+ ↓ find root heading (from head section, or first body heading) +/
+ if (doc.head.length > 0 && doc.head[0].is_a == "heading") {
+ auto root = doc.head[0];
+ lines ~= formatHeading(root, 0);
+ walkChildren(root, headings, lines, 1);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) {
+ foreach (child_ocn; parent.children) {
+ if (auto p = child_ocn in headings) {
+ lines ~= formatHeading(*p, depth);
+ walkChildren(*p, headings, lines, depth + 1);
+ }
+ }
+}
+
+private string formatHeading(OcdaObject obj, int depth) {
+ string indent = replicate(" ", depth);
+ string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?";
+ string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+ /+ ↓ strip internal markup from heading text for clean display +/
+ string text = stripInternalMarkup(obj.text);
+ return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text;
+}
+
+private string stripInternalMarkup(string text) {
+ /+ ↓ simple removal of common internal markers for display +/
+ import std.regex : replaceAll, regex;
+ import std.array : replace;
+ string result = text;
+ result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("┨", "").replace("┣", "")
+ .replace("┙", " ").replace("┘", "\n");
+ return result;
+}
+#+END_SRC
+
*** read html
#+HEADER: :tangle "../dlang/src/read/html.d"
generated by cgit v1.2.3 (git 2.25.1) at 2026-11-13 02:31:27 +0000