diff options
| -rw-r--r-- | dlang/dub.json | 16 | ||||
| -rw-r--r-- | dlang/flake.nix | 51 | ||||
| -rw-r--r-- | dlang/src/read/html.d | 388 | ||||
| -rw-r--r-- | dlang/src/read/images.d | 114 | ||||
| -rw-r--r-- | dlang/src/read/llm_jsonl.d | 121 | ||||
| -rw-r--r-- | dlang/src/read/llm_text.d | 182 | ||||
| -rw-r--r-- | dlang/src/read/main.d | 230 | ||||
| -rw-r--r-- | dlang/src/read/report_unknown.d | 41 | ||||
| -rw-r--r-- | dlang/src/read/tree.d | 69 | ||||
| -rw-r--r-- | dlang/src/read/validate.d | 265 | ||||
| -rw-r--r-- | dlang/src/read/verify.d | 212 |
11 files changed, 1670 insertions, 19 deletions
diff --git a/dlang/dub.json b/dlang/dub.json index 373e5bd..80be675 100644 --- a/dlang/dub.json +++ b/dlang/dub.json @@ -6,8 +6,20 @@ "description": "the object-centric document abstraction as a library: the model, and loaders for its two serialisations", "homepage": "https://sisudoc.org", "license": "AGPL-3.0+", - "targetType": "library", "sourcePaths": [ "./src" ], "importPaths": [ "./src" ], - "libs": [ "sqlite3", "zstd" ] + "libs": [ "sqlite3", "zstd" ], + "configurations": [ + { + "name": "library", + "targetType": "library", + "excludedSourceFiles": [ "src/read/*" ] + }, + { + "name": "ocda-read", + "targetType": "executable", + "targetName": "ocda-read", + "mainSourceFile": "src/read/main.d" + } + ] } diff --git a/dlang/flake.nix b/dlang/flake.nix index 2ad3b1c..09dce2e 100644 --- a/dlang/flake.nix +++ b/dlang/flake.nix @@ -8,14 +8,19 @@ system = "x86_64-linux"; # adjust for aarch64-linux / darwin pkgs = import nixpkgs { inherit system; }; - /* A library, so there is nothing to install as a binary. What is - built here are the two programs that put the library to the - question: the agreement check, which loads a document's .ssp and - its .ocda.db and compares the two models, and the dump, which + /* A library, so the library itself is not a build product: a program + that uses it compiles its modules alongside its own, as the writer's + flake does, or takes it as a dub dependency. What is built here are + programs over it: the reader, and the two that put the library to + the question - the agreement check, which loads a document's .ssp + and its .ocda.db and compares the two models, and the dump, which writes a loaded model out canonically so that an implementation in another language can be diffed against this one. + + `mains` is what is compiled besides the library: the checks are one + file each in test/, the reader is src/read/. */ - mkTool = { tool, binary, what, compiler, compilerBin }: + mkTool = { tool, binary, what, mains, compiler, compilerBin }: pkgs.stdenv.mkDerivation { pname = "ocda-lib-${tool}"; version = "0.1.0"; @@ -25,7 +30,7 @@ buildPhase = '' runHook preBuild export HOME=$TMPDIR - ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 -L-lzstd + ${compilerBin} -I=src -of=${tool} src/ocda/*.d ${mains} -L-lsqlite3 -L-lzstd runHook postBuild ''; installPhase = '' @@ -41,19 +46,31 @@ }; ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; + + ocda-read = mkTool (ldc // { + tool = "read"; binary = "ocda-read"; mains = "src/read/*.d"; + what = "reads the abstraction in any of its forms and says what is in it"; + }); + ocda-agree = mkTool (ldc // { + tool = "agree"; binary = "ocda-agree"; mains = "test/agree.d"; + what = "checks that the .ssp and .ocda.db loaders give one model"; + }); + ocda-dump = mkTool (ldc // { + tool = "dump"; binary = "ocda-dump"; mains = "test/dump.d"; + what = "writes a loaded abstraction out canonically, as an oracle"; + }); in { packages.${system} = { - default = self.packages.${system}.ocda-agree; - - ocda-agree = mkTool (ldc // { - tool = "agree"; binary = "ocda-agree"; - what = "checks that the .ssp and .ocda.db loaders give one model"; - }); - - ocda-dump = mkTool (ldc // { - tool = "dump"; binary = "ocda-dump"; - what = "writes a loaded abstraction out canonically, as an oracle"; - }); + /* All three in one bin/, so that one build leaves every program in + one result link. Built separately into the same link, each build + replaced the one before, and a script looking for ocda-dump beside + ocda-agree found nothing. + */ + default = pkgs.symlinkJoin { + name = "ocda-lib-programs-0.1.0"; + paths = [ ocda-read ocda-agree ocda-dump ]; + }; + inherit ocda-read ocda-agree ocda-dump; }; devShells.${system}.default = pkgs.mkShell { diff --git a/dlang/src/read/html.d b/dlang/src/read/html.d new file mode 100644 index 0000000..cd679e4 --- /dev/null +++ b/dlang/src/read/html.d @@ -0,0 +1,388 @@ +module read.html; + +import std.array : replace, split, join, appender; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, startsWith; +import ocda.model; + +void outputHtmlScroll(OcdaDocument doc, string output_file) { + auto buf = appender!(string[]); + + buf ~= htmlHead(doc); + + /+ ↓ title +/ + if (doc.head.length > 0) { + auto title_obj = doc.head[0]; + buf ~= `<div class="title">`; + buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`; + if (doc.meta("title.subtitle").length > 0) + buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`; + buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`; + buf ~= `</div>`; + } + + /+ ↓ table of contents +/ + if (doc.toc.length > 0) { + buf ~= `<div class="toc">`; + buf ~= `<h2>Table of Contents</h2>`; + foreach (obj; doc.toc) { + if (obj.is_a == "toc") { + string text = convertInlineMarkup(obj.text); + int indent = obj.indent_base; + string cls = (indent > 4) ? "toc-sub" : "toc-entry"; + buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`; + } + } + buf ~= `</div>`; + buf ~= `<hr/>`; + } + + /+ ↓ body +/ + buf ~= `<div class="body">`; + foreach (obj; doc.body) { + buf ~= renderObject(obj); + } + buf ~= `</div>`; + + /+ ↓ endnotes +/ + bool has_endnotes = false; + foreach (obj; doc.endnotes) { + if (obj.is_a == "endnote") { has_endnotes = true; break; } + } + if (has_endnotes) { + buf ~= `<div class="endnotes">`; + buf ~= `<hr/>`; + buf ~= `<h2>Endnotes</h2>`; + foreach (obj; doc.endnotes) { + if (obj.is_a == "endnote") { + string text = convertInlineMarkup(obj.text); + string parent_ref = (obj.parent_ocn > 0) + ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>` + : ""; + buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`; + } + } + buf ~= `</div>`; + } + + /+ ↓ glossary +/ + if (doc.glossary.length > 0) { + buf ~= `<div class="glossary">`; + buf ~= `<hr/>`; + buf ~= `<h2>Glossary</h2>`; + foreach (obj; doc.glossary) { + string text = convertInlineMarkup(obj.text); + buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`; + } + buf ~= `</div>`; + } + + /+ ↓ bookindex +/ + bool has_bookindex = false; + foreach (obj; doc.bookindex) { + if (obj.is_a == "bookindex") { has_bookindex = true; break; } + } + if (has_bookindex) { + buf ~= `<div class="bookindex">`; + buf ~= `<hr/>`; + buf ~= `<h2>Book Index</h2>`; + foreach (obj; doc.bookindex) { + if (obj.is_a == "bookindex") { + string text = convertInlineMarkup(obj.text); + buf ~= `<p class="index-entry">` ~ text ~ `</p>`; + } + } + buf ~= `</div>`; + } + + buf ~= htmlTail(); + + /+ ↓ write output +/ + string[] output = buf.data; + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; output) f.writeln(line); + } else { + foreach (line; output) writeln(line); + } +} + +private string[] renderObject(OcdaObject obj) { + /+ ↓ markup commentary is carried by the abstraction and is not output. + Rendering it would put the author notes into the document, which + spine itself does not do + +/ + if (!obj.renders) { return []; } + string[] lines; + string text = convertInlineMarkup(obj.text); + string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) + ? ` id="o` ~ obj.ocn.to!string ~ `"` + : ""; + string ocn_display = (obj.ocn > 0 && !obj.ocn_off) + ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>` + : ""; + + switch (obj.is_a) { + case "heading": + string tag = headingTag(obj.heading_level); + string seg_anchor = (obj.segment.length > 0) + ? ` id="` ~ htmlEscape(obj.segment) ~ `"` + : ocn_anchor; + lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`; + break; + case "para": + string indent_cls = ""; + if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`; + else if (obj.bullet) indent_cls = ` class="bullet"`; + lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`; + break; + case "code": + lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`; + break; + case "table": + lines ~= renderTable(obj, ocn_anchor); + break; + case "group": + lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`; + break; + case "block": + lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`; + break; + case "verse": + lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`; + break; + case "quote": + lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`; + break; + default: + if (text.length > 0) + lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`; + break; + } + return lines; +} + +private string headingTag(string level) { + switch (level) { + case "A": return "h1"; + case "B": return "h2"; + case "C": return "h3"; + case "D": return "h4"; + case "1": return "h3"; + case "2": return "h4"; + case "3": return "h5"; + case "4": return "h6"; + default: return "h4"; + } +} + +private string renderTable(OcdaObject obj, string ocn_anchor) { + string[] lines; + lines ~= `<table` ~ ocn_anchor ~ `>`; + /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/ + auto rows = obj.text.split("\n"); + bool first_row = obj.table_header; + foreach (row; rows) { + if (row.strip.length == 0) continue; + string tag = first_row ? "th" : "td"; + lines ~= `<tr>`; + foreach (cell; row.split("┊")) { + lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`; + } + lines ~= `</tr>`; + first_row = false; + } + lines ~= `</table>`; + return lines.join("\n"); +} + +/+ ↓ convert spine internal Unicode markup to HTML +/ +string convertInlineMarkup(string text) { + if (text.length == 0) return ""; + string result = htmlEscape(text); + + /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc. + Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X + The ┨┣ form is used within heading content +/ + + /+ ↓ bold +/ + result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>"); + result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>"); + + /+ ↓ emphasis +/ + result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>"); + result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>"); + + /+ ↓ italic +/ + result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>"); + result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>"); + + /+ ↓ underscore +/ + result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>"); + result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>"); + + /+ ↓ superscript +/ + result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>"); + result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>"); + + /+ ↓ subscript +/ + result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>"); + result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>"); + + /+ ↓ monospace +/ + result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>"); + + /+ ↓ links: ┥link text┝┤url├ +/ + result = replaceLinks(result); + + /+ ↓ footnote markers: 【N text】 +/ + result = replaceFootnotes(result); + + /+ ↓ line breaks +/ + result = result.replace("┘", "<br/>"); + result = result.replace("┙", "<br/>"); + + /+ ↓ non-breaking spaces +/ + result = result.replace("░", " "); + + /+ ↓ bullet markers +/ + result = result.replace("● ", "• "); + + /+ ↓ page break markers +/ + result = result.replace("┼", "").replace("┿", "").replace("╂", ""); + + return result; +} + +private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) { + /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/ + string result = text; + while (true) { + auto open_pos = result.indexOf(open_mark); + if (open_pos < 0) break; + auto after_open = open_pos + open_mark.length; + /+ ↓ look for closing ⑆X +/ + string close_mark = "⑆" ~ close_suffix; + auto close_pos = indexOf(result[after_open .. $], close_mark); + if (close_pos < 0) break; + close_pos += after_open; + string content = result[after_open .. close_pos]; + result = result[0 .. open_pos] ~ html_open ~ content ~ html_close + ~ result[close_pos + close_mark.length .. $]; + } + return result; +} + +private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) { + /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/ + string result = text; + string full_open = open_mark ~ bracket_open; + while (true) { + auto open_pos = result.indexOf(full_open); + if (open_pos < 0) break; + auto after_open = open_pos + full_open.length; + auto close_pos = indexOf(result[after_open .. $], bracket_close); + if (close_pos < 0) break; + close_pos += after_open; + string content = result[after_open .. close_pos]; + result = result[0 .. open_pos] ~ html_open ~ content ~ html_close + ~ result[close_pos + bracket_close.length .. $]; + } + return result; +} + +private string replaceLinks(string text) { + /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/ + string result = text; + while (true) { + auto link_open = result.indexOf("┥"); + if (link_open < 0) break; + auto link_mid = indexOf(result[link_open .. $], "┝┤"); + if (link_mid < 0) break; + link_mid += link_open; + auto link_close = indexOf(result[link_mid .. $], "├"); + if (link_close < 0) break; + link_close += link_mid; + string link_text = result[link_open + "┥".length .. link_mid]; + string url = result[link_mid + "┝┤".length .. link_close]; + /+ ↓ handle internal links (¤ prefix) +/ + if (url.length > 0 && url[0] == '¤') { + url = url[("¤").length .. $]; + } + /+ ↓ handle OCN-only links (#N) +/ + if (url.length > 0 && url[0] == '#') { + url = "#o" ~ url[1 .. $]; + } + result = result[0 .. link_open] + ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>` + ~ result[link_close + "├".length .. $]; + } + return result; +} + +private string replaceFootnotes(string text) { + /+ ↓ replace 【N content】 with superscript note number +/ + string result = text; + while (true) { + auto fn_open = result.indexOf("【"); + if (fn_open < 0) break; + auto fn_close = indexOf(result[fn_open .. $], "】"); + if (fn_close < 0) break; + fn_close += fn_open; + string fn_content = result[fn_open + "【".length .. fn_close]; + /+ ↓ extract note number (first token before space) +/ + auto space_pos = fn_content.indexOf(' '); + string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content; + result = result[0 .. fn_open] + ~ `<sup class="fn">` ~ note_num ~ `</sup>` + ~ result[fn_close + "】".length .. $]; + } + return result; +} + +string htmlEscape(string text) { + return text.replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\"", """); +} + +private string[] htmlHead(OcdaDocument doc) { + string[] lines; + lines ~= `<!DOCTYPE html>`; + lines ~= `<html>`; + lines ~= `<head>`; + lines ~= `<meta charset="utf-8"/>`; + lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`; + lines ~= `<style>`; + lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`; + lines ~= ` .title { text-align: center; margin-bottom: 2em; }`; + lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`; + lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`; + lines ~= ` .toc { margin-bottom: 2em; }`; + lines ~= ` .toc-entry { margin: 0.2em 0; }`; + lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`; + lines ~= ` .indent-1 { margin-left: 2em; }`; + lines ~= ` .indent-2 { margin-left: 4em; }`; + lines ~= ` .indent-3 { margin-left: 6em; }`; + lines ~= ` .bullet { margin-left: 2em; }`; + lines ~= ` .bullet:before { content: "\2022 "; }`; + lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`; + lines ~= ` .group { margin: 1em 2em; }`; + lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`; + lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`; + lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`; + lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`; + lines ~= ` th { background: #f0f0f0; }`; + lines ~= ` .endnote { font-size: 0.9em; }`; + lines ~= ` .fn { color: #666; }`; + lines ~= ` .index-entry { font-size: 0.9em; }`; + lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`; + lines ~= `</style>`; + lines ~= `</head>`; + lines ~= `<body>`; + return lines; +} + +private string[] htmlTail() { + return [`</body>`, `</html>`]; +} diff --git a/dlang/src/read/images.d b/dlang/src/read/images.d new file mode 100644 index 0000000..afddc74 --- /dev/null +++ b/dlang/src/read/images.d @@ -0,0 +1,114 @@ +module read.images; + +import std.array : join; +import std.conv : to; +import std.stdio : writeln, File; +import ocda.model; + +/+ ↓ the .image records of a document, gathered per distinct file, with the + objects referencing each file, in order of first appearance +/ +struct ImageEntry { + string name; + string sha256; + ulong bytes; + int width; + int height; + bool missing; + string[] refs; + string[] other_sha256; // same filename, differing digest +} + +struct ImageRecords { + int records; // .image lines, an image referenced twice counts twice + ImageEntry[] entries; // distinct files +} + +ImageRecords gatherImageRecords(OcdaDocument doc) { + ImageRecords ret; + size_t[string] seen; + foreach (section; sectionNames) { + auto objs = doc.sectionByName(section); + if (objs is null) continue; + foreach (obj; *objs) { + foreach (img; obj.images) { + ret.records += 1; + string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]"; + if (auto at = img.name in seen) { + if (ret.entries[*at].sha256 != img.sha256 + && img.sha256.length > 0 + && ret.entries[*at].sha256.length > 0) { + ret.entries[*at].other_sha256 ~= img.sha256; + } + ret.entries[*at].refs ~= _ref; + } else { + ImageEntry e_; + e_.name = img.name; + e_.sha256 = img.sha256; + e_.bytes = img.bytes; + e_.width = img.width; + e_.height = img.height; + e_.missing = img.missing; + e_.refs = [_ref]; + seen[img.name] = ret.entries.length; + ret.entries ~= e_; + } + } + } + } + return ret; +} + +/+ ↓ image manifest, line oriented and greppable: + + <sha256> <bytes> <px> <filename> refs: body[97] body[412] + MISSING <filename> refs: body[840] + + The point of the record in the .ssp is verification (is the image + shipped in this pod the image the abstraction was built from), so this + output is the shape a verifier wants: one line per distinct file. ++/ +void outputImages(OcdaDocument doc, string output_file) { + auto gathered = gatherImageRecords(doc); + string[] lines; + int missing = 0; + foreach (e; gathered.entries) { + if (e.missing) missing += 1; + } + lines ~= "Images: " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ ", " ~ missing.to!string ~ " missing"; + lines ~= ""; + foreach (e; gathered.entries) { + string px = (e.width > 0 && e.height > 0) + ? e.width.to!string ~ "x" ~ e.height.to!string : "-"; + lines ~= (e.missing) + ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ") + : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px + ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" "); + /+ ↓ one filename, more than one digest: the .ssp is inconsistent, + or two distinct files share a name +/ + foreach (other; e.other_sha256) { + lines ~= " WARNING differing digest for same filename: " ~ other; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +/+ ↓ count of .image records in a document, for the parse summary +/ +int imageRecordCount(OcdaDocument doc) { + return gatherImageRecords(doc).records; +} + +package immutable string[] sectionNames = [ + "head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", +]; diff --git a/dlang/src/read/llm_jsonl.d b/dlang/src/read/llm_jsonl.d new file mode 100644 index 0000000..5b81e68 --- /dev/null +++ b/dlang/src/read/llm_jsonl.d @@ -0,0 +1,121 @@ +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} diff --git a/dlang/src/read/llm_text.d b/dlang/src/read/llm_text.d new file mode 100644 index 0000000..e61f2b7 --- /dev/null +++ b/dlang/src/read/llm_text.d @@ -0,0 +1,182 @@ +module read.llm_text; + +import std.array : replace, split, join; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; + +/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption + format: + ## Document Title > Chapter > Section > [OCN] + + Clean paragraph text with inline markup stripped. ++/ +void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { + string[] lines; + string doc_title = doc.meta("title.main"); + + /+ ↓ document header +/ + lines ~= "# " ~ doc_title; + if (doc.meta("title.subtitle").length > 0) + lines ~= "# " ~ doc.meta("title.subtitle"); + lines ~= "# " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build heading text lookup from ancestors +/ + string[int] heading_text; + foreach (obj; doc.head) { + if (obj.is_a == "heading" && obj.ocn > 0) + heading_text[obj.ocn] = stripMarkup(obj.text); + } + foreach (obj; doc.body) { + if (obj.is_a == "heading") + heading_text[obj.ocn] = stripMarkup(obj.text); + } + + /+ ↓ body objects +/ + string current_breadcrumb = ""; + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); + lines ~= ""; + lines ~= "## " ~ current_breadcrumb; + lines ~= ""; + continue; + } + + /+ ↓ markup commentary is not content +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) + ? " [" ~ obj.ocn.to!string ~ "]" + : ""; + + switch (obj.is_a) { + case "para": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "code": + lines ~= "```"; + lines ~= obj.text; /+ preserve raw text for code +/ + lines ~= "```"; + lines ~= ""; + break; + case "table": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "verse": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "quote": + lines ~= "> " ~ text ~ ocn_ref; + lines ~= ""; + break; + case "group": + case "block": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + default: + if (text.length > 0) { + lines ~= text ~ ocn_ref; + lines ~= ""; + } + break; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { + /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + /+ ↓ truncate very long heading text for breadcrumb +/ + if (clean.length > 60) + clean = clean[0 .. 57] ~ "..."; + parts ~= clean; + } + } + } + /+ ↓ add the current heading itself +/ + string heading_text = stripMarkup(obj.text).strip; + if (heading_text.length > 60) + heading_text = heading_text[0 .. 57] ~ "..."; + + string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + + if (parts.length > 0) + return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; + else + return doc_title ~ " > " ~ heading_text ~ ocn_ref; +} + +/+ ↓ strip all internal Unicode markup to produce clean plain text +/ +string stripMarkup(string text) { + if (text.length == 0) return ""; + string r = text; + + /+ ↓ links: ┥link text┝┤url├ -> link text +/ + while (true) { + auto lo = r.indexOf("┥"); + if (lo < 0) break; + auto lm = indexOf(r[lo .. $], "┝┤"); + if (lm < 0) break; + lm += lo; + auto lc = indexOf(r[lm .. $], "├"); + if (lc < 0) break; + lc += lm; + string link_text = r[lo + "┥".length .. lm]; + r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; + } + + /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ + while (true) { + auto fo = r.indexOf("【"); + if (fo < 0) break; + auto fc = indexOf(r[fo .. $], "】"); + if (fc < 0) break; + fc += fo; + r = r[0 .. fo] ~ r[fc + "】".length .. $]; + } + + /+ ↓ font face markers +/ + r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("⑆■", "") + .replace("┨", "").replace("┣*", "").replace("┣!", "") + .replace("┣/", "").replace("┣_", "").replace("┣^", "") + .replace("┣,", "").replace("┣", ""); + + /+ ↓ line breaks and special chars +/ + r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); + r = r.replace("░", " "); + r = r.replace("● ", "- "); + r = r.replace("┼", "").replace("┿", "").replace("╂", ""); + + /+ ↓ table cell separators +/ + r = r.replace("┊", " | "); + + /+ ↓ clean up multiple spaces +/ + while (r.indexOf(" ") >= 0) + r = r.replace(" ", " "); + + return r.strip; +} diff --git a/dlang/src/read/main.d b/dlang/src/read/main.d new file mode 100644 index 0000000..7128b4c --- /dev/null +++ b/dlang/src/read/main.d @@ -0,0 +1,230 @@ +/++ + ocda-read - reads the object-centric document abstraction, in any of the + forms it is written in, and says what is in it. + + ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] + + One reader where there were two: sisudoc-ssp-reader read a .ssp and + sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is + built on the library, so every mode reads every form, and a mode given the + .ssp, the .ocda.db and the .sisupod of a document says the same thing of + each: it is told what the document is, not which file it came from. + + Two things are about the file rather than the document, and say so: + --files, what a .ocda.db carries besides the abstraction, and --validate, + which checks a .ssp as text. Everything else is the model's. + + The output modules are the .ssp reader's, moved onto the library's model. ++/ +module read.main; + +import std.algorithm : canFind, map; +import std.array : array, join; +import std.getopt; +import std.stdio : writeln, writefln, stderr; +import ocda.model; +import ocda.load; +import ocda.inventory; +import read.tree; +import read.html; +import read.llm_text; +import read.llm_jsonl; +import read.images; +import read.verify; +import read.report_unknown; +import read.validate; + +int main(string[] args) { + bool do_tree = false; + bool do_html = false; + bool do_llm_text = false; + bool do_llm_jsonl = false; + bool do_images = false; + bool do_verify = false; + bool do_report_unknown = false; + bool do_validate = false; + bool do_languages = false; + bool do_files = false; + bool any_version = false; + string image_dir = ""; + string output_file = ""; + string lang = ""; + + GetoptResult helpInfo; + try { + helpInfo = getopt(args, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, + "html", "the document as one html page", &do_html, + "llm-text", "plain text with heading breadcrumbs", &do_llm_text, + "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, + "images", "the image records: digest, size, pixels", &do_images, + "verify", "check the image records against --image-dir", &do_verify, + "image-dir", "the images to verify against", &image_dir, + "languages", "the languages the file holds", &do_languages, + "files", "what a .ocda.db carries besides the abstraction", &do_files, + "report-unknown", "what was read and not understood", &do_report_unknown, + "validate", "check a .ssp against its specification, as text", &do_validate, + "lang", "=<code> the language to read (default: the first)", &lang, + "output", "write the mode's output to a file", &output_file, + "any-version", "read a format major this reader does not know", &any_version, + ); + } catch (GetOptException e) { + stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); + return 2; + } + + if (helpInfo.helpWanted || args.length < 2) { + defaultGetoptPrinter( + "ocda-read - reads the document abstraction, in any of its forms\n\n" + ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n", + helpInfo.options, + ); + return 0; + } + + string input_file = args[1]; + if (lang.length == 0 && args.length > 2) { lang = args[2]; } + bool lang_named = (lang.length > 0); + + /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not + at all: there is no text form of a database to check, and a pod's + .ssp is checked by taking it out +/ + if (do_validate && sourceOf(input_file) != Source.ssp) { + stderr.writeln("--validate checks a .ssp as text, against the .ssp", + " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); + return 2; + } + + /+ ↓ which language. A .ocda.db and a pod may hold several; where none is + named, every form reads the first in the manifest's order and says + so, rather than the database taking one and the pod refusing +/ + OcdaLanguage[] held; + OcdaDocument doc; + try { + held = languages(input_file); + if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } + if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { + stderr.writeln("error: no language '", lang, "' in ", input_file, + "; it holds: ", held.map!(l => l.lang).array.join(" ")); + return 2; + } + doc = load(input_file, lang, any_version); + } catch (Exception e) { + stderr.writeln("Error reading ", input_file, ": ", e.msg); + return 2; + } + + /+ ↓ the version gate: the major part, and refuse what is not known +/ + if (doc.format_version.length == 0) { + stderr.writeln("warning: ", input_file, " declares no format version"); + } else if (doc.formatVersionMajor != ssp_format_major_read) { + if (any_version) { + stderr.writeln("warning: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); + } else { + stderr.writeln("error: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x"); + stderr.writeln(" a major version differs: a property may have", + " changed meaning or gone away."); + stderr.writeln(" --any-version reads it regardless."); + return 3; + } + } + + writeln("Read: ", input_file); + if (doc.format_version.length > 0) { + writeln(" Format: v", doc.format_version); + } + if (auto _lang = "language" in doc.source) { + writeln(" Source: ", *_lang, + ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", + ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); + } + if (held.length > 1 && !lang_named) { + writeln(" (reading ", lang, "; --lang=<code> for another)"); + } + writeln(" Title: ", doc.meta("title.main")); + writeln(" Author: ", doc.meta("creator.author")); + writeln(" Sections: head=", doc.head.length, + " toc=", doc.toc.length, + " body=", doc.body.length, + " endnotes=", doc.endnotes.length, + " glossary=", doc.glossary.length, + " bibliography=", doc.bibliography.length, + " bookindex=", doc.bookindex.length, + " blurb=", doc.blurb.length, + " tail=", doc.tail.length); + auto image_records = imageRecordCount(doc); + if (image_records > 0) { + writeln(" Images: ", image_records, " record", + (image_records == 1) ? "" : "s"); + } + + if (do_languages) { printLanguages(held); } + if (do_files) { printFiles(input_file); } + if (do_tree) { outputTree(doc, output_file); } + if (do_html) { outputHtmlScroll(doc, output_file); } + if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } + if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } + if (do_images) { outputImages(doc, output_file); } + + int _verify_failures = 0; + if (do_verify) { + if (image_dir.length == 0) { + stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); + return 2; + } + _verify_failures = verifyImages(doc, image_dir, output_file); + } + + if (do_report_unknown) { reportUnknown(doc); } + + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images + && !do_verify && !do_report_unknown && !do_validate + && !do_languages && !do_files) { + writeln("\nUse --tree, --html, --llm-text, --llm-jsonl, --images," + ~ " --verify --image-dir=<dir>, --languages, --files," + ~ " --report-unknown or --validate to produce output."); + } + + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; +} + +/+ ↓ the languages, each with the markup it was made from and its .ssp's + digest: the same from every form, the database storing the digest and + the others taking it from the bytes +/ +void printLanguages(OcdaLanguage[] held) { + writeln(); + writeln(" lang source ssp digest"); + foreach (l; held) { + writefln(" %-7s %-31s %s", l.lang, l.source_filename, + (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); + } +} + +/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries + nothing, and a pod's files are its archive entries rather than anything + the abstraction records, so for those it says so +/ +void printFiles(string path) { + writeln(); + if (!isCarrier(path)) { + writeln(" carried files: only a .ocda.db carries files; this is ", + sourceName(sourceOf(path))); + return; + } + auto rows = carried(path); + if (rows.length == 0) { writeln(" carries no files"); return; } + writeln(" carried files:"); + writeln(" role stored n original in the file"); + foreach (r; rows) { + writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); + } +} diff --git a/dlang/src/read/report_unknown.d b/dlang/src/read/report_unknown.d new file mode 100644 index 0000000..2f81702 --- /dev/null +++ b/dlang/src/read/report_unknown.d @@ -0,0 +1,41 @@ +module read.report_unknown; + +import std.algorithm : sort; +import std.array : array; +import std.stdio : writeln, writefln; +import ocda.model; + +/+ ↓ say what was read and not understood. + This is what makes the reader usable as a check on the format + specification: a reader that ignores what it does not know is + conforming, but one that cannot say what it ignored cannot tell you + whether the specification it was written from was complete. Silence + and understanding look the same from outside. ++/ +void reportUnknown(ref OcdaDocument doc) { + if (!doc.unknown.any) { + writeln("\nread and understood: every property, block and line was known"); + return; + } + writeln("\nread and NOT understood:"); + if (doc.unknown.property.length > 0) { + writeln(" properties ignored (", doc.unknown.property.length, " distinct):"); + foreach (name; doc.unknown.property.keys.sort) { + writefln(" .%-26s %8d", name, doc.unknown.property[name]); + } + } + if (doc.unknown.block.length > 0) { + writeln(" blocks ignored (", doc.unknown.block.length, " distinct):"); + foreach (name; doc.unknown.block.keys.sort) { + writefln(" @%-26s %8d", name, doc.unknown.block[name]); + } + } + if (doc.unknown.line.length > 0) { + writeln(" lines of no known shape, by first character:"); + foreach (c; doc.unknown.line.keys.sort) { + writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]); + } + } + writeln(" (ignoring these is conforming within a major version;", + " the count is so that it is not silent)"); +} diff --git a/dlang/src/read/tree.d b/dlang/src/read/tree.d new file mode 100644 index 0000000..9a45b02 --- /dev/null +++ b/dlang/src/read/tree.d @@ -0,0 +1,69 @@ +module read.tree; + +import std.conv : to; +import std.stdio : writeln, writef, writefln, File; +import std.array : replicate; +import ocda.model; + +void outputTree(OcdaDocument doc, string output_file) { + string[] lines; + + lines ~= "Document: " ~ doc.meta("title.main"); + if (doc.meta("title.subtitle").length > 0) + lines ~= " " ~ doc.meta("title.subtitle"); + lines ~= "Author: " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build OCN -> object lookup for body headings +/ + OcdaObject[int] headings; + foreach (obj; doc.body) { + if (obj.is_a == "heading" && obj.ocn > 0) { + headings[obj.ocn] = obj; + } + } + + /+ ↓ find root heading (from head section, or first body heading) +/ + if (doc.head.length > 0 && doc.head[0].is_a == "heading") { + auto root = doc.head[0]; + lines ~= formatHeading(root, 0); + walkChildren(root, headings, lines, 1); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { + foreach (child_ocn; parent.children) { + if (auto p = child_ocn in headings) { + lines ~= formatHeading(*p, depth); + walkChildren(*p, headings, lines, depth + 1); + } + } +} + +private string formatHeading(OcdaObject obj, int depth) { + string indent = replicate(" ", depth); + string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; + string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + /+ ↓ strip internal markup from heading text for clean display +/ + string text = stripInternalMarkup(obj.text); + return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; +} + +private string stripInternalMarkup(string text) { + /+ ↓ simple removal of common internal markers for display +/ + import std.regex : replaceAll, regex; + import std.array : replace; + string result = text; + result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("┨", "").replace("┣", "") + .replace("┙", " ").replace("┘", "\n"); + return result; +} diff --git a/dlang/src/read/validate.d b/dlang/src/read/validate.d new file mode 100644 index 0000000..4a77f8b --- /dev/null +++ b/dlang/src/read/validate.d @@ -0,0 +1,265 @@ +module read.validate; + +/+ ↓ check a .ssp against the format specification. + . + Written from specs/sisudoc-abstraction-format/ssp-format-specification.md + and from nothing else: the point of it is to be what the specification + says, executed, so that a file can be held against the document rather + than against an implementation. Each check names the section it comes + from, so a disagreement can be taken back to the text. ++/ + +import std.algorithm : canFind, startsWith; +import std.array : array, split; +import std.conv : to; +import std.file : readText; +import std.stdio : writeln, writefln; +import std.string : indexOf, strip; + +struct SspFinding { + size_t line; // 1-based + string section; // the specification section the rule is from + string detail; +} + +/+ ↓ specification section 6.3: the properties, in the order they are + emitted. A reader may rely on the order, so a validator can check it +/ +private immutable string[] property_order = [ + "identifier", "part", "section", "parent", "last_descendant", "ocn_range", + "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", + "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", + "table_header", "code_linenumbers", "stow_link", "segment", "anchor", + "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", + "segment_html_is", "segment_epub_is", "segment_lv4_is", + "heading_ancestors_text", "lev4_subtoc", "anchor_tag", +]; + +/+ ↓ section 6.1: the sixteen kinds +/ +private immutable string[] kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", + "blurb", "group", +]; + +/+ ↓ section 4: the nine sections, in the order they appear +/ +private immutable string[] section_order = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", + "bookindex", "blurb", "tail", +]; + +/+ ↓ section 4: the four preamble blocks, in the order they appear +/ +private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; + +/+ ↓ section 6.1: the heading levels +/ +private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; + +private int propertyRank(string name) { + foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } + return -1; +} + +SspFinding[] validateSspFile(string filename, string major_known) { + SspFinding[] found; + auto lines = readText(filename).split("\n"); + + bool seen_banner = false; + bool seen_source_line = false; + string block = ""; // the block currently open, "" when none + int preamble_at = -1; // how far through the preamble order + int section_at = -1; // how far through the section order + bool in_object = false; + int prev_rank = -1; + size_t doc_has_keys = 0; + bool in_text = false; // text lines come after properties + + foreach (i, line; lines) { + size_t no = i + 1; + + /+ ↓ section 2: the version, on the first line +/ + if (no == 1) { + enum banner = "% SiSU Document Abstraction v"; + if (!line.startsWith(banner)) { + found ~= SspFinding(no, "2", "first line is not the version banner"); + } else { + seen_banner = true; + string v = line[banner.length .. $].strip; + auto dot = v.indexOf('.'); + string major = (dot > 0) ? v[0 .. dot] : v; + if (major != major_known) { + found ~= SspFinding(no, "2.1", + "format major v" ~ major ~ ", this validator knows v" ~ major_known); + } + } + continue; + } + if (no == 2) { + if (!line.startsWith("% Source: ")) { + found ~= SspFinding(no, "4", "second line is not \"% Source: \""); + } else { seen_source_line = true; } + continue; + } + + if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } + + /+ ↓ section 3: a comment is the file header only +/ + if (line[0] == '%') { + found ~= SspFinding(no, "3", "a comment outside the two header lines"); + continue; + } + + /+ ↓ block open +/ + if (line[0] == '@') { + auto brace = line.indexOf(" {"); + if (brace < 1 || line[$ - 1] != '{') { + found ~= SspFinding(no, "3", "block open is not \"@name {\""); + continue; + } + string name = line[1 .. brace]; + if (preamble_order.canFind(name)) { + int at = 0; + foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } + if (at <= preamble_at) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); + } + preamble_at = at; + if (section_at >= 0) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); + } + } else if (section_order.canFind(name)) { + int at = 0; + foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } + if (at <= section_at) { + found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); + } + section_at = at; + } else { + found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); + } + block = name; + doc_has_keys = 0; + continue; + } + + if (line[0] == '}') { + if (line.length != 1) { + found ~= SspFinding(no, "3", "block close carries more than \"}\""); + } + /+ ↓ section 5.4: ten keys, always +/ + if (block == "doc_has" && doc_has_keys != 10) { + found ~= SspFinding(no, "5.4", + "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); + } + block = ""; + in_object = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 5: inside a preamble block, " key: value" +/ + if (preamble_order.canFind(block)) { + if (!line.startsWith(" ") || line.indexOf(": ") < 3) { + found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); + } else if (block == "doc_has") { + doc_has_keys++; + } + continue; + } + + /+ ↓ section 6.1: the object declaration line +/ + if (line[0] == '[') { + auto close = line.indexOf("] "); + if (close < 2) { + found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); + continue; + } + string ocn = line[1 .. close]; + foreach (c; ocn) { + if (c < '0' || c > '9') { + found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); + break; + } + } + auto rest = line[close + 2 .. $].split(" "); + if (rest.length == 0 || !kinds.canFind(rest[0])) { + found ~= SspFinding(no, "6.1", + "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); + } else if (rest[0] == "heading") { + if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { + found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); + } else if (!levels.canFind(rest[1][1 .. $])) { + found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); + } + } + in_object = true; + in_text = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 6.3: properties, known and in order +/ + if (line[0] == '.') { + if (!in_object) { + found ~= SspFinding(no, "6", "a property outside an object record"); + continue; + } + if (in_text) { + found ~= SspFinding(no, "6", "a property after the object text began"); + } + auto colon = line.indexOf(": "); + if (colon < 2) { + found ~= SspFinding(no, "3", "property is not \".name: value\""); + continue; + } + string name = line[1 .. colon]; + int rank = propertyRank(name); + if (rank < 0) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); + } else { + /+ ↓ the four repeatable properties may recur at the same rank +/ + if (rank < prev_rank) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); + } + prev_rank = rank; + } + continue; + } + + /+ ↓ section 6.2: text lines +/ + if (line[0] == '|') { + if (!in_object) { + found ~= SspFinding(no, "6.2", "a text line outside an object record"); + } + if (line.length < 2 || line[1] != ' ') { + found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); + } + in_text = true; + continue; + } + + found ~= SspFinding(no, "3", "line of no shape the specification defines"); + } + + if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } + if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } + return found; +} + +void reportValidation(string filename, SspFinding[] found) { + if (found.length == 0) { + writeln("\nvalid: ", filename, + " conforms to the .ssp specification as this validator reads it"); + return; + } + writefln("\n%d finding%s against the specification:", + found.length, (found.length == 1) ? "" : "s"); + foreach (f; found) { + if (f.line > 0) { + writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); + } else { + writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); + } + } +} diff --git a/dlang/src/read/verify.d b/dlang/src/read/verify.d new file mode 100644 index 0000000..ccce612 --- /dev/null +++ b/dlang/src/read/verify.d @@ -0,0 +1,212 @@ +module read.verify; + +import std.algorithm : sort; +import std.array : array, join; +import std.conv : to; +import std.digest : toHexString; +import std.digest.sha : SHA256, digest; +import std.file : dirEntries, exists, isDir, SpanMode, getSize; +import std.path : baseName, buildPath, extension; +import std.stdio : writeln, File; +import std.string : toLower; +import ocda.model; +import read.images : gatherImageRecords; + +/+ ↓ verification: recompute what the .image records claim, against the + images as they stand in a directory (a pod's media/image). + + ssp-reader --verify --image-dir=<pod>/media/image doc.ssp + + Reports one line per distinct file: + + OK won_benkler_2_1.png + MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... + ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) + PRESENT ffa.png (recorded missing, but the file is there) + EXTRA unused.png (in directory, no record in this .ssp) + + Returns the number of failures. EXTRA is reported but is not a failure: + a pod carries the images of every language version of a document, while + one .ssp covers one language. ++/ +int verifyImages(OcdaDocument doc, string image_dir, string output_file) { + string[] lines; + int failures = 0; + int ok = 0; + int mismatch = 0; + int absent = 0; + int extra = 0; + + if (!(image_dir.exists) || !(image_dir.isDir)) { + writeln("ERROR: image directory not found: ", image_dir); + return 1; + } + auto gathered = gatherImageRecords(doc); + bool[string] recorded; + + foreach (e; gathered.entries) { + recorded[e.name] = true; + string pth = buildPath(image_dir, e.name); + if (!(pth.exists)) { + if (e.missing) { + /+ recorded missing, and missing here too: consistent +/ + ok += 1; + lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; + } else { + absent += 1; + failures += 1; + lines ~= "ABSENT " ~ e.name + ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; + } + continue; + } + if (e.missing) { + mismatch += 1; + failures += 1; + lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; + continue; + } + auto file_ = readFileStat(pth); + string[] differs; + if (file_.sha256 != e.sha256) { + differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; + } + if (file_.bytes != e.bytes) { + differs ~= "bytes recorded " ~ e.bytes.to!string + ~ " file " ~ file_.bytes.to!string; + } + /+ px is written only where the dimensions could be read, and read here + only for the formats below, so compare only when both are known +/ + if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 + && (file_.width != e.width || file_.height != e.height)) { + differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string + ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; + } + if (differs.length > 0) { + mismatch += 1; + failures += 1; + lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); + } else { + ok += 1; + lines ~= "OK " ~ e.name; + } + } + + /+ ↓ files in the directory that no record names +/ + string[] in_dir; + foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { + if (!(entry.isFile)) continue; + string nm = entry.name.baseName; + switch (nm.extension.toLower) { + case ".png": case ".jpg": case ".jpeg": case ".gif": + in_dir ~= nm; + break; + default: break; + } + } + in_dir.sort; + foreach (nm; in_dir) { + if (nm !in recorded) { + extra += 1; + lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; + } + } + + string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ " in " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", against " ~ image_dir; + lines = [head, ""] ~ lines; + lines ~= ""; + lines ~= "verify: " ~ ok.to!string ~ " ok, " + ~ mismatch.to!string ~ " mismatch, " + ~ absent.to!string ~ " absent, " + ~ extra.to!string ~ " extra" + ~ ((failures == 0) ? " PASS" : " FAIL"); + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } + return failures; +} + +private struct FileStat_ { + string sha256; + ulong bytes; + int width; + int height; +} + +private FileStat_ readFileStat(string pth) { + FileStat_ ret; + auto f = File(pth, "rb"); + ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); + ret.sha256 = sha.toHexString.to!string; + ret.bytes = pth.getSize; + auto dim = readImageDimensions(pth); + ret.width = dim[0]; + ret.height = dim[1]; + return ret; +} + +/+ ↓ pixel dimensions from the file header, for the formats spine's markup + admits (png, gif, jpg). Header read only, no decoding, and no dependency + beyond phobos. Returns [0, 0] where the header cannot be read +/ +private int[2] readImageDimensions(string pth) { + int[2] ret = [0, 0]; + ubyte[] head; + try { + auto f = File(pth, "rb"); + head = new ubyte[](64 * 1024); + head = f.rawRead(head); + } catch (Exception ex) { + return ret; + } + if (head.length >= 24 + && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { + /+ png: IHDR width & height, big endian, at 16 and 20 +/ + ret[0] = beInt(head[16 .. 20]); + ret[1] = beInt(head[20 .. 24]); + return ret; + } + if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { + /+ gif: logical screen width & height, little endian, at 6 and 8 +/ + ret[0] = head[6] | (head[7] << 8); + ret[1] = head[8] | (head[9] << 8); + return ret; + } + if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { + /+ jpeg: walk the segments to a start of frame, height then width, + big endian, at offsets 5 and 7 of the segment +/ + size_t i = 2; + while (i + 9 < head.length) { + if (head[i] != 0xff) { i += 1; continue; } + ubyte marker = head[i + 1]; + if (marker == 0xd8 || marker == 0x01 + || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } + if (marker == 0xff) { i += 1; continue; } + size_t seg_len = (head[i + 2] << 8) | head[i + 3]; + bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf)); + if (is_sof) { + ret[1] = (head[i + 5] << 8) | head[i + 6]; + ret[0] = (head[i + 7] << 8) | head[i + 8]; + return ret; + } + if (seg_len < 2) break; + i += 2 + seg_len; + } + } + return ret; +} + +private int beInt(const(ubyte)[] b) { + return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); +} |
