diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-24 16:33:53 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-24 16:33:53 -0400 |
| commit | bbe505c8d48e822c012c9820dfe93b83dc61bcab (patch) | |
| tree | 3385f5420959ff20af295c3125a080297555a72b /src | |
| parent | dealing with six properties including .is_of_type (diff) | |
--validate, and a second reader, in Gleam
--validate: the specification, executed. A validator written from
ssp-format-specification.md and from nothing else, so that a file can
be held against the document rather than against an implementation.
Each finding names the specification section it comes from, so a
disagreement can be taken back to the text.
It checks the version banner and its major part, the file shape and
the order of the preamble blocks and the nine sections, the two
spaces of a preamble pair, the object declaration and its sixteen
kinds and eight heading levels, that a property is one of the 42 and
in the emitted order, that text lines carry their prefix, the ten
keys of doc_has, and that no line is of a shape the specification
does not define.
Every .ssp of the markup document samples validates clean. Five
classes of deliberate breakage are caught: an unknown major version, a
property that is not one of the 42, a property out of order, a kind
that is not one of the sixteen, and a block that is neither preamble
nor section. Writing it found two rules the specification implied
and did not state (repeated properties are consecutive; no property
follows the text), and both were written into it.
One repository, a reader per language. The readers are not separate
tools that share a name. They are the same specification implemented
more than once, and the interesting thing about them is that they
agree, so they are published as one thing with that claim at the top
of it. The D reader moves to dlang/; the Gleam reader arrives at
gleam/. The language directory is named for the language and not
for the tool: gleam refuses a hyphenated package name, so ssp-reader
cannot be a gleam package and must be ssp_reader.
A .gitignore for the repository, denying everything and whitelisting
what belongs, covering both languages.
Both build under nix, sandboxed and without network. There is no gleam
builder in nixpkgs and gleam build fetches from Hex, so the Gleam build
uses nix-gleam's builder, kept in gleam/nix-gleam/ (MIT), which fetches
each Hex package by the checksum manifest.toml already carries, with no
hash to maintain; the result is an escript wrapped with erlang. The D
reader depends on nothing beyond phobos, so --skip-registry=all is the
whole of what it takes: ldc by default, dmd available as ssp-reader-dmd
(given gcc14, GCC 15 having introduced nullptr in headers dmd's ImportC
does not yet parse).
(assisted by Claude-Code)
Diffstat (limited to 'src')
| -rw-r--r-- | src/main.d | 148 | ||||
| -rw-r--r-- | src/output_html.d | 388 | ||||
| -rw-r--r-- | src/output_images.d | 114 | ||||
| -rw-r--r-- | src/output_llm_breadcrumbs.d | 182 | ||||
| -rw-r--r-- | src/output_llm_jsonl.d | 121 | ||||
| -rw-r--r-- | src/output_tree.d | 69 | ||||
| -rw-r--r-- | src/report_unknown.d | 41 | ||||
| -rw-r--r-- | src/ssp_parser.d | 292 | ||||
| -rw-r--r-- | src/ssp_types.d | 142 | ||||
| -rw-r--r-- | src/verify_images.d | 212 |
10 files changed, 0 insertions, 1709 deletions
diff --git a/src/main.d b/src/main.d deleted file mode 100644 index 78ceb69..0000000 --- a/src/main.d +++ /dev/null @@ -1,148 +0,0 @@ -import std.getopt; -import std.stdio : writeln, stderr; -import ssp_parser; -import ssp_types; -import output_tree; -import output_html; -import output_llm_breadcrumbs; -import output_llm_jsonl; -import output_images; -import verify_images; -import report_unknown; - -int main(string[] args) { - bool do_tree = false; - bool do_html = false; - bool do_llm_text = false; - bool do_llm_jsonl = false; - bool do_images = false; - bool do_verify = false; - bool do_report_unknown = false; - bool any_version = false; - string image_dir = ""; - string output_file = ""; - - auto helpInfo = getopt(args, - "tree", "output document tree outline", &do_tree, - "html", "output HTML scroll", &do_html, - "llm-text", "output plain text with heading breadcrumbs", &do_llm_text, - "llm-jsonl", "output JSONL with embeddings-ready chunks", &do_llm_jsonl, - "images", "output image manifest (digest, size, pixels)", &do_images, - "verify", "verify .image records against --image-dir", &do_verify, - "image-dir", "image directory to verify against", &image_dir, - "output", "output file (default: stdout)", &output_file, - "report-unknown", "list what was read and not understood", &do_report_unknown, - "any-version", "read a format major version this reader does not know", &any_version, - ); - - if (helpInfo.helpWanted || args.length < 2) { - defaultGetoptPrinter( - "ssp-reader - reads .ssp files and produces output\n\n" - ~ "Usage: ssp-reader [options] <file.ssp>\n", - helpInfo.options, - ); - return 0; - } - - string input_file = args[1]; - - SspDocument doc; - try { - doc = parseSspFile(input_file); - } catch (Exception e) { - stderr.writeln("Error reading ", input_file, ": ", e.msg); - return 2; - } - - /+ ↓ the version gate. A reader checks the major part and refuses what it - does not know: an unknown major means a property may have changed - meaning or gone away, and loading it half populated in silence is the - failure this exists to prevent. --any-version overrides, for looking - at a file rather than trusting one. - +/ - if (doc.format_version.length == 0) { - stderr.writeln("warning: ", input_file, - " declares no format version (no \"% SiSU Document Abstraction v\" line)"); - } else if (doc.formatVersionMajor != ssp_format_major_read) { - if (any_version) { - stderr.writeln("warning: ", input_file, " is format v", doc.format_version, - ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); - } else { - stderr.writeln("error: ", input_file, " is format v", doc.format_version, - ", this reader knows v", ssp_format_major_read, ".x"); - stderr.writeln(" a major version differs: a property may have", - " changed meaning or gone away."); - stderr.writeln(" --any-version reads it regardless."); - return 3; - } - } - - writeln("Parsed: ", input_file); - if (doc.format_version.length > 0) { - writeln(" Format: v", doc.format_version); - } - if (auto _lang = "language" in doc.source) { - writeln(" Source: ", *_lang, - ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", - ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); - } - writeln(" Title: ", doc.meta("title.main")); - writeln(" Author: ", doc.meta("creator.author")); - writeln(" Sections: head=", doc.head.length, - " toc=", doc.toc.length, - " body=", doc.body.length, - " endnotes=", doc.endnotes.length, - " glossary=", doc.glossary.length, - " bibliography=", doc.bibliography.length, - " bookindex=", doc.bookindex.length, - " blurb=", doc.blurb.length, - " tail=", doc.tail.length); - auto image_records = imageRecordCount(doc); - if (image_records > 0) { - writeln(" Images: ", image_records, " record", - (image_records == 1) ? "" : "s"); - } - - if (do_tree) { - outputTree(doc, output_file); - } - - if (do_html) { - outputHtmlScroll(doc, output_file); - } - - if (do_llm_text) { - outputLlmBreadcrumbs(doc, output_file); - } - - if (do_llm_jsonl) { - outputLlmJsonl(doc, output_file); - } - - if (do_images) { - outputImages(doc, output_file); - } - - - int _verify_failures = 0; - if (do_verify) { - if (image_dir.length == 0) { - stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); - return 2; - } - _verify_failures = verifyImages(doc, image_dir, output_file); - } - - if (do_report_unknown) { - reportUnknown(doc); - } - - if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images - && !do_verify && !do_report_unknown) { - writeln("\nUse --tree, --html, --llm-text, --llm-jsonl, --images," - ~ " --verify --image-dir=<dir>" - ~ " or --report-unknown to produce output."); - } - - return (_verify_failures > 0) ? 1 : 0; -} diff --git a/src/output_html.d b/src/output_html.d deleted file mode 100644 index b1c711c..0000000 --- a/src/output_html.d +++ /dev/null @@ -1,388 +0,0 @@ -module output_html; - -import std.array : replace, split, join, appender; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, startsWith; -import ssp_types; - -void outputHtmlScroll(SspDocument doc, string output_file) { - auto buf = appender!(string[]); - - buf ~= htmlHead(doc); - - /+ ↓ title +/ - if (doc.head.length > 0) { - auto title_obj = doc.head[0]; - buf ~= `<div class="title">`; - buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`; - if (doc.meta("title.subtitle").length > 0) - buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`; - buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`; - buf ~= `</div>`; - } - - /+ ↓ table of contents +/ - if (doc.toc.length > 0) { - buf ~= `<div class="toc">`; - buf ~= `<h2>Table of Contents</h2>`; - foreach (obj; doc.toc) { - if (obj.is_a == "toc") { - string text = convertInlineMarkup(obj.text); - int indent = obj.indent_base; - string cls = (indent > 4) ? "toc-sub" : "toc-entry"; - buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`; - } - } - buf ~= `</div>`; - buf ~= `<hr/>`; - } - - /+ ↓ body +/ - buf ~= `<div class="body">`; - foreach (obj; doc.body) { - buf ~= renderObject(obj); - } - buf ~= `</div>`; - - /+ ↓ endnotes +/ - bool has_endnotes = false; - foreach (obj; doc.endnotes) { - if (obj.is_a == "endnote") { has_endnotes = true; break; } - } - if (has_endnotes) { - buf ~= `<div class="endnotes">`; - buf ~= `<hr/>`; - buf ~= `<h2>Endnotes</h2>`; - foreach (obj; doc.endnotes) { - if (obj.is_a == "endnote") { - string text = convertInlineMarkup(obj.text); - string parent_ref = (obj.parent_ocn > 0) - ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>` - : ""; - buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`; - } - } - buf ~= `</div>`; - } - - /+ ↓ glossary +/ - if (doc.glossary.length > 0) { - buf ~= `<div class="glossary">`; - buf ~= `<hr/>`; - buf ~= `<h2>Glossary</h2>`; - foreach (obj; doc.glossary) { - string text = convertInlineMarkup(obj.text); - buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`; - } - buf ~= `</div>`; - } - - /+ ↓ bookindex +/ - bool has_bookindex = false; - foreach (obj; doc.bookindex) { - if (obj.is_a == "bookindex") { has_bookindex = true; break; } - } - if (has_bookindex) { - buf ~= `<div class="bookindex">`; - buf ~= `<hr/>`; - buf ~= `<h2>Book Index</h2>`; - foreach (obj; doc.bookindex) { - if (obj.is_a == "bookindex") { - string text = convertInlineMarkup(obj.text); - buf ~= `<p class="index-entry">` ~ text ~ `</p>`; - } - } - buf ~= `</div>`; - } - - buf ~= htmlTail(); - - /+ ↓ write output +/ - string[] output = buf.data; - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; output) f.writeln(line); - } else { - foreach (line; output) writeln(line); - } -} - -private string[] renderObject(SspObject obj) { - /+ ↓ markup commentary is carried by the abstraction and is not output. - Rendering it would put the author notes into the document, which - spine itself does not do - +/ - if (!obj.renders) { return []; } - string[] lines; - string text = convertInlineMarkup(obj.text); - string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) - ? ` id="o` ~ obj.ocn.to!string ~ `"` - : ""; - string ocn_display = (obj.ocn > 0 && !obj.ocn_off) - ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>` - : ""; - - switch (obj.is_a) { - case "heading": - string tag = headingTag(obj.heading_level); - string seg_anchor = (obj.segment.length > 0) - ? ` id="` ~ htmlEscape(obj.segment) ~ `"` - : ocn_anchor; - lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`; - break; - case "para": - string indent_cls = ""; - if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`; - else if (obj.bullet) indent_cls = ` class="bullet"`; - lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`; - break; - case "code": - lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`; - break; - case "table": - lines ~= renderTable(obj, ocn_anchor); - break; - case "group": - lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`; - break; - case "block": - lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`; - break; - case "verse": - lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`; - break; - case "quote": - lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`; - break; - default: - if (text.length > 0) - lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`; - break; - } - return lines; -} - -private string headingTag(string level) { - switch (level) { - case "A": return "h1"; - case "B": return "h2"; - case "C": return "h3"; - case "D": return "h4"; - case "1": return "h3"; - case "2": return "h4"; - case "3": return "h5"; - case "4": return "h6"; - default: return "h4"; - } -} - -private string renderTable(SspObject obj, string ocn_anchor) { - string[] lines; - lines ~= `<table` ~ ocn_anchor ~ `>`; - /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/ - auto rows = obj.text.split("\n"); - bool first_row = obj.table_header; - foreach (row; rows) { - if (row.strip.length == 0) continue; - string tag = first_row ? "th" : "td"; - lines ~= `<tr>`; - foreach (cell; row.split("┊")) { - lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`; - } - lines ~= `</tr>`; - first_row = false; - } - lines ~= `</table>`; - return lines.join("\n"); -} - -/+ ↓ convert spine internal Unicode markup to HTML +/ -string convertInlineMarkup(string text) { - if (text.length == 0) return ""; - string result = htmlEscape(text); - - /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc. - Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X - The ┨┣ form is used within heading content +/ - - /+ ↓ bold +/ - result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>"); - result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>"); - - /+ ↓ emphasis +/ - result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>"); - result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>"); - - /+ ↓ italic +/ - result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>"); - result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>"); - - /+ ↓ underscore +/ - result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>"); - result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>"); - - /+ ↓ superscript +/ - result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>"); - result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>"); - - /+ ↓ subscript +/ - result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>"); - result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>"); - - /+ ↓ monospace +/ - result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>"); - - /+ ↓ links: ┥link text┝┤url├ +/ - result = replaceLinks(result); - - /+ ↓ footnote markers: 【N text】 +/ - result = replaceFootnotes(result); - - /+ ↓ line breaks +/ - result = result.replace("┘", "<br/>"); - result = result.replace("┙", "<br/>"); - - /+ ↓ non-breaking spaces +/ - result = result.replace("░", " "); - - /+ ↓ bullet markers +/ - result = result.replace("● ", "• "); - - /+ ↓ page break markers +/ - result = result.replace("┼", "").replace("┿", "").replace("╂", ""); - - return result; -} - -private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) { - /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/ - string result = text; - while (true) { - auto open_pos = result.indexOf(open_mark); - if (open_pos < 0) break; - auto after_open = open_pos + open_mark.length; - /+ ↓ look for closing ⑆X +/ - string close_mark = "⑆" ~ close_suffix; - auto close_pos = indexOf(result[after_open .. $], close_mark); - if (close_pos < 0) break; - close_pos += after_open; - string content = result[after_open .. close_pos]; - result = result[0 .. open_pos] ~ html_open ~ content ~ html_close - ~ result[close_pos + close_mark.length .. $]; - } - return result; -} - -private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) { - /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/ - string result = text; - string full_open = open_mark ~ bracket_open; - while (true) { - auto open_pos = result.indexOf(full_open); - if (open_pos < 0) break; - auto after_open = open_pos + full_open.length; - auto close_pos = indexOf(result[after_open .. $], bracket_close); - if (close_pos < 0) break; - close_pos += after_open; - string content = result[after_open .. close_pos]; - result = result[0 .. open_pos] ~ html_open ~ content ~ html_close - ~ result[close_pos + bracket_close.length .. $]; - } - return result; -} - -private string replaceLinks(string text) { - /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/ - string result = text; - while (true) { - auto link_open = result.indexOf("┥"); - if (link_open < 0) break; - auto link_mid = indexOf(result[link_open .. $], "┝┤"); - if (link_mid < 0) break; - link_mid += link_open; - auto link_close = indexOf(result[link_mid .. $], "├"); - if (link_close < 0) break; - link_close += link_mid; - string link_text = result[link_open + "┥".length .. link_mid]; - string url = result[link_mid + "┝┤".length .. link_close]; - /+ ↓ handle internal links (¤ prefix) +/ - if (url.length > 0 && url[0] == '¤') { - url = url[("¤").length .. $]; - } - /+ ↓ handle OCN-only links (#N) +/ - if (url.length > 0 && url[0] == '#') { - url = "#o" ~ url[1 .. $]; - } - result = result[0 .. link_open] - ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>` - ~ result[link_close + "├".length .. $]; - } - return result; -} - -private string replaceFootnotes(string text) { - /+ ↓ replace 【N content】 with superscript note number +/ - string result = text; - while (true) { - auto fn_open = result.indexOf("【"); - if (fn_open < 0) break; - auto fn_close = indexOf(result[fn_open .. $], "】"); - if (fn_close < 0) break; - fn_close += fn_open; - string fn_content = result[fn_open + "【".length .. fn_close]; - /+ ↓ extract note number (first token before space) +/ - auto space_pos = fn_content.indexOf(' '); - string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content; - result = result[0 .. fn_open] - ~ `<sup class="fn">` ~ note_num ~ `</sup>` - ~ result[fn_close + "】".length .. $]; - } - return result; -} - -string htmlEscape(string text) { - return text.replace("&", "&").replace("<", "<").replace(">", ">") - .replace("\"", """); -} - -private string[] htmlHead(SspDocument doc) { - string[] lines; - lines ~= `<!DOCTYPE html>`; - lines ~= `<html>`; - lines ~= `<head>`; - lines ~= `<meta charset="utf-8"/>`; - lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`; - lines ~= `<style>`; - lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`; - lines ~= ` .title { text-align: center; margin-bottom: 2em; }`; - lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`; - lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`; - lines ~= ` .toc { margin-bottom: 2em; }`; - lines ~= ` .toc-entry { margin: 0.2em 0; }`; - lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`; - lines ~= ` .indent-1 { margin-left: 2em; }`; - lines ~= ` .indent-2 { margin-left: 4em; }`; - lines ~= ` .indent-3 { margin-left: 6em; }`; - lines ~= ` .bullet { margin-left: 2em; }`; - lines ~= ` .bullet:before { content: "\2022 "; }`; - lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`; - lines ~= ` .group { margin: 1em 2em; }`; - lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`; - lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`; - lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`; - lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`; - lines ~= ` th { background: #f0f0f0; }`; - lines ~= ` .endnote { font-size: 0.9em; }`; - lines ~= ` .fn { color: #666; }`; - lines ~= ` .index-entry { font-size: 0.9em; }`; - lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`; - lines ~= `</style>`; - lines ~= `</head>`; - lines ~= `<body>`; - return lines; -} - -private string[] htmlTail() { - return [`</body>`, `</html>`]; -} diff --git a/src/output_images.d b/src/output_images.d deleted file mode 100644 index 69fb458..0000000 --- a/src/output_images.d +++ /dev/null @@ -1,114 +0,0 @@ -module output_images; - -import std.array : join; -import std.conv : to; -import std.stdio : writeln, File; -import ssp_types; - -/+ ↓ the .image records of a document, gathered per distinct file, with the - objects referencing each file, in order of first appearance +/ -struct ImageEntry { - string name; - string sha256; - ulong bytes; - int width; - int height; - bool missing; - string[] refs; - string[] other_sha256; // same filename, differing digest -} - -struct ImageRecords { - int records; // .image lines, an image referenced twice counts twice - ImageEntry[] entries; // distinct files -} - -ImageRecords gatherImageRecords(SspDocument doc) { - ImageRecords ret; - size_t[string] seen; - foreach (section; sectionNames) { - auto objs = doc.sectionByName(section); - if (objs is null) continue; - foreach (obj; *objs) { - foreach (img; obj.images) { - ret.records += 1; - string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]"; - if (auto at = img.name in seen) { - if (ret.entries[*at].sha256 != img.sha256 - && img.sha256.length > 0 - && ret.entries[*at].sha256.length > 0) { - ret.entries[*at].other_sha256 ~= img.sha256; - } - ret.entries[*at].refs ~= _ref; - } else { - ImageEntry e_; - e_.name = img.name; - e_.sha256 = img.sha256; - e_.bytes = img.bytes; - e_.width = img.width; - e_.height = img.height; - e_.missing = img.missing; - e_.refs = [_ref]; - seen[img.name] = ret.entries.length; - ret.entries ~= e_; - } - } - } - } - return ret; -} - -/+ ↓ image manifest, line oriented and greppable: - - <sha256> <bytes> <px> <filename> refs: body[97] body[412] - MISSING <filename> refs: body[840] - - The point of the record in the .ssp is verification (is the image - shipped in this pod the image the abstraction was built from), so this - output is the shape a verifier wants: one line per distinct file. -+/ -void outputImages(SspDocument doc, string output_file) { - auto gathered = gatherImageRecords(doc); - string[] lines; - int missing = 0; - foreach (e; gathered.entries) { - if (e.missing) missing += 1; - } - lines ~= "Images: " ~ gathered.records.to!string ~ " record" - ~ ((gathered.records == 1) ? "" : "s") - ~ ", " ~ gathered.entries.length.to!string ~ " distinct file" - ~ ((gathered.entries.length == 1) ? "" : "s") - ~ ", " ~ missing.to!string ~ " missing"; - lines ~= ""; - foreach (e; gathered.entries) { - string px = (e.width > 0 && e.height > 0) - ? e.width.to!string ~ "x" ~ e.height.to!string : "-"; - lines ~= (e.missing) - ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ") - : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px - ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" "); - /+ ↓ one filename, more than one digest: the .ssp is inconsistent, - or two distinct files share a name +/ - foreach (other; e.other_sha256) { - lines ~= " WARNING differing digest for same filename: " ~ other; - } - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -/+ ↓ count of .image records in a document, for the parse summary +/ -int imageRecordCount(SspDocument doc) { - return gatherImageRecords(doc).records; -} - -package immutable string[] sectionNames = [ - "head", "toc", "body", "endnotes", - "glossary", "bibliography", "bookindex", "blurb", -]; diff --git a/src/output_llm_breadcrumbs.d b/src/output_llm_breadcrumbs.d deleted file mode 100644 index 5b240d2..0000000 --- a/src/output_llm_breadcrumbs.d +++ /dev/null @@ -1,182 +0,0 @@ -module output_llm_breadcrumbs; - -import std.array : replace, split, join; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, stripRight; -import ssp_types; - -/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption - format: - ## Document Title > Chapter > Section > [OCN] - - Clean paragraph text with inline markup stripped. -+/ -void outputLlmBreadcrumbs(SspDocument doc, string output_file) { - string[] lines; - string doc_title = doc.meta("title.main"); - - /+ ↓ document header +/ - lines ~= "# " ~ doc_title; - if (doc.meta("title.subtitle").length > 0) - lines ~= "# " ~ doc.meta("title.subtitle"); - lines ~= "# " ~ doc.meta("creator.author"); - lines ~= ""; - - /+ ↓ build heading text lookup from ancestors +/ - string[int] heading_text; - foreach (obj; doc.head) { - if (obj.is_a == "heading" && obj.ocn > 0) - heading_text[obj.ocn] = stripMarkup(obj.text); - } - foreach (obj; doc.body) { - if (obj.is_a == "heading") - heading_text[obj.ocn] = stripMarkup(obj.text); - } - - /+ ↓ body objects +/ - string current_breadcrumb = ""; - foreach (obj; doc.body) { - if (obj.is_a == "heading") { - current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); - lines ~= ""; - lines ~= "## " ~ current_breadcrumb; - lines ~= ""; - continue; - } - - /+ ↓ markup commentary is not content +/ - if (!obj.renders) { continue; } - - string text = stripMarkup(obj.text); - if (text.strip.length == 0) continue; - - string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) - ? " [" ~ obj.ocn.to!string ~ "]" - : ""; - - switch (obj.is_a) { - case "para": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "code": - lines ~= "```"; - lines ~= obj.text; /+ preserve raw text for code +/ - lines ~= "```"; - lines ~= ""; - break; - case "table": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "verse": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - case "quote": - lines ~= "> " ~ text ~ ocn_ref; - lines ~= ""; - break; - case "group": - case "block": - lines ~= text ~ ocn_ref; - lines ~= ""; - break; - default: - if (text.length > 0) { - lines ~= text ~ ocn_ref; - lines ~= ""; - } - break; - } - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private string buildBreadcrumb(string doc_title, string ancestors_text, SspObject obj) { - /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ - string[] parts; - if (ancestors_text.length > 0) { - foreach (part; ancestors_text.split("|")) { - string clean = stripMarkup(part).strip; - if (clean.length > 0 && clean != doc_title) { - /+ ↓ truncate very long heading text for breadcrumb +/ - if (clean.length > 60) - clean = clean[0 .. 57] ~ "..."; - parts ~= clean; - } - } - } - /+ ↓ add the current heading itself +/ - string heading_text = stripMarkup(obj.text).strip; - if (heading_text.length > 60) - heading_text = heading_text[0 .. 57] ~ "..."; - - string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; - - if (parts.length > 0) - return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; - else - return doc_title ~ " > " ~ heading_text ~ ocn_ref; -} - -/+ ↓ strip all internal Unicode markup to produce clean plain text +/ -string stripMarkup(string text) { - if (text.length == 0) return ""; - string r = text; - - /+ ↓ links: ┥link text┝┤url├ -> link text +/ - while (true) { - auto lo = r.indexOf("┥"); - if (lo < 0) break; - auto lm = indexOf(r[lo .. $], "┝┤"); - if (lm < 0) break; - lm += lo; - auto lc = indexOf(r[lm .. $], "├"); - if (lc < 0) break; - lc += lm; - string link_text = r[lo + "┥".length .. lm]; - r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; - } - - /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ - while (true) { - auto fo = r.indexOf("【"); - if (fo < 0) break; - auto fc = indexOf(r[fo .. $], "】"); - if (fc < 0) break; - fc += fo; - r = r[0 .. fo] ~ r[fc + "】".length .. $]; - } - - /+ ↓ font face markers +/ - r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") - .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") - .replace("⑆■", "") - .replace("┨", "").replace("┣*", "").replace("┣!", "") - .replace("┣/", "").replace("┣_", "").replace("┣^", "") - .replace("┣,", "").replace("┣", ""); - - /+ ↓ line breaks and special chars +/ - r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); - r = r.replace("░", " "); - r = r.replace("● ", "- "); - r = r.replace("┼", "").replace("┿", "").replace("╂", ""); - - /+ ↓ table cell separators +/ - r = r.replace("┊", " | "); - - /+ ↓ clean up multiple spaces +/ - while (r.indexOf(" ") >= 0) - r = r.replace(" ", " "); - - return r.strip; -} diff --git a/src/output_llm_jsonl.d b/src/output_llm_jsonl.d deleted file mode 100644 index 3d34089..0000000 --- a/src/output_llm_jsonl.d +++ /dev/null @@ -1,121 +0,0 @@ -module output_llm_jsonl; - -import std.array : split, join, replace; -import std.conv : to; -import std.stdio : writeln, File; -import std.string : strip, indexOf, stripRight; -import ssp_types; -import output_llm_breadcrumbs : stripMarkup; - -/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines - one JSON object per line, each containing: - doc, ocn, path (breadcrumb), type, text (clean), - images (only where the object embeds one, so that a chunk whose - text reads around a figure carries the figure's identity), - index_terms (from bookindex entries associated with this object) -+/ -void outputLlmJsonl(SspDocument doc, string output_file) { - string[] lines; - string doc_id = doc.meta("title.main"); - string doc_author = doc.meta("creator.author"); - - /+ ↓ build bookindex -> OCN mapping for index_terms - bookindex objects in the .ssp have parent_ocn linking to body objects - but the index terms are in the text of bookindex objects, not - directly attached to body objects. We use a simpler approach: - headings carry heading_ancestors_text which gives context. +/ - - /+ ↓ track current heading path +/ - string current_path = ""; - - foreach (obj; doc.body) { - if (obj.is_a == "heading") { - current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); - /+ ↓ emit heading as a chunk too +/ - string heading_text = stripMarkup(obj.text); - if (heading_text.strip.length > 0) { - lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); - } - continue; - } - - /+ ↓ markup commentary is not content, and least of all is it a chunk - to embed: it would be indexed as though the author had written it - into the document - +/ - if (!obj.renders) { continue; } - - string text = stripMarkup(obj.text); - if (text.strip.length == 0) continue; - - lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, SspImage[] images) { - return `{"doc":` ~ jsonStr(doc_id) - ~ `,"author":` ~ jsonStr(doc_author) - ~ `,"ocn":` ~ ocn.to!string - ~ `,"path":` ~ jsonStr(path) - ~ `,"type":` ~ jsonStr(type) - ~ `,"text":` ~ jsonStr(text) - ~ jsonImages(images) - ~ `}`; -} - -private string jsonImages(SspImage[] images) { - /+ ↓ omitted entirely where the object embeds no image +/ - if (images.length == 0) return ""; - string[] items; - foreach (img; images) { - string item = `{"name":` ~ jsonStr(img.name); - if (img.missing) { - item ~= `,"missing":true`; - } else { - item ~= `,"sha256":` ~ jsonStr(img.sha256) - ~ `,"bytes":` ~ img.bytes.to!string; - if (img.width > 0 && img.height > 0) { - item ~= `,"width":` ~ img.width.to!string - ~ `,"height":` ~ img.height.to!string; - } - } - items ~= item ~ `}`; - } - return `,"images":[` ~ items.join(",") ~ `]`; -} - -private string buildPath(string doc_title, string ancestors_text, SspObject obj) { - string[] parts; - if (ancestors_text.length > 0) { - foreach (part; ancestors_text.split("|")) { - string clean = stripMarkup(part).strip; - if (clean.length > 0 && clean != doc_title) { - parts ~= clean; - } - } - } - string heading_text = stripMarkup(obj.text).strip; - if (parts.length > 0) - return parts.join(" > ") ~ " > " ~ heading_text; - else - return heading_text; -} - -private string jsonStr(string s) { - /+ ↓ JSON string escaping +/ - string r = s; - r = r.replace(`\`, `\\`); - r = r.replace(`"`, `\"`); - r = r.replace("\n", `\n`); - r = r.replace("\r", `\r`); - r = r.replace("\t", `\t`); - return `"` ~ r ~ `"`; -} diff --git a/src/output_tree.d b/src/output_tree.d deleted file mode 100644 index 88c1b4c..0000000 --- a/src/output_tree.d +++ /dev/null @@ -1,69 +0,0 @@ -module output_tree; - -import std.conv : to; -import std.stdio : writeln, writef, writefln, File; -import std.array : replicate; -import ssp_types; - -void outputTree(SspDocument doc, string output_file) { - string[] lines; - - lines ~= "Document: " ~ doc.meta("title.main"); - if (doc.meta("title.subtitle").length > 0) - lines ~= " " ~ doc.meta("title.subtitle"); - lines ~= "Author: " ~ doc.meta("creator.author"); - lines ~= ""; - - /+ ↓ build OCN -> object lookup for body headings +/ - SspObject[int] headings; - foreach (obj; doc.body) { - if (obj.is_a == "heading" && obj.ocn > 0) { - headings[obj.ocn] = obj; - } - } - - /+ ↓ find root heading (from head section, or first body heading) +/ - if (doc.head.length > 0 && doc.head[0].is_a == "heading") { - auto root = doc.head[0]; - lines ~= formatHeading(root, 0); - walkChildren(root, headings, lines, 1); - } - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } -} - -private void walkChildren(SspObject parent, ref SspObject[int] headings, ref string[] lines, int depth) { - foreach (child_ocn; parent.children) { - if (auto p = child_ocn in headings) { - lines ~= formatHeading(*p, depth); - walkChildren(*p, headings, lines, depth + 1); - } - } -} - -private string formatHeading(SspObject obj, int depth) { - string indent = replicate(" ", depth); - string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; - string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; - /+ ↓ strip internal markup from heading text for clean display +/ - string text = stripInternalMarkup(obj.text); - return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; -} - -private string stripInternalMarkup(string text) { - /+ ↓ simple removal of common internal markers for display +/ - import std.regex : replaceAll, regex; - import std.array : replace; - string result = text; - result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") - .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") - .replace("┨", "").replace("┣", "") - .replace("┙", " ").replace("┘", "\n"); - return result; -} diff --git a/src/report_unknown.d b/src/report_unknown.d deleted file mode 100644 index 1d5238e..0000000 --- a/src/report_unknown.d +++ /dev/null @@ -1,41 +0,0 @@ -module report_unknown; - -import std.algorithm : sort; -import std.array : array; -import std.stdio : writeln, writefln; -import ssp_types; - -/+ ↓ say what was read and not understood. - This is what makes the reader usable as a check on the format - specification: a reader that ignores what it does not know is - conforming, but one that cannot say what it ignored cannot tell you - whether the specification it was written from was complete. Silence - and understanding look the same from outside. -+/ -void reportUnknown(ref SspDocument doc) { - if (!doc.unknown.any) { - writeln("\nread and understood: every property, block and line was known"); - return; - } - writeln("\nread and NOT understood:"); - if (doc.unknown.property.length > 0) { - writeln(" properties ignored (", doc.unknown.property.length, " distinct):"); - foreach (name; doc.unknown.property.keys.sort) { - writefln(" .%-26s %8d", name, doc.unknown.property[name]); - } - } - if (doc.unknown.block.length > 0) { - writeln(" blocks ignored (", doc.unknown.block.length, " distinct):"); - foreach (name; doc.unknown.block.keys.sort) { - writefln(" @%-26s %8d", name, doc.unknown.block[name]); - } - } - if (doc.unknown.line.length > 0) { - writeln(" lines of no known shape, by first character:"); - foreach (c; doc.unknown.line.keys.sort) { - writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]); - } - } - writeln(" (ignoring these is conforming within a major version;", - " the count is so that it is not silent)"); -} diff --git a/src/ssp_parser.d b/src/ssp_parser.d deleted file mode 100644 index 1a1efe2..0000000 --- a/src/ssp_parser.d +++ /dev/null @@ -1,292 +0,0 @@ -module ssp_parser; - -import std.algorithm : splitter, startsWith; -import std.array : array, split; -import std.conv : to, ConvException; -import std.file : readText; -import std.stdio : writeln; -import std.string : strip, stripLeft, indexOf; -import ssp_types; - -SspDocument parseSspFile(string filename) { - string content = readText(filename); - return parseSsp(content); -} - -SspDocument parseSsp(string content) { - SspDocument doc; - SspObject current_obj; - bool in_object = false; - string current_section = ""; - string current_block = ""; // meta, make, doc_has, or section name - - foreach (line; content.splitter("\n")) { - /+ ↓ comment. The first line of a .ssp declares the format version, and - it is the one comment that carries meaning: a reader checks the - major part and refuses what it does not know rather than loading - an unfamiliar artefact half populated in silence. - +/ - if (line.length > 0 && line[0] == '%') { - enum _banner = "% SiSU Document Abstraction v"; - if (doc.format_version.length == 0 && line.startsWith(_banner)) { - doc.format_version = line[_banner.length .. $].strip; - } - continue; - } - - /+ ↓ blank line - finalize current object +/ - if (line.length == 0 || line.strip.length == 0) { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = SspObject.init; - in_object = false; - } - continue; - } - - /+ ↓ section/block open +/ - if (line[0] == '@') { - auto brace_pos = line.indexOf('{'); - if (brace_pos > 0) { - string name = line[1 .. brace_pos].strip; - current_block = name; - if (!isPreambleBlock(name)) { - current_section = name; - /+ ↓ a block that is neither preamble nor a section this reader - knows: counted, so that --report-unknown can say so +/ - if (doc.sectionByName(name) is null) { - doc.unknown.block[name]++; - } - } - } - continue; - } - - /+ ↓ section/block close +/ - if (line[0] == '}') { - if (in_object) { - commitObject(doc, current_section, current_obj); - current_obj = SspObject.init; - in_object = false; - } - current_block = ""; - continue; - } - - /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ - if (isPreambleBlock(current_block)) { - string stripped = line.stripLeft; - auto colon_pos = stripped.indexOf(':'); - if (colon_pos > 0) { - string key = stripped[0 .. colon_pos].strip; - string value = (colon_pos + 1 < stripped.length) - ? stripped[colon_pos + 1 .. $].strip : ""; - if (current_block == "meta") { doc.metadata[key] = value; } - else if (current_block == "make") { doc.make[key] = value; } - else if (current_block == "source") { doc.source[key] = value; } - else { doc.doc_has[key] = value; } - } - continue; - } - - /+ ↓ object declaration: [N] type ... +/ - if (line[0] == '[') { - if (in_object) { - commitObject(doc, current_section, current_obj); - } - current_obj = SspObject.init; - current_obj.section = current_section; - in_object = true; - parseObjectDeclaration(line, current_obj); - continue; - } - - /+ ↓ property: .name: value +/ - if (line[0] == '.') { - if (in_object) { - parseProperty(line, current_obj, doc.unknown); - } - continue; - } - - /+ ↓ text content: | content +/ - if (line.length >= 1 && line[0] == '|') { - if (in_object) { - string text_content = (line.length > 2) ? line[2 .. $] : ""; - if (current_obj.text.length > 0) { - current_obj.text ~= "\n" ~ text_content; - } else { - current_obj.text = text_content; - } - } - continue; - } - - /+ ↓ no known shape. Counted by its first character so that a reader - can be told the file held something this one did not expect - +/ - doc.unknown.line[line[0 .. 1].idup]++; - } - - /+ ↓ finalize last object +/ - if (in_object) { - commitObject(doc, current_section, current_obj); - } - - return doc; -} - -private void parseObjectDeclaration(string line, ref SspObject obj) { - /+ parse: [N] type :level identifier +/ - auto bracket_close = line.indexOf(']'); - if (bracket_close < 0) return; - try { - obj.ocn = line[1 .. bracket_close].strip.to!int; - } catch (ConvException) { - obj.ocn = 0; - } - string rest = (bracket_close + 1 < line.length) - ? line[bracket_close + 1 .. $].strip : ""; - if (rest.length == 0) return; - - auto parts = rest.split(" "); - if (parts.length == 0) return; - obj.is_a = parts[0]; - - if (obj.is_a == "heading" && parts.length >= 2) { - /+ parse :A, :B, :1, etc. +/ - string lev = parts[1]; - if (lev.length >= 2 && lev[0] == ':') { - obj.heading_level = lev[1 .. $]; - } - /+ optional identifier after level +/ - if (parts.length >= 3) { - obj.identifier = parts[2]; - } - } -} - -/+ ↓ the blocks that are document preamble rather than sections of objects +/ -private bool isPreambleBlock(string name) { - return name == "source" || name == "meta" || name == "make" || name == "doc_has"; -} - -private void parseProperty(string line, ref SspObject obj, ref SspUnknown unknown) { - /+ parse: .name: value +/ - auto colon_pos = line.indexOf(':'); - if (colon_pos < 1) return; - string name = line[1 .. colon_pos].strip; - string value = (colon_pos + 1 < line.length) - ? line[colon_pos + 1 .. $].strip : ""; - - switch (name) { - case "part": obj.part = value; break; - case "is_of_type": obj.is_of_type = value; break; - case "section": obj.own_section = value; break; - case "identifier": obj.identifier = value; break; - case "segment_html_is": obj.segment_html_is = value; break; - case "table_aligns": obj.table_aligns = value; break; - case "stow_link": obj.stow_link ~= value; break; - case "parent": obj.parent_ocn = toIntSafe(value); break; - case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; - case "children": obj.children = parseIntArray(value); break; - case "ancestors": obj.ancestors = parseIntArray(value); break; - case "dummy": obj.dummy_heading = (value == "true"); break; - case "ocn_off": obj.ocn_off = (value == "true"); break; - case "indent": - auto parts = value.split(" "); - if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); - if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); - break; - case "bullet": obj.bullet = (value == "true"); break; - case "lang": obj.lang = value; break; - case "has": - foreach (flag; value.split(" ")) { - switch (flag) { - case "links": obj.has_links = true; break; - case "notes_reg": obj.has_notes_reg = true; break; - case "notes_star": obj.has_notes_star = true; break; - case "images": obj.has_images = true; break; - default: break; - } - } - break; - case "segment": obj.segment = value; break; - case "segment_prev": obj.segment_prev = value; break; - case "segment_next": obj.segment_next = value; break; - case "anchor": obj.anchor = value; break; - case "table_cols": obj.table_cols = toIntSafe(value); break; - case "table_widths": obj.table_widths = value; break; - case "table_header": obj.table_header = (value == "true"); break; - case "code_syntax": obj.code_syntax = value; break; - case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; - case "heading_ancestors_text": obj.heading_ancestors_text = value; break; - case "sha256": obj.sha256 = value; break; - case "image": obj.images ~= parseImageRecord(value); break; - /+ ↓ a property this reader has no use for, or has never heard of. - Ignored, which is the conforming behaviour within a major - version, but counted rather than passed over in silence - +/ - default: unknown.property[name]++; break; - } -} - -private SspImage parseImageRecord(string value) { - /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] - filename first, remaining fields order-independent; unrecognised - fields are kept rather than discarded +/ - SspImage img; - bool first = true; - foreach (field; value.split(" ")) { - if (field.length == 0) continue; - if (first) { - img.name = field; - first = false; - continue; - } - auto colon_pos = field.indexOf(':'); - if (colon_pos < 1) continue; - string key = field[0 .. colon_pos]; - string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; - switch (key) { - case "sha256": img.sha256 = val; break; - case "bytes": img.bytes = toUlongSafe(val); break; - case "px": - auto dims = val.split("x"); - if (dims.length == 2) { - img.width = toIntSafe(dims[0]); - img.height = toIntSafe(dims[1]); - } - break; - case "missing": img.missing = (val == "true"); break; - default: img.fields[key] = val; break; - } - } - return img; -} - -private void commitObject(ref SspDocument doc, string section, ref SspObject obj) { - auto sec = doc.sectionByName(section); - if (sec !is null) { - *sec ~= obj; - } -} - -private int toIntSafe(string s) { - try { return s.strip.to!int; } - catch (ConvException) { return 0; } -} - -private ulong toUlongSafe(string s) { - try { return s.strip.to!ulong; } - catch (ConvException) { return 0; } -} - -private int[] parseIntArray(string s) { - int[] result; - foreach (part; s.split(" ")) { - auto v = toIntSafe(part); - result ~= v; - } - return result; -} diff --git a/src/ssp_types.d b/src/ssp_types.d deleted file mode 100644 index 3960df0..0000000 --- a/src/ssp_types.d +++ /dev/null @@ -1,142 +0,0 @@ -module ssp_types; - -struct SspImage { - string name; // image filename as referenced by the object - string sha256; // hex-encoded (uppercase) digest of the file - ulong bytes; // file size - int width; // file's own pixel width (not display width) - int height; // file's own pixel height - bool missing; // file absent or unreadable, no sha256/bytes - string[string] fields; // any further/unrecognised key:value fields -} - -struct SspObject { - int ocn; - string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb - string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb - string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) - string identifier; // heading: from the declaration line; otherwise .identifier - string part; // frontmatter, body, backmatter - string is_of_type; // para, block, comment: how the text behaves - string own_section; // .section, when it differs from the block it sits in - string segment_html_is;// the html segment this object sits in - string table_aligns; // l c r per column - string[] stow_link; // links extracted from the object - int parent_ocn; - int last_descendant_ocn; - int[] children; // direct child heading OCNs - int[] ancestors; // markedup_ancestors[8] - bool dummy_heading; - bool ocn_off; - int indent_base; - int indent_hang; - bool bullet; - string lang; - bool has_links; - bool has_notes_reg; - bool has_notes_star; - bool has_images; - string segment; - string segment_prev; - string segment_next; - string anchor; - int table_cols; - string table_widths; - bool table_header; - string code_syntax; - bool code_linenumbers; - string heading_ancestors_text; - string sha256; - SspImage[] images; // repeatable .image records, in order - string text; // content (lines joined with newline) - - /+ ↓ whether this object is rendered at all. - is_of_type "comment" is markup commentary: carried by the - abstraction so that nothing is lost, and not put into output. An - output that renders it leaks the author notes into the document, - which is what spine itself does not do. - +/ - bool renders() const { - return is_of_type != "comment"; - } - - /+ ↓ the section this object belongs to, which is the block it sits in - unless it says otherwise +/ - string sectionOf() const { - return (own_section.length > 0) ? own_section : section; - } -} - -/+ ↓ what the reader met and did not understand. - A reader that silently drops what it does not know cannot say whether - it understood the document, and cannot be used to check a - specification against an implementation: silence and understanding - look alike. So the parser counts what it ignored, and - --report-unknown prints it. -+/ -struct SspUnknown { - size_t[string] property; // .name -> times seen - size_t[string] block; // @name -> times seen - size_t[string] line; // first char -> times seen, lines of no known shape - - bool any() const { - return property.length > 0 || block.length > 0 || line.length > 0; - } -} - -struct SspDocument { - string format_version; // as declared on line 1, "" when absent - string[string] source; // @source: language, languages, digest - SspUnknown unknown; // what was met and not understood - string[string] metadata; // key -> value - string[string] make; // key -> value - string[string] doc_has; // key -> value - SspObject[] head; - SspObject[] toc; - SspObject[] body; - SspObject[] endnotes; - SspObject[] glossary; - SspObject[] bibliography; - SspObject[] bookindex; - SspObject[] blurb; - SspObject[] tail; - - /+ ↓ the major part of the declared version, "" when none was declared +/ - string formatVersionMajor() const { - import std.string : indexOf; - if (format_version.length == 0) { return ""; } - auto dot = format_version.indexOf('.'); - return (dot > 0) ? format_version[0 .. dot] : format_version; - } - - string meta(string key) { - if (auto p = key in metadata) return *p; - return ""; - } - - SspObject[]* sectionByName(string name) { - switch (name) { - case "head": return &head; - case "toc": return &toc; - case "body": return &body; - case "endnotes": return &endnotes; - case "glossary": return &glossary; - case "bibliography": return &bibliography; - case "bookindex": return &bookindex; - case "blurb": return &blurb; - case "tail": return &tail; - default: return null; - } - } -} - -/+ ↓ the format major version this reader knows. - The .ssp and the .ocda.db carry one version between them, declared in - band: the .ssp on its first line, the database in its schema.version - row. The minor part is raised when a property is added, and a reader of - the newer format still reads the older one, so only the major part is - checked here. Raise this when the reader has been brought to a new - major, not before: the point of the gate is to refuse what has not been - looked at. -+/ -enum ssp_format_major_read = "2"; diff --git a/src/verify_images.d b/src/verify_images.d deleted file mode 100644 index 216c02a..0000000 --- a/src/verify_images.d +++ /dev/null @@ -1,212 +0,0 @@ -module verify_images; - -import std.algorithm : sort; -import std.array : array, join; -import std.conv : to; -import std.digest : toHexString; -import std.digest.sha : SHA256, digest; -import std.file : dirEntries, exists, isDir, SpanMode, getSize; -import std.path : baseName, buildPath, extension; -import std.stdio : writeln, File; -import std.string : toLower; -import ssp_types; -import output_images : gatherImageRecords; - -/+ ↓ verification: recompute what the .image records claim, against the - images as they stand in a directory (a pod's media/image). - - ssp-reader --verify --image-dir=<pod>/media/image doc.ssp - - Reports one line per distinct file: - - OK won_benkler_2_1.png - MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... - ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) - PRESENT ffa.png (recorded missing, but the file is there) - EXTRA unused.png (in directory, no record in this .ssp) - - Returns the number of failures. EXTRA is reported but is not a failure: - a pod carries the images of every language version of a document, while - one .ssp covers one language. -+/ -int verifyImages(SspDocument doc, string image_dir, string output_file) { - string[] lines; - int failures = 0; - int ok = 0; - int mismatch = 0; - int absent = 0; - int extra = 0; - - if (!(image_dir.exists) || !(image_dir.isDir)) { - writeln("ERROR: image directory not found: ", image_dir); - return 1; - } - auto gathered = gatherImageRecords(doc); - bool[string] recorded; - - foreach (e; gathered.entries) { - recorded[e.name] = true; - string pth = buildPath(image_dir, e.name); - if (!(pth.exists)) { - if (e.missing) { - /+ recorded missing, and missing here too: consistent +/ - ok += 1; - lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; - } else { - absent += 1; - failures += 1; - lines ~= "ABSENT " ~ e.name - ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; - } - continue; - } - if (e.missing) { - mismatch += 1; - failures += 1; - lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; - continue; - } - auto file_ = readFileStat(pth); - string[] differs; - if (file_.sha256 != e.sha256) { - differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; - } - if (file_.bytes != e.bytes) { - differs ~= "bytes recorded " ~ e.bytes.to!string - ~ " file " ~ file_.bytes.to!string; - } - /+ px is written only where the dimensions could be read, and read here - only for the formats below, so compare only when both are known +/ - if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 - && (file_.width != e.width || file_.height != e.height)) { - differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string - ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; - } - if (differs.length > 0) { - mismatch += 1; - failures += 1; - lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); - } else { - ok += 1; - lines ~= "OK " ~ e.name; - } - } - - /+ ↓ files in the directory that no record names +/ - string[] in_dir; - foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { - if (!(entry.isFile)) continue; - string nm = entry.name.baseName; - switch (nm.extension.toLower) { - case ".png": case ".jpg": case ".jpeg": case ".gif": - in_dir ~= nm; - break; - default: break; - } - } - in_dir.sort; - foreach (nm; in_dir) { - if (nm !in recorded) { - extra += 1; - lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; - } - } - - string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" - ~ ((gathered.entries.length == 1) ? "" : "s") - ~ " in " ~ gathered.records.to!string ~ " record" - ~ ((gathered.records == 1) ? "" : "s") - ~ ", against " ~ image_dir; - lines = [head, ""] ~ lines; - lines ~= ""; - lines ~= "verify: " ~ ok.to!string ~ " ok, " - ~ mismatch.to!string ~ " mismatch, " - ~ absent.to!string ~ " absent, " - ~ extra.to!string ~ " extra" - ~ ((failures == 0) ? " PASS" : " FAIL"); - - /+ ↓ write output +/ - if (output_file.length > 0) { - auto f = File(output_file, "w"); - foreach (line; lines) f.writeln(line); - } else { - foreach (line; lines) writeln(line); - } - return failures; -} - -private struct FileStat_ { - string sha256; - ulong bytes; - int width; - int height; -} - -private FileStat_ readFileStat(string pth) { - FileStat_ ret; - auto f = File(pth, "rb"); - ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); - ret.sha256 = sha.toHexString.to!string; - ret.bytes = pth.getSize; - auto dim = readImageDimensions(pth); - ret.width = dim[0]; - ret.height = dim[1]; - return ret; -} - -/+ ↓ pixel dimensions from the file header, for the formats spine's markup - admits (png, gif, jpg). Header read only, no decoding, and no dependency - beyond phobos. Returns [0, 0] where the header cannot be read +/ -private int[2] readImageDimensions(string pth) { - int[2] ret = [0, 0]; - ubyte[] head; - try { - auto f = File(pth, "rb"); - head = new ubyte[](64 * 1024); - head = f.rawRead(head); - } catch (Exception ex) { - return ret; - } - if (head.length >= 24 - && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { - /+ png: IHDR width & height, big endian, at 16 and 20 +/ - ret[0] = beInt(head[16 .. 20]); - ret[1] = beInt(head[20 .. 24]); - return ret; - } - if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { - /+ gif: logical screen width & height, little endian, at 6 and 8 +/ - ret[0] = head[6] | (head[7] << 8); - ret[1] = head[8] | (head[9] << 8); - return ret; - } - if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { - /+ jpeg: walk the segments to a start of frame, height then width, - big endian, at offsets 5 and 7 of the segment +/ - size_t i = 2; - while (i + 9 < head.length) { - if (head[i] != 0xff) { i += 1; continue; } - ubyte marker = head[i + 1]; - if (marker == 0xd8 || marker == 0x01 - || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } - if (marker == 0xff) { i += 1; continue; } - size_t seg_len = (head[i + 2] << 8) | head[i + 3]; - bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) - || (marker >= 0xc5 && marker <= 0xc7) - || (marker >= 0xc9 && marker <= 0xcb) - || (marker >= 0xcd && marker <= 0xcf)); - if (is_sof) { - ret[1] = (head[i + 5] << 8) | head[i + 6]; - ret[0] = (head[i + 7] << 8) | head[i + 8]; - return ret; - } - if (seg_len < 2) break; - i += 2 + seg_len; - } - } - return ret; -} - -private int beInt(const(ubyte)[] b) { - return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); -} |
