summaryrefslogtreecommitdiffhomepage
path: root/dlang
diff options
context:
space:
mode:
Diffstat (limited to 'dlang')
-rw-r--r--dlang/README.md162
-rw-r--r--dlang/dub.json15
-rw-r--r--dlang/flake.lock27
-rw-r--r--dlang/flake.nix67
-rw-r--r--dlang/src/main.d158
-rw-r--r--dlang/src/output_html.d388
-rw-r--r--dlang/src/output_images.d114
-rw-r--r--dlang/src/output_llm_breadcrumbs.d182
-rw-r--r--dlang/src/output_llm_jsonl.d121
-rw-r--r--dlang/src/output_tree.d69
-rw-r--r--dlang/src/report_unknown.d41
-rw-r--r--dlang/src/ssp_parser.d292
-rw-r--r--dlang/src/ssp_types.d142
-rw-r--r--dlang/src/validate.d264
-rw-r--r--dlang/src/verify_images.d212
15 files changed, 2254 insertions, 0 deletions
diff --git a/dlang/README.md b/dlang/README.md
new file mode 100644
index 0000000..fa39eab
--- /dev/null
+++ b/dlang/README.md
@@ -0,0 +1,162 @@
+# ssp-reader
+
+Reader for the **`.ssp` document-abstraction text format** emitted
+by [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine).
+
+Reads a `.ssp` file (spine's PEG-parsable serialisation of its
+internal document abstraction) and renders it into a chosen output:
+an outline tree, scroll HTML, plain text with heading breadcrumbs,
+JSONL chunks suitable for LLM / embedding pipelines, or an image
+manifest for verifying the images a document embeds.
+
+## What this is for
+
+- Inspecting and debugging the abstraction stage of spine without
+ rebuilding spine itself.
+- Round-trip experiments: markup -> spine -> `.ssp` -> ssp-reader
+ -> downstream artefact.
+- Re-using spine's parsed document structure outside the spine
+ binary: feeding it to an indexer, an LLM pipeline, or a custom
+ renderer.
+- A worked example of consuming `.ssp` for anyone implementing
+ another reader in another language.
+
+## Audience
+
+Developers building tools that consume spine's output: search
+indexers, LLM ingestion pipelines, custom renderers, alternative
+output formats, and anyone studying the abstraction stage of
+spine itself.
+
+End-user document authors do **not** need this tool to publish a
+document. Run spine directly for that.
+
+## Relationship to spine
+
+Spine's processing pipeline has three stages:
+
+1. **Input:** parse `.sst` / `.ssm` markup.
+2. **Abstraction:** build a numbered object model (the "spine
+ abstraction"; one of spine's named outputs is `.ssp`, a
+ PEG-parsable text representation of this model).
+3. **Output:** render to HTML / EPUB / PDF / SQLite / etc.
+
+`ssp-reader` picks up the abstraction at the end of stage 2 (the
+`.ssp` file) and runs alternative stage-3 renderings on top of it.
+The `.ssp` format itself is documented in
+[`doc-abstraction-format`](https://git.sisudoc.org/tools/doc-abstraction-format).
+
+## Quickstart
+
+### Build
+
+```sh
+nix build # the reader, with ldc
+./result/bin/ssp-reader <file.ssp>
+```
+
+The reader depends on nothing beyond phobos, so `nix build` needs no
+network and works in a sandbox. `nix build .#ssp-reader-dmd` builds it with
+dmd instead, which is given gcc14 for the same reason spine gives it: GCC
+15 introduced `nullptr` in its headers and dmd's ImportC parser does not
+yet handle it.
+
+For development, or to build without nix:
+
+```sh
+nix develop # ldc, dub, dtools
+dub build --compiler=ldmd2
+```
+
+The binary then lands at `./bin/ssp-reader`.
+
+### Run
+
+```sh
+# generate a .ssp from a sample document via spine, then:
+ssp-reader --tree doc.ssp # outline view
+ssp-reader --html doc.ssp # HTML scroll output
+ssp-reader --llm-text doc.ssp # plain text with breadcrumbs
+ssp-reader --llm-jsonl doc.ssp # JSONL chunks for embeddings
+ssp-reader --images doc.ssp # image manifest (digest, size, pixels)
+ssp-reader --verify --image-dir=pod/media/image doc.ssp # check images against the records
+ssp-reader --report-unknown doc.ssp # what it read and did not understand
+ssp-reader --tree --output=out.txt doc.ssp
+ssp-reader --help
+```
+
+## Format version
+
+The reader knows abstraction format **2.x**. Every `.ssp` declares its
+version on its first line, and the reader checks the **major** part and
+refuses what it does not know: an unknown major means a property may have
+changed meaning or gone away, and loading such a file half populated in
+silence is the failure the check exists to prevent. `--any-version` reads
+it regardless, for looking at a file rather than trusting one.
+
+## What it does not understand
+
+Within a major version a reader **should** ignore properties it does not
+know, since the minor part is raised precisely by adding them. This reader
+does, and `--report-unknown` says what it ignored and how often:
+
+```
+read and NOT understood:
+ properties ignored (15 distinct):
+ .is_of_type 328
+ .segment_html_is 138
+ ...
+```
+
+### What it ignores, and why
+
+Twelve properties are ignored across the reference collection, and each is
+ignored for a reason rather than by oversight:
+
+| ignored | why |
+|--------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------|
+| `.ancestors_collapsed`, `.parent_lev`, `.heading_lev_collapsed`, `.dom_status`, `.dom_status_collapsed` | level bookkeeping; this reader works from `.ancestors` and the heading level |
+| `.segment_epub`, `.segment_epub_is`, `.segment_lv4_is`, `.heading_lev_anchor`, `.anchor_tag`, `.lev4_subtoc` | naming for segmented HTML and EPUB output, which this reader does not produce |
+| `.attrib` | one occurrence in the whole collection, and no output uses it |
+
+The list was longer. `--report-unknown` found that `.is_of_type` was
+ignored, and **that was a defect**: objects of type `comment` are markup
+commentary, carried by the abstraction and not put into output, and three
+of the four outputs were rendering them. A markup comment was appearing in
+the HTML, in the plain text, and as an embedding chunk in the JSONL.
+Fixed, and the reason the report exists.
+
+That is deliberate and is what makes the reader usable as a check on the
+[format specification](https://git.sisudoc.org/specs/doc-abstraction-format):
+a reader that drops what it does not know is conforming, but one that
+cannot say what it dropped cannot tell you whether the specification it was
+written from was complete. Silence and understanding look alike otherwise.
+
+## Documentation
+
+- The `.ssp` format itself: see
+ [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format).
+- Source: [`src/`](src/) - one module per output format
+ (`output_tree.d`, `output_html.d`, `output_llm_breadcrumbs.d`,
+ `output_llm_jsonl.d`, `output_images.d`, `verify_images.d`), plus
+ `ssp_parser.d` and `ssp_types.d`.
+
+## License
+
+AGPL-3.0-or-later. See [`dub.json`](dub.json) and the parent
+[sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine)
+project for full licence text.
+
+## Repository
+
+- **Canonical URL:** https://git.sisudoc.org/tools/ssp-reader
+- **Clone:** `git clone --depth=1 git://git.sisudoc.org/tools/ssp-reader`
+
+## Related projects
+
+- [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine) -
+ produces the `.ssp` files this tool consumes.
+- [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format) -
+ specification of the `.ssp` text format.
+- [tree-sitter-sisu](https://git.sisudoc.org/tools/tree-sitter-sisu) -
+ tree-sitter grammar for the upstream `.sst` / `.ssm` markup.
diff --git a/dlang/dub.json b/dlang/dub.json
new file mode 100644
index 0000000..51f7132
--- /dev/null
+++ b/dlang/dub.json
@@ -0,0 +1,15 @@
+{
+ "authors": [
+ "Ralph Amissah"
+ ],
+ "copyright": "Copyright (C) 2026 Ralph Amissah",
+ "name": "ssp-reader",
+ "version": "2.0.0",
+ "description": "reader for SiSU Document Abstraction (.ssp) files",
+ "homepage": "https://sisudoc.org",
+ "license": "AGPL-3.0+",
+ "targetPath": "./bin",
+ "sourcePaths": [ "./src" ],
+ "targetType": "executable",
+ "targetName": "ssp-reader"
+}
diff --git a/dlang/flake.lock b/dlang/flake.lock
new file mode 100644
index 0000000..5f236ab
--- /dev/null
+++ b/dlang/flake.lock
@@ -0,0 +1,27 @@
+{
+ "nodes": {
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1790185690,
+ "narHash": "sha256-xJ+X4hBtOcAFGBOe5nAMyMUeF9foJBmIOu3NjBqBycU=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "4975466d324710c576dc11ad614684e6bd8cad8e",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "ref": "nixos-unstable",
+ "repo": "nixpkgs",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "nixpkgs": "nixpkgs"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/dlang/flake.nix b/dlang/flake.nix
new file mode 100644
index 0000000..554b8e0
--- /dev/null
+++ b/dlang/flake.nix
@@ -0,0 +1,67 @@
+{
+ description = "ssp-reader: a reader for the .ssp document abstraction, in D";
+
+ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+ outputs = { self, nixpkgs }:
+ let
+ system = "x86_64-linux"; # adjust for aarch64-linux / darwin
+ pkgs = import nixpkgs { inherit system; };
+
+ name = "ssp-reader";
+
+ /* The reader has no dependencies beyond phobos, so there is nothing
+ to vendor and nothing to fetch: --skip-registry=all keeps dub from
+ reaching for a registry it does not need, which is also what lets
+ this build in a sandbox without network.
+ */
+ mkReader = { compiler, compilerBin }: pkgs.stdenv.mkDerivation {
+ pname = name;
+ version = "2.0.0";
+ src = self;
+ nativeBuildInputs = [ pkgs.dub compiler ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ dub build \
+ --cache=local \
+ --compiler=$(type -P ${compilerBin}) \
+ --build=release \
+ --skip-registry=all
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/bin
+ install -m755 bin/${name} $out/bin/${name}
+ runHook postInstall
+ '';
+ meta = {
+ description = "reads a .ssp document abstraction and says what is in it";
+ mainProgram = name;
+ };
+ };
+ in {
+ packages.${system} = {
+ default = self.packages.${system}.ssp-reader-ldc;
+
+ ssp-reader-ldc = mkReader {
+ compiler = pkgs.ldc;
+ compilerBin = "ldmd2";
+ };
+
+ /* dmd is given gcc14: GCC 15 introduced nullptr in its headers and
+ dmd's ImportC parser does not yet handle it. The same override
+ sisudoc-spine carries, for the same reason.
+ */
+ ssp-reader-dmd = mkReader {
+ compiler = pkgs.dmd.override { stdenv = pkgs.gcc14Stdenv; };
+ compilerBin = "dmd";
+ };
+ };
+
+ devShells.${system}.default = pkgs.mkShell {
+ packages = [ pkgs.ldc pkgs.dub pkgs.dtools ];
+ };
+ };
+}
diff --git a/dlang/src/main.d b/dlang/src/main.d
new file mode 100644
index 0000000..a7a6537
--- /dev/null
+++ b/dlang/src/main.d
@@ -0,0 +1,158 @@
+import std.getopt;
+import std.stdio : writeln, stderr;
+import ssp_parser;
+import ssp_types;
+import output_tree;
+import output_html;
+import output_llm_breadcrumbs;
+import output_llm_jsonl;
+import output_images;
+import verify_images;
+import report_unknown;
+import validate;
+
+int main(string[] args) {
+ bool do_tree = false;
+ bool do_html = false;
+ bool do_llm_text = false;
+ bool do_llm_jsonl = false;
+ bool do_images = false;
+ bool do_verify = false;
+ bool do_report_unknown = false;
+ bool do_validate = false;
+ bool any_version = false;
+ string image_dir = "";
+ string output_file = "";
+
+ auto helpInfo = getopt(args,
+ "tree", "output document tree outline", &do_tree,
+ "html", "output HTML scroll", &do_html,
+ "llm-text", "output plain text with heading breadcrumbs", &do_llm_text,
+ "llm-jsonl", "output JSONL with embeddings-ready chunks", &do_llm_jsonl,
+ "images", "output image manifest (digest, size, pixels)", &do_images,
+ "verify", "verify .image records against --image-dir", &do_verify,
+ "image-dir", "image directory to verify against", &image_dir,
+ "output", "output file (default: stdout)", &output_file,
+ "report-unknown", "list what was read and not understood", &do_report_unknown,
+ "validate", "check the file against the .ssp specification", &do_validate,
+ "any-version", "read a format major version this reader does not know", &any_version,
+ );
+
+ if (helpInfo.helpWanted || args.length < 2) {
+ defaultGetoptPrinter(
+ "ssp-reader - reads .ssp files and produces output\n\n"
+ ~ "Usage: ssp-reader [options] <file.ssp>\n",
+ helpInfo.options,
+ );
+ return 0;
+ }
+
+ string input_file = args[1];
+
+ SspDocument doc;
+ try {
+ doc = parseSspFile(input_file);
+ } catch (Exception e) {
+ stderr.writeln("Error reading ", input_file, ": ", e.msg);
+ return 2;
+ }
+
+ /+ ↓ the version gate. A reader checks the major part and refuses what it
+ does not know: an unknown major means a property may have changed
+ meaning or gone away, and loading it half populated in silence is the
+ failure this exists to prevent. --any-version overrides, for looking
+ at a file rather than trusting one.
+ +/
+ if (doc.format_version.length == 0) {
+ stderr.writeln("warning: ", input_file,
+ " declares no format version (no \"% SiSU Document Abstraction v\" line)");
+ } else if (doc.formatVersionMajor != ssp_format_major_read) {
+ if (any_version) {
+ stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
+ } else {
+ stderr.writeln("error: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x");
+ stderr.writeln(" a major version differs: a property may have",
+ " changed meaning or gone away.");
+ stderr.writeln(" --any-version reads it regardless.");
+ return 3;
+ }
+ }
+
+ writeln("Parsed: ", input_file);
+ if (doc.format_version.length > 0) {
+ writeln(" Format: v", doc.format_version);
+ }
+ if (auto _lang = "language" in doc.source) {
+ writeln(" Source: ", *_lang,
+ ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
+ ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
+ }
+ writeln(" Title: ", doc.meta("title.main"));
+ writeln(" Author: ", doc.meta("creator.author"));
+ writeln(" Sections: head=", doc.head.length,
+ " toc=", doc.toc.length,
+ " body=", doc.body.length,
+ " endnotes=", doc.endnotes.length,
+ " glossary=", doc.glossary.length,
+ " bibliography=", doc.bibliography.length,
+ " bookindex=", doc.bookindex.length,
+ " blurb=", doc.blurb.length,
+ " tail=", doc.tail.length);
+ auto image_records = imageRecordCount(doc);
+ if (image_records > 0) {
+ writeln(" Images: ", image_records, " record",
+ (image_records == 1) ? "" : "s");
+ }
+
+ if (do_tree) {
+ outputTree(doc, output_file);
+ }
+
+ if (do_html) {
+ outputHtmlScroll(doc, output_file);
+ }
+
+ if (do_llm_text) {
+ outputLlmBreadcrumbs(doc, output_file);
+ }
+
+ if (do_llm_jsonl) {
+ outputLlmJsonl(doc, output_file);
+ }
+
+ if (do_images) {
+ outputImages(doc, output_file);
+ }
+
+
+ int _verify_failures = 0;
+ if (do_verify) {
+ if (image_dir.length == 0) {
+ stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
+ return 2;
+ }
+ _verify_failures = verifyImages(doc, image_dir, output_file);
+ }
+
+ if (do_report_unknown) {
+ reportUnknown(doc);
+ }
+
+ int _validation_findings = 0;
+ if (do_validate) {
+ auto _found = validateSspFile(input_file, ssp_format_major_read);
+ reportValidation(input_file, _found);
+ _validation_findings = cast(int) _found.length;
+ }
+
+ if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
+ && !do_verify && !do_report_unknown && !do_validate) {
+ writeln("\nUse --tree, --html, --llm-text, --llm-jsonl, --images,"
+ ~ " --verify --image-dir=<dir>"
+ ~ " --report-unknown or --validate to produce output.");
+ }
+
+ return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
+}
diff --git a/dlang/src/output_html.d b/dlang/src/output_html.d
new file mode 100644
index 0000000..b1c711c
--- /dev/null
+++ b/dlang/src/output_html.d
@@ -0,0 +1,388 @@
+module output_html;
+
+import std.array : replace, split, join, appender;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, startsWith;
+import ssp_types;
+
+void outputHtmlScroll(SspDocument doc, string output_file) {
+ auto buf = appender!(string[]);
+
+ buf ~= htmlHead(doc);
+
+ /+ ↓ title +/
+ if (doc.head.length > 0) {
+ auto title_obj = doc.head[0];
+ buf ~= `<div class="title">`;
+ buf ~= `<h1>` ~ htmlEscape(doc.meta("title.main")) ~ `</h1>`;
+ if (doc.meta("title.subtitle").length > 0)
+ buf ~= `<h2 class="subtitle">` ~ htmlEscape(doc.meta("title.subtitle")) ~ `</h2>`;
+ buf ~= `<p class="author">` ~ htmlEscape(doc.meta("creator.author")) ~ `</p>`;
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ table of contents +/
+ if (doc.toc.length > 0) {
+ buf ~= `<div class="toc">`;
+ buf ~= `<h2>Table of Contents</h2>`;
+ foreach (obj; doc.toc) {
+ if (obj.is_a == "toc") {
+ string text = convertInlineMarkup(obj.text);
+ int indent = obj.indent_base;
+ string cls = (indent > 4) ? "toc-sub" : "toc-entry";
+ buf ~= `<p class="` ~ cls ~ `">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ buf ~= `<hr/>`;
+ }
+
+ /+ ↓ body +/
+ buf ~= `<div class="body">`;
+ foreach (obj; doc.body) {
+ buf ~= renderObject(obj);
+ }
+ buf ~= `</div>`;
+
+ /+ ↓ endnotes +/
+ bool has_endnotes = false;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") { has_endnotes = true; break; }
+ }
+ if (has_endnotes) {
+ buf ~= `<div class="endnotes">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Endnotes</h2>`;
+ foreach (obj; doc.endnotes) {
+ if (obj.is_a == "endnote") {
+ string text = convertInlineMarkup(obj.text);
+ string parent_ref = (obj.parent_ocn > 0)
+ ? ` <a href="#o` ~ obj.parent_ocn.to!string ~ `">[` ~ obj.parent_ocn.to!string ~ `]</a>`
+ : "";
+ buf ~= `<p class="endnote">` ~ text ~ parent_ref ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ glossary +/
+ if (doc.glossary.length > 0) {
+ buf ~= `<div class="glossary">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Glossary</h2>`;
+ foreach (obj; doc.glossary) {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="glossary-entry">` ~ text ~ `</p>`;
+ }
+ buf ~= `</div>`;
+ }
+
+ /+ ↓ bookindex +/
+ bool has_bookindex = false;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") { has_bookindex = true; break; }
+ }
+ if (has_bookindex) {
+ buf ~= `<div class="bookindex">`;
+ buf ~= `<hr/>`;
+ buf ~= `<h2>Book Index</h2>`;
+ foreach (obj; doc.bookindex) {
+ if (obj.is_a == "bookindex") {
+ string text = convertInlineMarkup(obj.text);
+ buf ~= `<p class="index-entry">` ~ text ~ `</p>`;
+ }
+ }
+ buf ~= `</div>`;
+ }
+
+ buf ~= htmlTail();
+
+ /+ ↓ write output +/
+ string[] output = buf.data;
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; output) f.writeln(line);
+ } else {
+ foreach (line; output) writeln(line);
+ }
+}
+
+private string[] renderObject(SspObject obj) {
+ /+ ↓ markup commentary is carried by the abstraction and is not output.
+ Rendering it would put the author notes into the document, which
+ spine itself does not do
+ +/
+ if (!obj.renders) { return []; }
+ string[] lines;
+ string text = convertInlineMarkup(obj.text);
+ string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off)
+ ? ` id="o` ~ obj.ocn.to!string ~ `"`
+ : "";
+ string ocn_display = (obj.ocn > 0 && !obj.ocn_off)
+ ? `<span class="ocn">` ~ obj.ocn.to!string ~ `</span>`
+ : "";
+
+ switch (obj.is_a) {
+ case "heading":
+ string tag = headingTag(obj.heading_level);
+ string seg_anchor = (obj.segment.length > 0)
+ ? ` id="` ~ htmlEscape(obj.segment) ~ `"`
+ : ocn_anchor;
+ lines ~= `<` ~ tag ~ seg_anchor ~ `>` ~ text ~ `</` ~ tag ~ `>`;
+ break;
+ case "para":
+ string indent_cls = "";
+ if (obj.indent_base > 0) indent_cls = ` class="indent-` ~ obj.indent_base.to!string ~ `"`;
+ else if (obj.bullet) indent_cls = ` class="bullet"`;
+ lines ~= `<p` ~ ocn_anchor ~ indent_cls ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ case "code":
+ lines ~= `<pre` ~ ocn_anchor ~ `><code>` ~ htmlEscape(obj.text) ~ `</code></pre>`;
+ break;
+ case "table":
+ lines ~= renderTable(obj, ocn_anchor);
+ break;
+ case "group":
+ lines ~= `<div` ~ ocn_anchor ~ ` class="group">` ~ text ~ `</div>`;
+ break;
+ case "block":
+ lines ~= `<pre` ~ ocn_anchor ~ ` class="block">` ~ htmlEscape(obj.text) ~ `</pre>`;
+ break;
+ case "verse":
+ lines ~= `<p` ~ ocn_anchor ~ ` class="verse">` ~ text.replace("\n", "<br/>\n") ~ ocn_display ~ `</p>`;
+ break;
+ case "quote":
+ lines ~= `<blockquote` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</blockquote>`;
+ break;
+ default:
+ if (text.length > 0)
+ lines ~= `<p` ~ ocn_anchor ~ `>` ~ text ~ ocn_display ~ `</p>`;
+ break;
+ }
+ return lines;
+}
+
+private string headingTag(string level) {
+ switch (level) {
+ case "A": return "h1";
+ case "B": return "h2";
+ case "C": return "h3";
+ case "D": return "h4";
+ case "1": return "h3";
+ case "2": return "h4";
+ case "3": return "h5";
+ case "4": return "h6";
+ default: return "h4";
+ }
+}
+
+private string renderTable(SspObject obj, string ocn_anchor) {
+ string[] lines;
+ lines ~= `<table` ~ ocn_anchor ~ `>`;
+ /+ ↓ table content uses ┊ as cell separator, rows separated by newlines +/
+ auto rows = obj.text.split("\n");
+ bool first_row = obj.table_header;
+ foreach (row; rows) {
+ if (row.strip.length == 0) continue;
+ string tag = first_row ? "th" : "td";
+ lines ~= `<tr>`;
+ foreach (cell; row.split("┊")) {
+ lines ~= `<` ~ tag ~ `>` ~ convertInlineMarkup(cell.strip) ~ `</` ~ tag ~ `>`;
+ }
+ lines ~= `</tr>`;
+ first_row = false;
+ }
+ lines ~= `</table>`;
+ return lines.join("\n");
+}
+
+/+ ↓ convert spine internal Unicode markup to HTML +/
+string convertInlineMarkup(string text) {
+ if (text.length == 0) return "";
+ string result = htmlEscape(text);
+
+ /+ ↓ font face markers: ⑆*bold⑆*, ⑆/italic⑆/, etc.
+ Pattern: ⑆X┨content┣X or ⑆Xcontent⑆X
+ The ┨┣ form is used within heading content +/
+
+ /+ ↓ bold +/
+ result = replaceInlineMarker(result, "⑆!", "!", "<b>", "</b>");
+ result = replaceInlineMarkerBracketed(result, "⑆!", "┨", "┣!", "<b>", "</b>");
+
+ /+ ↓ emphasis +/
+ result = replaceInlineMarker(result, "⑆*", "*", "<em>", "</em>");
+ result = replaceInlineMarkerBracketed(result, "⑆*", "┨", "┣*", "<em>", "</em>");
+
+ /+ ↓ italic +/
+ result = replaceInlineMarker(result, "⑆/", "/", "<i>", "</i>");
+ result = replaceInlineMarkerBracketed(result, "⑆/", "┨", "┣/", "<i>", "</i>");
+
+ /+ ↓ underscore +/
+ result = replaceInlineMarker(result, "⑆_", "_", "<u>", "</u>");
+ result = replaceInlineMarkerBracketed(result, "⑆_", "┨", "┣_", "<u>", "</u>");
+
+ /+ ↓ superscript +/
+ result = replaceInlineMarker(result, "⑆^", "^", "<sup>", "</sup>");
+ result = replaceInlineMarkerBracketed(result, "⑆^", "┨", "┣^", "<sup>", "</sup>");
+
+ /+ ↓ subscript +/
+ result = replaceInlineMarker(result, "⑆,", ",", "<sub>", "</sub>");
+ result = replaceInlineMarkerBracketed(result, "⑆,", "┨", "┣,", "<sub>", "</sub>");
+
+ /+ ↓ monospace +/
+ result = replaceInlineMarker(result, "⑆■", "■", "<code>", "</code>");
+
+ /+ ↓ links: ┥link text┝┤url├ +/
+ result = replaceLinks(result);
+
+ /+ ↓ footnote markers: 【N text】 +/
+ result = replaceFootnotes(result);
+
+ /+ ↓ line breaks +/
+ result = result.replace("┘", "<br/>");
+ result = result.replace("┙", "<br/>");
+
+ /+ ↓ non-breaking spaces +/
+ result = result.replace("░", "&nbsp;");
+
+ /+ ↓ bullet markers +/
+ result = result.replace("● ", "&#8226; ");
+
+ /+ ↓ page break markers +/
+ result = result.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ return result;
+}
+
+private string replaceInlineMarker(string text, string open_mark, string close_suffix, string html_open, string html_close) {
+ /+ ↓ replace ⑆Xcontent⑆X pattern (no bracket form) +/
+ string result = text;
+ while (true) {
+ auto open_pos = result.indexOf(open_mark);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + open_mark.length;
+ /+ ↓ look for closing ⑆X +/
+ string close_mark = "⑆" ~ close_suffix;
+ auto close_pos = indexOf(result[after_open .. $], close_mark);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + close_mark.length .. $];
+ }
+ return result;
+}
+
+private string replaceInlineMarkerBracketed(string text, string open_mark, string bracket_open, string bracket_close, string html_open, string html_close) {
+ /+ ↓ replace ⑆X┨content┣X pattern (bracketed form) +/
+ string result = text;
+ string full_open = open_mark ~ bracket_open;
+ while (true) {
+ auto open_pos = result.indexOf(full_open);
+ if (open_pos < 0) break;
+ auto after_open = open_pos + full_open.length;
+ auto close_pos = indexOf(result[after_open .. $], bracket_close);
+ if (close_pos < 0) break;
+ close_pos += after_open;
+ string content = result[after_open .. close_pos];
+ result = result[0 .. open_pos] ~ html_open ~ content ~ html_close
+ ~ result[close_pos + bracket_close.length .. $];
+ }
+ return result;
+}
+
+private string replaceLinks(string text) {
+ /+ ↓ replace ┥link text┝┤url├ with <a href="url">link text</a> +/
+ string result = text;
+ while (true) {
+ auto link_open = result.indexOf("┥");
+ if (link_open < 0) break;
+ auto link_mid = indexOf(result[link_open .. $], "┝┤");
+ if (link_mid < 0) break;
+ link_mid += link_open;
+ auto link_close = indexOf(result[link_mid .. $], "├");
+ if (link_close < 0) break;
+ link_close += link_mid;
+ string link_text = result[link_open + "┥".length .. link_mid];
+ string url = result[link_mid + "┝┤".length .. link_close];
+ /+ ↓ handle internal links (¤ prefix) +/
+ if (url.length > 0 && url[0] == '¤') {
+ url = url[("¤").length .. $];
+ }
+ /+ ↓ handle OCN-only links (#N) +/
+ if (url.length > 0 && url[0] == '#') {
+ url = "#o" ~ url[1 .. $];
+ }
+ result = result[0 .. link_open]
+ ~ `<a href="` ~ url ~ `">` ~ link_text ~ `</a>`
+ ~ result[link_close + "├".length .. $];
+ }
+ return result;
+}
+
+private string replaceFootnotes(string text) {
+ /+ ↓ replace 【N content】 with superscript note number +/
+ string result = text;
+ while (true) {
+ auto fn_open = result.indexOf("【");
+ if (fn_open < 0) break;
+ auto fn_close = indexOf(result[fn_open .. $], "】");
+ if (fn_close < 0) break;
+ fn_close += fn_open;
+ string fn_content = result[fn_open + "【".length .. fn_close];
+ /+ ↓ extract note number (first token before space) +/
+ auto space_pos = fn_content.indexOf(' ');
+ string note_num = (space_pos > 0) ? fn_content[0 .. space_pos] : fn_content;
+ result = result[0 .. fn_open]
+ ~ `<sup class="fn">` ~ note_num ~ `</sup>`
+ ~ result[fn_close + "】".length .. $];
+ }
+ return result;
+}
+
+string htmlEscape(string text) {
+ return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+ .replace("\"", "&quot;");
+}
+
+private string[] htmlHead(SspDocument doc) {
+ string[] lines;
+ lines ~= `<!DOCTYPE html>`;
+ lines ~= `<html>`;
+ lines ~= `<head>`;
+ lines ~= `<meta charset="utf-8"/>`;
+ lines ~= `<title>` ~ htmlEscape(doc.meta("title.main")) ~ `</title>`;
+ lines ~= `<style>`;
+ lines ~= ` body { max-width: 48em; margin: 2em auto; padding: 0 1em; font-family: serif; line-height: 1.5; }`;
+ lines ~= ` .title { text-align: center; margin-bottom: 2em; }`;
+ lines ~= ` .subtitle { font-weight: normal; font-style: italic; }`;
+ lines ~= ` .ocn { float: right; font-size: 0.7em; color: #999; }`;
+ lines ~= ` .toc { margin-bottom: 2em; }`;
+ lines ~= ` .toc-entry { margin: 0.2em 0; }`;
+ lines ~= ` .toc-sub { margin: 0.2em 0 0.2em 2em; }`;
+ lines ~= ` .indent-1 { margin-left: 2em; }`;
+ lines ~= ` .indent-2 { margin-left: 4em; }`;
+ lines ~= ` .indent-3 { margin-left: 6em; }`;
+ lines ~= ` .bullet { margin-left: 2em; }`;
+ lines ~= ` .bullet:before { content: "\2022 "; }`;
+ lines ~= ` .verse { white-space: pre-wrap; font-style: italic; }`;
+ lines ~= ` .group { margin: 1em 2em; }`;
+ lines ~= ` .block { margin: 1em 2em; background: #f8f8f8; padding: 1em; }`;
+ lines ~= ` pre code { display: block; background: #f4f4f4; padding: 1em; overflow-x: auto; }`;
+ lines ~= ` table { border-collapse: collapse; margin: 1em 0; }`;
+ lines ~= ` th, td { border: 1px solid #ccc; padding: 0.4em 0.8em; }`;
+ lines ~= ` th { background: #f0f0f0; }`;
+ lines ~= ` .endnote { font-size: 0.9em; }`;
+ lines ~= ` .fn { color: #666; }`;
+ lines ~= ` .index-entry { font-size: 0.9em; }`;
+ lines ~= ` blockquote { margin: 1em 2em; padding-left: 1em; border-left: 3px solid #ccc; }`;
+ lines ~= `</style>`;
+ lines ~= `</head>`;
+ lines ~= `<body>`;
+ return lines;
+}
+
+private string[] htmlTail() {
+ return [`</body>`, `</html>`];
+}
diff --git a/dlang/src/output_images.d b/dlang/src/output_images.d
new file mode 100644
index 0000000..69fb458
--- /dev/null
+++ b/dlang/src/output_images.d
@@ -0,0 +1,114 @@
+module output_images;
+
+import std.array : join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import ssp_types;
+
+/+ ↓ the .image records of a document, gathered per distinct file, with the
+ objects referencing each file, in order of first appearance +/
+struct ImageEntry {
+ string name;
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+ bool missing;
+ string[] refs;
+ string[] other_sha256; // same filename, differing digest
+}
+
+struct ImageRecords {
+ int records; // .image lines, an image referenced twice counts twice
+ ImageEntry[] entries; // distinct files
+}
+
+ImageRecords gatherImageRecords(SspDocument doc) {
+ ImageRecords ret;
+ size_t[string] seen;
+ foreach (section; sectionNames) {
+ auto objs = doc.sectionByName(section);
+ if (objs is null) continue;
+ foreach (obj; *objs) {
+ foreach (img; obj.images) {
+ ret.records += 1;
+ string _ref = section ~ "[" ~ obj.ocn.to!string ~ "]";
+ if (auto at = img.name in seen) {
+ if (ret.entries[*at].sha256 != img.sha256
+ && img.sha256.length > 0
+ && ret.entries[*at].sha256.length > 0) {
+ ret.entries[*at].other_sha256 ~= img.sha256;
+ }
+ ret.entries[*at].refs ~= _ref;
+ } else {
+ ImageEntry e_;
+ e_.name = img.name;
+ e_.sha256 = img.sha256;
+ e_.bytes = img.bytes;
+ e_.width = img.width;
+ e_.height = img.height;
+ e_.missing = img.missing;
+ e_.refs = [_ref];
+ seen[img.name] = ret.entries.length;
+ ret.entries ~= e_;
+ }
+ }
+ }
+ }
+ return ret;
+}
+
+/+ ↓ image manifest, line oriented and greppable:
+
+ <sha256> <bytes> <px> <filename> refs: body[97] body[412]
+ MISSING <filename> refs: body[840]
+
+ The point of the record in the .ssp is verification (is the image
+ shipped in this pod the image the abstraction was built from), so this
+ output is the shape a verifier wants: one line per distinct file.
++/
+void outputImages(SspDocument doc, string output_file) {
+ auto gathered = gatherImageRecords(doc);
+ string[] lines;
+ int missing = 0;
+ foreach (e; gathered.entries) {
+ if (e.missing) missing += 1;
+ }
+ lines ~= "Images: " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ ", " ~ missing.to!string ~ " missing";
+ lines ~= "";
+ foreach (e; gathered.entries) {
+ string px = (e.width > 0 && e.height > 0)
+ ? e.width.to!string ~ "x" ~ e.height.to!string : "-";
+ lines ~= (e.missing)
+ ? "MISSING" ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ")
+ : e.sha256 ~ " " ~ e.bytes.to!string ~ " " ~ px
+ ~ " " ~ e.name ~ " refs: " ~ e.refs.join(" ");
+ /+ ↓ one filename, more than one digest: the .ssp is inconsistent,
+ or two distinct files share a name +/
+ foreach (other; e.other_sha256) {
+ lines ~= " WARNING differing digest for same filename: " ~ other;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+/+ ↓ count of .image records in a document, for the parse summary +/
+int imageRecordCount(SspDocument doc) {
+ return gatherImageRecords(doc).records;
+}
+
+package immutable string[] sectionNames = [
+ "head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb",
+];
diff --git a/dlang/src/output_llm_breadcrumbs.d b/dlang/src/output_llm_breadcrumbs.d
new file mode 100644
index 0000000..5b240d2
--- /dev/null
+++ b/dlang/src/output_llm_breadcrumbs.d
@@ -0,0 +1,182 @@
+module output_llm_breadcrumbs;
+
+import std.array : replace, split, join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ssp_types;
+
+/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption
+ format:
+ ## Document Title > Chapter > Section > [OCN]
+
+ Clean paragraph text with inline markup stripped.
++/
+void outputLlmBreadcrumbs(SspDocument doc, string output_file) {
+ string[] lines;
+ string doc_title = doc.meta("title.main");
+
+ /+ ↓ document header +/
+ lines ~= "# " ~ doc_title;
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= "# " ~ doc.meta("title.subtitle");
+ lines ~= "# " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build heading text lookup from ancestors +/
+ string[int] heading_text;
+ foreach (obj; doc.head) {
+ if (obj.is_a == "heading" && obj.ocn > 0)
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading")
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+
+ /+ ↓ body objects +/
+ string current_breadcrumb = "";
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj);
+ lines ~= "";
+ lines ~= "## " ~ current_breadcrumb;
+ lines ~= "";
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ string ocn_ref = (obj.ocn > 0 && !obj.ocn_off)
+ ? " [" ~ obj.ocn.to!string ~ "]"
+ : "";
+
+ switch (obj.is_a) {
+ case "para":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "code":
+ lines ~= "```";
+ lines ~= obj.text; /+ preserve raw text for code +/
+ lines ~= "```";
+ lines ~= "";
+ break;
+ case "table":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "verse":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "quote":
+ lines ~= "> " ~ text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "group":
+ case "block":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ default:
+ if (text.length > 0) {
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ }
+ break;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string buildBreadcrumb(string doc_title, string ancestors_text, SspObject obj) {
+ /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ /+ ↓ truncate very long heading text for breadcrumb +/
+ if (clean.length > 60)
+ clean = clean[0 .. 57] ~ "...";
+ parts ~= clean;
+ }
+ }
+ }
+ /+ ↓ add the current heading itself +/
+ string heading_text = stripMarkup(obj.text).strip;
+ if (heading_text.length > 60)
+ heading_text = heading_text[0 .. 57] ~ "...";
+
+ string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+
+ if (parts.length > 0)
+ return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref;
+ else
+ return doc_title ~ " > " ~ heading_text ~ ocn_ref;
+}
+
+/+ ↓ strip all internal Unicode markup to produce clean plain text +/
+string stripMarkup(string text) {
+ if (text.length == 0) return "";
+ string r = text;
+
+ /+ ↓ links: ┥link text┝┤url├ -> link text +/
+ while (true) {
+ auto lo = r.indexOf("┥");
+ if (lo < 0) break;
+ auto lm = indexOf(r[lo .. $], "┝┤");
+ if (lm < 0) break;
+ lm += lo;
+ auto lc = indexOf(r[lm .. $], "├");
+ if (lc < 0) break;
+ lc += lm;
+ string link_text = r[lo + "┥".length .. lm];
+ r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $];
+ }
+
+ /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/
+ while (true) {
+ auto fo = r.indexOf("【");
+ if (fo < 0) break;
+ auto fc = indexOf(r[fo .. $], "】");
+ if (fc < 0) break;
+ fc += fo;
+ r = r[0 .. fo] ~ r[fc + "】".length .. $];
+ }
+
+ /+ ↓ font face markers +/
+ r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("⑆■", "")
+ .replace("┨", "").replace("┣*", "").replace("┣!", "")
+ .replace("┣/", "").replace("┣_", "").replace("┣^", "")
+ .replace("┣,", "").replace("┣", "");
+
+ /+ ↓ line breaks and special chars +/
+ r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n");
+ r = r.replace("░", " ");
+ r = r.replace("● ", "- ");
+ r = r.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ /+ ↓ table cell separators +/
+ r = r.replace("┊", " | ");
+
+ /+ ↓ clean up multiple spaces +/
+ while (r.indexOf(" ") >= 0)
+ r = r.replace(" ", " ");
+
+ return r.strip;
+}
diff --git a/dlang/src/output_llm_jsonl.d b/dlang/src/output_llm_jsonl.d
new file mode 100644
index 0000000..3d34089
--- /dev/null
+++ b/dlang/src/output_llm_jsonl.d
@@ -0,0 +1,121 @@
+module output_llm_jsonl;
+
+import std.array : split, join, replace;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ssp_types;
+import output_llm_breadcrumbs : stripMarkup;
+
+/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
+ one JSON object per line, each containing:
+ doc, ocn, path (breadcrumb), type, text (clean),
+ images (only where the object embeds one, so that a chunk whose
+ text reads around a figure carries the figure's identity),
+ index_terms (from bookindex entries associated with this object)
++/
+void outputLlmJsonl(SspDocument doc, string output_file) {
+ string[] lines;
+ string doc_id = doc.meta("title.main");
+ string doc_author = doc.meta("creator.author");
+
+ /+ ↓ build bookindex -> OCN mapping for index_terms
+ bookindex objects in the .ssp have parent_ocn linking to body objects
+ but the index terms are in the text of bookindex objects, not
+ directly attached to body objects. We use a simpler approach:
+ headings carry heading_ancestors_text which gives context. +/
+
+ /+ ↓ track current heading path +/
+ string current_path = "";
+
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
+ /+ ↓ emit heading as a chunk too +/
+ string heading_text = stripMarkup(obj.text);
+ if (heading_text.strip.length > 0) {
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
+ }
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content, and least of all is it a chunk
+ to embed: it would be indexed as though the author had written it
+ into the document
+ +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, SspImage[] images) {
+ return `{"doc":` ~ jsonStr(doc_id)
+ ~ `,"author":` ~ jsonStr(doc_author)
+ ~ `,"ocn":` ~ ocn.to!string
+ ~ `,"path":` ~ jsonStr(path)
+ ~ `,"type":` ~ jsonStr(type)
+ ~ `,"text":` ~ jsonStr(text)
+ ~ jsonImages(images)
+ ~ `}`;
+}
+
+private string jsonImages(SspImage[] images) {
+ /+ ↓ omitted entirely where the object embeds no image +/
+ if (images.length == 0) return "";
+ string[] items;
+ foreach (img; images) {
+ string item = `{"name":` ~ jsonStr(img.name);
+ if (img.missing) {
+ item ~= `,"missing":true`;
+ } else {
+ item ~= `,"sha256":` ~ jsonStr(img.sha256)
+ ~ `,"bytes":` ~ img.bytes.to!string;
+ if (img.width > 0 && img.height > 0) {
+ item ~= `,"width":` ~ img.width.to!string
+ ~ `,"height":` ~ img.height.to!string;
+ }
+ }
+ items ~= item ~ `}`;
+ }
+ return `,"images":[` ~ items.join(",") ~ `]`;
+}
+
+private string buildPath(string doc_title, string ancestors_text, SspObject obj) {
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ parts ~= clean;
+ }
+ }
+ }
+ string heading_text = stripMarkup(obj.text).strip;
+ if (parts.length > 0)
+ return parts.join(" > ") ~ " > " ~ heading_text;
+ else
+ return heading_text;
+}
+
+private string jsonStr(string s) {
+ /+ ↓ JSON string escaping +/
+ string r = s;
+ r = r.replace(`\`, `\\`);
+ r = r.replace(`"`, `\"`);
+ r = r.replace("\n", `\n`);
+ r = r.replace("\r", `\r`);
+ r = r.replace("\t", `\t`);
+ return `"` ~ r ~ `"`;
+}
diff --git a/dlang/src/output_tree.d b/dlang/src/output_tree.d
new file mode 100644
index 0000000..88c1b4c
--- /dev/null
+++ b/dlang/src/output_tree.d
@@ -0,0 +1,69 @@
+module output_tree;
+
+import std.conv : to;
+import std.stdio : writeln, writef, writefln, File;
+import std.array : replicate;
+import ssp_types;
+
+void outputTree(SspDocument doc, string output_file) {
+ string[] lines;
+
+ lines ~= "Document: " ~ doc.meta("title.main");
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= " " ~ doc.meta("title.subtitle");
+ lines ~= "Author: " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build OCN -> object lookup for body headings +/
+ SspObject[int] headings;
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading" && obj.ocn > 0) {
+ headings[obj.ocn] = obj;
+ }
+ }
+
+ /+ ↓ find root heading (from head section, or first body heading) +/
+ if (doc.head.length > 0 && doc.head[0].is_a == "heading") {
+ auto root = doc.head[0];
+ lines ~= formatHeading(root, 0);
+ walkChildren(root, headings, lines, 1);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private void walkChildren(SspObject parent, ref SspObject[int] headings, ref string[] lines, int depth) {
+ foreach (child_ocn; parent.children) {
+ if (auto p = child_ocn in headings) {
+ lines ~= formatHeading(*p, depth);
+ walkChildren(*p, headings, lines, depth + 1);
+ }
+ }
+}
+
+private string formatHeading(SspObject obj, int depth) {
+ string indent = replicate(" ", depth);
+ string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?";
+ string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+ /+ ↓ strip internal markup from heading text for clean display +/
+ string text = stripInternalMarkup(obj.text);
+ return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text;
+}
+
+private string stripInternalMarkup(string text) {
+ /+ ↓ simple removal of common internal markers for display +/
+ import std.regex : replaceAll, regex;
+ import std.array : replace;
+ string result = text;
+ result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("┨", "").replace("┣", "")
+ .replace("┙", " ").replace("┘", "\n");
+ return result;
+}
diff --git a/dlang/src/report_unknown.d b/dlang/src/report_unknown.d
new file mode 100644
index 0000000..1d5238e
--- /dev/null
+++ b/dlang/src/report_unknown.d
@@ -0,0 +1,41 @@
+module report_unknown;
+
+import std.algorithm : sort;
+import std.array : array;
+import std.stdio : writeln, writefln;
+import ssp_types;
+
+/+ ↓ say what was read and not understood.
+ This is what makes the reader usable as a check on the format
+ specification: a reader that ignores what it does not know is
+ conforming, but one that cannot say what it ignored cannot tell you
+ whether the specification it was written from was complete. Silence
+ and understanding look the same from outside.
++/
+void reportUnknown(ref SspDocument doc) {
+ if (!doc.unknown.any) {
+ writeln("\nread and understood: every property, block and line was known");
+ return;
+ }
+ writeln("\nread and NOT understood:");
+ if (doc.unknown.property.length > 0) {
+ writeln(" properties ignored (", doc.unknown.property.length, " distinct):");
+ foreach (name; doc.unknown.property.keys.sort) {
+ writefln(" .%-26s %8d", name, doc.unknown.property[name]);
+ }
+ }
+ if (doc.unknown.block.length > 0) {
+ writeln(" blocks ignored (", doc.unknown.block.length, " distinct):");
+ foreach (name; doc.unknown.block.keys.sort) {
+ writefln(" @%-26s %8d", name, doc.unknown.block[name]);
+ }
+ }
+ if (doc.unknown.line.length > 0) {
+ writeln(" lines of no known shape, by first character:");
+ foreach (c; doc.unknown.line.keys.sort) {
+ writefln(" %-27s %8d", "\"" ~ c ~ "\"", doc.unknown.line[c]);
+ }
+ }
+ writeln(" (ignoring these is conforming within a major version;",
+ " the count is so that it is not silent)");
+}
diff --git a/dlang/src/ssp_parser.d b/dlang/src/ssp_parser.d
new file mode 100644
index 0000000..1a1efe2
--- /dev/null
+++ b/dlang/src/ssp_parser.d
@@ -0,0 +1,292 @@
+module ssp_parser;
+
+import std.algorithm : splitter, startsWith;
+import std.array : array, split;
+import std.conv : to, ConvException;
+import std.file : readText;
+import std.stdio : writeln;
+import std.string : strip, stripLeft, indexOf;
+import ssp_types;
+
+SspDocument parseSspFile(string filename) {
+ string content = readText(filename);
+ return parseSsp(content);
+}
+
+SspDocument parseSsp(string content) {
+ SspDocument doc;
+ SspObject current_obj;
+ bool in_object = false;
+ string current_section = "";
+ string current_block = ""; // meta, make, doc_has, or section name
+
+ foreach (line; content.splitter("\n")) {
+ /+ ↓ comment. The first line of a .ssp declares the format version, and
+ it is the one comment that carries meaning: a reader checks the
+ major part and refuses what it does not know rather than loading
+ an unfamiliar artefact half populated in silence.
+ +/
+ if (line.length > 0 && line[0] == '%') {
+ enum _banner = "% SiSU Document Abstraction v";
+ if (doc.format_version.length == 0 && line.startsWith(_banner)) {
+ doc.format_version = line[_banner.length .. $].strip;
+ }
+ continue;
+ }
+
+ /+ ↓ blank line - finalize current object +/
+ if (line.length == 0 || line.strip.length == 0) {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = SspObject.init;
+ in_object = false;
+ }
+ continue;
+ }
+
+ /+ ↓ section/block open +/
+ if (line[0] == '@') {
+ auto brace_pos = line.indexOf('{');
+ if (brace_pos > 0) {
+ string name = line[1 .. brace_pos].strip;
+ current_block = name;
+ if (!isPreambleBlock(name)) {
+ current_section = name;
+ /+ ↓ a block that is neither preamble nor a section this reader
+ knows: counted, so that --report-unknown can say so +/
+ if (doc.sectionByName(name) is null) {
+ doc.unknown.block[name]++;
+ }
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ section/block close +/
+ if (line[0] == '}') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ current_obj = SspObject.init;
+ in_object = false;
+ }
+ current_block = "";
+ continue;
+ }
+
+ /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/
+ if (isPreambleBlock(current_block)) {
+ string stripped = line.stripLeft;
+ auto colon_pos = stripped.indexOf(':');
+ if (colon_pos > 0) {
+ string key = stripped[0 .. colon_pos].strip;
+ string value = (colon_pos + 1 < stripped.length)
+ ? stripped[colon_pos + 1 .. $].strip : "";
+ if (current_block == "meta") { doc.metadata[key] = value; }
+ else if (current_block == "make") { doc.make[key] = value; }
+ else if (current_block == "source") { doc.source[key] = value; }
+ else { doc.doc_has[key] = value; }
+ }
+ continue;
+ }
+
+ /+ ↓ object declaration: [N] type ... +/
+ if (line[0] == '[') {
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+ current_obj = SspObject.init;
+ current_obj.section = current_section;
+ in_object = true;
+ parseObjectDeclaration(line, current_obj);
+ continue;
+ }
+
+ /+ ↓ property: .name: value +/
+ if (line[0] == '.') {
+ if (in_object) {
+ parseProperty(line, current_obj, doc.unknown);
+ }
+ continue;
+ }
+
+ /+ ↓ text content: | content +/
+ if (line.length >= 1 && line[0] == '|') {
+ if (in_object) {
+ string text_content = (line.length > 2) ? line[2 .. $] : "";
+ if (current_obj.text.length > 0) {
+ current_obj.text ~= "\n" ~ text_content;
+ } else {
+ current_obj.text = text_content;
+ }
+ }
+ continue;
+ }
+
+ /+ ↓ no known shape. Counted by its first character so that a reader
+ can be told the file held something this one did not expect
+ +/
+ doc.unknown.line[line[0 .. 1].idup]++;
+ }
+
+ /+ ↓ finalize last object +/
+ if (in_object) {
+ commitObject(doc, current_section, current_obj);
+ }
+
+ return doc;
+}
+
+private void parseObjectDeclaration(string line, ref SspObject obj) {
+ /+ parse: [N] type :level identifier +/
+ auto bracket_close = line.indexOf(']');
+ if (bracket_close < 0) return;
+ try {
+ obj.ocn = line[1 .. bracket_close].strip.to!int;
+ } catch (ConvException) {
+ obj.ocn = 0;
+ }
+ string rest = (bracket_close + 1 < line.length)
+ ? line[bracket_close + 1 .. $].strip : "";
+ if (rest.length == 0) return;
+
+ auto parts = rest.split(" ");
+ if (parts.length == 0) return;
+ obj.is_a = parts[0];
+
+ if (obj.is_a == "heading" && parts.length >= 2) {
+ /+ parse :A, :B, :1, etc. +/
+ string lev = parts[1];
+ if (lev.length >= 2 && lev[0] == ':') {
+ obj.heading_level = lev[1 .. $];
+ }
+ /+ optional identifier after level +/
+ if (parts.length >= 3) {
+ obj.identifier = parts[2];
+ }
+ }
+}
+
+/+ ↓ the blocks that are document preamble rather than sections of objects +/
+private bool isPreambleBlock(string name) {
+ return name == "source" || name == "meta" || name == "make" || name == "doc_has";
+}
+
+private void parseProperty(string line, ref SspObject obj, ref SspUnknown unknown) {
+ /+ parse: .name: value +/
+ auto colon_pos = line.indexOf(':');
+ if (colon_pos < 1) return;
+ string name = line[1 .. colon_pos].strip;
+ string value = (colon_pos + 1 < line.length)
+ ? line[colon_pos + 1 .. $].strip : "";
+
+ switch (name) {
+ case "part": obj.part = value; break;
+ case "is_of_type": obj.is_of_type = value; break;
+ case "section": obj.own_section = value; break;
+ case "identifier": obj.identifier = value; break;
+ case "segment_html_is": obj.segment_html_is = value; break;
+ case "table_aligns": obj.table_aligns = value; break;
+ case "stow_link": obj.stow_link ~= value; break;
+ case "parent": obj.parent_ocn = toIntSafe(value); break;
+ case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break;
+ case "children": obj.children = parseIntArray(value); break;
+ case "ancestors": obj.ancestors = parseIntArray(value); break;
+ case "dummy": obj.dummy_heading = (value == "true"); break;
+ case "ocn_off": obj.ocn_off = (value == "true"); break;
+ case "indent":
+ auto parts = value.split(" ");
+ if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]);
+ if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]);
+ break;
+ case "bullet": obj.bullet = (value == "true"); break;
+ case "lang": obj.lang = value; break;
+ case "has":
+ foreach (flag; value.split(" ")) {
+ switch (flag) {
+ case "links": obj.has_links = true; break;
+ case "notes_reg": obj.has_notes_reg = true; break;
+ case "notes_star": obj.has_notes_star = true; break;
+ case "images": obj.has_images = true; break;
+ default: break;
+ }
+ }
+ break;
+ case "segment": obj.segment = value; break;
+ case "segment_prev": obj.segment_prev = value; break;
+ case "segment_next": obj.segment_next = value; break;
+ case "anchor": obj.anchor = value; break;
+ case "table_cols": obj.table_cols = toIntSafe(value); break;
+ case "table_widths": obj.table_widths = value; break;
+ case "table_header": obj.table_header = (value == "true"); break;
+ case "code_syntax": obj.code_syntax = value; break;
+ case "code_linenumbers": obj.code_linenumbers = (value == "true"); break;
+ case "heading_ancestors_text": obj.heading_ancestors_text = value; break;
+ case "sha256": obj.sha256 = value; break;
+ case "image": obj.images ~= parseImageRecord(value); break;
+ /+ ↓ a property this reader has no use for, or has never heard of.
+ Ignored, which is the conforming behaviour within a major
+ version, but counted rather than passed over in silence
+ +/
+ default: unknown.property[name]++; break;
+ }
+}
+
+private SspImage parseImageRecord(string value) {
+ /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...]
+ filename first, remaining fields order-independent; unrecognised
+ fields are kept rather than discarded +/
+ SspImage img;
+ bool first = true;
+ foreach (field; value.split(" ")) {
+ if (field.length == 0) continue;
+ if (first) {
+ img.name = field;
+ first = false;
+ continue;
+ }
+ auto colon_pos = field.indexOf(':');
+ if (colon_pos < 1) continue;
+ string key = field[0 .. colon_pos];
+ string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : "";
+ switch (key) {
+ case "sha256": img.sha256 = val; break;
+ case "bytes": img.bytes = toUlongSafe(val); break;
+ case "px":
+ auto dims = val.split("x");
+ if (dims.length == 2) {
+ img.width = toIntSafe(dims[0]);
+ img.height = toIntSafe(dims[1]);
+ }
+ break;
+ case "missing": img.missing = (val == "true"); break;
+ default: img.fields[key] = val; break;
+ }
+ }
+ return img;
+}
+
+private void commitObject(ref SspDocument doc, string section, ref SspObject obj) {
+ auto sec = doc.sectionByName(section);
+ if (sec !is null) {
+ *sec ~= obj;
+ }
+}
+
+private int toIntSafe(string s) {
+ try { return s.strip.to!int; }
+ catch (ConvException) { return 0; }
+}
+
+private ulong toUlongSafe(string s) {
+ try { return s.strip.to!ulong; }
+ catch (ConvException) { return 0; }
+}
+
+private int[] parseIntArray(string s) {
+ int[] result;
+ foreach (part; s.split(" ")) {
+ auto v = toIntSafe(part);
+ result ~= v;
+ }
+ return result;
+}
diff --git a/dlang/src/ssp_types.d b/dlang/src/ssp_types.d
new file mode 100644
index 0000000..3960df0
--- /dev/null
+++ b/dlang/src/ssp_types.d
@@ -0,0 +1,142 @@
+module ssp_types;
+
+struct SspImage {
+ string name; // image filename as referenced by the object
+ string sha256; // hex-encoded (uppercase) digest of the file
+ ulong bytes; // file size
+ int width; // file's own pixel width (not display width)
+ int height; // file's own pixel height
+ bool missing; // file absent or unreadable, no sha256/bytes
+ string[string] fields; // any further/unrecognised key:value fields
+}
+
+struct SspObject {
+ int ocn;
+ string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb
+ string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb
+ string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings)
+ string identifier; // heading: from the declaration line; otherwise .identifier
+ string part; // frontmatter, body, backmatter
+ string is_of_type; // para, block, comment: how the text behaves
+ string own_section; // .section, when it differs from the block it sits in
+ string segment_html_is;// the html segment this object sits in
+ string table_aligns; // l c r per column
+ string[] stow_link; // links extracted from the object
+ int parent_ocn;
+ int last_descendant_ocn;
+ int[] children; // direct child heading OCNs
+ int[] ancestors; // markedup_ancestors[8]
+ bool dummy_heading;
+ bool ocn_off;
+ int indent_base;
+ int indent_hang;
+ bool bullet;
+ string lang;
+ bool has_links;
+ bool has_notes_reg;
+ bool has_notes_star;
+ bool has_images;
+ string segment;
+ string segment_prev;
+ string segment_next;
+ string anchor;
+ int table_cols;
+ string table_widths;
+ bool table_header;
+ string code_syntax;
+ bool code_linenumbers;
+ string heading_ancestors_text;
+ string sha256;
+ SspImage[] images; // repeatable .image records, in order
+ string text; // content (lines joined with newline)
+
+ /+ ↓ whether this object is rendered at all.
+ is_of_type "comment" is markup commentary: carried by the
+ abstraction so that nothing is lost, and not put into output. An
+ output that renders it leaks the author notes into the document,
+ which is what spine itself does not do.
+ +/
+ bool renders() const {
+ return is_of_type != "comment";
+ }
+
+ /+ ↓ the section this object belongs to, which is the block it sits in
+ unless it says otherwise +/
+ string sectionOf() const {
+ return (own_section.length > 0) ? own_section : section;
+ }
+}
+
+/+ ↓ what the reader met and did not understand.
+ A reader that silently drops what it does not know cannot say whether
+ it understood the document, and cannot be used to check a
+ specification against an implementation: silence and understanding
+ look alike. So the parser counts what it ignored, and
+ --report-unknown prints it.
++/
+struct SspUnknown {
+ size_t[string] property; // .name -> times seen
+ size_t[string] block; // @name -> times seen
+ size_t[string] line; // first char -> times seen, lines of no known shape
+
+ bool any() const {
+ return property.length > 0 || block.length > 0 || line.length > 0;
+ }
+}
+
+struct SspDocument {
+ string format_version; // as declared on line 1, "" when absent
+ string[string] source; // @source: language, languages, digest
+ SspUnknown unknown; // what was met and not understood
+ string[string] metadata; // key -> value
+ string[string] make; // key -> value
+ string[string] doc_has; // key -> value
+ SspObject[] head;
+ SspObject[] toc;
+ SspObject[] body;
+ SspObject[] endnotes;
+ SspObject[] glossary;
+ SspObject[] bibliography;
+ SspObject[] bookindex;
+ SspObject[] blurb;
+ SspObject[] tail;
+
+ /+ ↓ the major part of the declared version, "" when none was declared +/
+ string formatVersionMajor() const {
+ import std.string : indexOf;
+ if (format_version.length == 0) { return ""; }
+ auto dot = format_version.indexOf('.');
+ return (dot > 0) ? format_version[0 .. dot] : format_version;
+ }
+
+ string meta(string key) {
+ if (auto p = key in metadata) return *p;
+ return "";
+ }
+
+ SspObject[]* sectionByName(string name) {
+ switch (name) {
+ case "head": return &head;
+ case "toc": return &toc;
+ case "body": return &body;
+ case "endnotes": return &endnotes;
+ case "glossary": return &glossary;
+ case "bibliography": return &bibliography;
+ case "bookindex": return &bookindex;
+ case "blurb": return &blurb;
+ case "tail": return &tail;
+ default: return null;
+ }
+ }
+}
+
+/+ ↓ the format major version this reader knows.
+ The .ssp and the .ocda.db carry one version between them, declared in
+ band: the .ssp on its first line, the database in its schema.version
+ row. The minor part is raised when a property is added, and a reader of
+ the newer format still reads the older one, so only the major part is
+ checked here. Raise this when the reader has been brought to a new
+ major, not before: the point of the gate is to refuse what has not been
+ looked at.
++/
+enum ssp_format_major_read = "2";
diff --git a/dlang/src/validate.d b/dlang/src/validate.d
new file mode 100644
index 0000000..7b48cd8
--- /dev/null
+++ b/dlang/src/validate.d
@@ -0,0 +1,264 @@
+module validate;
+
+/+ ↓ check a .ssp against the format specification.
+ .
+ Written from specs/doc-abstraction-format/ssp-format-specification.md
+ and from nothing else: the point of it is to be what the specification
+ says, executed, so that a file can be held against the document rather
+ than against an implementation. Each check names the section it comes
+ from, so a disagreement can be taken back to the text.
++/
+
+import std.algorithm : canFind, startsWith;
+import std.array : array, split;
+import std.conv : to;
+import std.file : readText;
+import std.stdio : writeln, writefln;
+import std.string : indexOf, strip;
+
+struct SspFinding {
+ size_t line; // 1-based
+ string section; // the specification section the rule is from
+ string detail;
+}
+
+/+ ↓ specification section 6.3: the properties, in the order they are
+ emitted. A reader may rely on the order, so a validator can check it +/
+private immutable string[] property_order = [
+ "identifier", "part", "section", "parent", "last_descendant", "children",
+ "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed",
+ "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type",
+ "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent",
+ "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns",
+ "table_header", "code_linenumbers", "stow_link", "segment", "anchor",
+ "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub",
+ "segment_html_is", "segment_epub_is", "segment_lv4_is",
+ "heading_ancestors_text", "lev4_subtoc", "anchor_tag",
+];
+
+/+ ↓ section 6.1: the sixteen kinds +/
+private immutable string[] kinds = [
+ "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography",
+ "comment", "verse", "glossary", "quote", "poem", "block", "table",
+ "blurb", "group",
+];
+
+/+ ↓ section 4: the nine sections, in the order they appear +/
+private immutable string[] section_order = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography",
+ "bookindex", "blurb", "tail",
+];
+
+/+ ↓ section 4: the four preamble blocks, in the order they appear +/
+private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"];
+
+/+ ↓ section 6.1: the heading levels +/
+private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"];
+
+private int propertyRank(string name) {
+ foreach (i, p; property_order) { if (p == name) { return cast(int) i; } }
+ return -1;
+}
+
+SspFinding[] validateSspFile(string filename, string major_known) {
+ SspFinding[] found;
+ auto lines = readText(filename).split("\n");
+
+ bool seen_banner = false;
+ bool seen_source_line = false;
+ string block = ""; // the block currently open, "" when none
+ int preamble_at = -1; // how far through the preamble order
+ int section_at = -1; // how far through the section order
+ bool in_object = false;
+ int prev_rank = -1;
+ size_t doc_has_keys = 0;
+ bool in_text = false; // text lines come after properties
+
+ foreach (i, line; lines) {
+ size_t no = i + 1;
+
+ /+ ↓ section 2: the version, on the first line +/
+ if (no == 1) {
+ enum banner = "% SiSU Document Abstraction v";
+ if (!line.startsWith(banner)) {
+ found ~= SspFinding(no, "2", "first line is not the version banner");
+ } else {
+ seen_banner = true;
+ string v = line[banner.length .. $].strip;
+ auto dot = v.indexOf('.');
+ string major = (dot > 0) ? v[0 .. dot] : v;
+ if (major != major_known) {
+ found ~= SspFinding(no, "2.1",
+ "format major v" ~ major ~ ", this validator knows v" ~ major_known);
+ }
+ }
+ continue;
+ }
+ if (no == 2) {
+ if (!line.startsWith("% Source: ")) {
+ found ~= SspFinding(no, "4", "second line is not \"% Source: \"");
+ } else { seen_source_line = true; }
+ continue;
+ }
+
+ if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; }
+
+ /+ ↓ section 3: a comment is the file header only +/
+ if (line[0] == '%') {
+ found ~= SspFinding(no, "3", "a comment outside the two header lines");
+ continue;
+ }
+
+ /+ ↓ block open +/
+ if (line[0] == '@') {
+ auto brace = line.indexOf(" {");
+ if (brace < 1 || line[$ - 1] != '{') {
+ found ~= SspFinding(no, "3", "block open is not \"@name {\"");
+ continue;
+ }
+ string name = line[1 .. brace];
+ if (preamble_order.canFind(name)) {
+ int at = 0;
+ foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } }
+ if (at <= preamble_at) {
+ found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order");
+ }
+ preamble_at = at;
+ if (section_at >= 0) {
+ found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section");
+ }
+ } else if (section_order.canFind(name)) {
+ int at = 0;
+ foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } }
+ if (at <= section_at) {
+ found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order");
+ }
+ section_at = at;
+ } else {
+ found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section");
+ }
+ block = name;
+ doc_has_keys = 0;
+ continue;
+ }
+
+ if (line[0] == '}') {
+ if (line.length != 1) {
+ found ~= SspFinding(no, "3", "block close carries more than \"}\"");
+ }
+ /+ ↓ section 5.4: ten keys, always +/
+ if (block == "doc_has" && doc_has_keys != 10) {
+ found ~= SspFinding(no, "5.4",
+ "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten");
+ }
+ block = "";
+ in_object = false;
+ prev_rank = -1;
+ continue;
+ }
+
+ /+ ↓ section 5: inside a preamble block, " key: value" +/
+ if (preamble_order.canFind(block)) {
+ if (!line.startsWith(" ") || line.indexOf(": ") < 3) {
+ found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block);
+ } else if (block == "doc_has") {
+ doc_has_keys++;
+ }
+ continue;
+ }
+
+ /+ ↓ section 6.1: the object declaration line +/
+ if (line[0] == '[') {
+ auto close = line.indexOf("] ");
+ if (close < 2) {
+ found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\"");
+ continue;
+ }
+ string ocn = line[1 .. close];
+ foreach (c; ocn) {
+ if (c < '0' || c > '9') {
+ found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number");
+ break;
+ }
+ }
+ auto rest = line[close + 2 .. $].split(" ");
+ if (rest.length == 0 || !kinds.canFind(rest[0])) {
+ found ~= SspFinding(no, "6.1",
+ "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds");
+ } else if (rest[0] == "heading") {
+ if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') {
+ found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\"");
+ } else if (!levels.canFind(rest[1][1 .. $])) {
+ found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight");
+ }
+ }
+ in_object = true;
+ in_text = false;
+ prev_rank = -1;
+ continue;
+ }
+
+ /+ ↓ section 6.3: properties, known and in order +/
+ if (line[0] == '.') {
+ if (!in_object) {
+ found ~= SspFinding(no, "6", "a property outside an object record");
+ continue;
+ }
+ if (in_text) {
+ found ~= SspFinding(no, "6", "a property after the object text began");
+ }
+ auto colon = line.indexOf(": ");
+ if (colon < 2) {
+ found ~= SspFinding(no, "3", "property is not \".name: value\"");
+ continue;
+ }
+ string name = line[1 .. colon];
+ int rank = propertyRank(name);
+ if (rank < 0) {
+ found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the 42 properties");
+ } else {
+ /+ ↓ the four repeatable properties may recur at the same rank +/
+ if (rank < prev_rank) {
+ found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order");
+ }
+ prev_rank = rank;
+ }
+ continue;
+ }
+
+ /+ ↓ section 6.2: text lines +/
+ if (line[0] == '|') {
+ if (!in_object) {
+ found ~= SspFinding(no, "6.2", "a text line outside an object record");
+ }
+ if (line.length < 2 || line[1] != ' ') {
+ found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text");
+ }
+ in_text = true;
+ continue;
+ }
+
+ found ~= SspFinding(no, "3", "line of no shape the specification defines");
+ }
+
+ if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); }
+ if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); }
+ return found;
+}
+
+void reportValidation(string filename, SspFinding[] found) {
+ if (found.length == 0) {
+ writeln("\nvalid: ", filename,
+ " conforms to the .ssp specification as this validator reads it");
+ return;
+ }
+ writefln("\n%d finding%s against the specification:",
+ found.length, (found.length == 1) ? "" : "s");
+ foreach (f; found) {
+ if (f.line > 0) {
+ writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail);
+ } else {
+ writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail);
+ }
+ }
+}
diff --git a/dlang/src/verify_images.d b/dlang/src/verify_images.d
new file mode 100644
index 0000000..216c02a
--- /dev/null
+++ b/dlang/src/verify_images.d
@@ -0,0 +1,212 @@
+module verify_images;
+
+import std.algorithm : sort;
+import std.array : array, join;
+import std.conv : to;
+import std.digest : toHexString;
+import std.digest.sha : SHA256, digest;
+import std.file : dirEntries, exists, isDir, SpanMode, getSize;
+import std.path : baseName, buildPath, extension;
+import std.stdio : writeln, File;
+import std.string : toLower;
+import ssp_types;
+import output_images : gatherImageRecords;
+
+/+ ↓ verification: recompute what the .image records claim, against the
+ images as they stand in a directory (a pod's media/image).
+
+ ssp-reader --verify --image-dir=<pod>/media/image doc.ssp
+
+ Reports one line per distinct file:
+
+ OK won_benkler_2_1.png
+ MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D...
+ ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory)
+ PRESENT ffa.png (recorded missing, but the file is there)
+ EXTRA unused.png (in directory, no record in this .ssp)
+
+ Returns the number of failures. EXTRA is reported but is not a failure:
+ a pod carries the images of every language version of a document, while
+ one .ssp covers one language.
++/
+int verifyImages(SspDocument doc, string image_dir, string output_file) {
+ string[] lines;
+ int failures = 0;
+ int ok = 0;
+ int mismatch = 0;
+ int absent = 0;
+ int extra = 0;
+
+ if (!(image_dir.exists) || !(image_dir.isDir)) {
+ writeln("ERROR: image directory not found: ", image_dir);
+ return 1;
+ }
+ auto gathered = gatherImageRecords(doc);
+ bool[string] recorded;
+
+ foreach (e; gathered.entries) {
+ recorded[e.name] = true;
+ string pth = buildPath(image_dir, e.name);
+ if (!(pth.exists)) {
+ if (e.missing) {
+ /+ recorded missing, and missing here too: consistent +/
+ ok += 1;
+ lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)";
+ } else {
+ absent += 1;
+ failures += 1;
+ lines ~= "ABSENT " ~ e.name
+ ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)";
+ }
+ continue;
+ }
+ if (e.missing) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)";
+ continue;
+ }
+ auto file_ = readFileStat(pth);
+ string[] differs;
+ if (file_.sha256 != e.sha256) {
+ differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256;
+ }
+ if (file_.bytes != e.bytes) {
+ differs ~= "bytes recorded " ~ e.bytes.to!string
+ ~ " file " ~ file_.bytes.to!string;
+ }
+ /+ px is written only where the dimensions could be read, and read here
+ only for the formats below, so compare only when both are known +/
+ if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0
+ && (file_.width != e.width || file_.height != e.height)) {
+ differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string
+ ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string;
+ }
+ if (differs.length > 0) {
+ mismatch += 1;
+ failures += 1;
+ lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; ");
+ } else {
+ ok += 1;
+ lines ~= "OK " ~ e.name;
+ }
+ }
+
+ /+ ↓ files in the directory that no record names +/
+ string[] in_dir;
+ foreach (entry; dirEntries(image_dir, SpanMode.shallow)) {
+ if (!(entry.isFile)) continue;
+ string nm = entry.name.baseName;
+ switch (nm.extension.toLower) {
+ case ".png": case ".jpg": case ".jpeg": case ".gif":
+ in_dir ~= nm;
+ break;
+ default: break;
+ }
+ }
+ in_dir.sort;
+ foreach (nm; in_dir) {
+ if (nm !in recorded) {
+ extra += 1;
+ lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)";
+ }
+ }
+
+ string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file"
+ ~ ((gathered.entries.length == 1) ? "" : "s")
+ ~ " in " ~ gathered.records.to!string ~ " record"
+ ~ ((gathered.records == 1) ? "" : "s")
+ ~ ", against " ~ image_dir;
+ lines = [head, ""] ~ lines;
+ lines ~= "";
+ lines ~= "verify: " ~ ok.to!string ~ " ok, "
+ ~ mismatch.to!string ~ " mismatch, "
+ ~ absent.to!string ~ " absent, "
+ ~ extra.to!string ~ " extra"
+ ~ ((failures == 0) ? " PASS" : " FAIL");
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+ return failures;
+}
+
+private struct FileStat_ {
+ string sha256;
+ ulong bytes;
+ int width;
+ int height;
+}
+
+private FileStat_ readFileStat(string pth) {
+ FileStat_ ret;
+ auto f = File(pth, "rb");
+ ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024));
+ ret.sha256 = sha.toHexString.to!string;
+ ret.bytes = pth.getSize;
+ auto dim = readImageDimensions(pth);
+ ret.width = dim[0];
+ ret.height = dim[1];
+ return ret;
+}
+
+/+ ↓ pixel dimensions from the file header, for the formats spine's markup
+ admits (png, gif, jpg). Header read only, no decoding, and no dependency
+ beyond phobos. Returns [0, 0] where the header cannot be read +/
+private int[2] readImageDimensions(string pth) {
+ int[2] ret = [0, 0];
+ ubyte[] head;
+ try {
+ auto f = File(pth, "rb");
+ head = new ubyte[](64 * 1024);
+ head = f.rawRead(head);
+ } catch (Exception ex) {
+ return ret;
+ }
+ if (head.length >= 24
+ && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
+ /+ png: IHDR width & height, big endian, at 16 and 20 +/
+ ret[0] = beInt(head[16 .. 20]);
+ ret[1] = beInt(head[20 .. 24]);
+ return ret;
+ }
+ if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) {
+ /+ gif: logical screen width & height, little endian, at 6 and 8 +/
+ ret[0] = head[6] | (head[7] << 8);
+ ret[1] = head[8] | (head[9] << 8);
+ return ret;
+ }
+ if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) {
+ /+ jpeg: walk the segments to a start of frame, height then width,
+ big endian, at offsets 5 and 7 of the segment +/
+ size_t i = 2;
+ while (i + 9 < head.length) {
+ if (head[i] != 0xff) { i += 1; continue; }
+ ubyte marker = head[i + 1];
+ if (marker == 0xd8 || marker == 0x01
+ || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }
+ if (marker == 0xff) { i += 1; continue; }
+ size_t seg_len = (head[i + 2] << 8) | head[i + 3];
+ bool is_sof = ((marker >= 0xc0 && marker <= 0xc3)
+ || (marker >= 0xc5 && marker <= 0xc7)
+ || (marker >= 0xc9 && marker <= 0xcb)
+ || (marker >= 0xcd && marker <= 0xcf));
+ if (is_sof) {
+ ret[1] = (head[i + 5] << 8) | head[i + 6];
+ ret[0] = (head[i + 7] << 8) | head[i + 8];
+ return ret;
+ }
+ if (seg_len < 2) break;
+ i += 2 + seg_len;
+ }
+ }
+ return ret;
+}
+
+private int beInt(const(ubyte)[] b) {
+ return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]);
+}