summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/output_llm_jsonl.d
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-24 16:33:53 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-24 16:33:53 -0400
commitbbe505c8d48e822c012c9820dfe93b83dc61bcab (patch)
tree3385f5420959ff20af295c3125a080297555a72b /dlang/src/output_llm_jsonl.d
parentdealing with six properties including .is_of_type (diff)
--validate, and a second reader, in Gleam
--validate: the specification, executed. A validator written from ssp-format-specification.md and from nothing else, so that a file can be held against the document rather than against an implementation. Each finding names the specification section it comes from, so a disagreement can be taken back to the text. It checks the version banner and its major part, the file shape and the order of the preamble blocks and the nine sections, the two spaces of a preamble pair, the object declaration and its sixteen kinds and eight heading levels, that a property is one of the 42 and in the emitted order, that text lines carry their prefix, the ten keys of doc_has, and that no line is of a shape the specification does not define. Every .ssp of the markup document samples validates clean. Five classes of deliberate breakage are caught: an unknown major version, a property that is not one of the 42, a property out of order, a kind that is not one of the sixteen, and a block that is neither preamble nor section. Writing it found two rules the specification implied and did not state (repeated properties are consecutive; no property follows the text), and both were written into it. One repository, a reader per language. The readers are not separate tools that share a name. They are the same specification implemented more than once, and the interesting thing about them is that they agree, so they are published as one thing with that claim at the top of it. The D reader moves to dlang/; the Gleam reader arrives at gleam/. The language directory is named for the language and not for the tool: gleam refuses a hyphenated package name, so ssp-reader cannot be a gleam package and must be ssp_reader. A .gitignore for the repository, denying everything and whitelisting what belongs, covering both languages. Both build under nix, sandboxed and without network. There is no gleam builder in nixpkgs and gleam build fetches from Hex, so the Gleam build uses nix-gleam's builder, kept in gleam/nix-gleam/ (MIT), which fetches each Hex package by the checksum manifest.toml already carries, with no hash to maintain; the result is an escript wrapped with erlang. The D reader depends on nothing beyond phobos, so --skip-registry=all is the whole of what it takes: ldc by default, dmd available as ssp-reader-dmd (given gcc14, GCC 15 having introduced nullptr in headers dmd's ImportC does not yet parse). (assisted by Claude-Code)
Diffstat (limited to 'dlang/src/output_llm_jsonl.d')
-rw-r--r--dlang/src/output_llm_jsonl.d121
1 files changed, 121 insertions, 0 deletions
diff --git a/dlang/src/output_llm_jsonl.d b/dlang/src/output_llm_jsonl.d
new file mode 100644
index 0000000..3d34089
--- /dev/null
+++ b/dlang/src/output_llm_jsonl.d
@@ -0,0 +1,121 @@
+module output_llm_jsonl;
+
+import std.array : split, join, replace;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ssp_types;
+import output_llm_breadcrumbs : stripMarkup;
+
+/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
+ one JSON object per line, each containing:
+ doc, ocn, path (breadcrumb), type, text (clean),
+ images (only where the object embeds one, so that a chunk whose
+ text reads around a figure carries the figure's identity),
+ index_terms (from bookindex entries associated with this object)
++/
+void outputLlmJsonl(SspDocument doc, string output_file) {
+ string[] lines;
+ string doc_id = doc.meta("title.main");
+ string doc_author = doc.meta("creator.author");
+
+ /+ ↓ build bookindex -> OCN mapping for index_terms
+ bookindex objects in the .ssp have parent_ocn linking to body objects
+ but the index terms are in the text of bookindex objects, not
+ directly attached to body objects. We use a simpler approach:
+ headings carry heading_ancestors_text which gives context. +/
+
+ /+ ↓ track current heading path +/
+ string current_path = "";
+
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
+ /+ ↓ emit heading as a chunk too +/
+ string heading_text = stripMarkup(obj.text);
+ if (heading_text.strip.length > 0) {
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
+ }
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content, and least of all is it a chunk
+ to embed: it would be indexed as though the author had written it
+ into the document
+ +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, SspImage[] images) {
+ return `{"doc":` ~ jsonStr(doc_id)
+ ~ `,"author":` ~ jsonStr(doc_author)
+ ~ `,"ocn":` ~ ocn.to!string
+ ~ `,"path":` ~ jsonStr(path)
+ ~ `,"type":` ~ jsonStr(type)
+ ~ `,"text":` ~ jsonStr(text)
+ ~ jsonImages(images)
+ ~ `}`;
+}
+
+private string jsonImages(SspImage[] images) {
+ /+ ↓ omitted entirely where the object embeds no image +/
+ if (images.length == 0) return "";
+ string[] items;
+ foreach (img; images) {
+ string item = `{"name":` ~ jsonStr(img.name);
+ if (img.missing) {
+ item ~= `,"missing":true`;
+ } else {
+ item ~= `,"sha256":` ~ jsonStr(img.sha256)
+ ~ `,"bytes":` ~ img.bytes.to!string;
+ if (img.width > 0 && img.height > 0) {
+ item ~= `,"width":` ~ img.width.to!string
+ ~ `,"height":` ~ img.height.to!string;
+ }
+ }
+ items ~= item ~ `}`;
+ }
+ return `,"images":[` ~ items.join(",") ~ `]`;
+}
+
+private string buildPath(string doc_title, string ancestors_text, SspObject obj) {
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ parts ~= clean;
+ }
+ }
+ }
+ string heading_text = stripMarkup(obj.text).strip;
+ if (parts.length > 0)
+ return parts.join(" > ") ~ " > " ~ heading_text;
+ else
+ return heading_text;
+}
+
+private string jsonStr(string s) {
+ /+ ↓ JSON string escaping +/
+ string r = s;
+ r = r.replace(`\`, `\\`);
+ r = r.replace(`"`, `\"`);
+ r = r.replace("\n", `\n`);
+ r = r.replace("\r", `\r`);
+ r = r.replace("\t", `\t`);
+ return `"` ~ r ~ `"`;
+}