diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-12 20:24:11 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-10-12 20:24:11 -0400 |
| commit | 4e4f2f8c2b2c043077390cb4ccf7ce97ffe4ac6b (patch) | |
| tree | bfafd6fb9cfa975db49fbcbcf7e3b23a4f13e0bc /dlang/src/read/llm_jsonl.d | |
| parent | model 2.1: ocn_range, the verses a poem holds (diff) | |
format 2.1: ocda-read, the readers folded in
ocda-read, one reader where there were two. The .ssp reader and the
.ocda.db reader each had their own parser; this one is built on the
library, so every mode reads a .ssp, a .ocda.db and a pod, and says
the same thing of each. Its output modules are the .ssp reader's,
copied (the reader repositories are kept, frozen) and moved onto the
library's model; --languages and --files come from the database
reader.
ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images|
--verify --image-dir=<d>|--languages|--files|
--report-unknown|--validate] <file> [lang]
In D every mode of both readers; in Gleam the modes the Gleam readers
had (the summary, --tree, --languages, --files, --report-unknown),
whose output is the D reader's, byte for byte. In each language it
lives in src/read/ and is not part of the library: dub configurations
keep it out of a dependant's build in D, and Gleam compiles all of
src/ but a dependant need not use it.
Checked over the markup document samples: against the .ssp reader on
a .ssp, byte for byte after the first line in every D mode but the
summary, whose closing hint differs; against the database reader on a
.ocda.db, --files and --languages (its id column left out); the same
output from the .ssp, the .ocda.db and the .sisupod; and D == Gleam.
The database reader's --outline is not reproduced: it ordered
headings within their own section, so that other sections' headings
fell among the body's. --outline here is --tree.
What writing it found in the library: the database loader could not
say what it had not understood. It asked for columns by name and so
never saw the rest, and reported that it had understood everything.
It now counts them on the conditions under which the .ssp writes the
same property, and --report-unknown is the same from every form.
ocda.inventory answers two questions about the file rather than the
document: its languages, in the manifest's order, each with the
markup it was made from and the sha256 of its .ssp, for every form
alike; and what a .ocda.db carries besides the abstraction. In Gleam
the sha256 comes from OTP's crypto through a four line Erlang FFI.
The flake's default is all three D programs in one result: built
separately into the same link, each replaced the one before.
(assisted by Claude-Code)
Diffstat (limited to 'dlang/src/read/llm_jsonl.d')
| -rw-r--r-- | dlang/src/read/llm_jsonl.d | 121 |
1 files changed, 121 insertions, 0 deletions
diff --git a/dlang/src/read/llm_jsonl.d b/dlang/src/read/llm_jsonl.d new file mode 100644 index 0000000..5b81e68 --- /dev/null +++ b/dlang/src/read/llm_jsonl.d @@ -0,0 +1,121 @@ +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} |
