diff options
Diffstat (limited to 'dlang/src/read/llm_jsonl.d')
| -rw-r--r-- | dlang/src/read/llm_jsonl.d | 121 |
1 files changed, 121 insertions, 0 deletions
diff --git a/dlang/src/read/llm_jsonl.d b/dlang/src/read/llm_jsonl.d new file mode 100644 index 0000000..5b81e68 --- /dev/null +++ b/dlang/src/read/llm_jsonl.d @@ -0,0 +1,121 @@ +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} |
