summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/llm_jsonl.d
diff options
context:
space:
mode:
Diffstat (limited to 'dlang/src/read/llm_jsonl.d')
-rw-r--r--dlang/src/read/llm_jsonl.d121
1 files changed, 121 insertions, 0 deletions
diff --git a/dlang/src/read/llm_jsonl.d b/dlang/src/read/llm_jsonl.d
new file mode 100644
index 0000000..5b81e68
--- /dev/null
+++ b/dlang/src/read/llm_jsonl.d
@@ -0,0 +1,121 @@
+module read.llm_jsonl;
+
+import std.array : split, join, replace;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ocda.model;
+import read.llm_text : stripMarkup;
+
+/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
+ one JSON object per line, each containing:
+ doc, ocn, path (breadcrumb), type, text (clean),
+ images (only where the object embeds one, so that a chunk whose
+ text reads around a figure carries the figure's identity),
+ index_terms (from bookindex entries associated with this object)
++/
+void outputLlmJsonl(OcdaDocument doc, string output_file) {
+ string[] lines;
+ string doc_id = doc.meta("title.main");
+ string doc_author = doc.meta("creator.author");
+
+ /+ ↓ build bookindex -> OCN mapping for index_terms
+ bookindex objects in the .ssp have parent_ocn linking to body objects
+ but the index terms are in the text of bookindex objects, not
+ directly attached to body objects. We use a simpler approach:
+ headings carry heading_ancestors_text which gives context. +/
+
+ /+ ↓ track current heading path +/
+ string current_path = "";
+
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
+ /+ ↓ emit heading as a chunk too +/
+ string heading_text = stripMarkup(obj.text);
+ if (heading_text.strip.length > 0) {
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
+ }
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content, and least of all is it a chunk
+ to embed: it would be indexed as though the author had written it
+ into the document
+ +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
+ return `{"doc":` ~ jsonStr(doc_id)
+ ~ `,"author":` ~ jsonStr(doc_author)
+ ~ `,"ocn":` ~ ocn.to!string
+ ~ `,"path":` ~ jsonStr(path)
+ ~ `,"type":` ~ jsonStr(type)
+ ~ `,"text":` ~ jsonStr(text)
+ ~ jsonImages(images)
+ ~ `}`;
+}
+
+private string jsonImages(OcdaImage[] images) {
+ /+ ↓ omitted entirely where the object embeds no image +/
+ if (images.length == 0) return "";
+ string[] items;
+ foreach (img; images) {
+ string item = `{"name":` ~ jsonStr(img.name);
+ if (img.missing) {
+ item ~= `,"missing":true`;
+ } else {
+ item ~= `,"sha256":` ~ jsonStr(img.sha256)
+ ~ `,"bytes":` ~ img.bytes.to!string;
+ if (img.width > 0 && img.height > 0) {
+ item ~= `,"width":` ~ img.width.to!string
+ ~ `,"height":` ~ img.height.to!string;
+ }
+ }
+ items ~= item ~ `}`;
+ }
+ return `,"images":[` ~ items.join(",") ~ `]`;
+}
+
+private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ parts ~= clean;
+ }
+ }
+ }
+ string heading_text = stripMarkup(obj.text).strip;
+ if (parts.length > 0)
+ return parts.join(" > ") ~ " > " ~ heading_text;
+ else
+ return heading_text;
+}
+
+private string jsonStr(string s) {
+ /+ ↓ JSON string escaping +/
+ string r = s;
+ r = r.replace(`\`, `\\`);
+ r = r.replace(`"`, `\"`);
+ r = r.replace("\n", `\n`);
+ r = r.replace("\r", `\r`);
+ r = r.replace("\t", `\t`);
+ return `"` ~ r ~ `"`;
+}