1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
module read.llm_jsonl;
import std.array : split, join, replace;
import std.conv : to;
import std.stdio : writeln, File;
import std.string : strip, indexOf, stripRight;
import ocda.model;
import read.llm_text : stripMarkup;
/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
one JSON object per line, each containing:
doc, ocn, path (breadcrumb), type, text (clean),
images (only where the object embeds one, so that a chunk whose
text reads around a figure carries the figure's identity),
index_terms (from bookindex entries associated with this object)
+/
void outputLlmJsonl(OcdaDocument doc, string output_file) {
string[] lines;
string doc_id = doc.meta("title.main");
string doc_author = doc.meta("creator.author");
/+ ↓ build bookindex -> OCN mapping for index_terms
bookindex objects in the .ssp have parent_ocn linking to body objects
but the index terms are in the text of bookindex objects, not
directly attached to body objects. We use a simpler approach:
headings carry heading_ancestors_text which gives context. +/
/+ ↓ track current heading path +/
string current_path = "";
foreach (obj; doc.body) {
if (obj.is_a == "heading") {
current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
/+ ↓ emit heading as a chunk too +/
string heading_text = stripMarkup(obj.text);
if (heading_text.strip.length > 0) {
lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
}
continue;
}
/+ ↓ markup commentary is not content, and least of all is it a chunk
to embed: it would be indexed as though the author had written it
into the document
+/
if (!obj.renders) { continue; }
string text = stripMarkup(obj.text);
if (text.strip.length == 0) continue;
lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
}
/+ ↓ write output +/
if (output_file.length > 0) {
auto f = File(output_file, "w");
foreach (line; lines) f.writeln(line);
} else {
foreach (line; lines) writeln(line);
}
}
private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
return `{"doc":` ~ jsonStr(doc_id)
~ `,"author":` ~ jsonStr(doc_author)
~ `,"ocn":` ~ ocn.to!string
~ `,"path":` ~ jsonStr(path)
~ `,"type":` ~ jsonStr(type)
~ `,"text":` ~ jsonStr(text)
~ jsonImages(images)
~ `}`;
}
private string jsonImages(OcdaImage[] images) {
/+ ↓ omitted entirely where the object embeds no image +/
if (images.length == 0) return "";
string[] items;
foreach (img; images) {
string item = `{"name":` ~ jsonStr(img.name);
if (img.missing) {
item ~= `,"missing":true`;
} else {
item ~= `,"sha256":` ~ jsonStr(img.sha256)
~ `,"bytes":` ~ img.bytes.to!string;
if (img.width > 0 && img.height > 0) {
item ~= `,"width":` ~ img.width.to!string
~ `,"height":` ~ img.height.to!string;
}
}
items ~= item ~ `}`;
}
return `,"images":[` ~ items.join(",") ~ `]`;
}
private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
string[] parts;
if (ancestors_text.length > 0) {
foreach (part; ancestors_text.split("|")) {
string clean = stripMarkup(part).strip;
if (clean.length > 0 && clean != doc_title) {
parts ~= clean;
}
}
}
string heading_text = stripMarkup(obj.text).strip;
if (parts.length > 0)
return parts.join(" > ") ~ " > " ~ heading_text;
else
return heading_text;
}
private string jsonStr(string s) {
/+ ↓ JSON string escaping +/
string r = s;
r = r.replace(`\`, `\\`);
r = r.replace(`"`, `\"`);
r = r.replace("\n", `\n`);
r = r.replace("\r", `\r`);
r = r.replace("\t", `\t`);
return `"` ~ r ~ `"`;
}
|