summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/llm_jsonl.d
blob: 5b81e68d52cb610d05a946ffd7433af81038643e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
module read.llm_jsonl;

import std.array : split, join, replace;
import std.conv : to;
import std.stdio : writeln, File;
import std.string : strip, indexOf, stripRight;
import ocda.model;
import read.llm_text : stripMarkup;

/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
   one JSON object per line, each containing:
     doc, ocn, path (breadcrumb), type, text (clean),
     images (only where the object embeds one, so that a chunk whose
     text reads around a figure carries the figure's identity),
     index_terms (from bookindex entries associated with this object)
+/
void outputLlmJsonl(OcdaDocument doc, string output_file) {
  string[] lines;
  string doc_id = doc.meta("title.main");
  string doc_author = doc.meta("creator.author");

  /+ ↓ build bookindex -> OCN mapping for index_terms
     bookindex objects in the .ssp have parent_ocn linking to body objects
     but the index terms are in the text of bookindex objects, not
     directly attached to body objects. We use a simpler approach:
     headings carry heading_ancestors_text which gives context. +/

  /+ ↓ track current heading path +/
  string current_path = "";

  foreach (obj; doc.body) {
    if (obj.is_a == "heading") {
      current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
      /+ ↓ emit heading as a chunk too +/
      string heading_text = stripMarkup(obj.text);
      if (heading_text.strip.length > 0) {
        lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
      }
      continue;
    }

    /+ ↓ markup commentary is not content, and least of all is it a chunk
         to embed: it would be indexed as though the author had written it
         into the document
    +/
    if (!obj.renders) { continue; }

    string text = stripMarkup(obj.text);
    if (text.strip.length == 0) continue;

    lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
  }

  /+ ↓ write output +/
  if (output_file.length > 0) {
    auto f = File(output_file, "w");
    foreach (line; lines) f.writeln(line);
  } else {
    foreach (line; lines) writeln(line);
  }
}

private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
  return `{"doc":` ~ jsonStr(doc_id)
    ~ `,"author":` ~ jsonStr(doc_author)
    ~ `,"ocn":` ~ ocn.to!string
    ~ `,"path":` ~ jsonStr(path)
    ~ `,"type":` ~ jsonStr(type)
    ~ `,"text":` ~ jsonStr(text)
    ~ jsonImages(images)
    ~ `}`;
}

private string jsonImages(OcdaImage[] images) {
  /+ ↓ omitted entirely where the object embeds no image +/
  if (images.length == 0) return "";
  string[] items;
  foreach (img; images) {
    string item = `{"name":` ~ jsonStr(img.name);
    if (img.missing) {
      item ~= `,"missing":true`;
    } else {
      item ~= `,"sha256":` ~ jsonStr(img.sha256)
        ~ `,"bytes":` ~ img.bytes.to!string;
      if (img.width > 0 && img.height > 0) {
        item ~= `,"width":` ~ img.width.to!string
          ~ `,"height":` ~ img.height.to!string;
      }
    }
    items ~= item ~ `}`;
  }
  return `,"images":[` ~ items.join(",") ~ `]`;
}

private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
  string[] parts;
  if (ancestors_text.length > 0) {
    foreach (part; ancestors_text.split("|")) {
      string clean = stripMarkup(part).strip;
      if (clean.length > 0 && clean != doc_title) {
        parts ~= clean;
      }
    }
  }
  string heading_text = stripMarkup(obj.text).strip;
  if (parts.length > 0)
    return parts.join(" > ") ~ " > " ~ heading_text;
  else
    return heading_text;
}

private string jsonStr(string s) {
  /+ ↓ JSON string escaping +/
  string r = s;
  r = r.replace(`\`, `\\`);
  r = r.replace(`"`, `\"`);
  r = r.replace("\n", `\n`);
  r = r.replace("\r", `\r`);
  r = r.replace("\t", `\t`);
  return `"` ~ r ~ `"`;
}