summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/llm_text.d
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-10-12 20:24:11 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-10-12 20:24:11 -0400
commit4e4f2f8c2b2c043077390cb4ccf7ce97ffe4ac6b (patch)
treebfafd6fb9cfa975db49fbcbcf7e3b23a4f13e0bc /dlang/src/read/llm_text.d
parentmodel 2.1: ocn_range, the verses a poem holds (diff)
format 2.1: ocda-read, the readers folded in
ocda-read, one reader where there were two. The .ssp reader and the .ocda.db reader each had their own parser; this one is built on the library, so every mode reads a .ssp, a .ocda.db and a pod, and says the same thing of each. Its output modules are the .ssp reader's, copied (the reader repositories are kept, frozen) and moved onto the library's model; --languages and --files come from the database reader. ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images| --verify --image-dir=<d>|--languages|--files| --report-unknown|--validate] <file> [lang] In D every mode of both readers; in Gleam the modes the Gleam readers had (the summary, --tree, --languages, --files, --report-unknown), whose output is the D reader's, byte for byte. In each language it lives in src/read/ and is not part of the library: dub configurations keep it out of a dependant's build in D, and Gleam compiles all of src/ but a dependant need not use it. Checked over the markup document samples: against the .ssp reader on a .ssp, byte for byte after the first line in every D mode but the summary, whose closing hint differs; against the database reader on a .ocda.db, --files and --languages (its id column left out); the same output from the .ssp, the .ocda.db and the .sisupod; and D == Gleam. The database reader's --outline is not reproduced: it ordered headings within their own section, so that other sections' headings fell among the body's. --outline here is --tree. What writing it found in the library: the database loader could not say what it had not understood. It asked for columns by name and so never saw the rest, and reported that it had understood everything. It now counts them on the conditions under which the .ssp writes the same property, and --report-unknown is the same from every form. ocda.inventory answers two questions about the file rather than the document: its languages, in the manifest's order, each with the markup it was made from and the sha256 of its .ssp, for every form alike; and what a .ocda.db carries besides the abstraction. In Gleam the sha256 comes from OTP's crypto through a four line Erlang FFI. The flake's default is all three D programs in one result: built separately into the same link, each replaced the one before. (assisted by Claude-Code)
Diffstat (limited to 'dlang/src/read/llm_text.d')
-rw-r--r--dlang/src/read/llm_text.d182
1 files changed, 182 insertions, 0 deletions
diff --git a/dlang/src/read/llm_text.d b/dlang/src/read/llm_text.d
new file mode 100644
index 0000000..e61f2b7
--- /dev/null
+++ b/dlang/src/read/llm_text.d
@@ -0,0 +1,182 @@
+module read.llm_text;
+
+import std.array : replace, split, join;
+import std.conv : to;
+import std.stdio : writeln, File;
+import std.string : strip, indexOf, stripRight;
+import ocda.model;
+
+/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption
+ format:
+ ## Document Title > Chapter > Section > [OCN]
+
+ Clean paragraph text with inline markup stripped.
++/
+void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) {
+ string[] lines;
+ string doc_title = doc.meta("title.main");
+
+ /+ ↓ document header +/
+ lines ~= "# " ~ doc_title;
+ if (doc.meta("title.subtitle").length > 0)
+ lines ~= "# " ~ doc.meta("title.subtitle");
+ lines ~= "# " ~ doc.meta("creator.author");
+ lines ~= "";
+
+ /+ ↓ build heading text lookup from ancestors +/
+ string[int] heading_text;
+ foreach (obj; doc.head) {
+ if (obj.is_a == "heading" && obj.ocn > 0)
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading")
+ heading_text[obj.ocn] = stripMarkup(obj.text);
+ }
+
+ /+ ↓ body objects +/
+ string current_breadcrumb = "";
+ foreach (obj; doc.body) {
+ if (obj.is_a == "heading") {
+ current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj);
+ lines ~= "";
+ lines ~= "## " ~ current_breadcrumb;
+ lines ~= "";
+ continue;
+ }
+
+ /+ ↓ markup commentary is not content +/
+ if (!obj.renders) { continue; }
+
+ string text = stripMarkup(obj.text);
+ if (text.strip.length == 0) continue;
+
+ string ocn_ref = (obj.ocn > 0 && !obj.ocn_off)
+ ? " [" ~ obj.ocn.to!string ~ "]"
+ : "";
+
+ switch (obj.is_a) {
+ case "para":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "code":
+ lines ~= "```";
+ lines ~= obj.text; /+ preserve raw text for code +/
+ lines ~= "```";
+ lines ~= "";
+ break;
+ case "table":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "verse":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "quote":
+ lines ~= "> " ~ text ~ ocn_ref;
+ lines ~= "";
+ break;
+ case "group":
+ case "block":
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ break;
+ default:
+ if (text.length > 0) {
+ lines ~= text ~ ocn_ref;
+ lines ~= "";
+ }
+ break;
+ }
+ }
+
+ /+ ↓ write output +/
+ if (output_file.length > 0) {
+ auto f = File(output_file, "w");
+ foreach (line; lines) f.writeln(line);
+ } else {
+ foreach (line; lines) writeln(line);
+ }
+}
+
+private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) {
+ /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/
+ string[] parts;
+ if (ancestors_text.length > 0) {
+ foreach (part; ancestors_text.split("|")) {
+ string clean = stripMarkup(part).strip;
+ if (clean.length > 0 && clean != doc_title) {
+ /+ ↓ truncate very long heading text for breadcrumb +/
+ if (clean.length > 60)
+ clean = clean[0 .. 57] ~ "...";
+ parts ~= clean;
+ }
+ }
+ }
+ /+ ↓ add the current heading itself +/
+ string heading_text = stripMarkup(obj.text).strip;
+ if (heading_text.length > 60)
+ heading_text = heading_text[0 .. 57] ~ "...";
+
+ string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : "";
+
+ if (parts.length > 0)
+ return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref;
+ else
+ return doc_title ~ " > " ~ heading_text ~ ocn_ref;
+}
+
+/+ ↓ strip all internal Unicode markup to produce clean plain text +/
+string stripMarkup(string text) {
+ if (text.length == 0) return "";
+ string r = text;
+
+ /+ ↓ links: ┥link text┝┤url├ -> link text +/
+ while (true) {
+ auto lo = r.indexOf("┥");
+ if (lo < 0) break;
+ auto lm = indexOf(r[lo .. $], "┝┤");
+ if (lm < 0) break;
+ lm += lo;
+ auto lc = indexOf(r[lm .. $], "├");
+ if (lc < 0) break;
+ lc += lm;
+ string link_text = r[lo + "┥".length .. lm];
+ r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $];
+ }
+
+ /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/
+ while (true) {
+ auto fo = r.indexOf("【");
+ if (fo < 0) break;
+ auto fc = indexOf(r[fo .. $], "】");
+ if (fc < 0) break;
+ fc += fo;
+ r = r[0 .. fo] ~ r[fc + "】".length .. $];
+ }
+
+ /+ ↓ font face markers +/
+ r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "")
+ .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "")
+ .replace("⑆■", "")
+ .replace("┨", "").replace("┣*", "").replace("┣!", "")
+ .replace("┣/", "").replace("┣_", "").replace("┣^", "")
+ .replace("┣,", "").replace("┣", "");
+
+ /+ ↓ line breaks and special chars +/
+ r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n");
+ r = r.replace("░", " ");
+ r = r.replace("● ", "- ");
+ r = r.replace("┼", "").replace("┿", "").replace("╂", "");
+
+ /+ ↓ table cell separators +/
+ r = r.replace("┊", " | ");
+
+ /+ ↓ clean up multiple spaces +/
+ while (r.indexOf(" ") >= 0)
+ r = r.replace(" ", " ");
+
+ return r.strip;
+}