module read.llm_text; import std.array : replace, split, join; import std.conv : to; import std.stdio : writeln, File; import std.string : strip, indexOf, stripRight; import ocda.model; /+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption format: ## Document Title > Chapter > Section > [OCN] Clean paragraph text with inline markup stripped. +/ void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { string[] lines; string doc_title = doc.meta("title.main"); /+ ↓ document header +/ lines ~= "# " ~ doc_title; if (doc.meta("title.subtitle").length > 0) lines ~= "# " ~ doc.meta("title.subtitle"); lines ~= "# " ~ doc.meta("creator.author"); lines ~= ""; /+ ↓ build heading text lookup from ancestors +/ string[int] heading_text; foreach (obj; doc.head) { if (obj.is_a == "heading" && obj.ocn > 0) heading_text[obj.ocn] = stripMarkup(obj.text); } foreach (obj; doc.body) { if (obj.is_a == "heading") heading_text[obj.ocn] = stripMarkup(obj.text); } /+ ↓ body objects +/ string current_breadcrumb = ""; foreach (obj; doc.body) { if (obj.is_a == "heading") { current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); lines ~= ""; lines ~= "## " ~ current_breadcrumb; lines ~= ""; continue; } /+ ↓ markup commentary is not content +/ if (!obj.renders) { continue; } string text = stripMarkup(obj.text); if (text.strip.length == 0) continue; string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) ? " [" ~ obj.ocn.to!string ~ "]" : ""; switch (obj.is_a) { case "para": lines ~= text ~ ocn_ref; lines ~= ""; break; case "code": lines ~= "```"; lines ~= obj.text; /+ preserve raw text for code +/ lines ~= "```"; lines ~= ""; break; case "table": lines ~= text ~ ocn_ref; lines ~= ""; break; case "verse": lines ~= text ~ ocn_ref; lines ~= ""; break; case "quote": lines ~= "> " ~ text ~ ocn_ref; lines ~= ""; break; case "group": case "block": lines ~= text ~ ocn_ref; lines ~= ""; break; default: if (text.length > 0) { lines ~= text ~ ocn_ref; lines ~= ""; } break; } } /+ ↓ write output +/ if (output_file.length > 0) { auto f = File(output_file, "w"); foreach (line; lines) f.writeln(line); } else { foreach (line; lines) writeln(line); } } private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ string[] parts; if (ancestors_text.length > 0) { foreach (part; ancestors_text.split("|")) { string clean = stripMarkup(part).strip; if (clean.length > 0 && clean != doc_title) { /+ ↓ truncate very long heading text for breadcrumb +/ if (clean.length > 60) clean = clean[0 .. 57] ~ "..."; parts ~= clean; } } } /+ ↓ add the current heading itself +/ string heading_text = stripMarkup(obj.text).strip; if (heading_text.length > 60) heading_text = heading_text[0 .. 57] ~ "..."; string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; if (parts.length > 0) return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; else return doc_title ~ " > " ~ heading_text ~ ocn_ref; } /+ ↓ strip all internal Unicode markup to produce clean plain text +/ string stripMarkup(string text) { if (text.length == 0) return ""; string r = text; /+ ↓ links: ┥link text┝┤url├ -> link text +/ while (true) { auto lo = r.indexOf("┥"); if (lo < 0) break; auto lm = indexOf(r[lo .. $], "┝┤"); if (lm < 0) break; lm += lo; auto lc = indexOf(r[lm .. $], "├"); if (lc < 0) break; lc += lm; string link_text = r[lo + "┥".length .. lm]; r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; } /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ while (true) { auto fo = r.indexOf("【"); if (fo < 0) break; auto fc = indexOf(r[fo .. $], "】"); if (fc < 0) break; fc += fo; r = r[0 .. fo] ~ r[fc + "】".length .. $]; } /+ ↓ font face markers +/ r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") .replace("⑆■", "") .replace("┨", "").replace("┣*", "").replace("┣!", "") .replace("┣/", "").replace("┣_", "").replace("┣^", "") .replace("┣,", "").replace("┣", ""); /+ ↓ line breaks and special chars +/ r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); r = r.replace("░", " "); r = r.replace("● ", "- "); r = r.replace("┼", "").replace("┿", "").replace("╂", ""); /+ ↓ table cell separators +/ r = r.replace("┊", " | "); /+ ↓ clean up multiple spaces +/ while (r.indexOf(" ") >= 0) r = r.replace(" ", " "); return r.strip; }