diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-24 11:12:28 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-24 11:12:28 -0400 |
| commit | da833f8b157ae501c9f6eec827dd4a965bfa7265 (patch) | |
| tree | 9368bbfe07dc130867dc8241d20bc3d125f3e510 /src/output_llm_breadcrumbs.d | |
| parent | commencement: a reader for the .ssp (diff) | |
format 2.0: the version gate
The reader as it stood when the format reached 2.0, in D: a parser of
the .ssp with no dependency beyond phobos, and outputs from it - a
heading tree, an html scroll, breadcrumbed text, chunked jsonl for
embedding pipelines, an image manifest, and a check of image records
against the images on disk.
Brought to abstraction format 2.0, and made able to say what it does
not understand, so a reader can check that its format specification is
complete. Previously the reader ignored what it did not recognise and
reported nothing, which made it seem it had understood a 2.0 file (and
drop things in silence).
Now:
- the banner is read, and the major part checked. A major this
reader does not know is refused, exit 3, because an unknown major
means a property may have changed meaning or gone away and loading
it half populated in silence is the failure the check exists to
prevent. --any-version reads it regardless, for looking at a file
rather than trusting one. A file with no banner warns and is read.
- @source is a preamble block beside @meta, @make and @doc_has, and
the summary line shows the language and the markup digest.
- tail is a section like the rest.
- --report-unknown lists every property, block and line shape met and
not understood, with counts. Ignoring them remains the conforming
behaviour within a major version; the counting is so that it is not
silent.
Over the markup document samples that report names 18 properties,
from .is_of_type at 56,882 occurrences down to .attrib at
1. Several are legitimately of no interest to a renderer and several
are not. Deciding which is which is the next pass, and it now has a
measured work list rather than a guess.
(assisted by Claude-Code)
Diffstat (limited to 'src/output_llm_breadcrumbs.d')
| -rw-r--r-- | src/output_llm_breadcrumbs.d | 179 |
1 files changed, 179 insertions, 0 deletions
diff --git a/src/output_llm_breadcrumbs.d b/src/output_llm_breadcrumbs.d new file mode 100644 index 0000000..0757d9f --- /dev/null +++ b/src/output_llm_breadcrumbs.d @@ -0,0 +1,179 @@ +module output_llm_breadcrumbs; + +import std.array : replace, split, join; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ssp_types; + +/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption + format: + ## Document Title > Chapter > Section > [OCN] + + Clean paragraph text with inline markup stripped. ++/ +void outputLlmBreadcrumbs(SspDocument doc, string output_file) { + string[] lines; + string doc_title = doc.meta("title.main"); + + /+ ↓ document header +/ + lines ~= "# " ~ doc_title; + if (doc.meta("title.subtitle").length > 0) + lines ~= "# " ~ doc.meta("title.subtitle"); + lines ~= "# " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build heading text lookup from ancestors +/ + string[int] heading_text; + foreach (obj; doc.head) { + if (obj.is_a == "heading" && obj.ocn > 0) + heading_text[obj.ocn] = stripMarkup(obj.text); + } + foreach (obj; doc.body) { + if (obj.is_a == "heading") + heading_text[obj.ocn] = stripMarkup(obj.text); + } + + /+ ↓ body objects +/ + string current_breadcrumb = ""; + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); + lines ~= ""; + lines ~= "## " ~ current_breadcrumb; + lines ~= ""; + continue; + } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) + ? " [" ~ obj.ocn.to!string ~ "]" + : ""; + + switch (obj.is_a) { + case "para": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "code": + lines ~= "```"; + lines ~= obj.text; /+ preserve raw text for code +/ + lines ~= "```"; + lines ~= ""; + break; + case "table": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "verse": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "quote": + lines ~= "> " ~ text ~ ocn_ref; + lines ~= ""; + break; + case "group": + case "block": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + default: + if (text.length > 0) { + lines ~= text ~ ocn_ref; + lines ~= ""; + } + break; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string buildBreadcrumb(string doc_title, string ancestors_text, SspObject obj) { + /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + /+ ↓ truncate very long heading text for breadcrumb +/ + if (clean.length > 60) + clean = clean[0 .. 57] ~ "..."; + parts ~= clean; + } + } + } + /+ ↓ add the current heading itself +/ + string heading_text = stripMarkup(obj.text).strip; + if (heading_text.length > 60) + heading_text = heading_text[0 .. 57] ~ "..."; + + string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + + if (parts.length > 0) + return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; + else + return doc_title ~ " > " ~ heading_text ~ ocn_ref; +} + +/+ ↓ strip all internal Unicode markup to produce clean plain text +/ +string stripMarkup(string text) { + if (text.length == 0) return ""; + string r = text; + + /+ ↓ links: ┥link text┝┤url├ -> link text +/ + while (true) { + auto lo = r.indexOf("┥"); + if (lo < 0) break; + auto lm = indexOf(r[lo .. $], "┝┤"); + if (lm < 0) break; + lm += lo; + auto lc = indexOf(r[lm .. $], "├"); + if (lc < 0) break; + lc += lm; + string link_text = r[lo + "┥".length .. lm]; + r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; + } + + /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ + while (true) { + auto fo = r.indexOf("【"); + if (fo < 0) break; + auto fc = indexOf(r[fo .. $], "】"); + if (fc < 0) break; + fc += fo; + r = r[0 .. fo] ~ r[fc + "】".length .. $]; + } + + /+ ↓ font face markers +/ + r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("⑆■", "") + .replace("┨", "").replace("┣*", "").replace("┣!", "") + .replace("┣/", "").replace("┣_", "").replace("┣^", "") + .replace("┣,", "").replace("┣", ""); + + /+ ↓ line breaks and special chars +/ + r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); + r = r.replace("░", " "); + r = r.replace("● ", "- "); + r = r.replace("┼", "").replace("┿", "").replace("╂", ""); + + /+ ↓ table cell separators +/ + r = r.replace("┊", " | "); + + /+ ↓ clean up multiple spaces +/ + while (r.indexOf(" ") >= 0) + r = r.replace(" ", " "); + + return r.strip; +} |
