diff options
| -rw-r--r-- | README.md | 18 | ||||
| -rw-r--r-- | src/output_html.d | 5 | ||||
| -rw-r--r-- | src/output_llm_breadcrumbs.d | 3 | ||||
| -rw-r--r-- | src/output_llm_jsonl.d | 6 | ||||
| -rw-r--r-- | src/ssp_parser.d | 6 | ||||
| -rw-r--r-- | src/ssp_types.d | 23 |
6 files changed, 60 insertions, 1 deletions
@@ -97,6 +97,24 @@ read and NOT understood: ... ``` +### What it ignores, and why + +Twelve properties are ignored across the reference collection, and each is +ignored for a reason rather than by oversight: + +| ignored | why | +|---------|-----| +| `.ancestors_collapsed`, `.parent_lev`, `.heading_lev_collapsed`, `.dom_status`, `.dom_status_collapsed` | level bookkeeping; this reader works from `.ancestors` and the heading level | +| `.segment_epub`, `.segment_epub_is`, `.segment_lv4_is`, `.heading_lev_anchor`, `.anchor_tag`, `.lev4_subtoc` | naming for segmented HTML and EPUB output, which this reader does not produce | +| `.attrib` | one occurrence in the whole collection, and no output uses it | + +The list was longer. `--report-unknown` found that `.is_of_type` was +ignored, and **that was a defect**: objects of type `comment` are markup +commentary, carried by the abstraction and not put into output, and three +of the four outputs were rendering them. A markup comment was appearing in +the HTML, in the plain text, and as an embedding chunk in the JSONL. +Fixed, and the reason the report exists. + That is deliberate and is what makes the reader usable as a check on the [format specification](https://git.sisudoc.org/specs/doc-abstraction-format): a reader that drops what it does not know is conforming, but one that diff --git a/src/output_html.d b/src/output_html.d index 14ae772..b1c711c 100644 --- a/src/output_html.d +++ b/src/output_html.d @@ -109,6 +109,11 @@ void outputHtmlScroll(SspDocument doc, string output_file) { } private string[] renderObject(SspObject obj) { + /+ ↓ markup commentary is carried by the abstraction and is not output. + Rendering it would put the author notes into the document, which + spine itself does not do + +/ + if (!obj.renders) { return []; } string[] lines; string text = convertInlineMarkup(obj.text); string ocn_anchor = (obj.ocn > 0 && !obj.ocn_off) diff --git a/src/output_llm_breadcrumbs.d b/src/output_llm_breadcrumbs.d index 0757d9f..5b240d2 100644 --- a/src/output_llm_breadcrumbs.d +++ b/src/output_llm_breadcrumbs.d @@ -45,6 +45,9 @@ void outputLlmBreadcrumbs(SspDocument doc, string output_file) { continue; } + /+ ↓ markup commentary is not content +/ + if (!obj.renders) { continue; } + string text = stripMarkup(obj.text); if (text.strip.length == 0) continue; diff --git a/src/output_llm_jsonl.d b/src/output_llm_jsonl.d index 6f7edbb..3d34089 100644 --- a/src/output_llm_jsonl.d +++ b/src/output_llm_jsonl.d @@ -39,6 +39,12 @@ void outputLlmJsonl(SspDocument doc, string output_file) { continue; } + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + string text = stripMarkup(obj.text); if (text.strip.length == 0) continue; diff --git a/src/ssp_parser.d b/src/ssp_parser.d index b118163..1a1efe2 100644 --- a/src/ssp_parser.d +++ b/src/ssp_parser.d @@ -181,6 +181,12 @@ private void parseProperty(string line, ref SspObject obj, ref SspUnknown unknow switch (name) { case "part": obj.part = value; break; + case "is_of_type": obj.is_of_type = value; break; + case "section": obj.own_section = value; break; + case "identifier": obj.identifier = value; break; + case "segment_html_is": obj.segment_html_is = value; break; + case "table_aligns": obj.table_aligns = value; break; + case "stow_link": obj.stow_link ~= value; break; case "parent": obj.parent_ocn = toIntSafe(value); break; case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; case "children": obj.children = parseIntArray(value); break; diff --git a/src/ssp_types.d b/src/ssp_types.d index f87da5b..3960df0 100644 --- a/src/ssp_types.d +++ b/src/ssp_types.d @@ -15,8 +15,13 @@ struct SspObject { string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) - string identifier; // named segment (for headings) + string identifier; // heading: from the declaration line; otherwise .identifier string part; // frontmatter, body, backmatter + string is_of_type; // para, block, comment: how the text behaves + string own_section; // .section, when it differs from the block it sits in + string segment_html_is;// the html segment this object sits in + string table_aligns; // l c r per column + string[] stow_link; // links extracted from the object int parent_ocn; int last_descendant_ocn; int[] children; // direct child heading OCNs @@ -44,6 +49,22 @@ struct SspObject { string sha256; SspImage[] images; // repeatable .image records, in order string text; // content (lines joined with newline) + + /+ ↓ whether this object is rendered at all. + is_of_type "comment" is markup commentary: carried by the + abstraction so that nothing is lost, and not put into output. An + output that renders it leaks the author notes into the document, + which is what spine itself does not do. + +/ + bool renders() const { + return is_of_type != "comment"; + } + + /+ ↓ the section this object belongs to, which is the block it sits in + unless it says otherwise +/ + string sectionOf() const { + return (own_section.length > 0) ? own_section : section; + } } /+ ↓ what the reader met and did not understand. |