diff options
Diffstat (limited to 'org/d-read.org')
| -rw-r--r-- | org/d-read.org | 1199 |
1 files changed, 1199 insertions, 0 deletions
diff --git a/org/d-read.org b/org/d-read.org new file mode 100644 index 0000000..9cf2416 --- /dev/null +++ b/org/d-read.org @@ -0,0 +1,1199 @@ +-*- mode: org -*- +#+TITLE: dlang load +#+DESCRIPTION: dlang lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* dlang :dlang: +** dlang read main + +#+HEADER: :tangle "../dlang/src/read/main.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + ocda-read - reads the object-centric document abstraction, in any of the + forms it is written in, and says what is in it. + + ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] + + One reader where there were two: sisudoc-ssp-reader read a .ssp and + sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is + built on the library, so every mode reads every form, and a mode given the + .ssp, the .ocda.db and the .sisupod of a document says the same thing of + each: it is told what the document is, not which file it came from. + + Two things are about the file rather than the document, and say so: + --files, what a .ocda.db carries besides the abstraction, and --validate, + which checks a .ssp as text. Everything else is the model's. + + The output modules are the .ssp reader's, moved onto the library's model. ++/ +module read.main; + +import std.algorithm : canFind, map; +import std.array : array, join; +import std.getopt; +import std.stdio : writeln, writefln, stderr; +import ocda.model; +import ocda.load; +import ocda.inventory; +import read.tree; +import read.html; +import read.llm_text; +import read.llm_jsonl; +import read.images; +import read.verify; +import read.report_unknown; +import read.validate; + +int main(string[] args) { + bool do_tree = false; + bool do_html = false; + bool do_llm_text = false; + bool do_llm_jsonl = false; + bool do_images = false; + bool do_verify = false; + bool do_report_unknown = false; + bool do_validate = false; + bool do_languages = false; + bool do_files = false; + bool any_version = false; + string image_dir = ""; + string output_file = ""; + string lang = ""; + + GetoptResult helpInfo; + try { + helpInfo = getopt(args, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, + "html", "the document as one html page", &do_html, + "llm-text", "plain text with heading breadcrumbs", &do_llm_text, + "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, + "images", "the image records: digest, size, pixels", &do_images, + "verify", "check the image records against --image-dir", &do_verify, + "image-dir", "the images to verify against", &image_dir, + "languages", "the languages the file holds", &do_languages, + "files", "what a .ocda.db carries besides the abstraction", &do_files, + "report-unknown", "what was read and not understood", &do_report_unknown, + "validate", "check a .ssp against its specification, as text", &do_validate, + "lang", "=<code> the language to read (default: the first)", &lang, + "output", "write the mode's output to a file", &output_file, + "any-version", "read a format major this reader does not know", &any_version, + ); + } catch (GetOptException e) { + stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); + return 2; + } + + if (helpInfo.helpWanted || args.length < 2) { + defaultGetoptPrinter( + "ocda-read - reads the document abstraction, in any of its forms\n\n" + ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n", + helpInfo.options, + ); + return 0; + } + + string input_file = args[1]; + if (lang.length == 0 && args.length > 2) { lang = args[2]; } + bool lang_named = (lang.length > 0); + + /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not + at all: there is no text form of a database to check, and a pod's + .ssp is checked by taking it out +/ + if (do_validate && sourceOf(input_file) != Source.ssp) { + stderr.writeln("--validate checks a .ssp as text, against the .ssp", + " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); + return 2; + } + + /+ ↓ which language. A .ocda.db and a pod may hold several; where none is + named, every form reads the first in the manifest's order and says + so, rather than the database taking one and the pod refusing +/ + OcdaLanguage[] held; + OcdaDocument doc; + try { + held = languages(input_file); + if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } + if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { + stderr.writeln("error: no language '", lang, "' in ", input_file, + "; it holds: ", held.map!(l => l.lang).array.join(" ")); + return 2; + } + doc = load(input_file, lang, any_version); + } catch (Exception e) { + stderr.writeln("Error reading ", input_file, ": ", e.msg); + return 2; + } + + /+ ↓ the version gate: the major part, and refuse what is not known +/ + if (doc.format_version.length == 0) { + stderr.writeln("warning: ", input_file, " declares no format version"); + } else if (doc.formatVersionMajor != ssp_format_major_read) { + if (any_version) { + stderr.writeln("warning: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); + } else { + stderr.writeln("error: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x"); + stderr.writeln(" a major version differs: a property may have", + " changed meaning or gone away."); + stderr.writeln(" --any-version reads it regardless."); + return 3; + } + } + + writeln("Read: ", input_file); + if (doc.format_version.length > 0) { + writeln(" Format: v", doc.format_version); + } + if (auto _lang = "language" in doc.source) { + writeln(" Source: ", *_lang, + ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", + ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); + } + if (held.length > 1 && !lang_named) { + writeln(" (reading ", lang, "; --lang=<code> for another)"); + } + writeln(" Title: ", doc.meta("title.main")); + writeln(" Author: ", doc.meta("creator.author")); + writeln(" Sections: head=", doc.head.length, + " toc=", doc.toc.length, + " body=", doc.body.length, + " endnotes=", doc.endnotes.length, + " glossary=", doc.glossary.length, + " bibliography=", doc.bibliography.length, + " bookindex=", doc.bookindex.length, + " blurb=", doc.blurb.length, + " tail=", doc.tail.length); + auto image_records = imageRecordCount(doc); + if (image_records > 0) { + writeln(" Images: ", image_records, " record", + (image_records == 1) ? "" : "s"); + } + + if (do_languages) { printLanguages(held); } + if (do_files) { printFiles(input_file); } + if (do_tree) { outputTree(doc, output_file); } + if (do_html) { outputHtmlScroll(doc, output_file); } + if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } + if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } + if (do_images) { outputImages(doc, output_file); } + + int _verify_failures = 0; + if (do_verify) { + if (image_dir.length == 0) { + stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); + return 2; + } + _verify_failures = verifyImages(doc, image_dir, output_file); + } + + if (do_report_unknown) { reportUnknown(doc); } + + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images + && !do_verify && !do_report_unknown && !do_validate + && !do_languages && !do_files) { + writeln("\n(this is the summary: --help lists the modes, each adding its output)"); + } + + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; +} + +/+ ↓ the languages, each with the markup it was made from and its .ssp's + digest: the same from every form, the database storing the digest and + the others taking it from the bytes +/ +void printLanguages(OcdaLanguage[] held) { + writeln(); + writeln(" lang source ssp digest"); + foreach (l; held) { + writefln(" %-7s %-31s %s", l.lang, l.source_filename, + (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); + } +} + +/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries + nothing, and a pod's files are its archive entries rather than anything + the abstraction records, so for those it says so +/ +void printFiles(string path) { + writeln(); + if (!isCarrier(path)) { + writeln(" carried files: only a .ocda.db carries files; this is ", + sourceName(sourceOf(path))); + return; + } + auto rows = carried(path); + if (rows.length == 0) { writeln(" carries no files"); return; } + writeln(" carried files:"); + writeln(" role stored n original in the file"); + foreach (r; rows) { + writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); + } +} +#+END_SRC + +** dlang read tree + +#+HEADER: :tangle "../dlang/src/read/tree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.tree; + +import std.conv : to; +import std.stdio : writeln, writef, writefln, File; +import std.array : replicate; +import ocda.model; + +void outputTree(OcdaDocument doc, string output_file) { + string[] lines; + + lines ~= "Document: " ~ doc.meta("title.main"); + if (doc.meta("title.subtitle").length > 0) + lines ~= " " ~ doc.meta("title.subtitle"); + lines ~= "Author: " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build OCN -> object lookup for body headings +/ + OcdaObject[int] headings; + foreach (obj; doc.body) { + if (obj.is_a == "heading" && obj.ocn > 0) { + headings[obj.ocn] = obj; + } + } + + /+ ↓ find root heading (from head section, or first body heading) +/ + if (doc.head.length > 0 && doc.head[0].is_a == "heading") { + auto root = doc.head[0]; + lines ~= formatHeading(root, 0); + walkChildren(root, headings, lines, 1); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { + foreach (child_ocn; parent.children) { + if (auto p = child_ocn in headings) { + lines ~= formatHeading(*p, depth); + walkChildren(*p, headings, lines, depth + 1); + } + } +} + +private string formatHeading(OcdaObject obj, int depth) { + string indent = replicate(" ", depth); + string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; + string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + /+ ↓ strip internal markup from heading text for clean display +/ + string text = stripInternalMarkup(obj.text); + return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; +} + +private string stripInternalMarkup(string text) { + /+ ↓ simple removal of common internal markers for display +/ + import std.regex : replaceAll, regex; + import std.array : replace; + string result = text; + result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("┨", "").replace("┣", "") + .replace("┙", " ").replace("┘", "\n"); + return result; +} +#+END_SRC + +** dlang read verify + +#+HEADER: :tangle "../dlang/src/read/verify.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.verify; + +import std.algorithm : sort; +import std.array : array, join; +import std.conv : to; +import std.digest : toHexString; +import std.digest.sha : SHA256, digest; +import std.file : dirEntries, exists, isDir, SpanMode, getSize; +import std.path : baseName, buildPath, extension; +import std.stdio : writeln, File; +import std.string : toLower; +import ocda.model; +import read.images : gatherImageRecords; + +/+ ↓ verification: recompute what the .image records claim, against the + images as they stand in a directory (a pod's media/image). + + ssp-reader --verify --image-dir=<pod>/media/image doc.ssp + + Reports one line per distinct file: + + OK won_benkler_2_1.png + MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... + ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) + PRESENT ffa.png (recorded missing, but the file is there) + EXTRA unused.png (in directory, no record in this .ssp) + + Returns the number of failures. EXTRA is reported but is not a failure: + a pod carries the images of every language version of a document, while + one .ssp covers one language. ++/ +int verifyImages(OcdaDocument doc, string image_dir, string output_file) { + string[] lines; + int failures = 0; + int ok = 0; + int mismatch = 0; + int absent = 0; + int extra = 0; + + if (!(image_dir.exists) || !(image_dir.isDir)) { + writeln("ERROR: image directory not found: ", image_dir); + return 1; + } + auto gathered = gatherImageRecords(doc); + bool[string] recorded; + + foreach (e; gathered.entries) { + recorded[e.name] = true; + string pth = buildPath(image_dir, e.name); + if (!(pth.exists)) { + if (e.missing) { + /+ recorded missing, and missing here too: consistent +/ + ok += 1; + lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; + } else { + absent += 1; + failures += 1; + lines ~= "ABSENT " ~ e.name + ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; + } + continue; + } + if (e.missing) { + mismatch += 1; + failures += 1; + lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; + continue; + } + auto file_ = readFileStat(pth); + string[] differs; + if (file_.sha256 != e.sha256) { + differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; + } + if (file_.bytes != e.bytes) { + differs ~= "bytes recorded " ~ e.bytes.to!string + ~ " file " ~ file_.bytes.to!string; + } + /+ px is written only where the dimensions could be read, and read here + only for the formats below, so compare only when both are known +/ + if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 + && (file_.width != e.width || file_.height != e.height)) { + differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string + ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; + } + if (differs.length > 0) { + mismatch += 1; + failures += 1; + lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); + } else { + ok += 1; + lines ~= "OK " ~ e.name; + } + } + + /+ ↓ files in the directory that no record names +/ + string[] in_dir; + foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { + if (!(entry.isFile)) continue; + string nm = entry.name.baseName; + switch (nm.extension.toLower) { + case ".png": case ".jpg": case ".jpeg": case ".gif": + in_dir ~= nm; + break; + default: break; + } + } + in_dir.sort; + foreach (nm; in_dir) { + if (nm !in recorded) { + extra += 1; + lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; + } + } + + string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ " in " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", against " ~ image_dir; + lines = [head, ""] ~ lines; + lines ~= ""; + lines ~= "verify: " ~ ok.to!string ~ " ok, " + ~ mismatch.to!string ~ " mismatch, " + ~ absent.to!string ~ " absent, " + ~ extra.to!string ~ " extra" + ~ ((failures == 0) ? " PASS" : " FAIL"); + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } + return failures; +} + +private struct FileStat_ { + string sha256; + ulong bytes; + int width; + int height; +} + +private FileStat_ readFileStat(string pth) { + FileStat_ ret; + auto f = File(pth, "rb"); + ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); + ret.sha256 = sha.toHexString.to!string; + ret.bytes = pth.getSize; + auto dim = readImageDimensions(pth); + ret.width = dim[0]; + ret.height = dim[1]; + return ret; +} + +/+ ↓ pixel dimensions from the file header, for the formats spine's markup + admits (png, gif, jpg). Header read only, no decoding, and no dependency + beyond phobos. Returns [0, 0] where the header cannot be read +/ +private int[2] readImageDimensions(string pth) { + int[2] ret = [0, 0]; + ubyte[] head; + try { + auto f = File(pth, "rb"); + head = new ubyte[](64 * 1024); + head = f.rawRead(head); + } catch (Exception ex) { + return ret; + } + if (head.length >= 24 + && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { + /+ png: IHDR width & height, big endian, at 16 and 20 +/ + ret[0] = beInt(head[16 .. 20]); + ret[1] = beInt(head[20 .. 24]); + return ret; + } + if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { + /+ gif: logical screen width & height, little endian, at 6 and 8 +/ + ret[0] = head[6] | (head[7] << 8); + ret[1] = head[8] | (head[9] << 8); + return ret; + } + if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { + /+ jpeg: walk the segments to a start of frame, height then width, + big endian, at offsets 5 and 7 of the segment +/ + size_t i = 2; + while (i + 9 < head.length) { + if (head[i] != 0xff) { i += 1; continue; } + ubyte marker = head[i + 1]; + if (marker == 0xd8 || marker == 0x01 + || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } + if (marker == 0xff) { i += 1; continue; } + size_t seg_len = (head[i + 2] << 8) | head[i + 3]; + bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf)); + if (is_sof) { + ret[1] = (head[i + 5] << 8) | head[i + 6]; + ret[0] = (head[i + 7] << 8) | head[i + 8]; + return ret; + } + if (seg_len < 2) break; + i += 2 + seg_len; + } + } + return ret; +} + +private int beInt(const(ubyte)[] b) { + return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); +} +#+END_SRC + +** dlang read validate + +#+HEADER: :tangle "../dlang/src/read/validate.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.validate; + +/+ ↓ check a .ssp against the format specification. + . + Written from specs/sisudoc-abstraction-format/ssp-format-specification.md + and from nothing else: the point of it is to be what the specification + says, executed, so that a file can be held against the document rather + than against an implementation. Each check names the section it comes + from, so a disagreement can be taken back to the text. ++/ + +import std.algorithm : canFind, startsWith; +import std.array : array, split; +import std.conv : to; +import std.file : readText; +import std.stdio : writeln, writefln; +import std.string : indexOf, strip; + +struct SspFinding { + size_t line; // 1-based + string section; // the specification section the rule is from + string detail; +} + +/+ ↓ specification section 6.3: the properties, in the order they are + emitted. A reader may rely on the order, so a validator can check it +/ +private immutable string[] property_order = [ + "identifier", "part", "section", "parent", "last_descendant", "ocn_range", + "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", + "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", + "table_header", "code_linenumbers", "stow_link", "segment", "anchor", + "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", + "segment_html_is", "segment_epub_is", "segment_lv4_is", + "heading_ancestors_text", "lev4_subtoc", "anchor_tag", +]; + +/+ ↓ section 6.1: the sixteen kinds +/ +private immutable string[] kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", + "blurb", "group", +]; + +/+ ↓ section 4: the nine sections, in the order they appear +/ +private immutable string[] section_order = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", + "bookindex", "blurb", "tail", +]; + +/+ ↓ section 4: the four preamble blocks, in the order they appear +/ +private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; + +/+ ↓ section 6.1: the heading levels +/ +private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; + +private int propertyRank(string name) { + foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } + return -1; +} + +SspFinding[] validateSspFile(string filename, string major_known) { + SspFinding[] found; + auto lines = readText(filename).split("\n"); + + bool seen_banner = false; + bool seen_source_line = false; + string block = ""; // the block currently open, "" when none + int preamble_at = -1; // how far through the preamble order + int section_at = -1; // how far through the section order + bool in_object = false; + int prev_rank = -1; + size_t doc_has_keys = 0; + bool in_text = false; // text lines come after properties + + foreach (i, line; lines) { + size_t no = i + 1; + + /+ ↓ section 2: the version, on the first line +/ + if (no == 1) { + enum banner = "% SiSU Document Abstraction v"; + if (!line.startsWith(banner)) { + found ~= SspFinding(no, "2", "first line is not the version banner"); + } else { + seen_banner = true; + string v = line[banner.length .. $].strip; + auto dot = v.indexOf('.'); + string major = (dot > 0) ? v[0 .. dot] : v; + if (major != major_known) { + found ~= SspFinding(no, "2.1", + "format major v" ~ major ~ ", this validator knows v" ~ major_known); + } + } + continue; + } + if (no == 2) { + if (!line.startsWith("% Source: ")) { + found ~= SspFinding(no, "4", "second line is not \"% Source: \""); + } else { seen_source_line = true; } + continue; + } + + if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } + + /+ ↓ section 3: a comment is the file header only +/ + if (line[0] == '%') { + found ~= SspFinding(no, "3", "a comment outside the two header lines"); + continue; + } + + /+ ↓ block open +/ + if (line[0] == '@') { + auto brace = line.indexOf(" {"); + if (brace < 1 || line[$ - 1] != '{') { + found ~= SspFinding(no, "3", "block open is not \"@name {\""); + continue; + } + string name = line[1 .. brace]; + if (preamble_order.canFind(name)) { + int at = 0; + foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } + if (at <= preamble_at) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); + } + preamble_at = at; + if (section_at >= 0) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); + } + } else if (section_order.canFind(name)) { + int at = 0; + foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } + if (at <= section_at) { + found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); + } + section_at = at; + } else { + found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); + } + block = name; + doc_has_keys = 0; + continue; + } + + if (line[0] == '}') { + if (line.length != 1) { + found ~= SspFinding(no, "3", "block close carries more than \"}\""); + } + /+ ↓ section 5.4: ten keys, always +/ + if (block == "doc_has" && doc_has_keys != 10) { + found ~= SspFinding(no, "5.4", + "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); + } + block = ""; + in_object = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 5: inside a preamble block, " key: value" +/ + if (preamble_order.canFind(block)) { + if (!line.startsWith(" ") || line.indexOf(": ") < 3) { + found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); + } else if (block == "doc_has") { + doc_has_keys++; + } + continue; + } + + /+ ↓ section 6.1: the object declaration line +/ + if (line[0] == '[') { + auto close = line.indexOf("] "); + if (close < 2) { + found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); + continue; + } + string ocn = line[1 .. close]; + foreach (c; ocn) { + if (c < '0' || c > '9') { + found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); + break; + } + } + auto rest = line[close + 2 .. $].split(" "); + if (rest.length == 0 || !kinds.canFind(rest[0])) { + found ~= SspFinding(no, "6.1", + "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); + } else if (rest[0] == "heading") { + if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { + found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); + } else if (!levels.canFind(rest[1][1 .. $])) { + found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); + } + } + in_object = true; + in_text = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 6.3: properties, known and in order +/ + if (line[0] == '.') { + if (!in_object) { + found ~= SspFinding(no, "6", "a property outside an object record"); + continue; + } + if (in_text) { + found ~= SspFinding(no, "6", "a property after the object text began"); + } + auto colon = line.indexOf(": "); + if (colon < 2) { + found ~= SspFinding(no, "3", "property is not \".name: value\""); + continue; + } + string name = line[1 .. colon]; + int rank = propertyRank(name); + if (rank < 0) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); + } else { + /+ ↓ the four repeatable properties may recur at the same rank +/ + if (rank < prev_rank) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); + } + prev_rank = rank; + } + continue; + } + + /+ ↓ section 6.2: text lines +/ + if (line[0] == '|') { + if (!in_object) { + found ~= SspFinding(no, "6.2", "a text line outside an object record"); + } + if (line.length < 2 || line[1] != ' ') { + found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); + } + in_text = true; + continue; + } + + found ~= SspFinding(no, "3", "line of no shape the specification defines"); + } + + if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } + if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } + return found; +} + +void reportValidation(string filename, SspFinding[] found) { + if (found.length == 0) { + writeln("\nvalid: ", filename, + " conforms to the .ssp specification as this validator reads it"); + return; + } + writefln("\n%d finding%s against the specification:", + found.length, (found.length == 1) ? "" : "s"); + foreach (f; found) { + if (f.line > 0) { + writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); + } else { + writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); + } + } +} +#+END_SRC + +** dlang read report unknown + +#+HEADER: :tangle "../dlang/src/read/report_unknown.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read zip + +#+HEADER: :tangle "../dlang/src/read/zip.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read zstd + +#+HEADER: :tangle "../dlang/src/read/zstd.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read html + +#+HEADER: :tangle "../dlang/src/read/html.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read images + +#+HEADER: :tangle "../dlang/src/read/images.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read llm text + +#+HEADER: :tangle "../dlang/src/read/llm_text.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_text; + +import std.array : replace, split, join; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; + +/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption + format: + ## Document Title > Chapter > Section > [OCN] + + Clean paragraph text with inline markup stripped. ++/ +void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { + string[] lines; + string doc_title = doc.meta("title.main"); + + /+ ↓ document header +/ + lines ~= "# " ~ doc_title; + if (doc.meta("title.subtitle").length > 0) + lines ~= "# " ~ doc.meta("title.subtitle"); + lines ~= "# " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build heading text lookup from ancestors +/ + string[int] heading_text; + foreach (obj; doc.head) { + if (obj.is_a == "heading" && obj.ocn > 0) + heading_text[obj.ocn] = stripMarkup(obj.text); + } + foreach (obj; doc.body) { + if (obj.is_a == "heading") + heading_text[obj.ocn] = stripMarkup(obj.text); + } + + /+ ↓ body objects +/ + string current_breadcrumb = ""; + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); + lines ~= ""; + lines ~= "## " ~ current_breadcrumb; + lines ~= ""; + continue; + } + + /+ ↓ markup commentary is not content +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) + ? " [" ~ obj.ocn.to!string ~ "]" + : ""; + + switch (obj.is_a) { + case "para": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "code": + lines ~= "```"; + lines ~= obj.text; /+ preserve raw text for code +/ + lines ~= "```"; + lines ~= ""; + break; + case "table": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "verse": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "quote": + lines ~= "> " ~ text ~ ocn_ref; + lines ~= ""; + break; + case "group": + case "block": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + default: + if (text.length > 0) { + lines ~= text ~ ocn_ref; + lines ~= ""; + } + break; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { + /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + /+ ↓ truncate very long heading text for breadcrumb +/ + if (clean.length > 60) + clean = clean[0 .. 57] ~ "..."; + parts ~= clean; + } + } + } + /+ ↓ add the current heading itself +/ + string heading_text = stripMarkup(obj.text).strip; + if (heading_text.length > 60) + heading_text = heading_text[0 .. 57] ~ "..."; + + string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + + if (parts.length > 0) + return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; + else + return doc_title ~ " > " ~ heading_text ~ ocn_ref; +} + +/+ ↓ strip all internal Unicode markup to produce clean plain text +/ +string stripMarkup(string text) { + if (text.length == 0) return ""; + string r = text; + + /+ ↓ links: ┥link text┝┤url├ -> link text +/ + while (true) { + auto lo = r.indexOf("┥"); + if (lo < 0) break; + auto lm = indexOf(r[lo .. $], "┝┤"); + if (lm < 0) break; + lm += lo; + auto lc = indexOf(r[lm .. $], "├"); + if (lc < 0) break; + lc += lm; + string link_text = r[lo + "┥".length .. lm]; + r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; + } + + /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ + while (true) { + auto fo = r.indexOf("【"); + if (fo < 0) break; + auto fc = indexOf(r[fo .. $], "】"); + if (fc < 0) break; + fc += fo; + r = r[0 .. fo] ~ r[fc + "】".length .. $]; + } + + /+ ↓ font face markers +/ + r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("⑆■", "") + .replace("┨", "").replace("┣*", "").replace("┣!", "") + .replace("┣/", "").replace("┣_", "").replace("┣^", "") + .replace("┣,", "").replace("┣", ""); + + /+ ↓ line breaks and special chars +/ + r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); + r = r.replace("░", " "); + r = r.replace("● ", "- "); + r = r.replace("┼", "").replace("┿", "").replace("╂", ""); + + /+ ↓ table cell separators +/ + r = r.replace("┊", " | "); + + /+ ↓ clean up multiple spaces +/ + while (r.indexOf(" ") >= 0) + r = r.replace(" ", " "); + + return r.strip; +} +#+END_SRC + +** dlang read llm jsonl + +#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ + |
