From 4e4f2f8c2b2c043077390cb4ccf7ce97ffe4ac6b Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 12 Oct 2026 20:24:11 -0400 Subject: format 2.1: ocda-read, the readers folded in ocda-read, one reader where there were two. The .ssp reader and the .ocda.db reader each had their own parser; this one is built on the library, so every mode reads a .ssp, a .ocda.db and a pod, and says the same thing of each. Its output modules are the .ssp reader's, copied (the reader repositories are kept, frozen) and moved onto the library's model; --languages and --files come from the database reader. ocda-read [--tree|--html|--llm-text|--llm-jsonl|--images| --verify --image-dir=|--languages|--files| --report-unknown|--validate] [lang] In D every mode of both readers; in Gleam the modes the Gleam readers had (the summary, --tree, --languages, --files, --report-unknown), whose output is the D reader's, byte for byte. In each language it lives in src/read/ and is not part of the library: dub configurations keep it out of a dependant's build in D, and Gleam compiles all of src/ but a dependant need not use it. Checked over the markup document samples: against the .ssp reader on a .ssp, byte for byte after the first line in every D mode but the summary, whose closing hint differs; against the database reader on a .ocda.db, --files and --languages (its id column left out); the same output from the .ssp, the .ocda.db and the .sisupod; and D == Gleam. The database reader's --outline is not reproduced: it ordered headings within their own section, so that other sections' headings fell among the body's. --outline here is --tree. What writing it found in the library: the database loader could not say what it had not understood. It asked for columns by name and so never saw the rest, and reported that it had understood everything. It now counts them on the conditions under which the .ssp writes the same property, and --report-unknown is the same from every form. ocda.inventory answers two questions about the file rather than the document: its languages, in the manifest's order, each with the markup it was made from and the sha256 of its .ssp, for every form alike; and what a .ocda.db carries besides the abstraction. In Gleam the sha256 comes from OTP's crypto through a four line Erlang FFI. The flake's default is all three D programs in one result: built separately into the same link, each replaced the one before. (assisted by Claude-Code) --- dlang/src/read/main.d | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 dlang/src/read/main.d (limited to 'dlang/src/read/main.d') diff --git a/dlang/src/read/main.d b/dlang/src/read/main.d new file mode 100644 index 0000000..ae0d318 --- /dev/null +++ b/dlang/src/read/main.d @@ -0,0 +1,228 @@ +/++ + ocda-read - reads the object-centric document abstraction, in any of the + forms it is written in, and says what is in it. + + ocda-read [options] .ssp | .ocda.db | .sisupod [lang] + + One reader where there were two: sisudoc-ssp-reader read a .ssp and + sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is + built on the library, so every mode reads every form, and a mode given the + .ssp, the .ocda.db and the .sisupod of a document says the same thing of + each: it is told what the document is, not which file it came from. + + Two things are about the file rather than the document, and say so: + --files, what a .ocda.db carries besides the abstraction, and --validate, + which checks a .ssp as text. Everything else is the model's. + + The output modules are the .ssp reader's, moved onto the library's model. ++/ +module read.main; + +import std.algorithm : canFind, map; +import std.array : array, join; +import std.getopt; +import std.stdio : writeln, writefln, stderr; +import ocda.model; +import ocda.load; +import ocda.inventory; +import read.tree; +import read.html; +import read.llm_text; +import read.llm_jsonl; +import read.images; +import read.verify; +import read.report_unknown; +import read.validate; + +int main(string[] args) { + bool do_tree = false; + bool do_html = false; + bool do_llm_text = false; + bool do_llm_jsonl = false; + bool do_images = false; + bool do_verify = false; + bool do_report_unknown = false; + bool do_validate = false; + bool do_languages = false; + bool do_files = false; + bool any_version = false; + string image_dir = ""; + string output_file = ""; + string lang = ""; + + GetoptResult helpInfo; + try { + helpInfo = getopt(args, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, + "html", "the document as one html page", &do_html, + "llm-text", "plain text with heading breadcrumbs", &do_llm_text, + "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, + "images", "the image records: digest, size, pixels", &do_images, + "verify", "check the image records against --image-dir", &do_verify, + "image-dir", "the images to verify against", &image_dir, + "languages", "the languages the file holds", &do_languages, + "files", "what a .ocda.db carries besides the abstraction", &do_files, + "report-unknown", "what was read and not understood", &do_report_unknown, + "validate", "check a .ssp against its specification, as text", &do_validate, + "lang", "= the language to read (default: the first)", &lang, + "output", "write the mode's output to a file", &output_file, + "any-version", "read a format major this reader does not know", &any_version, + ); + } catch (GetOptException e) { + stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); + return 2; + } + + if (helpInfo.helpWanted || args.length < 2) { + defaultGetoptPrinter( + "ocda-read - reads the document abstraction, in any of its forms\n\n" + ~ "Usage: ocda-read [options] .ssp|.ocda.db|.sisupod [lang]\n", + helpInfo.options, + ); + return 0; + } + + string input_file = args[1]; + if (lang.length == 0 && args.length > 2) { lang = args[2]; } + bool lang_named = (lang.length > 0); + + /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not + at all: there is no text form of a database to check, and a pod's + .ssp is checked by taking it out +/ + if (do_validate && sourceOf(input_file) != Source.ssp) { + stderr.writeln("--validate checks a .ssp as text, against the .ssp", + " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); + return 2; + } + + /+ ↓ which language. A .ocda.db and a pod may hold several; where none is + named, every form reads the first in the manifest's order and says + so, rather than the database taking one and the pod refusing +/ + OcdaLanguage[] held; + OcdaDocument doc; + try { + held = languages(input_file); + if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } + if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { + stderr.writeln("error: no language '", lang, "' in ", input_file, + "; it holds: ", held.map!(l => l.lang).array.join(" ")); + return 2; + } + doc = load(input_file, lang, any_version); + } catch (Exception e) { + stderr.writeln("Error reading ", input_file, ": ", e.msg); + return 2; + } + + /+ ↓ the version gate: the major part, and refuse what is not known +/ + if (doc.format_version.length == 0) { + stderr.writeln("warning: ", input_file, " declares no format version"); + } else if (doc.formatVersionMajor != ssp_format_major_read) { + if (any_version) { + stderr.writeln("warning: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); + } else { + stderr.writeln("error: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x"); + stderr.writeln(" a major version differs: a property may have", + " changed meaning or gone away."); + stderr.writeln(" --any-version reads it regardless."); + return 3; + } + } + + writeln("Read: ", input_file); + if (doc.format_version.length > 0) { + writeln(" Format: v", doc.format_version); + } + if (auto _lang = "language" in doc.source) { + writeln(" Source: ", *_lang, + ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", + ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); + } + if (held.length > 1 && !lang_named) { + writeln(" (reading ", lang, "; --lang= for another)"); + } + writeln(" Title: ", doc.meta("title.main")); + writeln(" Author: ", doc.meta("creator.author")); + writeln(" Sections: head=", doc.head.length, + " toc=", doc.toc.length, + " body=", doc.body.length, + " endnotes=", doc.endnotes.length, + " glossary=", doc.glossary.length, + " bibliography=", doc.bibliography.length, + " bookindex=", doc.bookindex.length, + " blurb=", doc.blurb.length, + " tail=", doc.tail.length); + auto image_records = imageRecordCount(doc); + if (image_records > 0) { + writeln(" Images: ", image_records, " record", + (image_records == 1) ? "" : "s"); + } + + if (do_languages) { printLanguages(held); } + if (do_files) { printFiles(input_file); } + if (do_tree) { outputTree(doc, output_file); } + if (do_html) { outputHtmlScroll(doc, output_file); } + if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } + if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } + if (do_images) { outputImages(doc, output_file); } + + int _verify_failures = 0; + if (do_verify) { + if (image_dir.length == 0) { + stderr.writeln("--verify needs --image-dir="); + return 2; + } + _verify_failures = verifyImages(doc, image_dir, output_file); + } + + if (do_report_unknown) { reportUnknown(doc); } + + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images + && !do_verify && !do_report_unknown && !do_validate + && !do_languages && !do_files) { + writeln("\n(this is the summary: --help lists the modes, each adding its output)"); + } + + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; +} + +/+ ↓ the languages, each with the markup it was made from and its .ssp's + digest: the same from every form, the database storing the digest and + the others taking it from the bytes +/ +void printLanguages(OcdaLanguage[] held) { + writeln(); + writeln(" lang source ssp digest"); + foreach (l; held) { + writefln(" %-7s %-31s %s", l.lang, l.source_filename, + (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); + } +} + +/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries + nothing, and a pod's files are its archive entries rather than anything + the abstraction records, so for those it says so +/ +void printFiles(string path) { + writeln(); + if (!isCarrier(path)) { + writeln(" carried files: only a .ocda.db carries files; this is ", + sourceName(sourceOf(path))); + return; + } + auto rows = carried(path); + if (rows.length == 0) { writeln(" carries no files"); return; } + writeln(" carried files:"); + writeln(" role stored n original in the file"); + foreach (r; rows) { + writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); + } +} -- cgit v1.2.3