summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/main.d
diff options
context:
space:
mode:
Diffstat (limited to 'dlang/src/read/main.d')
-rw-r--r--dlang/src/read/main.d228
1 files changed, 228 insertions, 0 deletions
diff --git a/dlang/src/read/main.d b/dlang/src/read/main.d
new file mode 100644
index 0000000..ae0d318
--- /dev/null
+++ b/dlang/src/read/main.d
@@ -0,0 +1,228 @@
+/++
+ ocda-read - reads the object-centric document abstraction, in any of the
+ forms it is written in, and says what is in it.
+
+ ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]
+
+ One reader where there were two: sisudoc-ssp-reader read a .ssp and
+ sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is
+ built on the library, so every mode reads every form, and a mode given the
+ .ssp, the .ocda.db and the .sisupod of a document says the same thing of
+ each: it is told what the document is, not which file it came from.
+
+ Two things are about the file rather than the document, and say so:
+ --files, what a .ocda.db carries besides the abstraction, and --validate,
+ which checks a .ssp as text. Everything else is the model's.
+
+ The output modules are the .ssp reader's, moved onto the library's model.
++/
+module read.main;
+
+import std.algorithm : canFind, map;
+import std.array : array, join;
+import std.getopt;
+import std.stdio : writeln, writefln, stderr;
+import ocda.model;
+import ocda.load;
+import ocda.inventory;
+import read.tree;
+import read.html;
+import read.llm_text;
+import read.llm_jsonl;
+import read.images;
+import read.verify;
+import read.report_unknown;
+import read.validate;
+
+int main(string[] args) {
+ bool do_tree = false;
+ bool do_html = false;
+ bool do_llm_text = false;
+ bool do_llm_jsonl = false;
+ bool do_images = false;
+ bool do_verify = false;
+ bool do_report_unknown = false;
+ bool do_validate = false;
+ bool do_languages = false;
+ bool do_files = false;
+ bool any_version = false;
+ string image_dir = "";
+ string output_file = "";
+ string lang = "";
+
+ GetoptResult helpInfo;
+ try {
+ helpInfo = getopt(args,
+ "tree", "the headings, as a tree", &do_tree,
+ "outline", "the same as --tree", &do_tree,
+ "html", "the document as one html page", &do_html,
+ "llm-text", "plain text with heading breadcrumbs", &do_llm_text,
+ "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl,
+ "images", "the image records: digest, size, pixels", &do_images,
+ "verify", "check the image records against --image-dir", &do_verify,
+ "image-dir", "the images to verify against", &image_dir,
+ "languages", "the languages the file holds", &do_languages,
+ "files", "what a .ocda.db carries besides the abstraction", &do_files,
+ "report-unknown", "what was read and not understood", &do_report_unknown,
+ "validate", "check a .ssp against its specification, as text", &do_validate,
+ "lang", "=<code> the language to read (default: the first)", &lang,
+ "output", "write the mode's output to a file", &output_file,
+ "any-version", "read a format major this reader does not know", &any_version,
+ );
+ } catch (GetOptException e) {
+ stderr.writeln("ocda-read: ", e.msg, "; --help lists the options");
+ return 2;
+ }
+
+ if (helpInfo.helpWanted || args.length < 2) {
+ defaultGetoptPrinter(
+ "ocda-read - reads the document abstraction, in any of its forms\n\n"
+ ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n",
+ helpInfo.options,
+ );
+ return 0;
+ }
+
+ string input_file = args[1];
+ if (lang.length == 0 && args.length > 2) { lang = args[2]; }
+ bool lang_named = (lang.length > 0);
+
+ /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not
+ at all: there is no text form of a database to check, and a pod's
+ .ssp is checked by taking it out +/
+ if (do_validate && sourceOf(input_file) != Source.ssp) {
+ stderr.writeln("--validate checks a .ssp as text, against the .ssp",
+ " specification; ", input_file, " is ", sourceName(sourceOf(input_file)));
+ return 2;
+ }
+
+ /+ ↓ which language. A .ocda.db and a pod may hold several; where none is
+ named, every form reads the first in the manifest's order and says
+ so, rather than the database taking one and the pod refusing +/
+ OcdaLanguage[] held;
+ OcdaDocument doc;
+ try {
+ held = languages(input_file);
+ if (lang.length == 0 && held.length > 0) { lang = held[0].lang; }
+ if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) {
+ stderr.writeln("error: no language '", lang, "' in ", input_file,
+ "; it holds: ", held.map!(l => l.lang).array.join(" "));
+ return 2;
+ }
+ doc = load(input_file, lang, any_version);
+ } catch (Exception e) {
+ stderr.writeln("Error reading ", input_file, ": ", e.msg);
+ return 2;
+ }
+
+ /+ ↓ the version gate: the major part, and refuse what is not known +/
+ if (doc.format_version.length == 0) {
+ stderr.writeln("warning: ", input_file, " declares no format version");
+ } else if (doc.formatVersionMajor != ssp_format_major_read) {
+ if (any_version) {
+ stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
+ } else {
+ stderr.writeln("error: ", input_file, " is format v", doc.format_version,
+ ", this reader knows v", ssp_format_major_read, ".x");
+ stderr.writeln(" a major version differs: a property may have",
+ " changed meaning or gone away.");
+ stderr.writeln(" --any-version reads it regardless.");
+ return 3;
+ }
+ }
+
+ writeln("Read: ", input_file);
+ if (doc.format_version.length > 0) {
+ writeln(" Format: v", doc.format_version);
+ }
+ if (auto _lang = "language" in doc.source) {
+ writeln(" Source: ", *_lang,
+ ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
+ ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
+ }
+ if (held.length > 1 && !lang_named) {
+ writeln(" (reading ", lang, "; --lang=<code> for another)");
+ }
+ writeln(" Title: ", doc.meta("title.main"));
+ writeln(" Author: ", doc.meta("creator.author"));
+ writeln(" Sections: head=", doc.head.length,
+ " toc=", doc.toc.length,
+ " body=", doc.body.length,
+ " endnotes=", doc.endnotes.length,
+ " glossary=", doc.glossary.length,
+ " bibliography=", doc.bibliography.length,
+ " bookindex=", doc.bookindex.length,
+ " blurb=", doc.blurb.length,
+ " tail=", doc.tail.length);
+ auto image_records = imageRecordCount(doc);
+ if (image_records > 0) {
+ writeln(" Images: ", image_records, " record",
+ (image_records == 1) ? "" : "s");
+ }
+
+ if (do_languages) { printLanguages(held); }
+ if (do_files) { printFiles(input_file); }
+ if (do_tree) { outputTree(doc, output_file); }
+ if (do_html) { outputHtmlScroll(doc, output_file); }
+ if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); }
+ if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); }
+ if (do_images) { outputImages(doc, output_file); }
+
+ int _verify_failures = 0;
+ if (do_verify) {
+ if (image_dir.length == 0) {
+ stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
+ return 2;
+ }
+ _verify_failures = verifyImages(doc, image_dir, output_file);
+ }
+
+ if (do_report_unknown) { reportUnknown(doc); }
+
+ int _validation_findings = 0;
+ if (do_validate) {
+ auto _found = validateSspFile(input_file, ssp_format_major_read);
+ reportValidation(input_file, _found);
+ _validation_findings = cast(int) _found.length;
+ }
+
+ if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
+ && !do_verify && !do_report_unknown && !do_validate
+ && !do_languages && !do_files) {
+ writeln("\n(this is the summary: --help lists the modes, each adding its output)");
+ }
+
+ return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
+}
+
+/+ ↓ the languages, each with the markup it was made from and its .ssp's
+ digest: the same from every form, the database storing the digest and
+ the others taking it from the bytes +/
+void printLanguages(OcdaLanguage[] held) {
+ writeln();
+ writeln(" lang source ssp digest");
+ foreach (l; held) {
+ writefln(" %-7s %-31s %s", l.lang, l.source_filename,
+ (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest);
+ }
+}
+
+/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries
+ nothing, and a pod's files are its archive entries rather than anything
+ the abstraction records, so for those it says so +/
+void printFiles(string path) {
+ writeln();
+ if (!isCarrier(path)) {
+ writeln(" carried files: only a .ocda.db carries files; this is ",
+ sourceName(sourceOf(path)));
+ return;
+ }
+ auto rows = carried(path);
+ if (rows.length == 0) { writeln(" carries no files"); return; }
+ writeln(" carried files:");
+ writeln(" role stored n original in the file");
+ foreach (r; rows) {
+ writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored);
+ }
+}