summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/main.d
blob: ae0d318a09f43cefa96d48e6f9038a9337fc8b9e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
/++
  ocda-read - reads the object-centric document abstraction, in any of the
  forms it is written in, and says what is in it.

    ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]

  One reader where there were two: sisudoc-ssp-reader read a .ssp and
  sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is
  built on the library, so every mode reads every form, and a mode given the
  .ssp, the .ocda.db and the .sisupod of a document says the same thing of
  each: it is told what the document is, not which file it came from.

  Two things are about the file rather than the document, and say so:
  --files, what a .ocda.db carries besides the abstraction, and --validate,
  which checks a .ssp as text. Everything else is the model's.

  The output modules are the .ssp reader's, moved onto the library's model.
+/
module read.main;

import std.algorithm : canFind, map;
import std.array : array, join;
import std.getopt;
import std.stdio : writeln, writefln, stderr;
import ocda.model;
import ocda.load;
import ocda.inventory;
import read.tree;
import read.html;
import read.llm_text;
import read.llm_jsonl;
import read.images;
import read.verify;
import read.report_unknown;
import read.validate;

int main(string[] args) {
  bool do_tree = false;
  bool do_html = false;
  bool do_llm_text = false;
  bool do_llm_jsonl = false;
  bool do_images = false;
  bool do_verify = false;
  bool do_report_unknown = false;
  bool do_validate = false;
  bool do_languages = false;
  bool do_files = false;
  bool any_version = false;
  string image_dir = "";
  string output_file = "";
  string lang = "";

  GetoptResult helpInfo;
  try {
    helpInfo = getopt(args,
      "tree",           "the headings, as a tree",                          &do_tree,
      "outline",        "the same as --tree",                               &do_tree,
      "html",           "the document as one html page",                    &do_html,
      "llm-text",       "plain text with heading breadcrumbs",              &do_llm_text,
      "llm-jsonl",      "JSONL, chunks ready for embedding",                &do_llm_jsonl,
      "images",         "the image records: digest, size, pixels",          &do_images,
      "verify",         "check the image records against --image-dir",      &do_verify,
      "image-dir",      "the images to verify against",                     &image_dir,
      "languages",      "the languages the file holds",                     &do_languages,
      "files",          "what a .ocda.db carries besides the abstraction",  &do_files,
      "report-unknown", "what was read and not understood",                 &do_report_unknown,
      "validate",       "check a .ssp against its specification, as text",  &do_validate,
      "lang",           "=<code> the language to read (default: the first)", &lang,
      "output",         "write the mode's output to a file",                &output_file,
      "any-version",    "read a format major this reader does not know",    &any_version,
    );
  } catch (GetOptException e) {
    stderr.writeln("ocda-read: ", e.msg, "; --help lists the options");
    return 2;
  }

  if (helpInfo.helpWanted || args.length < 2) {
    defaultGetoptPrinter(
      "ocda-read - reads the document abstraction, in any of its forms\n\n"
      ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n",
      helpInfo.options,
    );
    return 0;
  }

  string input_file = args[1];
  if (lang.length == 0 && args.length > 2) { lang = args[2]; }
  bool lang_named = (lang.length > 0);

  /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not
       at all: there is no text form of a database to check, and a pod's
       .ssp is checked by taking it out +/
  if (do_validate && sourceOf(input_file) != Source.ssp) {
    stderr.writeln("--validate checks a .ssp as text, against the .ssp",
      " specification; ", input_file, " is ", sourceName(sourceOf(input_file)));
    return 2;
  }

  /+ ↓ which language. A .ocda.db and a pod may hold several; where none is
       named, every form reads the first in the manifest's order and says
       so, rather than the database taking one and the pod refusing +/
  OcdaLanguage[] held;
  OcdaDocument doc;
  try {
    held = languages(input_file);
    if (lang.length == 0 && held.length > 0) { lang = held[0].lang; }
    if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) {
      stderr.writeln("error: no language '", lang, "' in ", input_file,
        "; it holds: ", held.map!(l => l.lang).array.join(" "));
      return 2;
    }
    doc = load(input_file, lang, any_version);
  } catch (Exception e) {
    stderr.writeln("Error reading ", input_file, ": ", e.msg);
    return 2;
  }

  /+ ↓ the version gate: the major part, and refuse what is not known +/
  if (doc.format_version.length == 0) {
    stderr.writeln("warning: ", input_file, " declares no format version");
  } else if (doc.formatVersionMajor != ssp_format_major_read) {
    if (any_version) {
      stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
        ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
    } else {
      stderr.writeln("error: ", input_file, " is format v", doc.format_version,
        ", this reader knows v", ssp_format_major_read, ".x");
      stderr.writeln("       a major version differs: a property may have",
        " changed meaning or gone away.");
      stderr.writeln("       --any-version reads it regardless.");
      return 3;
    }
  }

  writeln("Read: ", input_file);
  if (doc.format_version.length > 0) {
    writeln("  Format: v", doc.format_version);
  }
  if (auto _lang = "language" in doc.source) {
    writeln("  Source: ", *_lang,
      ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
      ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
  }
  if (held.length > 1 && !lang_named) {
    writeln("  (reading ", lang, "; --lang=<code> for another)");
  }
  writeln("  Title: ", doc.meta("title.main"));
  writeln("  Author: ", doc.meta("creator.author"));
  writeln("  Sections: head=", doc.head.length,
    " toc=", doc.toc.length,
    " body=", doc.body.length,
    " endnotes=", doc.endnotes.length,
    " glossary=", doc.glossary.length,
    " bibliography=", doc.bibliography.length,
    " bookindex=", doc.bookindex.length,
    " blurb=", doc.blurb.length,
    " tail=", doc.tail.length);
  auto image_records = imageRecordCount(doc);
  if (image_records > 0) {
    writeln("  Images: ", image_records, " record",
      (image_records == 1) ? "" : "s");
  }

  if (do_languages) { printLanguages(held); }
  if (do_files)     { printFiles(input_file); }
  if (do_tree)      { outputTree(doc, output_file); }
  if (do_html)      { outputHtmlScroll(doc, output_file); }
  if (do_llm_text)  { outputLlmBreadcrumbs(doc, output_file); }
  if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); }
  if (do_images)    { outputImages(doc, output_file); }

  int _verify_failures = 0;
  if (do_verify) {
    if (image_dir.length == 0) {
      stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
      return 2;
    }
    _verify_failures = verifyImages(doc, image_dir, output_file);
  }

  if (do_report_unknown) { reportUnknown(doc); }

  int _validation_findings = 0;
  if (do_validate) {
    auto _found = validateSspFile(input_file, ssp_format_major_read);
    reportValidation(input_file, _found);
    _validation_findings = cast(int) _found.length;
  }

  if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
    && !do_verify && !do_report_unknown && !do_validate
    && !do_languages && !do_files) {
    writeln("\n(this is the summary: --help lists the modes, each adding its output)");
  }

  return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
}

/+ ↓ the languages, each with the markup it was made from and its .ssp's
     digest: the same from every form, the database storing the digest and
     the others taking it from the bytes +/
void printLanguages(OcdaLanguage[] held) {
  writeln();
  writeln("  lang    source                          ssp digest");
  foreach (l; held) {
    writefln("  %-7s %-31s %s", l.lang, l.source_filename,
      (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest);
  }
}

/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries
     nothing, and a pod's files are its archive entries rather than anything
     the abstraction records, so for those it says so +/
void printFiles(string path) {
  writeln();
  if (!isCarrier(path)) {
    writeln("  carried files: only a .ocda.db carries files; this is ",
      sourceName(sourceOf(path)));
    return;
  }
  auto rows = carried(path);
  if (rows.length == 0) { writeln("  carries no files"); return; }
  writeln("  carried files:");
  writeln("    role      stored  n     original      in the file");
  foreach (r; rows) {
    writefln("    %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored);
  }
}