summaryrefslogtreecommitdiffhomepage
path: root/dlang/src/read/main.d
blob: 84690c5efd5ccbe32efa007155cbddd3f8e6b3b2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
/+
- Name: SisuDoc Spine, Doc Reform [a part of]
  - Description: documents, structuring, processing, publishing, search
    - static content generator

  - Author: Ralph Amissah
    [ralph.amissah@gmail.com]

  - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.

  - License: AGPL 3 or later:

    Spine (SiSU), a framework for document structuring, publishing and
    search

    Copyright (C) Ralph Amissah

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU AFERO General Public License as published by the
    Free Software Foundation, either version 3 of the License, or (at your
    option) any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program. If not, see [https://www.gnu.org/licenses/].

    If you have Internet connection, the latest version of the AGPL should be
    available at these locations:
    [https://www.fsf.org/licensing/licenses/agpl.html]
    [https://www.gnu.org/licenses/agpl.html]

  - Spine (by Doc Reform, related to SiSU) uses standard:
    - docReform markup syntax
      - standard SiSU markup syntax with modified headers and minor modifications
    - docReform object numbering
      - standard SiSU object citation numbering & system

  - Homepages:
    [https://www.sisudoc.org]
    [https://www.doc-reform.org]

  - Git
    [https://git.sisudoc.org/]

+/
/++
  ocda-read - reads the object-centric document abstraction, in any of the
  forms it is written in, and says what is in it.

    ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang]

  One reader where there were two: sisudoc-ssp-reader read a .ssp and
  sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is
  built on the library, so every mode reads every form, and a mode given the
  .ssp, the .ocda.db and the .sisupod of a document says the same thing of
  each: it is told what the document is, not which file it came from.

  Two things are about the file rather than the document, and say so:
  --files, what a .ocda.db carries besides the abstraction, and --validate,
  which checks a .ssp as text. Everything else is the model's.

  The output modules are the .ssp reader's, moved onto the library's model.
+/
module read.main;

import std.algorithm : canFind, map;
import std.array : array, join;
import std.getopt;
import std.stdio : writeln, writefln, stderr;
import ocda.model;
import ocda.load;
import ocda.inventory;
import read.tree;
import read.html;
import read.llm_text;
import read.llm_jsonl;
import read.images;
import read.verify;
import read.report_unknown;
import read.validate;

int main(string[] args) {
  bool do_tree = false;
  bool do_html = false;
  bool do_llm_text = false;
  bool do_llm_jsonl = false;
  bool do_images = false;
  bool do_verify = false;
  bool do_report_unknown = false;
  bool do_validate = false;
  bool do_languages = false;
  bool do_files = false;
  bool any_version = false;
  string image_dir = "";
  string output_file = "";
  string lang = "";

  GetoptResult helpInfo;
  try {
    helpInfo = getopt(args,
      "tree",           "the headings, as a tree",                          &do_tree,
      "outline",        "the same as --tree",                               &do_tree,
      "html",           "the document as one html page",                    &do_html,
      "llm-text",       "plain text with heading breadcrumbs",              &do_llm_text,
      "llm-jsonl",      "JSONL, chunks ready for embedding",                &do_llm_jsonl,
      "images",         "the image records: digest, size, pixels",          &do_images,
      "verify",         "check the image records against --image-dir",      &do_verify,
      "image-dir",      "the images to verify against",                     &image_dir,
      "languages",      "the languages the file holds",                     &do_languages,
      "files",          "what a .ocda.db carries besides the abstraction",  &do_files,
      "report-unknown", "what was read and not understood",                 &do_report_unknown,
      "validate",       "check a .ssp against its specification, as text",  &do_validate,
      "lang",           "=<code> the language to read (default: the first)", &lang,
      "output",         "write the mode's output to a file",                &output_file,
      "any-version",    "read a format major this reader does not know",    &any_version,
    );
  } catch (GetOptException e) {
    stderr.writeln("ocda-read: ", e.msg, "; --help lists the options");
    return 2;
  }

  if (helpInfo.helpWanted || args.length < 2) {
    defaultGetoptPrinter(
      "ocda-read - reads the document abstraction, in any of its forms\n\n"
      ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n",
      helpInfo.options,
    );
    return 0;
  }

  string input_file = args[1];
  if (lang.length == 0 && args.length > 2) { lang = args[2]; }
  bool lang_named = (lang.length > 0);

  /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not
       at all: there is no text form of a database to check, and a pod's
       .ssp is checked by taking it out +/
  if (do_validate && sourceOf(input_file) != Source.ssp) {
    stderr.writeln("--validate checks a .ssp as text, against the .ssp",
      " specification; ", input_file, " is ", sourceName(sourceOf(input_file)));
    return 2;
  }

  /+ ↓ which language. A .ocda.db and a pod may hold several; where none is
       named, every form reads the first in the manifest's order and says
       so, rather than the database taking one and the pod refusing +/
  OcdaLanguage[] held;
  OcdaDocument doc;
  try {
    held = languages(input_file);
    if (lang.length == 0 && held.length > 0) { lang = held[0].lang; }
    if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) {
      stderr.writeln("error: no language '", lang, "' in ", input_file,
        "; it holds: ", held.map!(l => l.lang).array.join(" "));
      return 2;
    }
    doc = load(input_file, lang, any_version);
  } catch (Exception e) {
    stderr.writeln("Error reading ", input_file, ": ", e.msg);
    return 2;
  }

  /+ ↓ the version gate: the major part, and refuse what is not known +/
  if (doc.format_version.length == 0) {
    stderr.writeln("warning: ", input_file, " declares no format version");
  } else if (doc.formatVersionMajor != ssp_format_major_read) {
    if (any_version) {
      stderr.writeln("warning: ", input_file, " is format v", doc.format_version,
        ", this reader knows v", ssp_format_major_read, ".x; reading anyway");
    } else {
      stderr.writeln("error: ", input_file, " is format v", doc.format_version,
        ", this reader knows v", ssp_format_major_read, ".x");
      stderr.writeln("       a major version differs: a property may have",
        " changed meaning or gone away.");
      stderr.writeln("       --any-version reads it regardless.");
      return 3;
    }
  }

  writeln("Read: ", input_file);
  if (doc.format_version.length > 0) {
    writeln("  Format: v", doc.format_version);
  }
  if (auto _lang = "language" in doc.source) {
    writeln("  Source: ", *_lang,
      ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "",
      ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : "");
  }
  if (held.length > 1 && !lang_named) {
    writeln("  (reading ", lang, "; --lang=<code> for another)");
  }
  writeln("  Title: ", doc.meta("title.main"));
  writeln("  Author: ", doc.meta("creator.author"));
  writeln("  Sections: head=", doc.head.length,
    " toc=", doc.toc.length,
    " body=", doc.body.length,
    " endnotes=", doc.endnotes.length,
    " glossary=", doc.glossary.length,
    " bibliography=", doc.bibliography.length,
    " bookindex=", doc.bookindex.length,
    " blurb=", doc.blurb.length,
    " tail=", doc.tail.length);
  auto image_records = imageRecordCount(doc);
  if (image_records > 0) {
    writeln("  Images: ", image_records, " record",
      (image_records == 1) ? "" : "s");
  }

  if (do_languages) { printLanguages(held); }
  if (do_files)     { printFiles(input_file); }
  if (do_tree)      { outputTree(doc, output_file); }
  if (do_html)      { outputHtmlScroll(doc, output_file); }
  if (do_llm_text)  { outputLlmBreadcrumbs(doc, output_file); }
  if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); }
  if (do_images)    { outputImages(doc, output_file); }

  int _verify_failures = 0;
  if (do_verify) {
    if (image_dir.length == 0) {
      stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>");
      return 2;
    }
    _verify_failures = verifyImages(doc, image_dir, output_file);
  }

  if (do_report_unknown) { reportUnknown(doc); }

  int _validation_findings = 0;
  if (do_validate) {
    auto _found = validateSspFile(input_file, ssp_format_major_read);
    reportValidation(input_file, _found);
    _validation_findings = cast(int) _found.length;
  }

  if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images
    && !do_verify && !do_report_unknown && !do_validate
    && !do_languages && !do_files) {
    writeln("\n(this is the summary: --help lists the modes, each adding its output)");
  }

  return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0;
}

/+ ↓ the languages, each with the markup it was made from and its .ssp's
     digest: the same from every form, the database storing the digest and
     the others taking it from the bytes +/
void printLanguages(OcdaLanguage[] held) {
  writeln();
  writeln("  lang    source                          ssp digest");
  foreach (l; held) {
    writefln("  %-7s %-31s %s", l.lang, l.source_filename,
      (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest);
  }
}

/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries
     nothing, and a pod's files are its archive entries rather than anything
     the abstraction records, so for those it says so +/
void printFiles(string path) {
  writeln();
  if (!isCarrier(path)) {
    writeln("  carried files: only a .ocda.db carries files; this is ",
      sourceName(sourceOf(path)));
    return;
  }
  auto rows = carried(path);
  if (rows.length == 0) { writeln("  carries no files"); return; }
  writeln("  carried files:");
  writeln("    role      stored  n     original      in the file");
  foreach (r; rows) {
    writefln("    %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored);
  }
}