diff options
Diffstat (limited to 'org')
| -rw-r--r-- | org/d-ocda-load.org | 953 | ||||
| -rw-r--r-- | org/d-read.org | 1199 | ||||
| -rw-r--r-- | org/gleam-ocda-load.org | 4 | ||||
| -rw-r--r-- | org/gleam-ocda_lib.org | 4 |
4 files changed, 2156 insertions, 4 deletions
diff --git a/org/d-ocda-load.org b/org/d-ocda-load.org new file mode 100644 index 0000000..59762a8 --- /dev/null +++ b/org/d-ocda-load.org @@ -0,0 +1,953 @@ +-*- mode: org -*- +#+TITLE: dlang load +#+DESCRIPTION: dlang lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* dlang :dlang: +** load dlang + +#+HEADER: :tangle "../dlang/src/ocda/load.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load; + +/+ ↓ what a path is, and loading it. + . + One place that decides, so that every reader and every writer accepts + the same inputs and a new input type is added once. This mirrors + spine's own ocda/abstraction/load.d, which names the sources it can + enter the pipeline from and tells them apart by suffix. + . + Deliberately *not* here: .sst and .ssm markup. Reading markup means + implementing the parser, which is spine's work and a far larger thing + than a reader or a sample writer needs. This library reads + abstractions. ++/ + +import std.algorithm : endsWith; +import ocda.model; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; + +enum Source { + ssp, // .ssp, the abstraction as text + ocda_db, // .ocda.db, the abstraction as sqlite + pod, // .sisupod or .zip, an archive carrying a .ssp + unknown, +} + +Source sourceOf(string path) { + if (path.endsWith(".ocda.db")) { return Source.ocda_db; } + if (path.endsWith(".ssp")) { return Source.ssp; } + /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote + before, and every pod already published carries that name, so both + are read. What is inside decides how it is opened, not the name. + +/ + if (path.endsWith(".sisupod")) { return Source.pod; } + if (path.endsWith(".zip")) { return Source.pod; } + if (path.endsWith(".db")) { return Source.ocda_db; } + return Source.unknown; +} + +string sourceName(Source s) { + final switch (s) { + case Source.ssp: return ".ssp (the abstraction as text)"; + case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; + case Source.pod: return "a pod archive carrying a .ssp"; + case Source.unknown: return "not an abstraction this library reads"; + } +} + +class OcdaLoadError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ load whatever the path is into the model. + lang names which language to take from a .ocda.db, which holds + every language of its document, and from a pod, which may carry more + than one .ssp. A .ssp is one language by construction and ignores it. ++/ +OcdaDocument load(string path, string lang = "", bool any_version = false) { + final switch (sourceOf(path)) { + case Source.ssp: return parseSspFile(path); + case Source.ocda_db: return loadDbFile(path, lang, any_version); + case Source.pod: return loadPodFile(path, lang); + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) + ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); + } +} +#+END_SRC + +** load db dlang + +#+HEADER: :tangle "../dlang/src/ocda/load_db.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_db; + +/+ ↓ build the model from a .ocda.db. + . + The point of this module is that it returns *the same value* + ocda.load_ssp returns from the same document's .ssp. A writer is then + a function from the model to bytes, and cannot tell which + serialisation it was given, exactly as spine's own writers cannot. + . + Written from the sisudoc-abstraction-format specification, + ocda-db-format-specification.md, whose section 4 gives the column by + column correspondence this module implements. ++/ + +import std.array : join, split; +import std.json : JSONType, parseJSON; +import std.conv : to; +import std.algorithm : startsWith; +import std.string : strip; +import ocda.model; +import ocda.sqlite; + +/+ ↓ the schema major this loader knows (specification section 2) +/ +enum schema_major_read = "2"; + +class OcdaDbError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ every language the database holds, in manifest order +/ +struct DbLanguage { + int doc_id; + string lang; + string source_filename; + string ssp_digest; +} + +DbLanguage[] dbLanguages(ref Db db) { + DbLanguage[] out_; + foreach (r; db.rows( + "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') + FROM documents ORDER BY id")) { + out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); + } + return out_; +} + +/+ ↓ load one language of a database as the model. + lang == "" takes the first language the manifest gave. ++/ +OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { + auto db = Db.open(path); + scope(exit) { db.close(); } + + if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { + throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" + ~ " it does not look like a .ocda.db"); + } + + /+ ↓ specification sections 2 and 3.2: the rows describing the file rather + than a document in it are the only ones with doc_id null +/ + string schema_version = db.one( + "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); + if (schema_version.length > 0 && !any_version) { + auto major = schema_version.split(".")[0]; + if (major != schema_major_read) { + throw new OcdaDbError(path ~ " is schema v" ~ schema_version + ~ ", this loader knows v" ~ schema_major_read ~ ".x"); + } + } + + auto languages = dbLanguages(db); + if (languages.length == 0) { + throw new OcdaDbError(path ~ ": no documents rows"); + } + DbLanguage chosen = languages[0]; + if (lang.length > 0) { + bool found = false; + foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } + if (!found) { + string have; + foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } + throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); + } + } + + OcdaDocument doc; + doc.format_version = schema_version; + doc.source_filename = chosen.source_filename; + + /+ ↓ @source of the .ssp corresponds to the documents row and the + source.digest metadata (specification section 5.1 of the .ssp, and + section 7 here) +/ + doc.source["language"] = chosen.lang; + string languages_joined; + foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } + doc.source["languages"] = languages_joined; + auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); + if (digest.length > 0) { doc.source["digest"] = digest; } + + /+ ↓ @meta: every metadata row of this language that is not a source.* + property. source.* describes the artefact rather than the document + and lives in @source in the text form. + +/ + foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " ORDER BY key")) { + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } + } + + /+ ↓ 2.1 added the container range as two columns. A 2.0 database has + neither, and reads as having no range, which is what 2.0 meant. +/ + bool has_range = false; + foreach (c; db.rows("PRAGMA table_info(objects)")) { + if (c[1] == "ocn_range_first") { has_range = true; } + } + string range_cols = has_range + ? "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" + : "0, 0"; + + /+ ↓ the objects, in document order within each section +/ + foreach (r; db.rows( + "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), + coalesce(is_of_section,''), coalesce(is_of_type,''), + coalesce(heading_level,-1), coalesce(identifier,''), + coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), + coalesce(children,''), coalesce(ancestors,''), + coalesce(dummy_heading,0), coalesce(object_number_off,0), + coalesce(indent_base,0), coalesce(indent_hang,0), + coalesce(bullet,0), coalesce(lang,''), + coalesce(has_links,0), coalesce(has_notes_reg,0), + coalesce(has_notes_star,0), coalesce(has_images,0), + coalesce(segment,''), coalesce(segment_prev,''), + coalesce(segment_next,''), coalesce(anchor,''), + coalesce(segment_html_is,''), + coalesce(table_cols,0), coalesce(table_widths,''), + coalesce(table_aligns,''), coalesce(table_header,0), + coalesce(code_linenumbers,0), + coalesce(heading_ancestors_text,''), coalesce(sha256,''), + coalesce(text,''), id, coalesce(heading_lev_anchor,''), + coalesce(segment_epub,''), coalesce(segment_epub_is,''), " ~ range_cols ~ " + FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string + ~ " ORDER BY CASE section + WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 + WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 + WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 + WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { + OcdaObject obj; + obj.section = r[0]; + obj.ocn = r[2].to!int; + obj.is_a = r[3]; + obj.part = r[4]; + obj.own_section = (r[5] == r[0]) ? "" : r[5]; + obj.is_of_type = r[6]; + /+ ↓ the database holds the level as a number, the text form as the + letter or digit the markup wrote. Model section 4 is the mapping. +/ + obj.heading_level = levelName(r[7].to!int); + obj.identifier = r[8]; + obj.parent_ocn = r[9].to!int; + obj.last_descendant_ocn = r[10].to!int; + obj.children = jsonInts(r[11]); + obj.ancestors = jsonInts(r[12]); + obj.dummy_heading = (r[13] != "0"); + obj.ocn_off = (r[14] != "0"); + obj.indent_base = r[15].to!int; + obj.indent_hang = r[16].to!int; + obj.bullet = (r[17] != "0"); + obj.lang = r[18]; + obj.has_links = (r[19] != "0"); + obj.has_notes_reg = (r[20] != "0"); + obj.has_notes_star = (r[21] != "0"); + obj.has_images = (r[22] != "0"); + obj.segment = r[23]; + obj.segment_prev = r[24]; + obj.segment_next = r[25]; + obj.anchor = r[26]; + obj.segment_html_is = r[27]; + obj.heading_lev_anchor = r[37]; + obj.segment_epub = r[38]; + obj.segment_epub_is = r[39]; + obj.ocn_range_first = r[40].to!int; + obj.ocn_range_last = r[41].to!int; + obj.table_cols = r[28].to!int; + obj.table_widths = jsonWords(r[29]); + obj.table_aligns = jsonWords(r[30]); + obj.table_header = (r[31] != "0"); + obj.code_linenumbers = (r[32] != "0"); + obj.heading_ancestors_text = jsonJoined(r[33]); + obj.sha256 = r[34]; + obj.text = r[35]; + + /+ ↓ the four repeatable properties of the text form are child tables + here (specification section 4) +/ + string object_id = r[36]; + foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), + coalesce(width,0), coalesce(height,0), + coalesce(missing,0) + FROM object_images WHERE object_id = " ~ object_id + ~ " ORDER BY seq")) { + OcdaImage img; + img.name = i[0]; + img.sha256 = i[1]; + img.bytes = i[2].to!ulong; + img.width = i[3].to!int; + img.height = i[4].to!int; + img.missing = (i[5] != "0"); + obj.images ~= img; + } + foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.stow_link ~= l[0]; + } + foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.anchor_tags ~= a[0]; + } + + auto section = doc.sectionByName(r[0]); + if (section !is null) { *section ~= obj; } + } + countUnread(db, chosen.doc_id, doc); + return doc; +} + +/+ ↓ what the database holds and the model does not. + . + The text loader meets every property line and counts those it does not + keep, so a reader can say what was read and not understood. This + loader asks for columns by name, and would never see the ones it did + not ask for: it would report that it had understood everything, which + is the silence the count exists to prevent. So it counts them, on the + conditions under which the .ssp writes the same property (specification + section 6.3), and the report is the same from either form: checked, + 36 of 36. ++/ +private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { + string id = doc_id.to!string; + void count(string property, string where) { + auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id + ~ " AND " ~ where).to!size_t; + if (n > 0) { doc.unknown.property[property] = n; } + } + count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); + count("dom_status", "coalesce(dom_status,'') != ''"); + count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); + count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); + count("parent_lev", "coalesce(parent_lev,0) != 0"); + count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); + count("attrib", "coalesce(attrib,'') != ''"); + if (db.has("object_subtoc")) { + auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" + ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; + if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } + } +} + +/+ ↓ model section 4: the eight levels and their indices +/ +private string levelName(int index) { + switch (index) { + case 0: return "A"; + case 1: return "B"; + case 2: return "C"; + case 3: return "D"; + case 4: return "1"; + case 5: return "2"; + case 6: return "3"; + case 7: return "4"; + default: return ""; + } +} + +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + +private int[] jsonInts(string v) { + int[] out_; + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} + } + return out_; +} + +/+ ↓ space separated in the text form: table_widths, table_aligns +/ +private string jsonWords(string v) { + return jsonStrings(v).join(" "); +} + +/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ +private string jsonJoined(string v) { + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); +} +#+END_SRC + +** load pod dlang + +#+HEADER: :tangle "../dlang/src/ocda/load_pod.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_pod; + +/+ ↓ the abstraction out of a pod archive. + . + A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same + archive unwrapped, which is what spine published before it wrapped them. + Either way, a pod built with --pod2 carries the abstraction inside it, + at media/abstraction/<uid>.<lang>.ssp, and that is what is read. + . + So this is a third input form and *not* a third model: the .ssp inside + is parsed by ocda.load_ssp and comes back as the same value. A writer + over the library gains .sisupod without being touched, which is the + whole reason the dispatch is in one place. + . + A pod built without --pod2 carries markup and no abstraction. That is + refused, and the refusal says which flag makes a pod this reader can + read, because "no abstraction in it" is a fact about the pod rather + than about the reader. ++/ + +import std.algorithm : endsWith, sort; +import std.array : array; +import std.file : read; +import std.string : indexOf; +import ocda.model; +import ocda.load_ssp; +import ocda.zip; +import ocda.zstd; + +/+ ↓ a limit on what a frame may expand to. + A reader of an artefact it did not make declines to be told how much + memory to allocate. 512 MiB is far above the largest pod in the + reference collection (4 MiB) and far below anything that matters. ++/ +enum size_t max_pod_size = 512 * 1024 * 1024; + +class OcdaPodError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ which languages of the abstraction the pod carries, in sorted order +/ +string[] podLanguages(string path) { + auto bytes = archiveBytes(path); + string[] out_; + foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { + string lang = langOfSspName(e.name); + if (lang.length > 0) { out_ ~= lang; } + } + return out_.sort.array; +} + +/+ ↓ load one language of the abstraction a pod carries. + lang == "" takes the only one there is, and where there is a choice it + says so rather than choosing, as the .ocda.db loader does. ++/ +OcdaDocument loadPodFile(string path, string lang = "") { + auto bytes = archiveBytes(path); + auto entries = zipEntries(bytes); + auto ssps = zipFindSuffix(entries, ".ssp"); + if (ssps.length == 0) { + throw new OcdaPodError(path ~ ": carries no .ssp." + ~ " A pod carries the abstraction only when spine was asked for it:" + ~ " spine --pod2 puts it in, spine --source does not."); + } + ZipEntry chosen = ssps[0]; + if (lang.length > 0) { + bool found = false; + foreach (e; ssps) { + if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } + } + if (!found) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": no language '" ~ lang + ~ "'; it holds: " ~ have); + } + } else if (ssps.length > 1) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": holds more than one language (" + ~ have ~ "); name the one wanted"); + } + auto ssp = zipRead(bytes, chosen); + return parseSsp(cast(string) ssp.idup); +} + +/+ ↓ the archive, unwrapped if it was wrapped. + The bytes decide, not the name: a .sisupod is a zstd frame and a .zip + is not, and a file misnamed either way still reads. ++/ +package ubyte[] archiveBytes(string path) { + auto bytes = cast(ubyte[]) read(path); + return zstdIsFrame(bytes) + ? zstdDecompress(bytes, max_pod_size) + : bytes; +} + +/+ ↓ <uid>.<lang>.ssp -> <lang> +/ +package string langOfSspName(string name) { + if (!name.endsWith(".ssp")) { return ""; } + string stem = name[0 .. $ - 4]; + auto dot = stem.lastIndexOf('.'); + return (dot > 0) ? stem[dot + 1 .. $] : ""; +} + +private ptrdiff_t lastIndexOf(string s, char c) { + for (ptrdiff_t i = s.length - 1; i >= 0; i--) { + if (s[i] == c) { return i; } + } + return -1; +} +#+END_SRC + +** load ssp dlang + +#+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.load_ssp; + +import std.algorithm : splitter, startsWith; +import std.array : array, split; +import std.conv : to, ConvException; +import std.file : readText; +import std.stdio : writeln; +import std.string : strip, stripLeft, indexOf; +import ocda.model; + +OcdaDocument parseSspFile(string filename) { + string content = readText(filename); + return parseSsp(content); +} + +OcdaDocument parseSsp(string content) { + OcdaDocument doc; + OcdaObject current_obj; + bool in_object = false; + /+ ↓ whether a text line has been seen for the object being read. + Not "is the text non-empty": an object whose first text line is blank + (a verse opening on an empty line, which the sample set has seven of) + would then lose it, the next line overwriting rather than appending. + +/ + bool seen_text = false; + string current_section = ""; + string current_block = ""; // meta, make, doc_has, or section name + + foreach (line; content.splitter("\n")) { + /+ ↓ comment. The first line of a .ssp declares the format version, and + it is the one comment that carries meaning: a reader checks the + major part and refuses what it does not know rather than loading + an unfamiliar artefact half populated in silence. + +/ + if (line.length > 0 && line[0] == '%') { + enum _banner = "% SiSU Document Abstraction v"; + enum _source = "% Source: "; + if (doc.format_version.length == 0 && line.startsWith(_banner)) { + doc.format_version = line[_banner.length .. $].strip; + } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { + doc.source_filename = line[_source.length .. $].strip; + } + continue; + } + + /+ ↓ blank line - finalize current object +/ + if (line.length == 0 || line.strip.length == 0) { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + continue; + } + + /+ ↓ section/block open +/ + if (line[0] == '@') { + auto brace_pos = line.indexOf('{'); + if (brace_pos > 0) { + string name = line[1 .. brace_pos].strip; + current_block = name; + if (!isPreambleBlock(name)) { + current_section = name; + /+ ↓ a block that is neither preamble nor a section this reader + knows: counted, so that --report-unknown can say so +/ + if (doc.sectionByName(name) is null) { + doc.unknown.block[name]++; + } + } + } + continue; + } + + /+ ↓ section/block close +/ + if (line[0] == '}') { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + current_block = ""; + continue; + } + + /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ + if (isPreambleBlock(current_block)) { + string stripped = line.stripLeft; + auto colon_pos = stripped.indexOf(':'); + if (colon_pos > 0) { + string key = stripped[0 .. colon_pos].strip; + string value = (colon_pos + 1 < stripped.length) + ? stripped[colon_pos + 1 .. $].strip : ""; + if (current_block == "meta") { doc.metadata[key] = value; } + else if (current_block == "make") { doc.make[key] = value; } + else if (current_block == "source") { doc.source[key] = value; } + else { doc.doc_has[key] = value; } + } + continue; + } + + /+ ↓ object declaration: [N] type ... +/ + if (line[0] == '[') { + if (in_object) { + commitObject(doc, current_section, current_obj); + } + current_obj = OcdaObject.init; + current_obj.section = current_section; + in_object = true; + seen_text = false; + parseObjectDeclaration(line, current_obj); + continue; + } + + /+ ↓ property: .name: value +/ + if (line[0] == '.') { + if (in_object) { + parseProperty(line, current_obj, doc.unknown); + } + continue; + } + + /+ ↓ text content: | content +/ + if (line.length >= 1 && line[0] == '|') { + if (in_object) { + string text_content = (line.length > 2) ? line[2 .. $] : ""; + if (seen_text) { + current_obj.text ~= "\n" ~ text_content; + } else { + current_obj.text = text_content; + seen_text = true; + } + } + continue; + } + + /+ ↓ no known shape. Counted by its first character so that a reader + can be told the file held something this one did not expect + +/ + doc.unknown.line[line[0 .. 1].idup]++; + } + + /+ ↓ finalize last object +/ + if (in_object) { + commitObject(doc, current_section, current_obj); + } + + return doc; +} + +private void parseObjectDeclaration(string line, ref OcdaObject obj) { + /+ parse: [N] type :level identifier +/ + auto bracket_close = line.indexOf(']'); + if (bracket_close < 0) return; + try { + obj.ocn = line[1 .. bracket_close].strip.to!int; + } catch (ConvException) { + obj.ocn = 0; + } + string rest = (bracket_close + 1 < line.length) + ? line[bracket_close + 1 .. $].strip : ""; + if (rest.length == 0) return; + + auto parts = rest.split(" "); + if (parts.length == 0) return; + obj.is_a = parts[0]; + + if (obj.is_a == "heading" && parts.length >= 2) { + /+ parse :A, :B, :1, etc. +/ + string lev = parts[1]; + if (lev.length >= 2 && lev[0] == ':') { + obj.heading_level = lev[1 .. $]; + } + /+ optional identifier after level +/ + if (parts.length >= 3) { + obj.identifier = parts[2]; + } + } +} + +/+ ↓ the blocks that are document preamble rather than sections of objects +/ +private bool isPreambleBlock(string name) { + return name == "source" || name == "meta" || name == "make" || name == "doc_has"; +} + +private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { + /+ parse: .name: value +/ + auto colon_pos = line.indexOf(':'); + if (colon_pos < 1) return; + string name = line[1 .. colon_pos].strip; + string value = (colon_pos + 1 < line.length) + ? line[colon_pos + 1 .. $].strip : ""; + + switch (name) { + case "part": obj.part = value; break; + case "is_of_type": obj.is_of_type = value; break; + case "section": obj.own_section = value; break; + case "identifier": obj.identifier = value; break; + case "segment_html_is": obj.segment_html_is = value; break; + /+ ↓ the epub segments at heading levels 1 to 4 where the html + segments at level 4 alone, so the two are different files and + the abstraction records both +/ + case "segment_epub": obj.segment_epub = value; break; + case "segment_epub_is": obj.segment_epub_is = value; break; + case "table_aligns": obj.table_aligns = value; break; + case "stow_link": obj.stow_link ~= value; break; + case "anchor_tag": obj.anchor_tags ~= value; break; + case "parent": obj.parent_ocn = toIntSafe(value); break; + case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; + case "ocn_range": { + auto r = parseIntArray(value); + if (r.length == 2) { obj.ocn_range_first = r[0]; obj.ocn_range_last = r[1]; } + break; + } + case "children": obj.children = parseIntArray(value); break; + case "ancestors": obj.ancestors = parseIntArray(value); break; + case "dummy": obj.dummy_heading = (value == "true"); break; + case "ocn_off": obj.ocn_off = (value == "true"); break; + case "indent": + auto parts = value.split(" "); + if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); + if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); + break; + case "bullet": obj.bullet = (value == "true"); break; + case "lang": obj.lang = value; break; + case "has": + foreach (flag; value.split(" ")) { + switch (flag) { + case "links": obj.has_links = true; break; + case "notes_reg": obj.has_notes_reg = true; break; + case "notes_star": obj.has_notes_star = true; break; + case "images": obj.has_images = true; break; + default: break; + } + } + break; + case "segment": obj.segment = value; break; + case "segment_prev": obj.segment_prev = value; break; + case "segment_next": obj.segment_next = value; break; + case "anchor": obj.anchor = value; break; + case "heading_lev_anchor": obj.heading_lev_anchor = value; break; + case "table_cols": obj.table_cols = toIntSafe(value); break; + case "table_widths": obj.table_widths = value; break; + case "table_header": obj.table_header = (value == "true"); break; + case "code_syntax": obj.code_syntax = value; break; + case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; + case "heading_ancestors_text": obj.heading_ancestors_text = value; break; + case "sha256": obj.sha256 = value; break; + case "image": obj.images ~= parseImageRecord(value); break; + /+ ↓ a property this reader has no use for, or has never heard of. + Ignored, which is the conforming behaviour within a major + version, but counted rather than passed over in silence + +/ + default: unknown.property[name]++; break; + } +} + +private OcdaImage parseImageRecord(string value) { + /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] + filename first, remaining fields order-independent; unrecognised + fields are kept rather than discarded +/ + OcdaImage img; + bool first = true; + foreach (field; value.split(" ")) { + if (field.length == 0) continue; + if (first) { + img.name = field; + first = false; + continue; + } + auto colon_pos = field.indexOf(':'); + if (colon_pos < 1) continue; + string key = field[0 .. colon_pos]; + string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; + switch (key) { + case "sha256": img.sha256 = val; break; + case "bytes": img.bytes = toUlongSafe(val); break; + case "px": + auto dims = val.split("x"); + if (dims.length == 2) { + img.width = toIntSafe(dims[0]); + img.height = toIntSafe(dims[1]); + } + break; + case "missing": img.missing = (val == "true"); break; + default: img.fields[key] = val; break; + } + } + return img; +} + +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + +private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); + auto sec = doc.sectionByName(section); + if (sec !is null) { + ,*sec ~= obj; + } +} + +private int toIntSafe(string s) { + try { return s.strip.to!int; } + catch (ConvException) { return 0; } +} + +private ulong toUlongSafe(string s) { + try { return s.strip.to!ulong; } + catch (ConvException) { return 0; } +} + +private int[] parseIntArray(string s) { + int[] result; + foreach (part; s.split(" ")) { + auto v = toIntSafe(part); + result ~= v; + } + return result; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/d-read.org b/org/d-read.org new file mode 100644 index 0000000..9cf2416 --- /dev/null +++ b/org/d-read.org @@ -0,0 +1,1199 @@ +-*- mode: org -*- +#+TITLE: dlang load +#+DESCRIPTION: dlang lib +#+FILETAGS: :spine:build:tools: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +* dlang :dlang: +** dlang read main + +#+HEADER: :tangle "../dlang/src/read/main.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + ocda-read - reads the object-centric document abstraction, in any of the + forms it is written in, and says what is in it. + + ocda-read [options] <file>.ssp | <file>.ocda.db | <file>.sisupod [lang] + + One reader where there were two: sisudoc-ssp-reader read a .ssp and + sisudoc-ocda-db-reader a .ocda.db, each with its own parser. This one is + built on the library, so every mode reads every form, and a mode given the + .ssp, the .ocda.db and the .sisupod of a document says the same thing of + each: it is told what the document is, not which file it came from. + + Two things are about the file rather than the document, and say so: + --files, what a .ocda.db carries besides the abstraction, and --validate, + which checks a .ssp as text. Everything else is the model's. + + The output modules are the .ssp reader's, moved onto the library's model. ++/ +module read.main; + +import std.algorithm : canFind, map; +import std.array : array, join; +import std.getopt; +import std.stdio : writeln, writefln, stderr; +import ocda.model; +import ocda.load; +import ocda.inventory; +import read.tree; +import read.html; +import read.llm_text; +import read.llm_jsonl; +import read.images; +import read.verify; +import read.report_unknown; +import read.validate; + +int main(string[] args) { + bool do_tree = false; + bool do_html = false; + bool do_llm_text = false; + bool do_llm_jsonl = false; + bool do_images = false; + bool do_verify = false; + bool do_report_unknown = false; + bool do_validate = false; + bool do_languages = false; + bool do_files = false; + bool any_version = false; + string image_dir = ""; + string output_file = ""; + string lang = ""; + + GetoptResult helpInfo; + try { + helpInfo = getopt(args, + "tree", "the headings, as a tree", &do_tree, + "outline", "the same as --tree", &do_tree, + "html", "the document as one html page", &do_html, + "llm-text", "plain text with heading breadcrumbs", &do_llm_text, + "llm-jsonl", "JSONL, chunks ready for embedding", &do_llm_jsonl, + "images", "the image records: digest, size, pixels", &do_images, + "verify", "check the image records against --image-dir", &do_verify, + "image-dir", "the images to verify against", &image_dir, + "languages", "the languages the file holds", &do_languages, + "files", "what a .ocda.db carries besides the abstraction", &do_files, + "report-unknown", "what was read and not understood", &do_report_unknown, + "validate", "check a .ssp against its specification, as text", &do_validate, + "lang", "=<code> the language to read (default: the first)", &lang, + "output", "write the mode's output to a file", &output_file, + "any-version", "read a format major this reader does not know", &any_version, + ); + } catch (GetOptException e) { + stderr.writeln("ocda-read: ", e.msg, "; --help lists the options"); + return 2; + } + + if (helpInfo.helpWanted || args.length < 2) { + defaultGetoptPrinter( + "ocda-read - reads the document abstraction, in any of its forms\n\n" + ~ "Usage: ocda-read [options] <file>.ssp|<file>.ocda.db|<file>.sisupod [lang]\n", + helpInfo.options, + ); + return 0; + } + + string input_file = args[1]; + if (lang.length == 0 && args.length > 2) { lang = args[2]; } + bool lang_named = (lang.length > 0); + + /+ ↓ --validate is about the .ssp as text, so it is asked of a .ssp or not + at all: there is no text form of a database to check, and a pod's + .ssp is checked by taking it out +/ + if (do_validate && sourceOf(input_file) != Source.ssp) { + stderr.writeln("--validate checks a .ssp as text, against the .ssp", + " specification; ", input_file, " is ", sourceName(sourceOf(input_file))); + return 2; + } + + /+ ↓ which language. A .ocda.db and a pod may hold several; where none is + named, every form reads the first in the manifest's order and says + so, rather than the database taking one and the pod refusing +/ + OcdaLanguage[] held; + OcdaDocument doc; + try { + held = languages(input_file); + if (lang.length == 0 && held.length > 0) { lang = held[0].lang; } + if (held.length > 0 && !held.map!(l => l.lang).canFind(lang)) { + stderr.writeln("error: no language '", lang, "' in ", input_file, + "; it holds: ", held.map!(l => l.lang).array.join(" ")); + return 2; + } + doc = load(input_file, lang, any_version); + } catch (Exception e) { + stderr.writeln("Error reading ", input_file, ": ", e.msg); + return 2; + } + + /+ ↓ the version gate: the major part, and refuse what is not known +/ + if (doc.format_version.length == 0) { + stderr.writeln("warning: ", input_file, " declares no format version"); + } else if (doc.formatVersionMajor != ssp_format_major_read) { + if (any_version) { + stderr.writeln("warning: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x; reading anyway"); + } else { + stderr.writeln("error: ", input_file, " is format v", doc.format_version, + ", this reader knows v", ssp_format_major_read, ".x"); + stderr.writeln(" a major version differs: a property may have", + " changed meaning or gone away."); + stderr.writeln(" --any-version reads it regardless."); + return 3; + } + } + + writeln("Read: ", input_file); + if (doc.format_version.length > 0) { + writeln(" Format: v", doc.format_version); + } + if (auto _lang = "language" in doc.source) { + writeln(" Source: ", *_lang, + ("languages" in doc.source) ? " of " ~ doc.source["languages"] : "", + ("digest" in doc.source) ? ", markup digest " ~ doc.source["digest"][0 .. 16] ~ "..." : ""); + } + if (held.length > 1 && !lang_named) { + writeln(" (reading ", lang, "; --lang=<code> for another)"); + } + writeln(" Title: ", doc.meta("title.main")); + writeln(" Author: ", doc.meta("creator.author")); + writeln(" Sections: head=", doc.head.length, + " toc=", doc.toc.length, + " body=", doc.body.length, + " endnotes=", doc.endnotes.length, + " glossary=", doc.glossary.length, + " bibliography=", doc.bibliography.length, + " bookindex=", doc.bookindex.length, + " blurb=", doc.blurb.length, + " tail=", doc.tail.length); + auto image_records = imageRecordCount(doc); + if (image_records > 0) { + writeln(" Images: ", image_records, " record", + (image_records == 1) ? "" : "s"); + } + + if (do_languages) { printLanguages(held); } + if (do_files) { printFiles(input_file); } + if (do_tree) { outputTree(doc, output_file); } + if (do_html) { outputHtmlScroll(doc, output_file); } + if (do_llm_text) { outputLlmBreadcrumbs(doc, output_file); } + if (do_llm_jsonl) { outputLlmJsonl(doc, output_file); } + if (do_images) { outputImages(doc, output_file); } + + int _verify_failures = 0; + if (do_verify) { + if (image_dir.length == 0) { + stderr.writeln("--verify needs --image-dir=<path to the pod's media/image>"); + return 2; + } + _verify_failures = verifyImages(doc, image_dir, output_file); + } + + if (do_report_unknown) { reportUnknown(doc); } + + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images + && !do_verify && !do_report_unknown && !do_validate + && !do_languages && !do_files) { + writeln("\n(this is the summary: --help lists the modes, each adding its output)"); + } + + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; +} + +/+ ↓ the languages, each with the markup it was made from and its .ssp's + digest: the same from every form, the database storing the digest and + the others taking it from the bytes +/ +void printLanguages(OcdaLanguage[] held) { + writeln(); + writeln(" lang source ssp digest"); + foreach (l; held) { + writefln(" %-7s %-31s %s", l.lang, l.source_filename, + (l.ssp_digest.length > 16) ? l.ssp_digest[0 .. 16] ~ "..." : l.ssp_digest); + } +} + +/+ ↓ what a .ocda.db carries besides the abstraction. A .ssp carries + nothing, and a pod's files are its archive entries rather than anything + the abstraction records, so for those it says so +/ +void printFiles(string path) { + writeln(); + if (!isCarrier(path)) { + writeln(" carried files: only a .ocda.db carries files; this is ", + sourceName(sourceOf(path))); + return; + } + auto rows = carried(path); + if (rows.length == 0) { writeln(" carries no files"); return; } + writeln(" carried files:"); + writeln(" role stored n original in the file"); + foreach (r; rows) { + writefln(" %-9s %-7s %-5s %-13s %s", r.role, r.compression, r.count, r.bytes, r.stored); + } +} +#+END_SRC + +** dlang read tree + +#+HEADER: :tangle "../dlang/src/read/tree.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.tree; + +import std.conv : to; +import std.stdio : writeln, writef, writefln, File; +import std.array : replicate; +import ocda.model; + +void outputTree(OcdaDocument doc, string output_file) { + string[] lines; + + lines ~= "Document: " ~ doc.meta("title.main"); + if (doc.meta("title.subtitle").length > 0) + lines ~= " " ~ doc.meta("title.subtitle"); + lines ~= "Author: " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build OCN -> object lookup for body headings +/ + OcdaObject[int] headings; + foreach (obj; doc.body) { + if (obj.is_a == "heading" && obj.ocn > 0) { + headings[obj.ocn] = obj; + } + } + + /+ ↓ find root heading (from head section, or first body heading) +/ + if (doc.head.length > 0 && doc.head[0].is_a == "heading") { + auto root = doc.head[0]; + lines ~= formatHeading(root, 0); + walkChildren(root, headings, lines, 1); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private void walkChildren(OcdaObject parent, ref OcdaObject[int] headings, ref string[] lines, int depth) { + foreach (child_ocn; parent.children) { + if (auto p = child_ocn in headings) { + lines ~= formatHeading(*p, depth); + walkChildren(*p, headings, lines, depth + 1); + } + } +} + +private string formatHeading(OcdaObject obj, int depth) { + string indent = replicate(" ", depth); + string level_tag = (obj.heading_level.length > 0) ? obj.heading_level : "?"; + string ocn_str = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + /+ ↓ strip internal markup from heading text for clean display +/ + string text = stripInternalMarkup(obj.text); + return indent ~ level_tag ~ "~" ~ ocn_str ~ " " ~ text; +} + +private string stripInternalMarkup(string text) { + /+ ↓ simple removal of common internal markers for display +/ + import std.regex : replaceAll, regex; + import std.array : replace; + string result = text; + result = result.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("┨", "").replace("┣", "") + .replace("┙", " ").replace("┘", "\n"); + return result; +} +#+END_SRC + +** dlang read verify + +#+HEADER: :tangle "../dlang/src/read/verify.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.verify; + +import std.algorithm : sort; +import std.array : array, join; +import std.conv : to; +import std.digest : toHexString; +import std.digest.sha : SHA256, digest; +import std.file : dirEntries, exists, isDir, SpanMode, getSize; +import std.path : baseName, buildPath, extension; +import std.stdio : writeln, File; +import std.string : toLower; +import ocda.model; +import read.images : gatherImageRecords; + +/+ ↓ verification: recompute what the .image records claim, against the + images as they stand in a directory (a pod's media/image). + + ssp-reader --verify --image-dir=<pod>/media/image doc.ssp + + Reports one line per distinct file: + + OK won_benkler_2_1.png + MISMATCH won_benkler_2_1.png sha256 recorded 5BA4... file 1C3D... + ABSENT won_benkler_2_1.png (recorded sha256 5BA4..., not in directory) + PRESENT ffa.png (recorded missing, but the file is there) + EXTRA unused.png (in directory, no record in this .ssp) + + Returns the number of failures. EXTRA is reported but is not a failure: + a pod carries the images of every language version of a document, while + one .ssp covers one language. ++/ +int verifyImages(OcdaDocument doc, string image_dir, string output_file) { + string[] lines; + int failures = 0; + int ok = 0; + int mismatch = 0; + int absent = 0; + int extra = 0; + + if (!(image_dir.exists) || !(image_dir.isDir)) { + writeln("ERROR: image directory not found: ", image_dir); + return 1; + } + auto gathered = gatherImageRecords(doc); + bool[string] recorded; + + foreach (e; gathered.entries) { + recorded[e.name] = true; + string pth = buildPath(image_dir, e.name); + if (!(pth.exists)) { + if (e.missing) { + /+ recorded missing, and missing here too: consistent +/ + ok += 1; + lines ~= "OK " ~ e.name ~ " (recorded missing, absent here)"; + } else { + absent += 1; + failures += 1; + lines ~= "ABSENT " ~ e.name + ~ " (recorded sha256 " ~ e.sha256 ~ ", not in directory)"; + } + continue; + } + if (e.missing) { + mismatch += 1; + failures += 1; + lines ~= "PRESENT " ~ e.name ~ " (recorded missing, but the file is there)"; + continue; + } + auto file_ = readFileStat(pth); + string[] differs; + if (file_.sha256 != e.sha256) { + differs ~= "sha256 recorded " ~ e.sha256 ~ " file " ~ file_.sha256; + } + if (file_.bytes != e.bytes) { + differs ~= "bytes recorded " ~ e.bytes.to!string + ~ " file " ~ file_.bytes.to!string; + } + /+ px is written only where the dimensions could be read, and read here + only for the formats below, so compare only when both are known +/ + if (e.width > 0 && e.height > 0 && file_.width > 0 && file_.height > 0 + && (file_.width != e.width || file_.height != e.height)) { + differs ~= "px recorded " ~ e.width.to!string ~ "x" ~ e.height.to!string + ~ " file " ~ file_.width.to!string ~ "x" ~ file_.height.to!string; + } + if (differs.length > 0) { + mismatch += 1; + failures += 1; + lines ~= "MISMATCH " ~ e.name ~ " " ~ differs.join("; "); + } else { + ok += 1; + lines ~= "OK " ~ e.name; + } + } + + /+ ↓ files in the directory that no record names +/ + string[] in_dir; + foreach (entry; dirEntries(image_dir, SpanMode.shallow)) { + if (!(entry.isFile)) continue; + string nm = entry.name.baseName; + switch (nm.extension.toLower) { + case ".png": case ".jpg": case ".jpeg": case ".gif": + in_dir ~= nm; + break; + default: break; + } + } + in_dir.sort; + foreach (nm; in_dir) { + if (nm !in recorded) { + extra += 1; + lines ~= "EXTRA " ~ nm ~ " (in directory, no record in this .ssp)"; + } + } + + string head = "Verify: " ~ gathered.entries.length.to!string ~ " distinct file" + ~ ((gathered.entries.length == 1) ? "" : "s") + ~ " in " ~ gathered.records.to!string ~ " record" + ~ ((gathered.records == 1) ? "" : "s") + ~ ", against " ~ image_dir; + lines = [head, ""] ~ lines; + lines ~= ""; + lines ~= "verify: " ~ ok.to!string ~ " ok, " + ~ mismatch.to!string ~ " mismatch, " + ~ absent.to!string ~ " absent, " + ~ extra.to!string ~ " extra" + ~ ((failures == 0) ? " PASS" : " FAIL"); + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } + return failures; +} + +private struct FileStat_ { + string sha256; + ulong bytes; + int width; + int height; +} + +private FileStat_ readFileStat(string pth) { + FileStat_ ret; + auto f = File(pth, "rb"); + ubyte[32] sha = digest!SHA256(f.byChunk(4096 * 1024)); + ret.sha256 = sha.toHexString.to!string; + ret.bytes = pth.getSize; + auto dim = readImageDimensions(pth); + ret.width = dim[0]; + ret.height = dim[1]; + return ret; +} + +/+ ↓ pixel dimensions from the file header, for the formats spine's markup + admits (png, gif, jpg). Header read only, no decoding, and no dependency + beyond phobos. Returns [0, 0] where the header cannot be read +/ +private int[2] readImageDimensions(string pth) { + int[2] ret = [0, 0]; + ubyte[] head; + try { + auto f = File(pth, "rb"); + head = new ubyte[](64 * 1024); + head = f.rawRead(head); + } catch (Exception ex) { + return ret; + } + if (head.length >= 24 + && head[0 .. 8] == cast(ubyte[])[0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a]) { + /+ png: IHDR width & height, big endian, at 16 and 20 +/ + ret[0] = beInt(head[16 .. 20]); + ret[1] = beInt(head[20 .. 24]); + return ret; + } + if (head.length >= 10 && (head[0 .. 3] == cast(ubyte[])['G', 'I', 'F'])) { + /+ gif: logical screen width & height, little endian, at 6 and 8 +/ + ret[0] = head[6] | (head[7] << 8); + ret[1] = head[8] | (head[9] << 8); + return ret; + } + if (head.length >= 4 && head[0] == 0xff && head[1] == 0xd8) { + /+ jpeg: walk the segments to a start of frame, height then width, + big endian, at offsets 5 and 7 of the segment +/ + size_t i = 2; + while (i + 9 < head.length) { + if (head[i] != 0xff) { i += 1; continue; } + ubyte marker = head[i + 1]; + if (marker == 0xd8 || marker == 0x01 + || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } + if (marker == 0xff) { i += 1; continue; } + size_t seg_len = (head[i + 2] << 8) | head[i + 3]; + bool is_sof = ((marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf)); + if (is_sof) { + ret[1] = (head[i + 5] << 8) | head[i + 6]; + ret[0] = (head[i + 7] << 8) | head[i + 8]; + return ret; + } + if (seg_len < 2) break; + i += 2 + seg_len; + } + } + return ret; +} + +private int beInt(const(ubyte)[] b) { + return cast(int)((b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3]); +} +#+END_SRC + +** dlang read validate + +#+HEADER: :tangle "../dlang/src/read/validate.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.validate; + +/+ ↓ check a .ssp against the format specification. + . + Written from specs/sisudoc-abstraction-format/ssp-format-specification.md + and from nothing else: the point of it is to be what the specification + says, executed, so that a file can be held against the document rather + than against an implementation. Each check names the section it comes + from, so a disagreement can be taken back to the text. ++/ + +import std.algorithm : canFind, startsWith; +import std.array : array, split; +import std.conv : to; +import std.file : readText; +import std.stdio : writeln, writefln; +import std.string : indexOf, strip; + +struct SspFinding { + size_t line; // 1-based + string section; // the specification section the rule is from + string detail; +} + +/+ ↓ specification section 6.3: the properties, in the order they are + emitted. A reader may rely on the order, so a validator can check it +/ +private immutable string[] property_order = [ + "identifier", "part", "section", "parent", "last_descendant", "ocn_range", + "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", + "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", + "table_header", "code_linenumbers", "stow_link", "segment", "anchor", + "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", + "segment_html_is", "segment_epub_is", "segment_lv4_is", + "heading_ancestors_text", "lev4_subtoc", "anchor_tag", +]; + +/+ ↓ section 6.1: the sixteen kinds +/ +private immutable string[] kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", + "blurb", "group", +]; + +/+ ↓ section 4: the nine sections, in the order they appear +/ +private immutable string[] section_order = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", + "bookindex", "blurb", "tail", +]; + +/+ ↓ section 4: the four preamble blocks, in the order they appear +/ +private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; + +/+ ↓ section 6.1: the heading levels +/ +private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; + +private int propertyRank(string name) { + foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } + return -1; +} + +SspFinding[] validateSspFile(string filename, string major_known) { + SspFinding[] found; + auto lines = readText(filename).split("\n"); + + bool seen_banner = false; + bool seen_source_line = false; + string block = ""; // the block currently open, "" when none + int preamble_at = -1; // how far through the preamble order + int section_at = -1; // how far through the section order + bool in_object = false; + int prev_rank = -1; + size_t doc_has_keys = 0; + bool in_text = false; // text lines come after properties + + foreach (i, line; lines) { + size_t no = i + 1; + + /+ ↓ section 2: the version, on the first line +/ + if (no == 1) { + enum banner = "% SiSU Document Abstraction v"; + if (!line.startsWith(banner)) { + found ~= SspFinding(no, "2", "first line is not the version banner"); + } else { + seen_banner = true; + string v = line[banner.length .. $].strip; + auto dot = v.indexOf('.'); + string major = (dot > 0) ? v[0 .. dot] : v; + if (major != major_known) { + found ~= SspFinding(no, "2.1", + "format major v" ~ major ~ ", this validator knows v" ~ major_known); + } + } + continue; + } + if (no == 2) { + if (!line.startsWith("% Source: ")) { + found ~= SspFinding(no, "4", "second line is not \"% Source: \""); + } else { seen_source_line = true; } + continue; + } + + if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } + + /+ ↓ section 3: a comment is the file header only +/ + if (line[0] == '%') { + found ~= SspFinding(no, "3", "a comment outside the two header lines"); + continue; + } + + /+ ↓ block open +/ + if (line[0] == '@') { + auto brace = line.indexOf(" {"); + if (brace < 1 || line[$ - 1] != '{') { + found ~= SspFinding(no, "3", "block open is not \"@name {\""); + continue; + } + string name = line[1 .. brace]; + if (preamble_order.canFind(name)) { + int at = 0; + foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } + if (at <= preamble_at) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); + } + preamble_at = at; + if (section_at >= 0) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); + } + } else if (section_order.canFind(name)) { + int at = 0; + foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } + if (at <= section_at) { + found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); + } + section_at = at; + } else { + found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); + } + block = name; + doc_has_keys = 0; + continue; + } + + if (line[0] == '}') { + if (line.length != 1) { + found ~= SspFinding(no, "3", "block close carries more than \"}\""); + } + /+ ↓ section 5.4: ten keys, always +/ + if (block == "doc_has" && doc_has_keys != 10) { + found ~= SspFinding(no, "5.4", + "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); + } + block = ""; + in_object = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 5: inside a preamble block, " key: value" +/ + if (preamble_order.canFind(block)) { + if (!line.startsWith(" ") || line.indexOf(": ") < 3) { + found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); + } else if (block == "doc_has") { + doc_has_keys++; + } + continue; + } + + /+ ↓ section 6.1: the object declaration line +/ + if (line[0] == '[') { + auto close = line.indexOf("] "); + if (close < 2) { + found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); + continue; + } + string ocn = line[1 .. close]; + foreach (c; ocn) { + if (c < '0' || c > '9') { + found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); + break; + } + } + auto rest = line[close + 2 .. $].split(" "); + if (rest.length == 0 || !kinds.canFind(rest[0])) { + found ~= SspFinding(no, "6.1", + "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); + } else if (rest[0] == "heading") { + if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { + found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); + } else if (!levels.canFind(rest[1][1 .. $])) { + found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); + } + } + in_object = true; + in_text = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 6.3: properties, known and in order +/ + if (line[0] == '.') { + if (!in_object) { + found ~= SspFinding(no, "6", "a property outside an object record"); + continue; + } + if (in_text) { + found ~= SspFinding(no, "6", "a property after the object text began"); + } + auto colon = line.indexOf(": "); + if (colon < 2) { + found ~= SspFinding(no, "3", "property is not \".name: value\""); + continue; + } + string name = line[1 .. colon]; + int rank = propertyRank(name); + if (rank < 0) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the " ~ property_order.length.to!string ~ " properties"); + } else { + /+ ↓ the four repeatable properties may recur at the same rank +/ + if (rank < prev_rank) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); + } + prev_rank = rank; + } + continue; + } + + /+ ↓ section 6.2: text lines +/ + if (line[0] == '|') { + if (!in_object) { + found ~= SspFinding(no, "6.2", "a text line outside an object record"); + } + if (line.length < 2 || line[1] != ' ') { + found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); + } + in_text = true; + continue; + } + + found ~= SspFinding(no, "3", "line of no shape the specification defines"); + } + + if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } + if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } + return found; +} + +void reportValidation(string filename, SspFinding[] found) { + if (found.length == 0) { + writeln("\nvalid: ", filename, + " conforms to the .ssp specification as this validator reads it"); + return; + } + writefln("\n%d finding%s against the specification:", + found.length, (found.length == 1) ? "" : "s"); + foreach (f; found) { + if (f.line > 0) { + writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); + } else { + writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); + } + } +} +#+END_SRC + +** dlang read report unknown + +#+HEADER: :tangle "../dlang/src/read/report_unknown.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read zip + +#+HEADER: :tangle "../dlang/src/read/zip.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read zstd + +#+HEADER: :tangle "../dlang/src/read/zstd.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read html + +#+HEADER: :tangle "../dlang/src/read/html.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read images + +#+HEADER: :tangle "../dlang/src/read/images.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +#+END_SRC + +** dlang read llm text + +#+HEADER: :tangle "../dlang/src/read/llm_text.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_text; + +import std.array : replace, split, join; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; + +/+ ↓ plain structured text with heading breadcrumbs for direct LLM consumption + format: + ## Document Title > Chapter > Section > [OCN] + + Clean paragraph text with inline markup stripped. ++/ +void outputLlmBreadcrumbs(OcdaDocument doc, string output_file) { + string[] lines; + string doc_title = doc.meta("title.main"); + + /+ ↓ document header +/ + lines ~= "# " ~ doc_title; + if (doc.meta("title.subtitle").length > 0) + lines ~= "# " ~ doc.meta("title.subtitle"); + lines ~= "# " ~ doc.meta("creator.author"); + lines ~= ""; + + /+ ↓ build heading text lookup from ancestors +/ + string[int] heading_text; + foreach (obj; doc.head) { + if (obj.is_a == "heading" && obj.ocn > 0) + heading_text[obj.ocn] = stripMarkup(obj.text); + } + foreach (obj; doc.body) { + if (obj.is_a == "heading") + heading_text[obj.ocn] = stripMarkup(obj.text); + } + + /+ ↓ body objects +/ + string current_breadcrumb = ""; + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_breadcrumb = buildBreadcrumb(doc_title, obj.heading_ancestors_text, obj); + lines ~= ""; + lines ~= "## " ~ current_breadcrumb; + lines ~= ""; + continue; + } + + /+ ↓ markup commentary is not content +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + string ocn_ref = (obj.ocn > 0 && !obj.ocn_off) + ? " [" ~ obj.ocn.to!string ~ "]" + : ""; + + switch (obj.is_a) { + case "para": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "code": + lines ~= "```"; + lines ~= obj.text; /+ preserve raw text for code +/ + lines ~= "```"; + lines ~= ""; + break; + case "table": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "verse": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + case "quote": + lines ~= "> " ~ text ~ ocn_ref; + lines ~= ""; + break; + case "group": + case "block": + lines ~= text ~ ocn_ref; + lines ~= ""; + break; + default: + if (text.length > 0) { + lines ~= text ~ ocn_ref; + lines ~= ""; + } + break; + } + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string buildBreadcrumb(string doc_title, string ancestors_text, OcdaObject obj) { + /+ ↓ ancestors_text is pipe-separated: "Title|Part||Chapter|||" +/ + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + /+ ↓ truncate very long heading text for breadcrumb +/ + if (clean.length > 60) + clean = clean[0 .. 57] ~ "..."; + parts ~= clean; + } + } + } + /+ ↓ add the current heading itself +/ + string heading_text = stripMarkup(obj.text).strip; + if (heading_text.length > 60) + heading_text = heading_text[0 .. 57] ~ "..."; + + string ocn_ref = (obj.ocn > 0) ? " [" ~ obj.ocn.to!string ~ "]" : ""; + + if (parts.length > 0) + return doc_title ~ " > " ~ parts.join(" > ") ~ " > " ~ heading_text ~ ocn_ref; + else + return doc_title ~ " > " ~ heading_text ~ ocn_ref; +} + +/+ ↓ strip all internal Unicode markup to produce clean plain text +/ +string stripMarkup(string text) { + if (text.length == 0) return ""; + string r = text; + + /+ ↓ links: ┥link text┝┤url├ -> link text +/ + while (true) { + auto lo = r.indexOf("┥"); + if (lo < 0) break; + auto lm = indexOf(r[lo .. $], "┝┤"); + if (lm < 0) break; + lm += lo; + auto lc = indexOf(r[lm .. $], "├"); + if (lc < 0) break; + lc += lm; + string link_text = r[lo + "┥".length .. lm]; + r = r[0 .. lo] ~ link_text ~ r[lc + "├".length .. $]; + } + + /+ ↓ footnotes: 【N content】 -> (remove entirely for clean text) +/ + while (true) { + auto fo = r.indexOf("【"); + if (fo < 0) break; + auto fc = indexOf(r[fo .. $], "】"); + if (fc < 0) break; + fc += fo; + r = r[0 .. fo] ~ r[fc + "】".length .. $]; + } + + /+ ↓ font face markers +/ + r = r.replace("⑆*", "").replace("⑆!", "").replace("⑆/", "") + .replace("⑆_", "").replace("⑆^", "").replace("⑆,", "") + .replace("⑆■", "") + .replace("┨", "").replace("┣*", "").replace("┣!", "") + .replace("┣/", "").replace("┣_", "").replace("┣^", "") + .replace("┣,", "").replace("┣", ""); + + /+ ↓ line breaks and special chars +/ + r = r.replace("┘", "\n").replace("┙", " ").replace("┚", "\n"); + r = r.replace("░", " "); + r = r.replace("● ", "- "); + r = r.replace("┼", "").replace("┿", "").replace("╂", ""); + + /+ ↓ table cell separators +/ + r = r.replace("┊", " | "); + + /+ ↓ clean up multiple spaces +/ + while (r.indexOf(" ") >= 0) + r = r.replace(" ", " "); + + return r.strip; +} +#+END_SRC + +** dlang read llm jsonl + +#+HEADER: :tangle "../dlang/src/read/llm_jsonl.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module read.llm_jsonl; + +import std.array : split, join, replace; +import std.conv : to; +import std.stdio : writeln, File; +import std.string : strip, indexOf, stripRight; +import ocda.model; +import read.llm_text : stripMarkup; + +/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines + one JSON object per line, each containing: + doc, ocn, path (breadcrumb), type, text (clean), + images (only where the object embeds one, so that a chunk whose + text reads around a figure carries the figure's identity), + index_terms (from bookindex entries associated with this object) ++/ +void outputLlmJsonl(OcdaDocument doc, string output_file) { + string[] lines; + string doc_id = doc.meta("title.main"); + string doc_author = doc.meta("creator.author"); + + /+ ↓ build bookindex -> OCN mapping for index_terms + bookindex objects in the .ssp have parent_ocn linking to body objects + but the index terms are in the text of bookindex objects, not + directly attached to body objects. We use a simpler approach: + headings carry heading_ancestors_text which gives context. +/ + + /+ ↓ track current heading path +/ + string current_path = ""; + + foreach (obj; doc.body) { + if (obj.is_a == "heading") { + current_path = buildPath(doc_id, obj.heading_ancestors_text, obj); + /+ ↓ emit heading as a chunk too +/ + string heading_text = stripMarkup(obj.text); + if (heading_text.strip.length > 0) { + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images); + } + continue; + } + + /+ ↓ markup commentary is not content, and least of all is it a chunk + to embed: it would be indexed as though the author had written it + into the document + +/ + if (!obj.renders) { continue; } + + string text = stripMarkup(obj.text); + if (text.strip.length == 0) continue; + + lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images); + } + + /+ ↓ write output +/ + if (output_file.length > 0) { + auto f = File(output_file, "w"); + foreach (line; lines) f.writeln(line); + } else { + foreach (line; lines) writeln(line); + } +} + +private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) { + return `{"doc":` ~ jsonStr(doc_id) + ~ `,"author":` ~ jsonStr(doc_author) + ~ `,"ocn":` ~ ocn.to!string + ~ `,"path":` ~ jsonStr(path) + ~ `,"type":` ~ jsonStr(type) + ~ `,"text":` ~ jsonStr(text) + ~ jsonImages(images) + ~ `}`; +} + +private string jsonImages(OcdaImage[] images) { + /+ ↓ omitted entirely where the object embeds no image +/ + if (images.length == 0) return ""; + string[] items; + foreach (img; images) { + string item = `{"name":` ~ jsonStr(img.name); + if (img.missing) { + item ~= `,"missing":true`; + } else { + item ~= `,"sha256":` ~ jsonStr(img.sha256) + ~ `,"bytes":` ~ img.bytes.to!string; + if (img.width > 0 && img.height > 0) { + item ~= `,"width":` ~ img.width.to!string + ~ `,"height":` ~ img.height.to!string; + } + } + items ~= item ~ `}`; + } + return `,"images":[` ~ items.join(",") ~ `]`; +} + +private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) { + string[] parts; + if (ancestors_text.length > 0) { + foreach (part; ancestors_text.split("|")) { + string clean = stripMarkup(part).strip; + if (clean.length > 0 && clean != doc_title) { + parts ~= clean; + } + } + } + string heading_text = stripMarkup(obj.text).strip; + if (parts.length > 0) + return parts.join(" > ") ~ " > " ~ heading_text; + else + return heading_text; +} + +private string jsonStr(string s) { + /+ ↓ JSON string escaping +/ + string r = s; + r = r.replace(`\`, `\\`); + r = r.replace(`"`, `\"`); + r = r.replace("\n", `\n`); + r = r.replace("\r", `\r`); + r = r.replace("\t", `\t`); + return `"` ~ r ~ `"`; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ + diff --git a/org/gleam-ocda-load.org b/org/gleam-ocda-load.org index dbf5611..f3f111f 100644 --- a/org/gleam-ocda-load.org +++ b/org/gleam-ocda-load.org @@ -288,7 +288,7 @@ pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { //// apart by its first character, so the loader is a fold over the lines //// carrying the state the shapes need. //// -//// Written against `ssp-format-specification.md`. Where it and the D +//// Written against `ssp-format-specification.org`. Where it and the D //// library differ the D library is the bug, or this is: they are meant to //// give the same model, and `ocda-dump` says whether they do. @@ -792,7 +792,7 @@ fn numbers(value: String) -> List(Int) { //// then a function from the model to bytes and cannot tell which //// serialisation it was given, exactly as spine's own writers cannot. //// -//// Written against `ocda-db-format-specification.md`, whose section 4 +//// Written against `ocda-db-format-specification.org`, whose section 4 //// gives the column by column correspondence this module implements. import gleam/dict diff --git a/org/gleam-ocda_lib.org b/org/gleam-ocda_lib.org index a359818..e19715b 100644 --- a/org/gleam-ocda_lib.org +++ b/org/gleam-ocda_lib.org @@ -141,8 +141,8 @@ one and the other must change with it. ## Format -Specified in **sisudoc-abstraction-format**: `ocda-model.md` for the model, -`ssp-format-specification.md` and `ocda-db-format-specification.md` for the +Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, +`ssp-format-specification.org` and `ocda-db-format-specification.org` for the two serialisations. At format **2.x**. #+END_SRC |
