From 64571ec14438e85d740ede5810a6f3a5cca88fe3 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 25 Sep 2026 21:02:39 -0400 Subject: ocda-dump, and the four disagreements it found The agreement check compared ten fields of each object. That was enough to find the blank-first-line defect and not enough for anything subtler, so this adds ocda-dump: the whole loaded model written out canonically, every field, sorted dictionaries, sections in format order, defaults omitted. It is an oracle, not an output format. Two implementations of this model can now be compared with diff. Run against the reference collection it found four disagreements between the two loaders, none of which the ten-field check could see. 1. .identifier and .anchor. The .ssp emits a property only when it carries information, so the identifier is absent when it is simply the ocn and the anchor is absent when it is the segment name; the database stores both unconditionally. Over one document that is 1,314 identifiers and 69 anchors. The loader now derives them, which is what the model means: an object's identifier is its ocn unless it says otherwise. 2. make and doc_has. The database has one metadata table where the text form has three blocks, and tells them apart by a key prefix. The loader now unprefixes them, so a writer asks doc.make["footer"] whichever serialisation it was handed and never doc.metadata["make.footer"]. 3. The JSON columns were read by a character scan that stripped brackets and quotes, which is wrong the moment a value holds an escape: JSONValue.toString escapes "/" as "\/", so every heading containing a slash came back with a backslash in it. All 36 document-languages differed on heading_ancestors_text. Now parsed as JSON. 4. translation.ocn_aligned is in the database and not in the .ssp, legitimately: it says whether the languages share one numbering, and a .ssp is one language and cannot know. It is a fact about the file rather than the document, so the loader leaves it out rather than letting a writer tell which serialisation it was given. Metadata values are also stripped now, as the text loader strips them: one topic_register value carried surrounding spaces in the database that the text form cannot represent. After all four: 36 of 36 document-languages give byte identical dumps from their .ssp and from their .ocda.db. (assisted by Claude-Code) --- dlang/src/ocda/dump.d | 160 ++++++++++++++++++++++++++++++++++++++++++++++ dlang/src/ocda/load_db.d | 115 ++++++++++++++++++++++----------- dlang/src/ocda/load_ssp.d | 25 ++++++++ 3 files changed, 261 insertions(+), 39 deletions(-) create mode 100644 dlang/src/ocda/dump.d (limited to 'dlang/src/ocda') diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d new file mode 100644 index 0000000..e308db2 --- /dev/null +++ b/dlang/src/ocda/dump.d @@ -0,0 +1,160 @@ +module ocda.dump; + +/+ ↓ the model, written out canonically. + . + Not an output format: an *oracle*. Two libraries in two languages + implement the same model from the same specification, and the cheap + way to ask whether they agree is to have each write the whole loaded + document as text and diff the two. That is only useful if the text is + fully determined by the model, so: + . + - every line is " ", the value running to end of line, + and only one free-form value per line; + - a field at its default is *omitted*, so the output says what the + document has rather than what the struct declares; + - dictionaries are emitted in sorted key order, because neither + language's hash map has an order worth relying on; + - sections come in the fixed order of the format, not + alphabetically. + . + Anything that reads this to reconstruct a document has misunderstood + it. Load the .ssp or the .ocda.db. ++/ + +import std.algorithm : sort, splitter; +import std.array : appender, array, join; +import std.conv : to; +import ocda.model; + +/+ ↓ the nine sections, in format order +/ +static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + +string dumpDocument(ref OcdaDocument doc) { + auto out_ = appender!string; + + void kv(string key, string value) { + if (value.length == 0) { return; } + out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; + } + void kn(string key, long value) { + if (value == 0) { return; } + kv(key, value.to!string); + } + void kb(string key, bool value) { + if (!value) { return; } + kv(key, "1"); + } + void dict(string prefix, string[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key]; out_ ~= "\n"; + } + } + + kv("version", doc.format_version); + kv("source_filename", doc.source_filename); + dict("source", doc.source); + dict("meta", doc.metadata); + dict("make", doc.make); + dict("doc_has", doc.doc_has); + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null || (*sec).length == 0) { continue; } + kv("section", name ~ " " ~ (*sec).length.to!string); + } + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null) { continue; } + foreach (i, obj; *sec) { + out_ ~= "object "; out_ ~= name; out_ ~= " "; + out_ ~= i.to!string; out_ ~= "\n"; + kn("ocn", obj.ocn); + kv("is_a", obj.is_a); + kv("own_section", obj.own_section); + kv("level", obj.heading_level); + kv("identifier", obj.identifier); + kv("part", obj.part); + kv("is_of_type", obj.is_of_type); + kv("segment_html_is", obj.segment_html_is); + kn("parent", obj.parent_ocn); + kn("last_descendant", obj.last_descendant_ocn); + kv("children", ints(obj.children)); + kv("ancestors", ints(obj.ancestors)); + kb("dummy", obj.dummy_heading); + kb("ocn_off", obj.ocn_off); + kn("indent_base", obj.indent_base); + kn("indent_hang", obj.indent_hang); + kb("bullet", obj.bullet); + kv("lang", obj.lang); + kb("has_links", obj.has_links); + kb("has_notes_reg", obj.has_notes_reg); + kb("has_notes_star", obj.has_notes_star); + kb("has_images", obj.has_images); + kv("segment", obj.segment); + kv("segment_prev", obj.segment_prev); + kv("segment_next", obj.segment_next); + kv("anchor", obj.anchor); + kn("table_cols", obj.table_cols); + kv("table_widths", obj.table_widths); + kv("table_aligns", obj.table_aligns); + kb("table_header", obj.table_header); + kv("code_syntax", obj.code_syntax); + kb("code_linenumbers", obj.code_linenumbers); + kv("heading_ancestors_text", obj.heading_ancestors_text); + kv("sha256", obj.sha256); + foreach (url; obj.stow_link) { kv("stow_link", url); } + foreach (img; obj.images) { + /+ ↓ the name last, because a filename may hold a space where + none of the other fields can +/ + kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " + ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " + ~ (img.missing ? "1" : "0") ~ " " ~ img.name); + foreach (key; img.fields.keys.dup.sort) { + kv("image_field", key ~ " " ~ img.fields[key]); + } + } + /+ ↓ text, when there is any. An object whose only text line is + blank holds "" here, indistinguishable from no text at all, + because the model joins the lines into one string. Both + libraries lose the same distinction in the same place, which is + what makes them comparable; it is recorded rather than hidden. + +/ + if (obj.text.length > 0) { + auto lines = obj.text.splitter("\n").array; + kv("text", lines.length.to!string); + foreach (line; lines) { + out_ ~= "| "; out_ ~= line; out_ ~= "\n"; + } + } + } + } + return out_.data; +} + +/+ ↓ what the reader met and did not understand, sorted. Separate from the + document dump: it describes the reader, not the document, and the + .ocda.db loader has nothing to report where the .ssp loader does. ++/ +string dumpUnknown(ref OcdaDocument doc) { + auto out_ = appender!string; + void counts(string prefix, size_t[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key].to!string; out_ ~= "\n"; + } + } + counts("property", doc.unknown.property); + counts("block", doc.unknown.block); + counts("line", doc.unknown.line); + return out_.data; +} + +private string ints(int[] xs) { + if (xs.length == 0) { return ""; } + string[] parts; + foreach (x; xs) { parts ~= x.to!string; } + return parts.join(","); +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index ffc28a8..f602da9 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -12,8 +12,10 @@ module ocda.load_db; column correspondence this module implements. +/ -import std.array : split; +import std.array : join, split; +import std.json : JSONType, parseJSON; import std.conv : to; +import std.algorithm : startsWith; import std.string : strip; import ocda.model; import ocda.sqlite; @@ -103,8 +105,31 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) +/ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY key")) { - if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; } - doc.metadata[r[0]] = r[1]; + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } } /+ ↓ the objects, in document order within each section +/ @@ -215,51 +240,63 @@ private string levelName(int index) { } } -/+ ↓ the fixed width arrays are JSON in the database and space separated in - the text form (specification section 3.3) +/ +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + private int[] jsonInts(string v) { int[] out_; - foreach (w; jsonWords(v).split(" ")) { - if (w.length == 0) { continue; } - try { out_ ~= w.to!int; } catch (Exception) {} + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} } return out_; } +/+ ↓ space separated in the text form: table_widths, table_aligns +/ private string jsonWords(string v) { - string out_; - bool in_string = false; - foreach (c; v) { - switch (c) { - case '[': case ']': case ' ': break; - case '"': in_string = !in_string; break; - case ',': out_ ~= " "; break; - default: out_ ~= c; break; - } - } - return out_.strip; + return jsonStrings(v).join(" "); } /+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text - form and a JSON array here +/ + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ private string jsonJoined(string v) { - if (v.length == 0) { return ""; } - string out_; - bool in_string = false; - bool first = true; - string cur; - foreach (c; v) { - if (c == '"') { - in_string = !in_string; - if (!in_string) { - if (!first) { out_ ~= "|"; } - out_ ~= cur; - cur = ""; - first = false; - } - continue; - } - if (in_string) { cur ~= c; } - } - return out_; + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); } diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d index 83e3572..37872ea 100644 --- a/dlang/src/ocda/load_ssp.d +++ b/dlang/src/ocda/load_ssp.d @@ -278,7 +278,32 @@ private OcdaImage parseImageRecord(string value) { return img; } +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); auto sec = doc.sectionByName(section); if (sec !is null) { *sec ~= obj; -- cgit v1.2.3