diff options
| -rw-r--r-- | dlang/flake.nix | 66 | ||||
| -rw-r--r-- | dlang/src/ocda/dump.d | 160 | ||||
| -rw-r--r-- | dlang/src/ocda/load_db.d | 115 | ||||
| -rw-r--r-- | dlang/src/ocda/load_ssp.d | 25 | ||||
| -rw-r--r-- | dlang/test/dump.d | 35 | ||||
| -rw-r--r-- | gleam/gleam.toml | 13 | ||||
| -rw-r--r-- | gleam/src/ocda/model.gleam | 270 |
7 files changed, 619 insertions, 65 deletions
diff --git a/dlang/flake.nix b/dlang/flake.nix index 954cad9..26dbe2d 100644 --- a/dlang/flake.nix +++ b/dlang/flake.nix @@ -9,37 +9,51 @@ pkgs = import nixpkgs { inherit system; }; /* A library, so there is nothing to install as a binary. What is - built here is the agreement check, which is the library's own - reason to exist put to the question: load a document's .ssp and - its .ocda.db and compare the two models. + built here are the two programs that put the library to the + question: the agreement check, which loads a document's .ssp and + its .ocda.db and compares the two models, and the dump, which + writes a loaded model out canonically so that an implementation in + another language can be diffed against this one. */ - mkAgree = { compiler, compilerBin }: pkgs.stdenv.mkDerivation { - pname = "ocda-lib-agree"; - version = "0.1.0"; - src = self; - nativeBuildInputs = [ compiler ]; - buildInputs = [ pkgs.sqlite ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - ${compilerBin} -I=src -of=agree src/ocda/*.d test/agree.d -L-lsqlite3 - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/bin - install -m755 agree $out/bin/ocda-agree - runHook postInstall - ''; - meta = { - description = "checks that the .ssp and .ocda.db loaders give one model"; - mainProgram = "ocda-agree"; + mkTool = { tool, binary, what, compiler, compilerBin }: + pkgs.stdenv.mkDerivation { + pname = "ocda-lib-${tool}"; + version = "0.1.0"; + src = self; + nativeBuildInputs = [ compiler ]; + buildInputs = [ pkgs.sqlite ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ${tool} $out/bin/${binary} + runHook postInstall + ''; + meta = { + description = what; + mainProgram = binary; + }; }; - }; + + ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; in { packages.${system} = { default = self.packages.${system}.ocda-agree; - ocda-agree = mkAgree { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; + + ocda-agree = mkTool (ldc // { + tool = "agree"; binary = "ocda-agree"; + what = "checks that the .ssp and .ocda.db loaders give one model"; + }); + + ocda-dump = mkTool (ldc // { + tool = "dump"; binary = "ocda-dump"; + what = "writes a loaded abstraction out canonically, as an oracle"; + }); }; devShells.${system}.default = pkgs.mkShell { diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d new file mode 100644 index 0000000..e308db2 --- /dev/null +++ b/dlang/src/ocda/dump.d @@ -0,0 +1,160 @@ +module ocda.dump; + +/+ ↓ the model, written out canonically. + . + Not an output format: an *oracle*. Two libraries in two languages + implement the same model from the same specification, and the cheap + way to ask whether they agree is to have each write the whole loaded + document as text and diff the two. That is only useful if the text is + fully determined by the model, so: + . + - every line is "<key> <value>", the value running to end of line, + and only one free-form value per line; + - a field at its default is *omitted*, so the output says what the + document has rather than what the struct declares; + - dictionaries are emitted in sorted key order, because neither + language's hash map has an order worth relying on; + - sections come in the fixed order of the format, not + alphabetically. + . + Anything that reads this to reconstruct a document has misunderstood + it. Load the .ssp or the .ocda.db. ++/ + +import std.algorithm : sort, splitter; +import std.array : appender, array, join; +import std.conv : to; +import ocda.model; + +/+ ↓ the nine sections, in format order +/ +static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + +string dumpDocument(ref OcdaDocument doc) { + auto out_ = appender!string; + + void kv(string key, string value) { + if (value.length == 0) { return; } + out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; + } + void kn(string key, long value) { + if (value == 0) { return; } + kv(key, value.to!string); + } + void kb(string key, bool value) { + if (!value) { return; } + kv(key, "1"); + } + void dict(string prefix, string[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key]; out_ ~= "\n"; + } + } + + kv("version", doc.format_version); + kv("source_filename", doc.source_filename); + dict("source", doc.source); + dict("meta", doc.metadata); + dict("make", doc.make); + dict("doc_has", doc.doc_has); + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null || (*sec).length == 0) { continue; } + kv("section", name ~ " " ~ (*sec).length.to!string); + } + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null) { continue; } + foreach (i, obj; *sec) { + out_ ~= "object "; out_ ~= name; out_ ~= " "; + out_ ~= i.to!string; out_ ~= "\n"; + kn("ocn", obj.ocn); + kv("is_a", obj.is_a); + kv("own_section", obj.own_section); + kv("level", obj.heading_level); + kv("identifier", obj.identifier); + kv("part", obj.part); + kv("is_of_type", obj.is_of_type); + kv("segment_html_is", obj.segment_html_is); + kn("parent", obj.parent_ocn); + kn("last_descendant", obj.last_descendant_ocn); + kv("children", ints(obj.children)); + kv("ancestors", ints(obj.ancestors)); + kb("dummy", obj.dummy_heading); + kb("ocn_off", obj.ocn_off); + kn("indent_base", obj.indent_base); + kn("indent_hang", obj.indent_hang); + kb("bullet", obj.bullet); + kv("lang", obj.lang); + kb("has_links", obj.has_links); + kb("has_notes_reg", obj.has_notes_reg); + kb("has_notes_star", obj.has_notes_star); + kb("has_images", obj.has_images); + kv("segment", obj.segment); + kv("segment_prev", obj.segment_prev); + kv("segment_next", obj.segment_next); + kv("anchor", obj.anchor); + kn("table_cols", obj.table_cols); + kv("table_widths", obj.table_widths); + kv("table_aligns", obj.table_aligns); + kb("table_header", obj.table_header); + kv("code_syntax", obj.code_syntax); + kb("code_linenumbers", obj.code_linenumbers); + kv("heading_ancestors_text", obj.heading_ancestors_text); + kv("sha256", obj.sha256); + foreach (url; obj.stow_link) { kv("stow_link", url); } + foreach (img; obj.images) { + /+ ↓ the name last, because a filename may hold a space where + none of the other fields can +/ + kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " + ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " + ~ (img.missing ? "1" : "0") ~ " " ~ img.name); + foreach (key; img.fields.keys.dup.sort) { + kv("image_field", key ~ " " ~ img.fields[key]); + } + } + /+ ↓ text, when there is any. An object whose only text line is + blank holds "" here, indistinguishable from no text at all, + because the model joins the lines into one string. Both + libraries lose the same distinction in the same place, which is + what makes them comparable; it is recorded rather than hidden. + +/ + if (obj.text.length > 0) { + auto lines = obj.text.splitter("\n").array; + kv("text", lines.length.to!string); + foreach (line; lines) { + out_ ~= "| "; out_ ~= line; out_ ~= "\n"; + } + } + } + } + return out_.data; +} + +/+ ↓ what the reader met and did not understand, sorted. Separate from the + document dump: it describes the reader, not the document, and the + .ocda.db loader has nothing to report where the .ssp loader does. ++/ +string dumpUnknown(ref OcdaDocument doc) { + auto out_ = appender!string; + void counts(string prefix, size_t[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key].to!string; out_ ~= "\n"; + } + } + counts("property", doc.unknown.property); + counts("block", doc.unknown.block); + counts("line", doc.unknown.line); + return out_.data; +} + +private string ints(int[] xs) { + if (xs.length == 0) { return ""; } + string[] parts; + foreach (x; xs) { parts ~= x.to!string; } + return parts.join(","); +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index ffc28a8..f602da9 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -12,8 +12,10 @@ module ocda.load_db; column correspondence this module implements. +/ -import std.array : split; +import std.array : join, split; +import std.json : JSONType, parseJSON; import std.conv : to; +import std.algorithm : startsWith; import std.string : strip; import ocda.model; import ocda.sqlite; @@ -103,8 +105,31 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) +/ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY key")) { - if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; } - doc.metadata[r[0]] = r[1]; + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } } /+ ↓ the objects, in document order within each section +/ @@ -215,51 +240,63 @@ private string levelName(int index) { } } -/+ ↓ the fixed width arrays are JSON in the database and space separated in - the text form (specification section 3.3) +/ +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + private int[] jsonInts(string v) { int[] out_; - foreach (w; jsonWords(v).split(" ")) { - if (w.length == 0) { continue; } - try { out_ ~= w.to!int; } catch (Exception) {} + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} } return out_; } +/+ ↓ space separated in the text form: table_widths, table_aligns +/ private string jsonWords(string v) { - string out_; - bool in_string = false; - foreach (c; v) { - switch (c) { - case '[': case ']': case ' ': break; - case '"': in_string = !in_string; break; - case ',': out_ ~= " "; break; - default: out_ ~= c; break; - } - } - return out_.strip; + return jsonStrings(v).join(" "); } /+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text - form and a JSON array here +/ + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ private string jsonJoined(string v) { - if (v.length == 0) { return ""; } - string out_; - bool in_string = false; - bool first = true; - string cur; - foreach (c; v) { - if (c == '"') { - in_string = !in_string; - if (!in_string) { - if (!first) { out_ ~= "|"; } - out_ ~= cur; - cur = ""; - first = false; - } - continue; - } - if (in_string) { cur ~= c; } - } - return out_; + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); } diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d index 83e3572..37872ea 100644 --- a/dlang/src/ocda/load_ssp.d +++ b/dlang/src/ocda/load_ssp.d @@ -278,7 +278,32 @@ private OcdaImage parseImageRecord(string value) { return img; } +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); auto sec = doc.sectionByName(section); if (sec !is null) { *sec ~= obj; diff --git a/dlang/test/dump.d b/dlang/test/dump.d new file mode 100644 index 0000000..17aa9bb --- /dev/null +++ b/dlang/test/dump.d @@ -0,0 +1,35 @@ +/++ + Write a loaded abstraction out canonically. + + The oracle for everything that claims to implement this model: load a + .ssp or a .ocda.db and print the model, deterministically, so that two + implementations can be compared with diff. + + Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang] ++/ +import std.stdio; +import ocda.model; +import ocda.load; +import ocda.dump; + +int main(string[] args) { + bool unknown = false; + string[] rest; + foreach (a; args[1 .. $]) { + if (a == "--unknown") { unknown = true; } else { rest ~= a; } + } + if (rest.length < 1) { + stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]"); + return 2; + } + string lang = (rest.length > 1) ? rest[1] : ""; + OcdaDocument doc; + try { + doc = load(rest[0], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); + return 0; +} diff --git a/gleam/gleam.toml b/gleam/gleam.toml new file mode 100644 index 0000000..d1a7a57 --- /dev/null +++ b/gleam/gleam.toml @@ -0,0 +1,13 @@ +name = "ocda_lib" +version = "0.1.0" +description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" +licences = ["AGPL-3.0-or-later"] + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +argv = ">= 1.1.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +sqlight = ">= 1.0.0 and < 2.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/gleam/src/ocda/model.gleam b/gleam/src/ocda/model.gleam new file mode 100644 index 0000000..20d7fba --- /dev/null +++ b/gleam/src/ocda/model.gleam @@ -0,0 +1,270 @@ +//// The object-centric document abstraction, as Gleam types. +//// +//// A deliberate mirror of the D library's `ocda.model`, field for field. +//// The two are meant to agree, and the cheapest way to make that testable +//// is for them to hold the same things under the same names: the dump of a +//// loaded document is then identical by construction rather than by +//// negotiation. +//// +//// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for +//// the model, the two format specifications for what each serialisation +//// says about it. + +import gleam/dict.{type Dict} +import gleam/list +import gleam/string + +/// The format major version this library knows. +/// +/// The `.ssp` and the `.ocda.db` carry one version between them, declared +/// in band. The minor part rises when a property is added and a reader of +/// the newer format still reads the older, so only the major is checked. +pub const format_major_read = "2" + +/// The nine sections, in the order the format gives them, which is not the +/// order a sorted key list or a `group by` would give. +pub const section_names = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", + "blurb", "tail", +] + +/// The four blocks that are document preamble rather than sections of +/// objects. +pub const preamble_blocks = ["source", "meta", "make", "doc_has"] + +/// A repeatable `.image` record: the file as the abstraction found it. +pub type Image { + Image( + /// filename as the object refers to it + name: String, + /// hex, uppercase + sha256: String, + bytes: Int, + /// the file's own pixel dimensions, not any display width + width: Int, + height: Int, + /// absent or unreadable, so no digest and no size + missing: Bool, + /// any further key:value fields, kept rather than dropped + fields: Dict(String, String), + ) +} + +pub fn new_image() -> Image { + Image( + name: "", + sha256: "", + bytes: 0, + width: 0, + height: 0, + missing: False, + fields: dict.new(), + ) +} + +/// One object of the abstraction. +/// +/// `text` is one string with the lines joined by newline, as in the D +/// library and for the same reason: an object whose only text line is +/// blank is then indistinguishable from an object with no text, in both +/// libraries, in the same place. A difference that exists in one language +/// and not the other would be worse than a loss both share. +pub type Object { + Object( + ocn: Int, + /// one of the sixteen kinds + is_a: String, + /// the section this object was found in + section: String, + /// headings: A B C D 1 2 3 4 + heading_level: String, + identifier: String, + part: String, + /// how the text behaves: para, block, comment + is_of_type: String, + /// the object's own section, when it differs from the block it sits in + own_section: String, + segment_html_is: String, + table_aligns: String, + stow_link: List(String), + parent_ocn: Int, + last_descendant_ocn: Int, + children: List(Int), + ancestors: List(Int), + dummy_heading: Bool, + ocn_off: Bool, + indent_base: Int, + indent_hang: Int, + bullet: Bool, + lang: String, + has_links: Bool, + has_notes_reg: Bool, + has_notes_star: Bool, + has_images: Bool, + segment: String, + segment_prev: String, + segment_next: String, + anchor: String, + table_cols: Int, + table_widths: String, + table_header: Bool, + code_syntax: String, + code_linenumbers: Bool, + heading_ancestors_text: String, + sha256: String, + images: List(Image), + text: String, + ) +} + +pub fn new_object() -> Object { + Object( + ocn: 0, + is_a: "", + section: "", + heading_level: "", + identifier: "", + part: "", + is_of_type: "", + own_section: "", + segment_html_is: "", + table_aligns: "", + stow_link: [], + parent_ocn: 0, + last_descendant_ocn: 0, + children: [], + ancestors: [], + dummy_heading: False, + ocn_off: False, + indent_base: 0, + indent_hang: 0, + bullet: False, + lang: "", + has_links: False, + has_notes_reg: False, + has_notes_star: False, + has_images: False, + segment: "", + segment_prev: "", + segment_next: "", + anchor: "", + table_cols: 0, + table_widths: "", + table_header: False, + code_syntax: "", + code_linenumbers: False, + heading_ancestors_text: "", + sha256: "", + images: [], + text: "", + ) +} + +/// Whether this object is rendered at all. +/// +/// `is_of_type` "comment" is markup commentary: carried by the abstraction +/// so that nothing is lost, and not put into output. An output that +/// renders it leaks the author's notes into the document, which is what +/// spine itself does not do. +pub fn renders(object: Object) -> Bool { + object.is_of_type != "comment" +} + +/// The section this object belongs to: the block it sits in, unless it +/// says otherwise. +pub fn section_of(object: Object) -> String { + case object.own_section { + "" -> object.section + own -> own + } +} + +/// What the loader met and did not understand. +/// +/// A loader that silently drops what it does not know cannot say whether +/// it understood the document, and so cannot be used to check a +/// specification against an implementation: silence and understanding look +/// alike. +pub type Unknown { + Unknown( + /// `.name` -> times seen + property: Dict(String, Int), + /// `@name` -> times seen + block: Dict(String, Int), + /// first character -> times seen, for lines of no known shape + line: Dict(String, Int), + ) +} + +pub fn new_unknown() -> Unknown { + Unknown(property: dict.new(), block: dict.new(), line: dict.new()) +} + +pub fn any_unknown(unknown: Unknown) -> Bool { + dict.size(unknown.property) > 0 + || dict.size(unknown.block) > 0 + || dict.size(unknown.line) > 0 +} + +pub type Document { + Document( + /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when + /// absent + format_version: String, + source_filename: String, + /// language, languages, digest + source: Dict(String, String), + unknown: Unknown, + metadata: Dict(String, String), + make: Dict(String, String), + doc_has: Dict(String, String), + /// section name -> objects, in document order + sections: Dict(String, List(Object)), + ) +} + +pub fn new_document() -> Document { + Document( + format_version: "", + source_filename: "", + source: dict.new(), + unknown: new_unknown(), + metadata: dict.new(), + make: dict.new(), + doc_has: dict.new(), + sections: dict.new(), + ) +} + +/// The major part of the declared version, "" when none was declared. +pub fn version_major(document: Document) -> String { + case string.split_once(document.format_version, ".") { + Ok(#(major, _)) -> major + Error(_) -> document.format_version + } +} + +pub fn meta(document: Document, key: String) -> String { + case dict.get(document.metadata, key) { + Ok(value) -> value + Error(_) -> "" + } +} + +/// The objects of a named section, empty when the document has none. +pub fn section(document: Document, name: String) -> List(Object) { + case dict.get(document.sections, name) { + Ok(objects) -> objects + Error(_) -> [] + } +} + +/// Is this one of the nine sections of the format? +pub fn is_section(name: String) -> Bool { + list.contains(section_names, name) +} + +/// Is this one of the four preamble blocks? +pub fn is_preamble_block(name: String) -> Bool { + list.contains(preamble_blocks, name) +} |
