diff options
Diffstat (limited to 'dlang')
| -rw-r--r-- | dlang/flake.nix | 66 | ||||
| -rw-r--r-- | dlang/src/ocda/dump.d | 160 | ||||
| -rw-r--r-- | dlang/src/ocda/load_db.d | 115 | ||||
| -rw-r--r-- | dlang/src/ocda/load_ssp.d | 25 | ||||
| -rw-r--r-- | dlang/test/dump.d | 35 |
5 files changed, 336 insertions, 65 deletions
diff --git a/dlang/flake.nix b/dlang/flake.nix index 954cad9..26dbe2d 100644 --- a/dlang/flake.nix +++ b/dlang/flake.nix @@ -9,37 +9,51 @@ pkgs = import nixpkgs { inherit system; }; /* A library, so there is nothing to install as a binary. What is - built here is the agreement check, which is the library's own - reason to exist put to the question: load a document's .ssp and - its .ocda.db and compare the two models. + built here are the two programs that put the library to the + question: the agreement check, which loads a document's .ssp and + its .ocda.db and compares the two models, and the dump, which + writes a loaded model out canonically so that an implementation in + another language can be diffed against this one. */ - mkAgree = { compiler, compilerBin }: pkgs.stdenv.mkDerivation { - pname = "ocda-lib-agree"; - version = "0.1.0"; - src = self; - nativeBuildInputs = [ compiler ]; - buildInputs = [ pkgs.sqlite ]; - buildPhase = '' - runHook preBuild - export HOME=$TMPDIR - ${compilerBin} -I=src -of=agree src/ocda/*.d test/agree.d -L-lsqlite3 - runHook postBuild - ''; - installPhase = '' - runHook preInstall - mkdir -p $out/bin - install -m755 agree $out/bin/ocda-agree - runHook postInstall - ''; - meta = { - description = "checks that the .ssp and .ocda.db loaders give one model"; - mainProgram = "ocda-agree"; + mkTool = { tool, binary, what, compiler, compilerBin }: + pkgs.stdenv.mkDerivation { + pname = "ocda-lib-${tool}"; + version = "0.1.0"; + src = self; + nativeBuildInputs = [ compiler ]; + buildInputs = [ pkgs.sqlite ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ${tool} $out/bin/${binary} + runHook postInstall + ''; + meta = { + description = what; + mainProgram = binary; + }; }; - }; + + ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; in { packages.${system} = { default = self.packages.${system}.ocda-agree; - ocda-agree = mkAgree { compiler = pkgs.ldc; compilerBin = "ldmd2"; }; + + ocda-agree = mkTool (ldc // { + tool = "agree"; binary = "ocda-agree"; + what = "checks that the .ssp and .ocda.db loaders give one model"; + }); + + ocda-dump = mkTool (ldc // { + tool = "dump"; binary = "ocda-dump"; + what = "writes a loaded abstraction out canonically, as an oracle"; + }); }; devShells.${system}.default = pkgs.mkShell { diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d new file mode 100644 index 0000000..e308db2 --- /dev/null +++ b/dlang/src/ocda/dump.d @@ -0,0 +1,160 @@ +module ocda.dump; + +/+ ↓ the model, written out canonically. + . + Not an output format: an *oracle*. Two libraries in two languages + implement the same model from the same specification, and the cheap + way to ask whether they agree is to have each write the whole loaded + document as text and diff the two. That is only useful if the text is + fully determined by the model, so: + . + - every line is "<key> <value>", the value running to end of line, + and only one free-form value per line; + - a field at its default is *omitted*, so the output says what the + document has rather than what the struct declares; + - dictionaries are emitted in sorted key order, because neither + language's hash map has an order worth relying on; + - sections come in the fixed order of the format, not + alphabetically. + . + Anything that reads this to reconstruct a document has misunderstood + it. Load the .ssp or the .ocda.db. ++/ + +import std.algorithm : sort, splitter; +import std.array : appender, array, join; +import std.conv : to; +import ocda.model; + +/+ ↓ the nine sections, in format order +/ +static immutable string[] dump_sections = ["head", "toc", "body", "endnotes", + "glossary", "bibliography", "bookindex", "blurb", "tail"]; + +string dumpDocument(ref OcdaDocument doc) { + auto out_ = appender!string; + + void kv(string key, string value) { + if (value.length == 0) { return; } + out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n"; + } + void kn(string key, long value) { + if (value == 0) { return; } + kv(key, value.to!string); + } + void kb(string key, bool value) { + if (!value) { return; } + kv(key, "1"); + } + void dict(string prefix, string[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key]; out_ ~= "\n"; + } + } + + kv("version", doc.format_version); + kv("source_filename", doc.source_filename); + dict("source", doc.source); + dict("meta", doc.metadata); + dict("make", doc.make); + dict("doc_has", doc.doc_has); + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null || (*sec).length == 0) { continue; } + kv("section", name ~ " " ~ (*sec).length.to!string); + } + + foreach (name; dump_sections) { + auto sec = doc.sectionByName(name); + if (sec is null) { continue; } + foreach (i, obj; *sec) { + out_ ~= "object "; out_ ~= name; out_ ~= " "; + out_ ~= i.to!string; out_ ~= "\n"; + kn("ocn", obj.ocn); + kv("is_a", obj.is_a); + kv("own_section", obj.own_section); + kv("level", obj.heading_level); + kv("identifier", obj.identifier); + kv("part", obj.part); + kv("is_of_type", obj.is_of_type); + kv("segment_html_is", obj.segment_html_is); + kn("parent", obj.parent_ocn); + kn("last_descendant", obj.last_descendant_ocn); + kv("children", ints(obj.children)); + kv("ancestors", ints(obj.ancestors)); + kb("dummy", obj.dummy_heading); + kb("ocn_off", obj.ocn_off); + kn("indent_base", obj.indent_base); + kn("indent_hang", obj.indent_hang); + kb("bullet", obj.bullet); + kv("lang", obj.lang); + kb("has_links", obj.has_links); + kb("has_notes_reg", obj.has_notes_reg); + kb("has_notes_star", obj.has_notes_star); + kb("has_images", obj.has_images); + kv("segment", obj.segment); + kv("segment_prev", obj.segment_prev); + kv("segment_next", obj.segment_next); + kv("anchor", obj.anchor); + kn("table_cols", obj.table_cols); + kv("table_widths", obj.table_widths); + kv("table_aligns", obj.table_aligns); + kb("table_header", obj.table_header); + kv("code_syntax", obj.code_syntax); + kb("code_linenumbers", obj.code_linenumbers); + kv("heading_ancestors_text", obj.heading_ancestors_text); + kv("sha256", obj.sha256); + foreach (url; obj.stow_link) { kv("stow_link", url); } + foreach (img; obj.images) { + /+ ↓ the name last, because a filename may hold a space where + none of the other fields can +/ + kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " " + ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " " + ~ (img.missing ? "1" : "0") ~ " " ~ img.name); + foreach (key; img.fields.keys.dup.sort) { + kv("image_field", key ~ " " ~ img.fields[key]); + } + } + /+ ↓ text, when there is any. An object whose only text line is + blank holds "" here, indistinguishable from no text at all, + because the model joins the lines into one string. Both + libraries lose the same distinction in the same place, which is + what makes them comparable; it is recorded rather than hidden. + +/ + if (obj.text.length > 0) { + auto lines = obj.text.splitter("\n").array; + kv("text", lines.length.to!string); + foreach (line; lines) { + out_ ~= "| "; out_ ~= line; out_ ~= "\n"; + } + } + } + } + return out_.data; +} + +/+ ↓ what the reader met and did not understand, sorted. Separate from the + document dump: it describes the reader, not the document, and the + .ocda.db loader has nothing to report where the .ssp loader does. ++/ +string dumpUnknown(ref OcdaDocument doc) { + auto out_ = appender!string; + void counts(string prefix, size_t[string] d) { + foreach (key; d.keys.dup.sort) { + out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " "; + out_ ~= d[key].to!string; out_ ~= "\n"; + } + } + counts("property", doc.unknown.property); + counts("block", doc.unknown.block); + counts("line", doc.unknown.line); + return out_.data; +} + +private string ints(int[] xs) { + if (xs.length == 0) { return ""; } + string[] parts; + foreach (x; xs) { parts ~= x.to!string; } + return parts.join(","); +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index ffc28a8..f602da9 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -12,8 +12,10 @@ module ocda.load_db; column correspondence this module implements. +/ -import std.array : split; +import std.array : join, split; +import std.json : JSONType, parseJSON; import std.conv : to; +import std.algorithm : startsWith; import std.string : strip; import ocda.model; import ocda.sqlite; @@ -103,8 +105,31 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) +/ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY key")) { - if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; } - doc.metadata[r[0]] = r[1]; + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } } /+ ↓ the objects, in document order within each section +/ @@ -215,51 +240,63 @@ private string levelName(int index) { } } -/+ ↓ the fixed width arrays are JSON in the database and space separated in - the text form (specification section 3.3) +/ +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + private int[] jsonInts(string v) { int[] out_; - foreach (w; jsonWords(v).split(" ")) { - if (w.length == 0) { continue; } - try { out_ ~= w.to!int; } catch (Exception) {} + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} } return out_; } +/+ ↓ space separated in the text form: table_widths, table_aligns +/ private string jsonWords(string v) { - string out_; - bool in_string = false; - foreach (c; v) { - switch (c) { - case '[': case ']': case ' ': break; - case '"': in_string = !in_string; break; - case ',': out_ ~= " "; break; - default: out_ ~= c; break; - } - } - return out_.strip; + return jsonStrings(v).join(" "); } /+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text - form and a JSON array here +/ + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ private string jsonJoined(string v) { - if (v.length == 0) { return ""; } - string out_; - bool in_string = false; - bool first = true; - string cur; - foreach (c; v) { - if (c == '"') { - in_string = !in_string; - if (!in_string) { - if (!first) { out_ ~= "|"; } - out_ ~= cur; - cur = ""; - first = false; - } - continue; - } - if (in_string) { cur ~= c; } - } - return out_; + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); } diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d index 83e3572..37872ea 100644 --- a/dlang/src/ocda/load_ssp.d +++ b/dlang/src/ocda/load_ssp.d @@ -278,7 +278,32 @@ private OcdaImage parseImageRecord(string value) { return img; } +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); auto sec = doc.sectionByName(section); if (sec !is null) { *sec ~= obj; diff --git a/dlang/test/dump.d b/dlang/test/dump.d new file mode 100644 index 0000000..17aa9bb --- /dev/null +++ b/dlang/test/dump.d @@ -0,0 +1,35 @@ +/++ + Write a loaded abstraction out canonically. + + The oracle for everything that claims to implement this model: load a + .ssp or a .ocda.db and print the model, deterministically, so that two + implementations can be compared with diff. + + Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang] ++/ +import std.stdio; +import ocda.model; +import ocda.load; +import ocda.dump; + +int main(string[] args) { + bool unknown = false; + string[] rest; + foreach (a; args[1 .. $]) { + if (a == "--unknown") { unknown = true; } else { rest ~= a; } + } + if (rest.length < 1) { + stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]"); + return 2; + } + string lang = (rest.length > 1) ? rest[1] : ""; + OcdaDocument doc; + try { + doc = load(rest[0], lang); + } catch (Exception e) { + stderr.writeln("error: ", e.msg); + return 2; + } + write(unknown ? dumpUnknown(doc) : dumpDocument(doc)); + return 0; +} |
