summaryrefslogtreecommitdiffhomepage
path: root/dlang
diff options
context:
space:
mode:
Diffstat (limited to 'dlang')
-rw-r--r--dlang/flake.nix66
-rw-r--r--dlang/src/ocda/dump.d160
-rw-r--r--dlang/src/ocda/load_db.d115
-rw-r--r--dlang/src/ocda/load_ssp.d25
-rw-r--r--dlang/test/dump.d35
5 files changed, 336 insertions, 65 deletions
diff --git a/dlang/flake.nix b/dlang/flake.nix
index 954cad9..26dbe2d 100644
--- a/dlang/flake.nix
+++ b/dlang/flake.nix
@@ -9,37 +9,51 @@
pkgs = import nixpkgs { inherit system; };
/* A library, so there is nothing to install as a binary. What is
- built here is the agreement check, which is the library's own
- reason to exist put to the question: load a document's .ssp and
- its .ocda.db and compare the two models.
+ built here are the two programs that put the library to the
+ question: the agreement check, which loads a document's .ssp and
+ its .ocda.db and compares the two models, and the dump, which
+ writes a loaded model out canonically so that an implementation in
+ another language can be diffed against this one.
*/
- mkAgree = { compiler, compilerBin }: pkgs.stdenv.mkDerivation {
- pname = "ocda-lib-agree";
- version = "0.1.0";
- src = self;
- nativeBuildInputs = [ compiler ];
- buildInputs = [ pkgs.sqlite ];
- buildPhase = ''
- runHook preBuild
- export HOME=$TMPDIR
- ${compilerBin} -I=src -of=agree src/ocda/*.d test/agree.d -L-lsqlite3
- runHook postBuild
- '';
- installPhase = ''
- runHook preInstall
- mkdir -p $out/bin
- install -m755 agree $out/bin/ocda-agree
- runHook postInstall
- '';
- meta = {
- description = "checks that the .ssp and .ocda.db loaders give one model";
- mainProgram = "ocda-agree";
+ mkTool = { tool, binary, what, compiler, compilerBin }:
+ pkgs.stdenv.mkDerivation {
+ pname = "ocda-lib-${tool}";
+ version = "0.1.0";
+ src = self;
+ nativeBuildInputs = [ compiler ];
+ buildInputs = [ pkgs.sqlite ];
+ buildPhase = ''
+ runHook preBuild
+ export HOME=$TMPDIR
+ ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3
+ runHook postBuild
+ '';
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/bin
+ install -m755 ${tool} $out/bin/${binary}
+ runHook postInstall
+ '';
+ meta = {
+ description = what;
+ mainProgram = binary;
+ };
};
- };
+
+ ldc = { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
in {
packages.${system} = {
default = self.packages.${system}.ocda-agree;
- ocda-agree = mkAgree { compiler = pkgs.ldc; compilerBin = "ldmd2"; };
+
+ ocda-agree = mkTool (ldc // {
+ tool = "agree"; binary = "ocda-agree";
+ what = "checks that the .ssp and .ocda.db loaders give one model";
+ });
+
+ ocda-dump = mkTool (ldc // {
+ tool = "dump"; binary = "ocda-dump";
+ what = "writes a loaded abstraction out canonically, as an oracle";
+ });
};
devShells.${system}.default = pkgs.mkShell {
diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d
new file mode 100644
index 0000000..e308db2
--- /dev/null
+++ b/dlang/src/ocda/dump.d
@@ -0,0 +1,160 @@
+module ocda.dump;
+
+/+ ↓ the model, written out canonically.
+ .
+ Not an output format: an *oracle*. Two libraries in two languages
+ implement the same model from the same specification, and the cheap
+ way to ask whether they agree is to have each write the whole loaded
+ document as text and diff the two. That is only useful if the text is
+ fully determined by the model, so:
+ .
+ - every line is "<key> <value>", the value running to end of line,
+ and only one free-form value per line;
+ - a field at its default is *omitted*, so the output says what the
+ document has rather than what the struct declares;
+ - dictionaries are emitted in sorted key order, because neither
+ language's hash map has an order worth relying on;
+ - sections come in the fixed order of the format, not
+ alphabetically.
+ .
+ Anything that reads this to reconstruct a document has misunderstood
+ it. Load the .ssp or the .ocda.db.
++/
+
+import std.algorithm : sort, splitter;
+import std.array : appender, array, join;
+import std.conv : to;
+import ocda.model;
+
+/+ ↓ the nine sections, in format order +/
+static immutable string[] dump_sections = ["head", "toc", "body", "endnotes",
+ "glossary", "bibliography", "bookindex", "blurb", "tail"];
+
+string dumpDocument(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+
+ void kv(string key, string value) {
+ if (value.length == 0) { return; }
+ out_ ~= key; out_ ~= " "; out_ ~= value; out_ ~= "\n";
+ }
+ void kn(string key, long value) {
+ if (value == 0) { return; }
+ kv(key, value.to!string);
+ }
+ void kb(string key, bool value) {
+ if (!value) { return; }
+ kv(key, "1");
+ }
+ void dict(string prefix, string[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key]; out_ ~= "\n";
+ }
+ }
+
+ kv("version", doc.format_version);
+ kv("source_filename", doc.source_filename);
+ dict("source", doc.source);
+ dict("meta", doc.metadata);
+ dict("make", doc.make);
+ dict("doc_has", doc.doc_has);
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null || (*sec).length == 0) { continue; }
+ kv("section", name ~ " " ~ (*sec).length.to!string);
+ }
+
+ foreach (name; dump_sections) {
+ auto sec = doc.sectionByName(name);
+ if (sec is null) { continue; }
+ foreach (i, obj; *sec) {
+ out_ ~= "object "; out_ ~= name; out_ ~= " ";
+ out_ ~= i.to!string; out_ ~= "\n";
+ kn("ocn", obj.ocn);
+ kv("is_a", obj.is_a);
+ kv("own_section", obj.own_section);
+ kv("level", obj.heading_level);
+ kv("identifier", obj.identifier);
+ kv("part", obj.part);
+ kv("is_of_type", obj.is_of_type);
+ kv("segment_html_is", obj.segment_html_is);
+ kn("parent", obj.parent_ocn);
+ kn("last_descendant", obj.last_descendant_ocn);
+ kv("children", ints(obj.children));
+ kv("ancestors", ints(obj.ancestors));
+ kb("dummy", obj.dummy_heading);
+ kb("ocn_off", obj.ocn_off);
+ kn("indent_base", obj.indent_base);
+ kn("indent_hang", obj.indent_hang);
+ kb("bullet", obj.bullet);
+ kv("lang", obj.lang);
+ kb("has_links", obj.has_links);
+ kb("has_notes_reg", obj.has_notes_reg);
+ kb("has_notes_star", obj.has_notes_star);
+ kb("has_images", obj.has_images);
+ kv("segment", obj.segment);
+ kv("segment_prev", obj.segment_prev);
+ kv("segment_next", obj.segment_next);
+ kv("anchor", obj.anchor);
+ kn("table_cols", obj.table_cols);
+ kv("table_widths", obj.table_widths);
+ kv("table_aligns", obj.table_aligns);
+ kb("table_header", obj.table_header);
+ kv("code_syntax", obj.code_syntax);
+ kb("code_linenumbers", obj.code_linenumbers);
+ kv("heading_ancestors_text", obj.heading_ancestors_text);
+ kv("sha256", obj.sha256);
+ foreach (url; obj.stow_link) { kv("stow_link", url); }
+ foreach (img; obj.images) {
+ /+ ↓ the name last, because a filename may hold a space where
+ none of the other fields can +/
+ kv("image", img.sha256 ~ " " ~ img.bytes.to!string ~ " "
+ ~ img.width.to!string ~ "x" ~ img.height.to!string ~ " "
+ ~ (img.missing ? "1" : "0") ~ " " ~ img.name);
+ foreach (key; img.fields.keys.dup.sort) {
+ kv("image_field", key ~ " " ~ img.fields[key]);
+ }
+ }
+ /+ ↓ text, when there is any. An object whose only text line is
+ blank holds "" here, indistinguishable from no text at all,
+ because the model joins the lines into one string. Both
+ libraries lose the same distinction in the same place, which is
+ what makes them comparable; it is recorded rather than hidden.
+ +/
+ if (obj.text.length > 0) {
+ auto lines = obj.text.splitter("\n").array;
+ kv("text", lines.length.to!string);
+ foreach (line; lines) {
+ out_ ~= "| "; out_ ~= line; out_ ~= "\n";
+ }
+ }
+ }
+ }
+ return out_.data;
+}
+
+/+ ↓ what the reader met and did not understand, sorted. Separate from the
+ document dump: it describes the reader, not the document, and the
+ .ocda.db loader has nothing to report where the .ssp loader does.
++/
+string dumpUnknown(ref OcdaDocument doc) {
+ auto out_ = appender!string;
+ void counts(string prefix, size_t[string] d) {
+ foreach (key; d.keys.dup.sort) {
+ out_ ~= prefix; out_ ~= " "; out_ ~= key; out_ ~= " ";
+ out_ ~= d[key].to!string; out_ ~= "\n";
+ }
+ }
+ counts("property", doc.unknown.property);
+ counts("block", doc.unknown.block);
+ counts("line", doc.unknown.line);
+ return out_.data;
+}
+
+private string ints(int[] xs) {
+ if (xs.length == 0) { return ""; }
+ string[] parts;
+ foreach (x; xs) { parts ~= x.to!string; }
+ return parts.join(",");
+}
diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d
index ffc28a8..f602da9 100644
--- a/dlang/src/ocda/load_db.d
+++ b/dlang/src/ocda/load_db.d
@@ -12,8 +12,10 @@ module ocda.load_db;
column correspondence this module implements.
+/
-import std.array : split;
+import std.array : join, split;
+import std.json : JSONType, parseJSON;
import std.conv : to;
+import std.algorithm : startsWith;
import std.string : strip;
import ocda.model;
import ocda.sqlite;
@@ -103,8 +105,31 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false)
+/
foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = "
~ chosen.doc_id.to!string ~ " ORDER BY key")) {
- if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; }
- doc.metadata[r[0]] = r[1];
+ string key = r[0];
+ if (key.startsWith("source.")) { continue; }
+ /+ ↓ translation.ocn_aligned says whether the languages of this
+ document share one numbering. A .ocda.db can know it because it
+ holds every language; a .ssp is one language and cannot. So it is
+ a fact about the *file* rather than about the document, and putting
+ it in the document's metadata would let a writer tell which
+ serialisation it was handed, which is the one thing this library
+ exists to prevent. Left out until the model has a place for
+ collection-wide facts.
+ +/
+ if (key.startsWith("translation.")) { continue; }
+ /+ ↓ the database has one metadata table where the text form has three
+ blocks, and tells them apart by a key prefix. Unprefixing them
+ here is what makes the two loaders give one model: a writer asks
+ doc.make["footer"] whichever serialisation it was handed, and
+ never doc.metadata["make.footer"].
+ +/
+ if (key.startsWith("make.")) {
+ doc.make[key["make.".length .. $]] = r[1].strip;
+ } else if (key.startsWith("doc_has.")) {
+ doc.doc_has[key["doc_has.".length .. $]] = r[1].strip;
+ } else {
+ doc.metadata[key] = r[1].strip;
+ }
}
/+ ↓ the objects, in document order within each section +/
@@ -215,51 +240,63 @@ private string levelName(int index) {
}
}
-/+ ↓ the fixed width arrays are JSON in the database and space separated in
- the text form (specification section 3.3) +/
+/+ ↓ the JSON columns, read as JSON.
+ .
+ The fixed width arrays are JSON in the database and space separated
+ (or "|" separated) in the text form. This was first written as a
+ character scan that threw away brackets and quotes, which is wrong the
+ moment a value holds an escape: Phobos' JSONValue.toString escapes
+ "/" as "\/", so every heading containing a slash came back with a
+ backslash in it. 36 of 36 documents differed on
+ heading_ancestors_text and nothing else had shown it.
+ .
+ So: parse. The values are small, there are few of them, and a real
+ parser is shorter than the scan it replaces.
++/
+private string[] jsonStrings(string v) {
+ if (v.length == 0) { return []; }
+ string[] out_;
+ try {
+ auto j = parseJSON(v);
+ if (j.type != JSONType.array) { return []; }
+ foreach (e; j.array) {
+ switch (e.type) {
+ case JSONType.string: out_ ~= e.str; break;
+ case JSONType.integer: out_ ~= e.integer.to!string; break;
+ case JSONType.uinteger: out_ ~= e.uinteger.to!string; break;
+ case JSONType.float_: out_ ~= e.floating.to!string; break;
+ case JSONType.true_: out_ ~= "true"; break;
+ case JSONType.false_: out_ ~= "false"; break;
+ default: out_ ~= ""; break;
+ }
+ }
+ } catch (Exception) {
+ return [];
+ }
+ return out_;
+}
+
private int[] jsonInts(string v) {
int[] out_;
- foreach (w; jsonWords(v).split(" ")) {
- if (w.length == 0) { continue; }
- try { out_ ~= w.to!int; } catch (Exception) {}
+ foreach (s; jsonStrings(v)) {
+ if (s.length == 0) { continue; }
+ try { out_ ~= s.to!int; } catch (Exception) {}
}
return out_;
}
+/+ ↓ space separated in the text form: table_widths, table_aligns +/
private string jsonWords(string v) {
- string out_;
- bool in_string = false;
- foreach (c; v) {
- switch (c) {
- case '[': case ']': case ' ': break;
- case '"': in_string = !in_string; break;
- case ',': out_ ~= " "; break;
- default: out_ ~= c; break;
- }
- }
- return out_.strip;
+ return jsonStrings(v).join(" ");
}
/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text
- form and a JSON array here +/
+ form and a JSON array here. The empty slots are significant: they say
+ which levels the heading sits under, so the separators carry as much
+ as the text does.
++/
private string jsonJoined(string v) {
- if (v.length == 0) { return ""; }
- string out_;
- bool in_string = false;
- bool first = true;
- string cur;
- foreach (c; v) {
- if (c == '"') {
- in_string = !in_string;
- if (!in_string) {
- if (!first) { out_ ~= "|"; }
- out_ ~= cur;
- cur = "";
- first = false;
- }
- continue;
- }
- if (in_string) { cur ~= c; }
- }
- return out_;
+ auto parts = jsonStrings(v);
+ if (parts.length == 0) { return ""; }
+ return parts.join("|");
}
diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d
index 83e3572..37872ea 100644
--- a/dlang/src/ocda/load_ssp.d
+++ b/dlang/src/ocda/load_ssp.d
@@ -278,7 +278,32 @@ private OcdaImage parseImageRecord(string value) {
return img;
}
+/+ ↓ two values the text form leaves out because they can be worked out.
+ .
+ The .ssp emits a property only when it carries information, so
+ .identifier is absent when it is simply the ocn, and .anchor is absent
+ when it is the segment name. The .ocda.db, being columns rather than
+ lines, stores both unconditionally. So a loader that takes the text
+ form literally gives a *different model* from the one the database
+ gives, and the difference is not small: in the reference collection
+ the identifier is omitted on all but a handful of objects.
+ .
+ The model is the thing both serialisations describe, so the loader
+ fills them in. Found by dumping both models and diffing them, which no
+ amount of comparing section counts would have shown.
++/
+private void deriveOmitted(ref OcdaObject obj) {
+ import std.conv : to;
+ if (obj.identifier.length == 0 && obj.ocn != 0) {
+ obj.identifier = obj.ocn.to!string;
+ }
+ if (obj.anchor.length == 0 && obj.segment.length > 0) {
+ obj.anchor = obj.segment;
+ }
+}
+
private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) {
+ deriveOmitted(obj);
auto sec = doc.sectionByName(section);
if (sec !is null) {
*sec ~= obj;
diff --git a/dlang/test/dump.d b/dlang/test/dump.d
new file mode 100644
index 0000000..17aa9bb
--- /dev/null
+++ b/dlang/test/dump.d
@@ -0,0 +1,35 @@
+/++
+ Write a loaded abstraction out canonically.
+
+ The oracle for everything that claims to implement this model: load a
+ .ssp or a .ocda.db and print the model, deterministically, so that two
+ implementations can be compared with diff.
+
+ Usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
++/
+import std.stdio;
+import ocda.model;
+import ocda.load;
+import ocda.dump;
+
+int main(string[] args) {
+ bool unknown = false;
+ string[] rest;
+ foreach (a; args[1 .. $]) {
+ if (a == "--unknown") { unknown = true; } else { rest ~= a; }
+ }
+ if (rest.length < 1) {
+ stderr.writeln("usage: ocda-dump [--unknown] <file>.ssp|<file>.ocda.db [lang]");
+ return 2;
+ }
+ string lang = (rest.length > 1) ? rest[1] : "";
+ OcdaDocument doc;
+ try {
+ doc = load(rest[0], lang);
+ } catch (Exception e) {
+ stderr.writeln("error: ", e.msg);
+ return 2;
+ }
+ write(unknown ? dumpUnknown(doc) : dumpDocument(doc));
+ return 0;
+}