diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-25 16:34:44 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-25 16:34:44 -0400 |
| commit | 0e2c9fa40ed98ceb7a26c2af69807cf577b6e32a (patch) | |
| tree | c9422586349ffbcafb93aad5a4e5f57f15ddf091 /dlang/src/ocda | |
the abstraction as a library, and a defect it found at once
The readers each read one serialisation and print. A writer needs
neither: it needs the document. So the model and both loaders live
here, and a writer cannot tell which serialisation it was handed,
exactly as spine own writers cannot.
ocda.model imports nothing, which is the property that makes writers
possible: they depend on the model and on neither loader. The model
and the .ssp loader are the ssp-reader code, moved and renamed to the
level they belong to: the model is neither the .ssp nor the .ocda.db.
ocda.load_db is new, and returns the same value from sqlite that
load_ssp returns from text. ocda.load is the one place that decides
what a path is.
The claim is only true if the two loaders agree, so test/agree.d asks:
load a document both ways and compare the models object by object.
It found a real defect on its first run. Eight of thirty-six
disagreed, and the cause was the .ssp loader, not the formats: an
object whose first text line is blank lost it, the loader having
decided whether to start a new line by asking whether the accumulated
text was empty, which is a different question from whether a line has
been seen. Seven documents of the sample set have verse opening on a
blank line and every one of them was wrong.
Nothing else had caught it: the two .ssp readers were compared on
section counts, which the bug does not disturb. It took loading one
document two ways and comparing the objects. Fixed here, and in
ssp-reader/dlang which carried the same code. 36 of 36 now agree.
(assisted by Claude-Code)
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'dlang/src/ocda')
| -rw-r--r-- | dlang/src/ocda/load.d | 58 | ||||
| -rw-r--r-- | dlang/src/ocda/load_db.d | 265 | ||||
| -rw-r--r-- | dlang/src/ocda/load_ssp.d | 305 | ||||
| -rw-r--r-- | dlang/src/ocda/model.d | 143 | ||||
| -rw-r--r-- | dlang/src/ocda/sqlite.d | 101 |
5 files changed, 872 insertions, 0 deletions
diff --git a/dlang/src/ocda/load.d b/dlang/src/ocda/load.d new file mode 100644 index 0000000..0f630bc --- /dev/null +++ b/dlang/src/ocda/load.d @@ -0,0 +1,58 @@ +module ocda.load; + +/+ ↓ what a path is, and loading it. + . + One place that decides, so that every reader and every writer accepts + the same inputs and a new input type is added once. This mirrors + spine's own ocda/abstraction/load.d, which names the sources it can + enter the pipeline from and tells them apart by suffix. + . + Deliberately *not* here: .sst and .ssm markup. Reading markup means + implementing the parser, which is spine's work and a far larger thing + than a reader or a sample writer needs. This library reads + abstractions. ++/ + +import std.algorithm : endsWith; +import ocda.model; +import ocda.load_ssp; +import ocda.load_db; + +enum Source { + ssp, // .ssp, the abstraction as text + ocda_db, // .ocda.db, the abstraction as sqlite + unknown, +} + +Source sourceOf(string path) { + if (path.endsWith(".ocda.db")) { return Source.ocda_db; } + if (path.endsWith(".ssp")) { return Source.ssp; } + if (path.endsWith(".db")) { return Source.ocda_db; } + return Source.unknown; +} + +string sourceName(Source s) { + final switch (s) { + case Source.ssp: return ".ssp (the abstraction as text)"; + case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; + case Source.unknown: return "not an abstraction this library reads"; + } +} + +class OcdaLoadError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ load whatever the path is into the model. + lang applies only to a .ocda.db, which holds every language of its + document; a .ssp is one language and the argument is ignored. ++/ +OcdaDocument load(string path, string lang = "", bool any_version = false) { + final switch (sourceOf(path)) { + case Source.ssp: return parseSspFile(path); + case Source.ocda_db: return loadDbFile(path, lang, any_version); + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) + ~ " (expected .ssp or .ocda.db)"); + } +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d new file mode 100644 index 0000000..ffc28a8 --- /dev/null +++ b/dlang/src/ocda/load_db.d @@ -0,0 +1,265 @@ +module ocda.load_db; + +/+ ↓ build the model from a .ocda.db. + . + The point of this module is that it returns *the same value* + ocda.load_ssp returns from the same document's .ssp. A writer is then + a function from the model to bytes, and cannot tell which + serialisation it was given, exactly as spine's own writers cannot. + . + Written from the sisudoc-abstraction-format specification, + ocda-db-format-specification.md, whose section 4 gives the column by + column correspondence this module implements. ++/ + +import std.array : split; +import std.conv : to; +import std.string : strip; +import ocda.model; +import ocda.sqlite; + +/+ ↓ the schema major this loader knows (specification section 2) +/ +enum schema_major_read = "2"; + +class OcdaDbError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ every language the database holds, in manifest order +/ +struct DbLanguage { + int doc_id; + string lang; + string source_filename; + string ssp_digest; +} + +DbLanguage[] dbLanguages(ref Db db) { + DbLanguage[] out_; + foreach (r; db.rows( + "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') + FROM documents ORDER BY id")) { + out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); + } + return out_; +} + +/+ ↓ load one language of a database as the model. + lang == "" takes the first language the manifest gave. ++/ +OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { + auto db = Db.open(path); + scope(exit) { db.close(); } + + if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { + throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" + ~ " it does not look like a .ocda.db"); + } + + /+ ↓ specification sections 2 and 3.2: the rows describing the file rather + than a document in it are the only ones with doc_id null +/ + string schema_version = db.one( + "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); + if (schema_version.length > 0 && !any_version) { + auto major = schema_version.split(".")[0]; + if (major != schema_major_read) { + throw new OcdaDbError(path ~ " is schema v" ~ schema_version + ~ ", this loader knows v" ~ schema_major_read ~ ".x"); + } + } + + auto languages = dbLanguages(db); + if (languages.length == 0) { + throw new OcdaDbError(path ~ ": no documents rows"); + } + DbLanguage chosen = languages[0]; + if (lang.length > 0) { + bool found = false; + foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } + if (!found) { + string have; + foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } + throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); + } + } + + OcdaDocument doc; + doc.format_version = schema_version; + doc.source_filename = chosen.source_filename; + + /+ ↓ @source of the .ssp corresponds to the documents row and the + source.digest metadata (specification section 5.1 of the .ssp, and + section 7 here) +/ + doc.source["language"] = chosen.lang; + string languages_joined; + foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } + doc.source["languages"] = languages_joined; + auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); + if (digest.length > 0) { doc.source["digest"] = digest; } + + /+ ↓ @meta: every metadata row of this language that is not a source.* + property. source.* describes the artefact rather than the document + and lives in @source in the text form. + +/ + foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " ORDER BY key")) { + if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; } + doc.metadata[r[0]] = r[1]; + } + + /+ ↓ the objects, in document order within each section +/ + foreach (r; db.rows( + "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), + coalesce(is_of_section,''), coalesce(is_of_type,''), + coalesce(heading_level,-1), coalesce(identifier,''), + coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), + coalesce(children,''), coalesce(ancestors,''), + coalesce(dummy_heading,0), coalesce(object_number_off,0), + coalesce(indent_base,0), coalesce(indent_hang,0), + coalesce(bullet,0), coalesce(lang,''), + coalesce(has_links,0), coalesce(has_notes_reg,0), + coalesce(has_notes_star,0), coalesce(has_images,0), + coalesce(segment,''), coalesce(segment_prev,''), + coalesce(segment_next,''), coalesce(anchor,''), + coalesce(segment_html_is,''), + coalesce(table_cols,0), coalesce(table_widths,''), + coalesce(table_aligns,''), coalesce(table_header,0), + coalesce(code_linenumbers,0), + coalesce(heading_ancestors_text,''), coalesce(sha256,''), + coalesce(text,''), id + FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string + ~ " ORDER BY CASE section + WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 + WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 + WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 + WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { + OcdaObject obj; + obj.section = r[0]; + obj.ocn = r[2].to!int; + obj.is_a = r[3]; + obj.part = r[4]; + obj.own_section = (r[5] == r[0]) ? "" : r[5]; + obj.is_of_type = r[6]; + /+ ↓ the database holds the level as a number, the text form as the + letter or digit the markup wrote. Model section 4 is the mapping. +/ + obj.heading_level = levelName(r[7].to!int); + obj.identifier = r[8]; + obj.parent_ocn = r[9].to!int; + obj.last_descendant_ocn = r[10].to!int; + obj.children = jsonInts(r[11]); + obj.ancestors = jsonInts(r[12]); + obj.dummy_heading = (r[13] != "0"); + obj.ocn_off = (r[14] != "0"); + obj.indent_base = r[15].to!int; + obj.indent_hang = r[16].to!int; + obj.bullet = (r[17] != "0"); + obj.lang = r[18]; + obj.has_links = (r[19] != "0"); + obj.has_notes_reg = (r[20] != "0"); + obj.has_notes_star = (r[21] != "0"); + obj.has_images = (r[22] != "0"); + obj.segment = r[23]; + obj.segment_prev = r[24]; + obj.segment_next = r[25]; + obj.anchor = r[26]; + obj.segment_html_is = r[27]; + obj.table_cols = r[28].to!int; + obj.table_widths = jsonWords(r[29]); + obj.table_aligns = jsonWords(r[30]); + obj.table_header = (r[31] != "0"); + obj.code_linenumbers = (r[32] != "0"); + obj.heading_ancestors_text = jsonJoined(r[33]); + obj.sha256 = r[34]; + obj.text = r[35]; + + /+ ↓ the four repeatable properties of the text form are child tables + here (specification section 4) +/ + string object_id = r[36]; + foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), + coalesce(width,0), coalesce(height,0), + coalesce(missing,0) + FROM object_images WHERE object_id = " ~ object_id + ~ " ORDER BY seq")) { + OcdaImage img; + img.name = i[0]; + img.sha256 = i[1]; + img.bytes = i[2].to!ulong; + img.width = i[3].to!int; + img.height = i[4].to!int; + img.missing = (i[5] != "0"); + obj.images ~= img; + } + foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.stow_link ~= l[0]; + } + + auto section = doc.sectionByName(r[0]); + if (section !is null) { *section ~= obj; } + } + return doc; +} + +/+ ↓ model section 4: the eight levels and their indices +/ +private string levelName(int index) { + switch (index) { + case 0: return "A"; + case 1: return "B"; + case 2: return "C"; + case 3: return "D"; + case 4: return "1"; + case 5: return "2"; + case 6: return "3"; + case 7: return "4"; + default: return ""; + } +} + +/+ ↓ the fixed width arrays are JSON in the database and space separated in + the text form (specification section 3.3) +/ +private int[] jsonInts(string v) { + int[] out_; + foreach (w; jsonWords(v).split(" ")) { + if (w.length == 0) { continue; } + try { out_ ~= w.to!int; } catch (Exception) {} + } + return out_; +} + +private string jsonWords(string v) { + string out_; + bool in_string = false; + foreach (c; v) { + switch (c) { + case '[': case ']': case ' ': break; + case '"': in_string = !in_string; break; + case ',': out_ ~= " "; break; + default: out_ ~= c; break; + } + } + return out_.strip; +} + +/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text + form and a JSON array here +/ +private string jsonJoined(string v) { + if (v.length == 0) { return ""; } + string out_; + bool in_string = false; + bool first = true; + string cur; + foreach (c; v) { + if (c == '"') { + in_string = !in_string; + if (!in_string) { + if (!first) { out_ ~= "|"; } + out_ ~= cur; + cur = ""; + first = false; + } + continue; + } + if (in_string) { cur ~= c; } + } + return out_; +} diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d new file mode 100644 index 0000000..ccaa0f4 --- /dev/null +++ b/dlang/src/ocda/load_ssp.d @@ -0,0 +1,305 @@ +module ocda.load_ssp; + +import std.algorithm : splitter, startsWith; +import std.array : array, split; +import std.conv : to, ConvException; +import std.file : readText; +import std.stdio : writeln; +import std.string : strip, stripLeft, indexOf; +import ocda.model; + +OcdaDocument parseSspFile(string filename) { + string content = readText(filename); + return parseSsp(content); +} + +OcdaDocument parseSsp(string content) { + OcdaDocument doc; + OcdaObject current_obj; + bool in_object = false; + /+ ↓ whether a text line has been seen for the object being read. + Not "is the text non-empty": an object whose first text line is blank + (a verse opening on an empty line, which the sample set has seven of) + would then lose it, the next line overwriting rather than appending. + +/ + bool seen_text = false; + string current_section = ""; + string current_block = ""; // meta, make, doc_has, or section name + + foreach (line; content.splitter("\n")) { + /+ â comment. The first line of a .ssp declares the format version, and + it is the one comment that carries meaning: a reader checks the + major part and refuses what it does not know rather than loading + an unfamiliar artefact half populated in silence. + +/ + if (line.length > 0 && line[0] == '%') { + enum _banner = "% SiSU Document Abstraction v"; + enum _source = "% Source: "; + if (doc.format_version.length == 0 && line.startsWith(_banner)) { + doc.format_version = line[_banner.length .. $].strip; + } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { + doc.source_filename = line[_source.length .. $].strip; + } + continue; + } + + /+ â blank line - finalize current object +/ + if (line.length == 0 || line.strip.length == 0) { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + continue; + } + + /+ â section/block open +/ + if (line[0] == '@') { + auto brace_pos = line.indexOf('{'); + if (brace_pos > 0) { + string name = line[1 .. brace_pos].strip; + current_block = name; + if (!isPreambleBlock(name)) { + current_section = name; + /+ â a block that is neither preamble nor a section this reader + knows: counted, so that --report-unknown can say so +/ + if (doc.sectionByName(name) is null) { + doc.unknown.block[name]++; + } + } + } + continue; + } + + /+ â section/block close +/ + if (line[0] == '}') { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + current_block = ""; + continue; + } + + /+ â inside meta/make/doc_has blocks: key-value pairs +/ + if (isPreambleBlock(current_block)) { + string stripped = line.stripLeft; + auto colon_pos = stripped.indexOf(':'); + if (colon_pos > 0) { + string key = stripped[0 .. colon_pos].strip; + string value = (colon_pos + 1 < stripped.length) + ? stripped[colon_pos + 1 .. $].strip : ""; + if (current_block == "meta") { doc.metadata[key] = value; } + else if (current_block == "make") { doc.make[key] = value; } + else if (current_block == "source") { doc.source[key] = value; } + else { doc.doc_has[key] = value; } + } + continue; + } + + /+ â object declaration: [N] type ... +/ + if (line[0] == '[') { + if (in_object) { + commitObject(doc, current_section, current_obj); + } + current_obj = OcdaObject.init; + current_obj.section = current_section; + in_object = true; + seen_text = false; + parseObjectDeclaration(line, current_obj); + continue; + } + + /+ â property: .name: value +/ + if (line[0] == '.') { + if (in_object) { + parseProperty(line, current_obj, doc.unknown); + } + continue; + } + + /+ â text content: | content +/ + if (line.length >= 1 && line[0] == '|') { + if (in_object) { + string text_content = (line.length > 2) ? line[2 .. $] : ""; + if (seen_text) { + current_obj.text ~= "\n" ~ text_content; + } else { + current_obj.text = text_content; + seen_text = true; + } + } + continue; + } + + /+ â no known shape. Counted by its first character so that a reader + can be told the file held something this one did not expect + +/ + doc.unknown.line[line[0 .. 1].idup]++; + } + + /+ â finalize last object +/ + if (in_object) { + commitObject(doc, current_section, current_obj); + } + + return doc; +} + +private void parseObjectDeclaration(string line, ref OcdaObject obj) { + /+ parse: [N] type :level identifier +/ + auto bracket_close = line.indexOf(']'); + if (bracket_close < 0) return; + try { + obj.ocn = line[1 .. bracket_close].strip.to!int; + } catch (ConvException) { + obj.ocn = 0; + } + string rest = (bracket_close + 1 < line.length) + ? line[bracket_close + 1 .. $].strip : ""; + if (rest.length == 0) return; + + auto parts = rest.split(" "); + if (parts.length == 0) return; + obj.is_a = parts[0]; + + if (obj.is_a == "heading" && parts.length >= 2) { + /+ parse :A, :B, :1, etc. +/ + string lev = parts[1]; + if (lev.length >= 2 && lev[0] == ':') { + obj.heading_level = lev[1 .. $]; + } + /+ optional identifier after level +/ + if (parts.length >= 3) { + obj.identifier = parts[2]; + } + } +} + +/+ â the blocks that are document preamble rather than sections of objects +/ +private bool isPreambleBlock(string name) { + return name == "source" || name == "meta" || name == "make" || name == "doc_has"; +} + +private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { + /+ parse: .name: value +/ + auto colon_pos = line.indexOf(':'); + if (colon_pos < 1) return; + string name = line[1 .. colon_pos].strip; + string value = (colon_pos + 1 < line.length) + ? line[colon_pos + 1 .. $].strip : ""; + + switch (name) { + case "part": obj.part = value; break; + case "is_of_type": obj.is_of_type = value; break; + case "section": obj.own_section = value; break; + case "identifier": obj.identifier = value; break; + case "segment_html_is": obj.segment_html_is = value; break; + case "table_aligns": obj.table_aligns = value; break; + case "stow_link": obj.stow_link ~= value; break; + case "parent": obj.parent_ocn = toIntSafe(value); break; + case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; + case "children": obj.children = parseIntArray(value); break; + case "ancestors": obj.ancestors = parseIntArray(value); break; + case "dummy": obj.dummy_heading = (value == "true"); break; + case "ocn_off": obj.ocn_off = (value == "true"); break; + case "indent": + auto parts = value.split(" "); + if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); + if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); + break; + case "bullet": obj.bullet = (value == "true"); break; + case "lang": obj.lang = value; break; + case "has": + foreach (flag; value.split(" ")) { + switch (flag) { + case "links": obj.has_links = true; break; + case "notes_reg": obj.has_notes_reg = true; break; + case "notes_star": obj.has_notes_star = true; break; + case "images": obj.has_images = true; break; + default: break; + } + } + break; + case "segment": obj.segment = value; break; + case "segment_prev": obj.segment_prev = value; break; + case "segment_next": obj.segment_next = value; break; + case "anchor": obj.anchor = value; break; + case "table_cols": obj.table_cols = toIntSafe(value); break; + case "table_widths": obj.table_widths = value; break; + case "table_header": obj.table_header = (value == "true"); break; + case "code_syntax": obj.code_syntax = value; break; + case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; + case "heading_ancestors_text": obj.heading_ancestors_text = value; break; + case "sha256": obj.sha256 = value; break; + case "image": obj.images ~= parseImageRecord(value); break; + /+ â a property this reader has no use for, or has never heard of. + Ignored, which is the conforming behaviour within a major + version, but counted rather than passed over in silence + +/ + default: unknown.property[name]++; break; + } +} + +private OcdaImage parseImageRecord(string value) { + /+ parse: <filename> sha256:<HEX> bytes:<n> [key:value ...] + filename first, remaining fields order-independent; unrecognised + fields are kept rather than discarded +/ + OcdaImage img; + bool first = true; + foreach (field; value.split(" ")) { + if (field.length == 0) continue; + if (first) { + img.name = field; + first = false; + continue; + } + auto colon_pos = field.indexOf(':'); + if (colon_pos < 1) continue; + string key = field[0 .. colon_pos]; + string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; + switch (key) { + case "sha256": img.sha256 = val; break; + case "bytes": img.bytes = toUlongSafe(val); break; + case "px": + auto dims = val.split("x"); + if (dims.length == 2) { + img.width = toIntSafe(dims[0]); + img.height = toIntSafe(dims[1]); + } + break; + case "missing": img.missing = (val == "true"); break; + default: img.fields[key] = val; break; + } + } + return img; +} + +private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + auto sec = doc.sectionByName(section); + if (sec !is null) { + *sec ~= obj; + } +} + +private int toIntSafe(string s) { + try { return s.strip.to!int; } + catch (ConvException) { return 0; } +} + +private ulong toUlongSafe(string s) { + try { return s.strip.to!ulong; } + catch (ConvException) { return 0; } +} + +private int[] parseIntArray(string s) { + int[] result; + foreach (part; s.split(" ")) { + auto v = toIntSafe(part); + result ~= v; + } + return result; +} diff --git a/dlang/src/ocda/model.d b/dlang/src/ocda/model.d new file mode 100644 index 0000000..0ddefa1 --- /dev/null +++ b/dlang/src/ocda/model.d @@ -0,0 +1,143 @@ +module ocda.model; + +struct OcdaImage { + string name; // image filename as referenced by the object + string sha256; // hex-encoded (uppercase) digest of the file + ulong bytes; // file size + int width; // file's own pixel width (not display width) + int height; // file's own pixel height + bool missing; // file absent or unreadable, no sha256/bytes + string[string] fields; // any further/unrecognised key:value fields +} + +struct OcdaObject { + int ocn; + string is_a; // heading, para, toc, table, code, group, block, verse, quote, endnote, glossary, bibliography, bookindex, blurb + string section; // head, toc, body, endnotes, glossary, bibliography, bookindex, blurb + string heading_level; // A, B, C, D, 1, 2, 3, 4 (for headings) + string identifier; // heading: from the declaration line; otherwise .identifier + string part; // frontmatter, body, backmatter + string is_of_type; // para, block, comment: how the text behaves + string own_section; // .section, when it differs from the block it sits in + string segment_html_is;// the html segment this object sits in + string table_aligns; // l c r per column + string[] stow_link; // links extracted from the object + int parent_ocn; + int last_descendant_ocn; + int[] children; // direct child heading OCNs + int[] ancestors; // markedup_ancestors[8] + bool dummy_heading; + bool ocn_off; + int indent_base; + int indent_hang; + bool bullet; + string lang; + bool has_links; + bool has_notes_reg; + bool has_notes_star; + bool has_images; + string segment; + string segment_prev; + string segment_next; + string anchor; + int table_cols; + string table_widths; + bool table_header; + string code_syntax; + bool code_linenumbers; + string heading_ancestors_text; + string sha256; + OcdaImage[] images; // repeatable .image records, in order + string text; // content (lines joined with newline) + + /+ ↓ whether this object is rendered at all. + is_of_type "comment" is markup commentary: carried by the + abstraction so that nothing is lost, and not put into output. An + output that renders it leaks the author notes into the document, + which is what spine itself does not do. + +/ + bool renders() const { + return is_of_type != "comment"; + } + + /+ ↓ the section this object belongs to, which is the block it sits in + unless it says otherwise +/ + string sectionOf() const { + return (own_section.length > 0) ? own_section : section; + } +} + +/+ ↓ what the reader met and did not understand. + A reader that silently drops what it does not know cannot say whether + it understood the document, and cannot be used to check a + specification against an implementation: silence and understanding + look alike. So the parser counts what it ignored, and + --report-unknown prints it. ++/ +struct OcdaUnknown { + size_t[string] property; // .name -> times seen + size_t[string] block; // @name -> times seen + size_t[string] line; // first char -> times seen, lines of no known shape + + bool any() const { + return property.length > 0 || block.length > 0 || line.length > 0; + } +} + +struct OcdaDocument { + string format_version; // as declared on line 1, "" when absent + string source_filename; // from the "% Source:" line + string[string] source; // @source: language, languages, digest + OcdaUnknown unknown; // what was met and not understood + string[string] metadata; // key -> value + string[string] make; // key -> value + string[string] doc_has; // key -> value + OcdaObject[] head; + OcdaObject[] toc; + OcdaObject[] body; + OcdaObject[] endnotes; + OcdaObject[] glossary; + OcdaObject[] bibliography; + OcdaObject[] bookindex; + OcdaObject[] blurb; + OcdaObject[] tail; + + /+ ↓ the major part of the declared version, "" when none was declared +/ + string formatVersionMajor() const { + import std.string : indexOf; + if (format_version.length == 0) { return ""; } + auto dot = format_version.indexOf('.'); + return (dot > 0) ? format_version[0 .. dot] : format_version; + } + + string meta(string key) { + if (auto p = key in metadata) return *p; + return ""; + } + + OcdaObject[]* sectionByName(string name) { + switch (name) { + case "head": return &head; + case "toc": return &toc; + case "body": return &body; + case "endnotes": return &endnotes; + case "glossary": return &glossary; + case "bibliography": return &bibliography; + case "bookindex": return &bookindex; + case "blurb": return &blurb; + case "tail": return &tail; + default: return null; + } + } +} + +/+ ↓ the format major version this reader knows. + The .ssp and the .ocda.db carry one version between them, declared in + band: the .ssp on its first line, the database in its schema.version + row. The minor part is raised when a property is added, and a reader of + the newer format still reads the older one, so only the major part is + checked here. Raise this when the reader has been brought to a new + major, not before: the point of the gate is to refuse what has not been + looked at. ++/ +enum ssp_format_major_read = "2"; diff --git a/dlang/src/ocda/sqlite.d b/dlang/src/ocda/sqlite.d new file mode 100644 index 0000000..1552fc4 --- /dev/null +++ b/dlang/src/ocda/sqlite.d @@ -0,0 +1,101 @@ +module ocda.sqlite; + +/+ ↓ the sqlite3 C API, declared rather than depended upon. + . + A binding from the dub registry would be a dependency to fetch, and + the point of this reader is that it builds in a sandbox with no + network. The dozen functions a reader needs are declared here and the + library is linked directly, which costs a page of extern(C) and saves + a dependency. ++/ + +import std.conv : to; +import std.string : fromStringz, toStringz; + +extern (C) { + struct sqlite3; + struct sqlite3_stmt; + + int sqlite3_open_v2(const(char)* filename, sqlite3** db, int flags, const(char)* vfs); + int sqlite3_close(sqlite3* db); + int sqlite3_prepare_v2(sqlite3* db, const(char)* sql, int nbyte, + sqlite3_stmt** stmt, const(char)** tail); + int sqlite3_step(sqlite3_stmt* stmt); + int sqlite3_finalize(sqlite3_stmt* stmt); + int sqlite3_column_count(sqlite3_stmt* stmt); + const(char)* sqlite3_column_name(sqlite3_stmt* stmt, int col); + const(ubyte)* sqlite3_column_text(sqlite3_stmt* stmt, int col); + int sqlite3_column_int(sqlite3_stmt* stmt, int col); + int sqlite3_column_type(sqlite3_stmt* stmt, int col); + const(char)* sqlite3_errmsg(sqlite3* db); +} + +enum SQLITE_OK = 0; +enum SQLITE_ROW = 100; +enum SQLITE_DONE = 101; +enum SQLITE_NULL = 5; +enum SQLITE_OPEN_READONLY = 0x00000001; + +class SqliteError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ a database, opened read only. + Read only is deliberate: a reader has no business writing, and a + request that opens a database read-write can create an empty one where + a path is mistyped. ++/ +struct Db { + private sqlite3* _handle; + + static Db open(string path) { + sqlite3* handle; + auto rc = sqlite3_open_v2(path.toStringz, &handle, SQLITE_OPEN_READONLY, null); + if (rc != SQLITE_OK) { + string msg = (handle is null) + ? "cannot open " ~ path + : "cannot open " ~ path ~ ": " ~ sqlite3_errmsg(handle).fromStringz.idup; + if (handle !is null) { sqlite3_close(handle); } + throw new SqliteError(msg); + } + Db db; + db._handle = handle; + return db; + } + + void close() { + if (_handle !is null) { sqlite3_close(_handle); _handle = null; } + } + + /+ ↓ every row of a query, each row as its columns in order +/ + string[][] rows(string sql) { + sqlite3_stmt* stmt; + auto rc = sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null); + if (rc != SQLITE_OK) { + throw new SqliteError(sqlite3_errmsg(_handle).fromStringz.idup ~ " in: " ~ sql); + } + scope(exit) { sqlite3_finalize(stmt); } + string[][] out_rows; + int cols = sqlite3_column_count(stmt); + while (sqlite3_step(stmt) == SQLITE_ROW) { + string[] row; + foreach (c; 0 .. cols) { + row ~= (sqlite3_column_type(stmt, c) == SQLITE_NULL) + ? "" : (cast(const(char)*) sqlite3_column_text(stmt, c)).fromStringz.idup; + } + out_rows ~= row; + } + return out_rows; + } + + /+ ↓ the first column of the first row, "" when there is none +/ + string one(string sql) { + auto r = rows(sql); + return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; + } + + /+ ↓ does a table or view of this name exist +/ + bool has(string name) { + return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; + } +} |
