diff options
Diffstat (limited to 'dlang/src/ocda/load_db.d')
| -rw-r--r-- | dlang/src/ocda/load_db.d | 310 |
1 files changed, 310 insertions, 0 deletions
diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d new file mode 100644 index 0000000..dec0626 --- /dev/null +++ b/dlang/src/ocda/load_db.d @@ -0,0 +1,310 @@ +module ocda.load_db; + +/+ ↓ build the model from a .ocda.db. + . + The point of this module is that it returns *the same value* + ocda.load_ssp returns from the same document's .ssp. A writer is then + a function from the model to bytes, and cannot tell which + serialisation it was given, exactly as spine's own writers cannot. + . + Written from the sisudoc-abstraction-format specification, + ocda-db-format-specification.md, whose section 4 gives the column by + column correspondence this module implements. ++/ + +import std.array : join, split; +import std.json : JSONType, parseJSON; +import std.conv : to; +import std.algorithm : startsWith; +import std.string : strip; +import ocda.model; +import ocda.sqlite; + +/+ ↓ the schema major this loader knows (specification section 2) +/ +enum schema_major_read = "2"; + +class OcdaDbError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ every language the database holds, in manifest order +/ +struct DbLanguage { + int doc_id; + string lang; + string source_filename; + string ssp_digest; +} + +DbLanguage[] dbLanguages(ref Db db) { + DbLanguage[] out_; + foreach (r; db.rows( + "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') + FROM documents ORDER BY id")) { + out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); + } + return out_; +} + +/+ ↓ load one language of a database as the model. + lang == "" takes the first language the manifest gave. ++/ +OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { + auto db = Db.open(path); + scope(exit) { db.close(); } + + if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { + throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" + ~ " it does not look like a .ocda.db"); + } + + /+ ↓ specification sections 2 and 3.2: the rows describing the file rather + than a document in it are the only ones with doc_id null +/ + string schema_version = db.one( + "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); + if (schema_version.length > 0 && !any_version) { + auto major = schema_version.split(".")[0]; + if (major != schema_major_read) { + throw new OcdaDbError(path ~ " is schema v" ~ schema_version + ~ ", this loader knows v" ~ schema_major_read ~ ".x"); + } + } + + auto languages = dbLanguages(db); + if (languages.length == 0) { + throw new OcdaDbError(path ~ ": no documents rows"); + } + DbLanguage chosen = languages[0]; + if (lang.length > 0) { + bool found = false; + foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } + if (!found) { + string have; + foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } + throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); + } + } + + OcdaDocument doc; + doc.format_version = schema_version; + doc.source_filename = chosen.source_filename; + + /+ ↓ @source of the .ssp corresponds to the documents row and the + source.digest metadata (specification section 5.1 of the .ssp, and + section 7 here) +/ + doc.source["language"] = chosen.lang; + string languages_joined; + foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } + doc.source["languages"] = languages_joined; + auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); + if (digest.length > 0) { doc.source["digest"] = digest; } + + /+ ↓ @meta: every metadata row of this language that is not a source.* + property. source.* describes the artefact rather than the document + and lives in @source in the text form. + +/ + foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " + ~ chosen.doc_id.to!string ~ " ORDER BY key")) { + string key = r[0]; + if (key.startsWith("source.")) { continue; } + /+ ↓ translation.ocn_aligned says whether the languages of this + document share one numbering. A .ocda.db can know it because it + holds every language; a .ssp is one language and cannot. So it is + a fact about the *file* rather than about the document, and putting + it in the document's metadata would let a writer tell which + serialisation it was handed, which is the one thing this library + exists to prevent. Left out until the model has a place for + collection-wide facts. + +/ + if (key.startsWith("translation.")) { continue; } + /+ ↓ the database has one metadata table where the text form has three + blocks, and tells them apart by a key prefix. Unprefixing them + here is what makes the two loaders give one model: a writer asks + doc.make["footer"] whichever serialisation it was handed, and + never doc.metadata["make.footer"]. + +/ + if (key.startsWith("make.")) { + doc.make[key["make.".length .. $]] = r[1].strip; + } else if (key.startsWith("doc_has.")) { + doc.doc_has[key["doc_has.".length .. $]] = r[1].strip; + } else { + doc.metadata[key] = r[1].strip; + } + } + + /+ ↓ the objects, in document order within each section +/ + foreach (r; db.rows( + "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), + coalesce(is_of_section,''), coalesce(is_of_type,''), + coalesce(heading_level,-1), coalesce(identifier,''), + coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), + coalesce(children,''), coalesce(ancestors,''), + coalesce(dummy_heading,0), coalesce(object_number_off,0), + coalesce(indent_base,0), coalesce(indent_hang,0), + coalesce(bullet,0), coalesce(lang,''), + coalesce(has_links,0), coalesce(has_notes_reg,0), + coalesce(has_notes_star,0), coalesce(has_images,0), + coalesce(segment,''), coalesce(segment_prev,''), + coalesce(segment_next,''), coalesce(anchor,''), + coalesce(segment_html_is,''), + coalesce(table_cols,0), coalesce(table_widths,''), + coalesce(table_aligns,''), coalesce(table_header,0), + coalesce(code_linenumbers,0), + coalesce(heading_ancestors_text,''), coalesce(sha256,''), + coalesce(text,''), id, coalesce(heading_lev_anchor,''), + coalesce(segment_epub,''), coalesce(segment_epub_is,'') + FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string + ~ " ORDER BY CASE section + WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 + WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 + WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 + WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { + OcdaObject obj; + obj.section = r[0]; + obj.ocn = r[2].to!int; + obj.is_a = r[3]; + obj.part = r[4]; + obj.own_section = (r[5] == r[0]) ? "" : r[5]; + obj.is_of_type = r[6]; + /+ ↓ the database holds the level as a number, the text form as the + letter or digit the markup wrote. Model section 4 is the mapping. +/ + obj.heading_level = levelName(r[7].to!int); + obj.identifier = r[8]; + obj.parent_ocn = r[9].to!int; + obj.last_descendant_ocn = r[10].to!int; + obj.children = jsonInts(r[11]); + obj.ancestors = jsonInts(r[12]); + obj.dummy_heading = (r[13] != "0"); + obj.ocn_off = (r[14] != "0"); + obj.indent_base = r[15].to!int; + obj.indent_hang = r[16].to!int; + obj.bullet = (r[17] != "0"); + obj.lang = r[18]; + obj.has_links = (r[19] != "0"); + obj.has_notes_reg = (r[20] != "0"); + obj.has_notes_star = (r[21] != "0"); + obj.has_images = (r[22] != "0"); + obj.segment = r[23]; + obj.segment_prev = r[24]; + obj.segment_next = r[25]; + obj.anchor = r[26]; + obj.segment_html_is = r[27]; + obj.heading_lev_anchor = r[37]; + obj.segment_epub = r[38]; + obj.segment_epub_is = r[39]; + obj.table_cols = r[28].to!int; + obj.table_widths = jsonWords(r[29]); + obj.table_aligns = jsonWords(r[30]); + obj.table_header = (r[31] != "0"); + obj.code_linenumbers = (r[32] != "0"); + obj.heading_ancestors_text = jsonJoined(r[33]); + obj.sha256 = r[34]; + obj.text = r[35]; + + /+ ↓ the four repeatable properties of the text form are child tables + here (specification section 4) +/ + string object_id = r[36]; + foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), + coalesce(width,0), coalesce(height,0), + coalesce(missing,0) + FROM object_images WHERE object_id = " ~ object_id + ~ " ORDER BY seq")) { + OcdaImage img; + img.name = i[0]; + img.sha256 = i[1]; + img.bytes = i[2].to!ulong; + img.width = i[3].to!int; + img.height = i[4].to!int; + img.missing = (i[5] != "0"); + obj.images ~= img; + } + foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.stow_link ~= l[0]; + } + foreach (a; db.rows("SELECT anchor FROM object_anchors WHERE object_id = " + ~ object_id ~ " ORDER BY seq")) { + obj.anchor_tags ~= a[0]; + } + + auto section = doc.sectionByName(r[0]); + if (section !is null) { *section ~= obj; } + } + return doc; +} + +/+ ↓ model section 4: the eight levels and their indices +/ +private string levelName(int index) { + switch (index) { + case 0: return "A"; + case 1: return "B"; + case 2: return "C"; + case 3: return "D"; + case 4: return "1"; + case 5: return "2"; + case 6: return "3"; + case 7: return "4"; + default: return ""; + } +} + +/+ ↓ the JSON columns, read as JSON. + . + The fixed width arrays are JSON in the database and space separated + (or "|" separated) in the text form. This was first written as a + character scan that threw away brackets and quotes, which is wrong the + moment a value holds an escape: Phobos' JSONValue.toString escapes + "/" as "\/", so every heading containing a slash came back with a + backslash in it. 36 of 36 documents differed on + heading_ancestors_text and nothing else had shown it. + . + So: parse. The values are small, there are few of them, and a real + parser is shorter than the scan it replaces. ++/ +private string[] jsonStrings(string v) { + if (v.length == 0) { return []; } + string[] out_; + try { + auto j = parseJSON(v); + if (j.type != JSONType.array) { return []; } + foreach (e; j.array) { + switch (e.type) { + case JSONType.string: out_ ~= e.str; break; + case JSONType.integer: out_ ~= e.integer.to!string; break; + case JSONType.uinteger: out_ ~= e.uinteger.to!string; break; + case JSONType.float_: out_ ~= e.floating.to!string; break; + case JSONType.true_: out_ ~= "true"; break; + case JSONType.false_: out_ ~= "false"; break; + default: out_ ~= ""; break; + } + } + } catch (Exception) { + return []; + } + return out_; +} + +private int[] jsonInts(string v) { + int[] out_; + foreach (s; jsonStrings(v)) { + if (s.length == 0) { continue; } + try { out_ ~= s.to!int; } catch (Exception) {} + } + return out_; +} + +/+ ↓ space separated in the text form: table_widths, table_aligns +/ +private string jsonWords(string v) { + return jsonStrings(v).join(" "); +} + +/+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text + form and a JSON array here. The empty slots are significant: they say + which levels the heading sits under, so the separators carry as much + as the text does. ++/ +private string jsonJoined(string v) { + auto parts = jsonStrings(v); + if (parts.length == 0) { return ""; } + return parts.join("|"); +} |
