module ocda.load_db; /+ ↓ build the model from a .ocda.db. . The point of this module is that it returns *the same value* ocda.load_ssp returns from the same document's .ssp. A writer is then a function from the model to bytes, and cannot tell which serialisation it was given, exactly as spine's own writers cannot. . Written from the sisudoc-abstraction-format specification, ocda-db-format-specification.md, whose section 4 gives the column by column correspondence this module implements. +/ import std.array : split; import std.conv : to; import std.string : strip; import ocda.model; import ocda.sqlite; /+ ↓ the schema major this loader knows (specification section 2) +/ enum schema_major_read = "2"; class OcdaDbError : Exception { this(string msg) { super(msg); } } /+ ↓ every language the database holds, in manifest order +/ struct DbLanguage { int doc_id; string lang; string source_filename; string ssp_digest; } DbLanguage[] dbLanguages(ref Db db) { DbLanguage[] out_; foreach (r; db.rows( "SELECT id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') FROM documents ORDER BY id")) { out_ ~= DbLanguage(r[0].to!int, r[1], r[2], r[3]); } return out_; } /+ ↓ load one language of a database as the model. lang == "" takes the first language the manifest gave. +/ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) { auto db = Db.open(path); scope(exit) { db.close(); } if (!db.has("documents") || !db.has("objects") || !db.has("metadata")) { throw new OcdaDbError(path ~ ": no documents/objects/metadata tables;" ~ " it does not look like a .ocda.db"); } /+ ↓ specification sections 2 and 3.2: the rows describing the file rather than a document in it are the only ones with doc_id null +/ string schema_version = db.one( "SELECT value FROM metadata WHERE doc_id IS NULL AND key = 'schema.version'"); if (schema_version.length > 0 && !any_version) { auto major = schema_version.split(".")[0]; if (major != schema_major_read) { throw new OcdaDbError(path ~ " is schema v" ~ schema_version ~ ", this loader knows v" ~ schema_major_read ~ ".x"); } } auto languages = dbLanguages(db); if (languages.length == 0) { throw new OcdaDbError(path ~ ": no documents rows"); } DbLanguage chosen = languages[0]; if (lang.length > 0) { bool found = false; foreach (l; languages) { if (l.lang == lang) { chosen = l; found = true; break; } } if (!found) { string have; foreach (i, l; languages) { have ~= (i > 0 ? " " : "") ~ l.lang; } throw new OcdaDbError(path ~ ": no language '" ~ lang ~ "'; it holds: " ~ have); } } OcdaDocument doc; doc.format_version = schema_version; doc.source_filename = chosen.source_filename; /+ ↓ @source of the .ssp corresponds to the documents row and the source.digest metadata (specification section 5.1 of the .ssp, and section 7 here) +/ doc.source["language"] = chosen.lang; string languages_joined; foreach (i, l; languages) { languages_joined ~= (i > 0 ? " " : "") ~ l.lang; } doc.source["languages"] = languages_joined; auto digest = db.one("SELECT value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " AND key = 'source.digest'"); if (digest.length > 0) { doc.source["digest"] = digest; } /+ ↓ @meta: every metadata row of this language that is not a source.* property. source.* describes the artefact rather than the document and lives in @source in the text form. +/ foreach (r; db.rows("SELECT key, value FROM metadata WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY key")) { if (r[0].length > 7 && r[0][0 .. 7] == "source.") { continue; } doc.metadata[r[0]] = r[1]; } /+ ↓ the objects, in document order within each section +/ foreach (r; db.rows( "SELECT section, seq, ocn, is_a, coalesce(is_of_part,''), coalesce(is_of_section,''), coalesce(is_of_type,''), coalesce(heading_level,-1), coalesce(identifier,''), coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), coalesce(children,''), coalesce(ancestors,''), coalesce(dummy_heading,0), coalesce(object_number_off,0), coalesce(indent_base,0), coalesce(indent_hang,0), coalesce(bullet,0), coalesce(lang,''), coalesce(has_links,0), coalesce(has_notes_reg,0), coalesce(has_notes_star,0), coalesce(has_images,0), coalesce(segment,''), coalesce(segment_prev,''), coalesce(segment_next,''), coalesce(anchor,''), coalesce(segment_html_is,''), coalesce(table_cols,0), coalesce(table_widths,''), coalesce(table_aligns,''), coalesce(table_header,0), coalesce(code_linenumbers,0), coalesce(heading_ancestors_text,''), coalesce(sha256,''), coalesce(text,''), id FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY CASE section WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 WHEN 'endnotes' THEN 4 WHEN 'glossary' THEN 5 WHEN 'bibliography' THEN 6 WHEN 'bookindex' THEN 7 WHEN 'blurb' THEN 8 WHEN 'tail' THEN 9 ELSE 10 END, seq")) { OcdaObject obj; obj.section = r[0]; obj.ocn = r[2].to!int; obj.is_a = r[3]; obj.part = r[4]; obj.own_section = (r[5] == r[0]) ? "" : r[5]; obj.is_of_type = r[6]; /+ ↓ the database holds the level as a number, the text form as the letter or digit the markup wrote. Model section 4 is the mapping. +/ obj.heading_level = levelName(r[7].to!int); obj.identifier = r[8]; obj.parent_ocn = r[9].to!int; obj.last_descendant_ocn = r[10].to!int; obj.children = jsonInts(r[11]); obj.ancestors = jsonInts(r[12]); obj.dummy_heading = (r[13] != "0"); obj.ocn_off = (r[14] != "0"); obj.indent_base = r[15].to!int; obj.indent_hang = r[16].to!int; obj.bullet = (r[17] != "0"); obj.lang = r[18]; obj.has_links = (r[19] != "0"); obj.has_notes_reg = (r[20] != "0"); obj.has_notes_star = (r[21] != "0"); obj.has_images = (r[22] != "0"); obj.segment = r[23]; obj.segment_prev = r[24]; obj.segment_next = r[25]; obj.anchor = r[26]; obj.segment_html_is = r[27]; obj.table_cols = r[28].to!int; obj.table_widths = jsonWords(r[29]); obj.table_aligns = jsonWords(r[30]); obj.table_header = (r[31] != "0"); obj.code_linenumbers = (r[32] != "0"); obj.heading_ancestors_text = jsonJoined(r[33]); obj.sha256 = r[34]; obj.text = r[35]; /+ ↓ the four repeatable properties of the text form are child tables here (specification section 4) +/ string object_id = r[36]; foreach (i; db.rows("SELECT name, coalesce(sha256,''), coalesce(bytes,0), coalesce(width,0), coalesce(height,0), coalesce(missing,0) FROM object_images WHERE object_id = " ~ object_id ~ " ORDER BY seq")) { OcdaImage img; img.name = i[0]; img.sha256 = i[1]; img.bytes = i[2].to!ulong; img.width = i[3].to!int; img.height = i[4].to!int; img.missing = (i[5] != "0"); obj.images ~= img; } foreach (l; db.rows("SELECT url FROM object_links WHERE object_id = " ~ object_id ~ " ORDER BY seq")) { obj.stow_link ~= l[0]; } auto section = doc.sectionByName(r[0]); if (section !is null) { *section ~= obj; } } return doc; } /+ ↓ model section 4: the eight levels and their indices +/ private string levelName(int index) { switch (index) { case 0: return "A"; case 1: return "B"; case 2: return "C"; case 3: return "D"; case 4: return "1"; case 5: return "2"; case 6: return "3"; case 7: return "4"; default: return ""; } } /+ ↓ the fixed width arrays are JSON in the database and space separated in the text form (specification section 3.3) +/ private int[] jsonInts(string v) { int[] out_; foreach (w; jsonWords(v).split(" ")) { if (w.length == 0) { continue; } try { out_ ~= w.to!int; } catch (Exception) {} } return out_; } private string jsonWords(string v) { string out_; bool in_string = false; foreach (c; v) { switch (c) { case '[': case ']': case ' ': break; case '"': in_string = !in_string; break; case ',': out_ ~= " "; break; default: out_ ~= c; break; } } return out_.strip; } /+ ↓ heading_ancestors_text is eight slots, joined with "|" in the text form and a JSON array here +/ private string jsonJoined(string v) { if (v.length == 0) { return ""; } string out_; bool in_string = false; bool first = true; string cur; foreach (c; v) { if (c == '"') { in_string = !in_string; if (!in_string) { if (!first) { out_ ~= "|"; } out_ ~= cur; cur = ""; first = false; } continue; } if (in_string) { cur ~= c; } } return out_; }