diff options
Diffstat (limited to 'dlang/src/ocda')
| -rw-r--r-- | dlang/src/ocda/inventory.d | 117 | ||||
| -rw-r--r-- | dlang/src/ocda/load_db.d | 33 | ||||
| -rw-r--r-- | dlang/src/ocda/load_pod.d | 4 |
3 files changed, 152 insertions, 2 deletions
diff --git a/dlang/src/ocda/inventory.d b/dlang/src/ocda/inventory.d new file mode 100644 index 0000000..dba2156 --- /dev/null +++ b/dlang/src/ocda/inventory.d @@ -0,0 +1,117 @@ +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index eb3d71b..2f59437 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -242,9 +242,42 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) auto section = doc.sectionByName(r[0]); if (section !is null) { *section ~= obj; } } + countUnread(db, chosen.doc_id, doc); return doc; } +/+ ↓ what the database holds and the model does not. + . + The text loader meets every property line and counts those it does not + keep, so a reader can say what was read and not understood. This + loader asks for columns by name, and would never see the ones it did + not ask for: it would report that it had understood everything, which + is the silence the count exists to prevent. So it counts them, on the + conditions under which the .ssp writes the same property (specification + section 6.3), and the report is the same from either form: checked, + 36 of 36. ++/ +private void countUnread(ref Db db, int doc_id, ref OcdaDocument doc) { + string id = doc_id.to!string; + void count(string property, string where) { + auto n = db.one("SELECT count(*) FROM objects WHERE doc_id = " ~ id + ~ " AND " ~ where).to!size_t; + if (n > 0) { doc.unknown.property[property] = n; } + } + count("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"); + count("dom_status", "coalesce(dom_status,'') != ''"); + count("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"); + count("heading_lev_collapsed", "heading_lev_collapsed IS NOT NULL"); + count("parent_lev", "coalesce(parent_lev,0) != 0"); + count("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"); + count("attrib", "coalesce(attrib,'') != ''"); + if (db.has("object_subtoc")) { + auto n = db.one("SELECT count(*) FROM object_subtoc s JOIN objects o" + ~ " ON o.id = s.object_id WHERE o.doc_id = " ~ id).to!size_t; + if (n > 0) { doc.unknown.property["lev4_subtoc"] = n; } + } +} + /+ ↓ model section 4: the eight levels and their indices +/ private string levelName(int index) { switch (index) { diff --git a/dlang/src/ocda/load_pod.d b/dlang/src/ocda/load_pod.d index 75993be..8f59fe3 100644 --- a/dlang/src/ocda/load_pod.d +++ b/dlang/src/ocda/load_pod.d @@ -92,7 +92,7 @@ OcdaDocument loadPodFile(string path, string lang = "") { The bytes decide, not the name: a .sisupod is a zstd frame and a .zip is not, and a file misnamed either way still reads. +/ -private ubyte[] archiveBytes(string path) { +package ubyte[] archiveBytes(string path) { auto bytes = cast(ubyte[]) read(path); return zstdIsFrame(bytes) ? zstdDecompress(bytes, max_pod_size) @@ -100,7 +100,7 @@ private ubyte[] archiveBytes(string path) { } /+ ↓ <uid>.<lang>.ssp -> <lang> +/ -private string langOfSspName(string name) { +package string langOfSspName(string name) { if (!name.endsWith(".ssp")) { return ""; } string stem = name[0 .. $ - 4]; auto dot = stem.lastIndexOf('.'); |
