diff options
Diffstat (limited to 'org/d-ocda.org')
| -rw-r--r-- | org/d-ocda.org | 258 |
1 files changed, 130 insertions, 128 deletions
diff --git a/org/d-ocda.org b/org/d-ocda.org index dd50bde..c4510af 100644 --- a/org/d-ocda.org +++ b/org/d-ocda.org @@ -173,7 +173,133 @@ enum ssp_format_major_read = "2"; #+END_SRC ** load +** what to load +*** inventory + +#+HEADER: :tangle "../dlang/src/ocda/inventory.d" +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +module ocda.inventory; + +/+ ↓ what a source holds, beyond the one document the model is loaded from. + . + The model is one language of one document. Two questions a reader asks + are about the file instead: which languages it holds, and, for a + database, what else it carries. They are here rather than in a reader + because the answer depends on the input form, which is the library's + to know, and so that a reader asking them does not learn which form it + was handed. + . + languages() answers for every form, and answers the same: the + languages in the order the manifest gave them (which every .ssp + records, in @source languages), each with the markup file it was made + from and the sha256 of its .ssp. The database stores that digest; for a + .ssp and a pod it is taken from the bytes, which are the same bytes. + . + carried() is the database's: a .ocda.db holds the markup, images, + configuration and catalogues it was built from, a .ssp holds none of + them, and a pod's are its archive entries rather than anything the + abstraction says. So it answers for a database and says so otherwise. ++/ + +import std.algorithm : endsWith, countUntil; +import std.array : array, split; +import std.conv : to; +import std.digest : toHexString, LetterCase; +import std.digest.sha : sha256Of; +import std.file : read; +import std.string : strip; +import ocda.model; +import ocda.load; +import ocda.load_ssp; +import ocda.load_db; +import ocda.load_pod; +import ocda.sqlite; +import ocda.zip; + +struct OcdaLanguage { + string lang; + string source_filename; // the markup file the abstraction was made from + string ssp_digest; // sha256 of that language's .ssp, upper case hex +} + +struct OcdaCarried { + string role; // source, image, conf, manifest, tools + string compression; // how it is stored: none, zstd + string count; + string bytes; // the files' own size, whatever the storage + string stored; // what they take in the database +} + +OcdaLanguage[] languages(string path) { + final switch (sourceOf(path)) { + case Source.ocda_db: + auto db = Db.open(path); + scope(exit) { db.close(); } + OcdaLanguage[] out_; + foreach (l; dbLanguages(db)) { + out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); + } + return out_; + case Source.ssp: + auto bytes = cast(ubyte[]) read(path); + auto doc = parseSsp(cast(string) bytes.idup); + return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; + case Source.pod: + auto archive = archiveBytes(path); + OcdaLanguage[string] found; + string[] order; + foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { + auto bytes = zipRead(archive, e); + auto doc = parseSsp(cast(string) bytes.idup); + string lang = langOfSspName(e.name); + found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); + if (order.length == 0) { + order = sourceValue(doc, "languages").split; + } + } + /+ ↓ the manifest's order, and anything it did not name after it +/ + OcdaLanguage[] out_; + foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } + foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } + return out_; + case Source.unknown: + throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); + } +} + +/+ ↓ what a database carries, by role and storage; empty for any other form, + which isCarrier() says in advance +/ +bool isCarrier(string path) { + return sourceOf(path) == Source.ocda_db; +} + +OcdaCarried[] carried(string path) { + if (!isCarrier(path)) { return []; } + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + OcdaCarried[] out_; + foreach (r; db.rows( + "SELECT role, compression, count(*), sum(bytes), sum(length(data)) + FROM files GROUP BY role, compression ORDER BY role")) { + out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); + } + return out_; +} + +private string sourceValue(ref OcdaDocument doc, string key) { + if (auto p = key in doc.source) { return *p; } + return ""; +} + +private string hexOf(const(ubyte)[] bytes) { + return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; +} +#+END_SRC + *** load +**** load #+HEADER: :tangle "../dlang/src/ocda/load.d" #+BEGIN_SRC d @@ -249,7 +375,7 @@ OcdaDocument load(string path, string lang = "", bool any_version = false) { } #+END_SRC -*** load db +**** load db #+HEADER: :tangle "../dlang/src/ocda/load_db.d" #+BEGIN_SRC d @@ -611,7 +737,7 @@ private string jsonJoined(string v) { } #+END_SRC -*** load pod +**** load pod #+HEADER: :tangle "../dlang/src/ocda/load_pod.d" #+BEGIN_SRC d @@ -733,7 +859,7 @@ private ptrdiff_t lastIndexOf(string s, char c) { } #+END_SRC -*** load ssp +**** load ssp #+HEADER: :tangle "../dlang/src/ocda/load_ssp.d" #+BEGIN_SRC d @@ -1082,131 +1208,7 @@ private int[] parseIntArray(string s) { } #+END_SRC -** inventory - -#+HEADER: :tangle "../dlang/src/ocda/inventory.d" -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module ocda.inventory; - -/+ ↓ what a source holds, beyond the one document the model is loaded from. - . - The model is one language of one document. Two questions a reader asks - are about the file instead: which languages it holds, and, for a - database, what else it carries. They are here rather than in a reader - because the answer depends on the input form, which is the library's - to know, and so that a reader asking them does not learn which form it - was handed. - . - languages() answers for every form, and answers the same: the - languages in the order the manifest gave them (which every .ssp - records, in @source languages), each with the markup file it was made - from and the sha256 of its .ssp. The database stores that digest; for a - .ssp and a pod it is taken from the bytes, which are the same bytes. - . - carried() is the database's: a .ocda.db holds the markup, images, - configuration and catalogues it was built from, a .ssp holds none of - them, and a pod's are its archive entries rather than anything the - abstraction says. So it answers for a database and says so otherwise. -+/ - -import std.algorithm : endsWith, countUntil; -import std.array : array, split; -import std.conv : to; -import std.digest : toHexString, LetterCase; -import std.digest.sha : sha256Of; -import std.file : read; -import std.string : strip; -import ocda.model; -import ocda.load; -import ocda.load_ssp; -import ocda.load_db; -import ocda.load_pod; -import ocda.sqlite; -import ocda.zip; - -struct OcdaLanguage { - string lang; - string source_filename; // the markup file the abstraction was made from - string ssp_digest; // sha256 of that language's .ssp, upper case hex -} - -struct OcdaCarried { - string role; // source, image, conf, manifest, tools - string compression; // how it is stored: none, zstd - string count; - string bytes; // the files' own size, whatever the storage - string stored; // what they take in the database -} - -OcdaLanguage[] languages(string path) { - final switch (sourceOf(path)) { - case Source.ocda_db: - auto db = Db.open(path); - scope(exit) { db.close(); } - OcdaLanguage[] out_; - foreach (l; dbLanguages(db)) { - out_ ~= OcdaLanguage(l.lang, l.source_filename, l.ssp_digest); - } - return out_; - case Source.ssp: - auto bytes = cast(ubyte[]) read(path); - auto doc = parseSsp(cast(string) bytes.idup); - return [ OcdaLanguage(sourceValue(doc, "language"), doc.source_filename, hexOf(bytes)) ]; - case Source.pod: - auto archive = archiveBytes(path); - OcdaLanguage[string] found; - string[] order; - foreach (e; zipFindSuffix(zipEntries(archive), ".ssp")) { - auto bytes = zipRead(archive, e); - auto doc = parseSsp(cast(string) bytes.idup); - string lang = langOfSspName(e.name); - found[lang] = OcdaLanguage(lang, doc.source_filename, hexOf(bytes)); - if (order.length == 0) { - order = sourceValue(doc, "languages").split; - } - } - /+ ↓ the manifest's order, and anything it did not name after it +/ - OcdaLanguage[] out_; - foreach (l; order) { if (auto p = l in found) { out_ ~= *p; } } - foreach (l, v; found) { if (order.countUntil(l) < 0) { out_ ~= v; } } - return out_; - case Source.unknown: - throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)); - } -} - -/+ ↓ what a database carries, by role and storage; empty for any other form, - which isCarrier() says in advance +/ -bool isCarrier(string path) { - return sourceOf(path) == Source.ocda_db; -} - -OcdaCarried[] carried(string path) { - if (!isCarrier(path)) { return []; } - auto db = Db.open(path); - scope(exit) { db.close(); } - if (!db.has("files")) { return []; } - OcdaCarried[] out_; - foreach (r; db.rows( - "SELECT role, compression, count(*), sum(bytes), sum(length(data)) - FROM files GROUP BY role, compression ORDER BY role")) { - out_ ~= OcdaCarried(r[0], r[1], r[2], r[3], r[4]); - } - return out_; -} - -private string sourceValue(ref OcdaDocument doc, string key) { - if (auto p = key in doc.source) { return *p; } - return ""; -} - -private string hexOf(const(ubyte)[] bytes) { - return toHexString!(LetterCase.upper)(sha256Of(bytes)).idup; -} -#+END_SRC - -** images +*** images #+HEADER: :tangle "../dlang/src/ocda/images.d" #+BEGIN_SRC d |
