diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | gleam/src/ocda/inventory.gleam | 138 | ||||
| -rw-r--r-- | gleam/src/ocda/load_db.gleam | 106 | ||||
| -rw-r--r-- | gleam/src/ocda/load_pod.gleam | 6 | ||||
| -rw-r--r-- | gleam/src/ocda_ffi.erl | 7 |
5 files changed, 256 insertions, 3 deletions
@@ -50,6 +50,8 @@ result-* # ── gleam ──────────────────────────────────────────────────────────────── !**/*.gleam +# Erlang, for what Gleam reaches through FFI (a sha256, from OTP crypto) +!**/src/*.erl # gleam.toml is the manifest, manifest.toml is the lockfile: both belong !**/*.toml *.beam diff --git a/gleam/src/ocda/inventory.gleam b/gleam/src/ocda/inventory.gleam new file mode 100644 index 0000000..6047313 --- /dev/null +++ b/gleam/src/ocda/inventory.gleam @@ -0,0 +1,138 @@ +//// What a source holds, beyond the one document the model is loaded from. +//// +//// The model is one language of one document. Two questions a reader asks +//// are about the file instead: which languages it holds, and, for a +//// database, what else it carries. They are here rather than in a reader +//// because the answer depends on the input form, which is the library's to +//// know, and so that a reader asking them does not learn which form it was +//// handed. A mirror of the D library's `ocda.inventory`. +//// +//// `languages` answers for every form, and answers the same: the languages +//// in the order the manifest gave them (which every `.ssp` records, in +//// `@source languages`), each with the markup file it was made from and the +//// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a +//// pod it is taken from the bytes, which are the same bytes. +//// +//// `carried` is the database's: a `.ocda.db` holds the markup, images, +//// configuration and catalogues it was built from, a `.ssp` holds none of +//// them, and a pod's are its archive entries rather than anything the +//// abstraction says. + +import gleam/bit_array +import gleam/dict +import gleam/list +import gleam/result +import gleam/string +import ocda/load +import ocda/load_db +import ocda/load_pod +import ocda/load_ssp +import ocda/zip +import simplifile + +pub type Language { + Language(lang: String, source_filename: String, ssp_digest: String) +} + +pub type Carried { + Carried( + role: String, + compression: String, + count: String, + bytes: String, + stored: String, + ) +} + +pub fn languages(path: String) -> Result(List(Language), String) { + case load.source_of(path) { + load.OcdaDb -> { + use held <- result.try(load_db.languages_of_file(path)) + Ok( + list.map(held, fn(l) { + Language(l.lang, l.source_filename, l.ssp_digest) + }), + ) + } + load.Ssp -> { + use bytes <- result.try( + simplifile.read_bits(path) + |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }), + ) + use text <- result.try( + bit_array.to_string(bytes) + |> result.replace_error(path <> ": not valid UTF-8"), + ) + let doc = load_ssp.parse(text) + Ok([ + Language( + source_value(doc.source, "language"), + doc.source_filename, + sha256_hex(bytes), + ), + ]) + } + load.Pod -> { + use archive <- result.try(load_pod.archive_bytes(path)) + use entries <- result.try(zip.entries(archive)) + use found <- result.try( + zip.find_suffix(entries, ".ssp") + |> list.try_map(fn(e) { + use bytes <- result.try(zip.read(archive, e)) + use text <- result.try( + bit_array.to_string(bytes) + |> result.replace_error(e.name <> ": not valid UTF-8"), + ) + let doc = load_ssp.parse(text) + let lang = result.unwrap(load_pod.lang_of(e.name), "") + Ok(#( + Language(lang, doc.source_filename, sha256_hex(bytes)), + string.split(source_value(doc.source, "languages"), " "), + )) + }), + ) + let order = case found { + [#(_, o), ..] -> list.filter(o, fn(l) { l != "" }) + [] -> [] + } + let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list + // the manifest's order, and anything it did not name after it + let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) }) + let rest = + list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) }) + |> list.map(fn(f) { f.0 }) + Ok(list.append(named, rest)) + } + load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown)) + } +} + +/// Only a `.ocda.db` carries files. +pub fn is_carrier(path: String) -> Bool { + load.source_of(path) == load.OcdaDb +} + +pub fn carried(path: String) -> Result(List(Carried), String) { + case is_carrier(path) { + False -> Ok([]) + True -> { + use rows <- result.try(load_db.carried_of_file(path)) + Ok( + list.filter_map(rows, fn(r) { + case r { + [role, compression, n, bytes, stored] -> + Ok(Carried(role, compression, n, bytes, stored)) + _ -> Error(Nil) + } + }), + ) + } + } +} + +fn source_value(source: dict.Dict(String, String), key: String) -> String { + result.unwrap(dict.get(source, key), "") +} + +@external(erlang, "ocda_ffi", "sha256_hex") +fn sha256_hex(bytes: BitArray) -> String diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam index 641923e..18a5335 100644 --- a/gleam/src/ocda/load_db.gleam +++ b/gleam/src/ocda/load_db.gleam @@ -112,6 +112,7 @@ pub fn load_file( ) let document = metadata(conn, chosen.doc_id, document) let document = objects(conn, chosen.doc_id, document) + let document = count_unread(conn, chosen.doc_id, document) Ok(document) } @@ -561,3 +562,108 @@ fn json_joined(value: String) -> String { parts -> string.join(parts, "|") } } + +/// What the database holds and the model does not. +/// +/// The text loader meets every property line and counts those it does not +/// keep, so a reader can say what was read and not understood. This loader +/// asks for columns by name, and would never see the ones it did not ask +/// for: it would report that it had understood everything, which is the +/// silence the count exists to prevent. So it counts them, on the +/// conditions under which the `.ssp` writes the same property +/// (specification section 6.3), and the report is the same from either +/// form, as the D loader's is. +fn count_unread( + conn: sqlight.Connection, + doc_id: Int, + document: Document, +) -> Document { + let id = int.to_string(doc_id) + let counts = [ + #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"), + #("dom_status", "coalesce(dom_status,'') != ''"), + #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"), + #("heading_lev_collapsed", "heading_lev_collapsed is not null"), + #("parent_lev", "coalesce(parent_lev,0) != 0"), + #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"), + #("attrib", "coalesce(attrib,'') != ''"), + ] + let property = + list.fold(counts, document.unknown.property, fn(acc, c) { + let n = + count( + conn, + "select count(*) from objects where doc_id = " <> id <> " and " <> c.1, + ) + case n > 0 { + True -> dict.insert(acc, c.0, n) + False -> acc + } + }) + let property = case has(conn, "object_subtoc") { + False -> property + True -> { + let n = + count( + conn, + "select count(*) from object_subtoc s join objects o + on o.id = s.object_id where o.doc_id = " <> id, + ) + case n > 0 { + True -> dict.insert(property, "lev4_subtoc", n) + False -> property + } + } + } + model.Document( + ..document, + unknown: model.Unknown(..document.unknown, property: property), + ) +} + +fn count(conn: sqlight.Connection, sql: String) -> Int { + let d = decode.field(0, decode.int, decode.success) + case sqlight.query(sql, on: conn, with: [], expecting: d) { + Ok([n, ..]) -> n + _ -> 0 + } +} + +/// Every language the database at `path` holds, in manifest order. +pub fn languages_of_file(path: String) -> Result(List(Language), String) { + use conn <- with_connection("file:" <> path <> "?mode=ro", path) + Ok(languages(conn)) +} + +/// What the database at `path` carries besides the abstraction, by role +/// and storage: role, compression, count, original bytes, stored bytes. +pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { + use conn <- with_connection("file:" <> path <> "?mode=ro", path) + case has(conn, "files") { + False -> Ok([]) + True -> { + let d = { + use role <- decode.field(0, decode.string) + use compression <- decode.field(1, decode.string) + use n <- decode.field(2, decode.int) + use bytes <- decode.field(3, decode.int) + use stored <- decode.field(4, decode.int) + decode.success([ + role, + compression, + int.to_string(n), + int.to_string(bytes), + int.to_string(stored), + ]) + } + sqlight.query( + "select role, compression, count(*), sum(bytes), sum(length(data)) + from files group by role, compression order by role", + on: conn, + with: [], + expecting: d, + ) + |> result.map_error(fn(e) { path <> ": " <> e.message }) + } + } +} diff --git a/gleam/src/ocda/load_pod.gleam b/gleam/src/ocda/load_pod.gleam index 35483c6..05f7add 100644 --- a/gleam/src/ocda/load_pod.gleam +++ b/gleam/src/ocda/load_pod.gleam @@ -89,7 +89,7 @@ fn langs(entries: List(zip.Entry)) -> List(String) { } /// `<uid>.<lang>.ssp` -> `<lang>` -fn lang_of(name: String) -> Result(String, Nil) { +pub fn lang_of(name: String) -> Result(String, Nil) { case string.ends_with(name, ".ssp") { False -> Error(Nil) True -> { @@ -104,7 +104,7 @@ fn lang_of(name: String) -> Result(String, Nil) { /// The bytes decide, not the name: a file misnamed either way is treated by /// what it holds. -fn archive_bytes(path: String) -> Result(BitArray, String) { +pub fn archive_bytes(path: String) -> Result(BitArray, String) { case simplifile.read_bits(path) { Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) Ok(bytes) -> @@ -133,6 +133,6 @@ fn is_zstd_frame(bytes: BitArray) -> Bool { @external(erlang, "unicode", "characters_to_binary") fn bit_array_to_string_ffi(b: BitArray) -> String -fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { +pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { Ok(bit_array_to_string_ffi(b)) } diff --git a/gleam/src/ocda_ffi.erl b/gleam/src/ocda_ffi.erl new file mode 100644 index 0000000..2e395cd --- /dev/null +++ b/gleam/src/ocda_ffi.erl @@ -0,0 +1,7 @@ +%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's +%% crypto has and the Gleam standard library does not. Upper case hex, as +%% the database stores it and the D library writes it. +-module(ocda_ffi). +-export([sha256_hex/1]). + +sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). |