Gleam: what a source holds, and what a database holds unread

The mirror of the D commit before it. ocda/inventory gives the
languages of any form, in the manifest's order, each with its markup
file and the sha256 of its .ssp (stored by the database, taken from
the bytes for a .ssp or a pod), and what a .ocda.db carries. The
sha256 comes from OTP's crypto through a four line Erlang FFI file,
since the standard library has none and a Hex package would be one
more fixed output hash; the whitelist admits src/*.erl for it.

The database loader counts the columns it does not read, on the
conditions the .ssp writer uses, so that it can say what it did not
understand; it could not before. The pod loader's archive, language
and text helpers become public for the inventory.

Checked: the four dumps, D and Gleam of .ssp and .ocda.db, identical
36 of 36, and with --unknown also 36 of 36; --languages of a database
== of its pod, 27 of 27.

(assisted by Claude-Code)

4 files changed, 254 insertions, 3 deletions

diff --git a/gleam/src/ocda/inventory.gleam b/gleam/src/ocda/inventory.gleam
new file mode 100644
index 0000000..6047313
--- /dev/null
+++ b/gleam/src/ocda/inventory.gleam
@@ -0,0 +1,138 @@
+//// What a source holds, beyond the one document the model is loaded from.
+////
+//// The model is one language of one document. Two questions a reader asks
+//// are about the file instead: which languages it holds, and, for a
+//// database, what else it carries. They are here rather than in a reader
+//// because the answer depends on the input form, which is the library's to
+//// know, and so that a reader asking them does not learn which form it was
+//// handed. A mirror of the D library's `ocda.inventory`.
+////
+//// `languages` answers for every form, and answers the same: the languages
+//// in the order the manifest gave them (which every `.ssp` records, in
+//// `@source languages`), each with the markup file it was made from and the
+//// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a
+//// pod it is taken from the bytes, which are the same bytes.
+////
+//// `carried` is the database's: a `.ocda.db` holds the markup, images,
+//// configuration and catalogues it was built from, a `.ssp` holds none of
+//// them, and a pod's are its archive entries rather than anything the
+//// abstraction says.
+
+import gleam/bit_array
+import gleam/dict
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/load_db
+import ocda/load_pod
+import ocda/load_ssp
+import ocda/zip
+import simplifile
+
+pub type Language {
+  Language(lang: String, source_filename: String, ssp_digest: String)
+}
+
+pub type Carried {
+  Carried(
+    role: String,
+    compression: String,
+    count: String,
+    bytes: String,
+    stored: String,
+  )
+}
+
+pub fn languages(path: String) -> Result(List(Language), String) {
+  case load.source_of(path) {
+    load.OcdaDb -> {
+      use held <- result.try(load_db.languages_of_file(path))
+      Ok(
+        list.map(held, fn(l) {
+          Language(l.lang, l.source_filename, l.ssp_digest)
+        }),
+      )
+    }
+    load.Ssp -> {
+      use bytes <- result.try(
+        simplifile.read_bits(path)
+        |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }),
+      )
+      use text <- result.try(
+        bit_array.to_string(bytes)
+        |> result.replace_error(path <> ": not valid UTF-8"),
+      )
+      let doc = load_ssp.parse(text)
+      Ok([
+        Language(
+          source_value(doc.source, "language"),
+          doc.source_filename,
+          sha256_hex(bytes),
+        ),
+      ])
+    }
+    load.Pod -> {
+      use archive <- result.try(load_pod.archive_bytes(path))
+      use entries <- result.try(zip.entries(archive))
+      use found <- result.try(
+        zip.find_suffix(entries, ".ssp")
+        |> list.try_map(fn(e) {
+          use bytes <- result.try(zip.read(archive, e))
+          use text <- result.try(
+            bit_array.to_string(bytes)
+            |> result.replace_error(e.name <> ": not valid UTF-8"),
+          )
+          let doc = load_ssp.parse(text)
+          let lang = result.unwrap(load_pod.lang_of(e.name), "")
+          Ok(#(
+            Language(lang, doc.source_filename, sha256_hex(bytes)),
+            string.split(source_value(doc.source, "languages"), " "),
+          ))
+        }),
+      )
+      let order = case found {
+        [#(_, o), ..] -> list.filter(o, fn(l) { l != "" })
+        [] -> []
+      }
+      let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list
+      // the manifest's order, and anything it did not name after it
+      let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) })
+      let rest =
+        list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) })
+        |> list.map(fn(f) { f.0 })
+      Ok(list.append(named, rest))
+    }
+    load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown))
+  }
+}
+
+/// Only a `.ocda.db` carries files.
+pub fn is_carrier(path: String) -> Bool {
+  load.source_of(path) == load.OcdaDb
+}
+
+pub fn carried(path: String) -> Result(List(Carried), String) {
+  case is_carrier(path) {
+    False -> Ok([])
+    True -> {
+      use rows <- result.try(load_db.carried_of_file(path))
+      Ok(
+        list.filter_map(rows, fn(r) {
+          case r {
+            [role, compression, n, bytes, stored] ->
+              Ok(Carried(role, compression, n, bytes, stored))
+            _ -> Error(Nil)
+          }
+        }),
+      )
+    }
+  }
+}
+
+fn source_value(source: dict.Dict(String, String), key: String) -> String {
+  result.unwrap(dict.get(source, key), "")
+}
+
+@external(erlang, "ocda_ffi", "sha256_hex")
+fn sha256_hex(bytes: BitArray) -> String
diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam
index 641923e..18a5335 100644
--- a/gleam/src/ocda/load_db.gleam
+++ b/gleam/src/ocda/load_db.gleam
@@ -112,6 +112,7 @@ pub fn load_file(
     )
   let document = metadata(conn, chosen.doc_id, document)
   let document = objects(conn, chosen.doc_id, document)
+  let document = count_unread(conn, chosen.doc_id, document)
   Ok(document)
 }
 
@@ -561,3 +562,108 @@ fn json_joined(value: String) -> String {
     parts -> string.join(parts, "|")
   }
 }
+
+/// What the database holds and the model does not.
+///
+/// The text loader meets every property line and counts those it does not
+/// keep, so a reader can say what was read and not understood. This loader
+/// asks for columns by name, and would never see the ones it did not ask
+/// for: it would report that it had understood everything, which is the
+/// silence the count exists to prevent. So it counts them, on the
+/// conditions under which the `.ssp` writes the same property
+/// (specification section 6.3), and the report is the same from either
+/// form, as the D loader's is.
+fn count_unread(
+  conn: sqlight.Connection,
+  doc_id: Int,
+  document: Document,
+) -> Document {
+  let id = int.to_string(doc_id)
+  let counts = [
+    #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"),
+    #("dom_status", "coalesce(dom_status,'') != ''"),
+    #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"),
+    #("heading_lev_collapsed", "heading_lev_collapsed is not null"),
+    #("parent_lev", "coalesce(parent_lev,0) != 0"),
+    #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"),
+    #("attrib", "coalesce(attrib,'') != ''"),
+  ]
+  let property =
+    list.fold(counts, document.unknown.property, fn(acc, c) {
+      let n =
+        count(
+          conn,
+          "select count(*) from objects where doc_id = " <> id <> " and " <> c.1,
+        )
+      case n > 0 {
+        True -> dict.insert(acc, c.0, n)
+        False -> acc
+      }
+    })
+  let property = case has(conn, "object_subtoc") {
+    False -> property
+    True -> {
+      let n =
+        count(
+          conn,
+          "select count(*) from object_subtoc s join objects o
+             on o.id = s.object_id where o.doc_id = " <> id,
+        )
+      case n > 0 {
+        True -> dict.insert(property, "lev4_subtoc", n)
+        False -> property
+      }
+    }
+  }
+  model.Document(
+    ..document,
+    unknown: model.Unknown(..document.unknown, property: property),
+  )
+}
+
+fn count(conn: sqlight.Connection, sql: String) -> Int {
+  let d = decode.field(0, decode.int, decode.success)
+  case sqlight.query(sql, on: conn, with: [], expecting: d) {
+    Ok([n, ..]) -> n
+    _ -> 0
+  }
+}
+
+/// Every language the database at `path` holds, in manifest order.
+pub fn languages_of_file(path: String) -> Result(List(Language), String) {
+  use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+  Ok(languages(conn))
+}
+
+/// What the database at `path` carries besides the abstraction, by role
+/// and storage: role, compression, count, original bytes, stored bytes.
+pub fn carried_of_file(path: String) -> Result(List(List(String)), String) {
+  use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+  case has(conn, "files") {
+    False -> Ok([])
+    True -> {
+      let d = {
+        use role <- decode.field(0, decode.string)
+        use compression <- decode.field(1, decode.string)
+        use n <- decode.field(2, decode.int)
+        use bytes <- decode.field(3, decode.int)
+        use stored <- decode.field(4, decode.int)
+        decode.success([
+          role,
+          compression,
+          int.to_string(n),
+          int.to_string(bytes),
+          int.to_string(stored),
+        ])
+      }
+      sqlight.query(
+        "select role, compression, count(*), sum(bytes), sum(length(data))
+           from files group by role, compression order by role",
+        on: conn,
+        with: [],
+        expecting: d,
+      )
+      |> result.map_error(fn(e) { path <> ": " <> e.message })
+    }
+  }
+}
diff --git a/gleam/src/ocda/load_pod.gleam b/gleam/src/ocda/load_pod.gleam
index 35483c6..05f7add 100644
--- a/gleam/src/ocda/load_pod.gleam
+++ b/gleam/src/ocda/load_pod.gleam
@@ -89,7 +89,7 @@ fn langs(entries: List(zip.Entry)) -> List(String) {
 }
 
 /// `<uid>.<lang>.ssp` -> `<lang>`
-fn lang_of(name: String) -> Result(String, Nil) {
+pub fn lang_of(name: String) -> Result(String, Nil) {
   case string.ends_with(name, ".ssp") {
     False -> Error(Nil)
     True -> {
@@ -104,7 +104,7 @@ fn lang_of(name: String) -> Result(String, Nil) {
 
 /// The bytes decide, not the name: a file misnamed either way is treated by
 /// what it holds.
-fn archive_bytes(path: String) -> Result(BitArray, String) {
+pub fn archive_bytes(path: String) -> Result(BitArray, String) {
   case simplifile.read_bits(path) {
     Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
     Ok(bytes) ->
@@ -133,6 +133,6 @@ fn is_zstd_frame(bytes: BitArray) -> Bool {
 @external(erlang, "unicode", "characters_to_binary")
 fn bit_array_to_string_ffi(b: BitArray) -> String
 
-fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
+pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
   Ok(bit_array_to_string_ffi(b))
 }
diff --git a/gleam/src/ocda_ffi.erl b/gleam/src/ocda_ffi.erl
new file mode 100644
index 0000000..2e395cd
--- /dev/null
+++ b/gleam/src/ocda_ffi.erl
@@ -0,0 +1,7 @@
+%% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's
+%% crypto has and the Gleam standard library does not. Upper case hex, as
+%% the database stores it and the D library writes it.
+-module(ocda_ffi).
+-export([sha256_hex/1]).
+
+sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)).