summaryrefslogtreecommitdiffhomepage
path: root/gleam/src
diff options
context:
space:
mode:
Diffstat (limited to 'gleam/src')
-rw-r--r--gleam/src/COPYRIGHT45
-rw-r--r--gleam/src/ocda/dump.gleam247
-rw-r--r--gleam/src/ocda/images.gleam135
-rw-r--r--gleam/src/ocda/load.gleam79
-rw-r--r--gleam/src/ocda/load_db.gleam532
-rw-r--r--gleam/src/ocda/load_pod.gleam138
-rw-r--r--gleam/src/ocda/load_ssp.gleam478
-rw-r--r--gleam/src/ocda/model.gleam278
-rw-r--r--gleam/src/ocda/zip.gleam170
-rw-r--r--gleam/src/ocda_lib.gleam136
10 files changed, 2238 insertions, 0 deletions
diff --git a/gleam/src/COPYRIGHT b/gleam/src/COPYRIGHT
new file mode 100644
index 0000000..7e7dd56
--- /dev/null
+++ b/gleam/src/COPYRIGHT
@@ -0,0 +1,45 @@
+- Name: sisudoc-ocda-lib
+ - Description: part of SiSU spine (Doc Reform): documents, structuring,
+ processing, publishing and search
+ - [https://www.sisudoc.org]
+
+ - Author: Ralph Amissah
+ [ralph.amissah@gmail.com]
+
+ - Copyright: (C) 2026 Ralph Amissah, All Rights Reserved.
+
+ - License: AGPL 3 or later, for everything in this repository unless a file or
+ a directory specifies otherwise.:
+
+ sisudoc-ocda-lib
+
+ part of SiSU spine (Doc Reform): documents, structuring,
+ processing, publishing and search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU Affero General Public License (AGPLv3) as
+ published by the Free Software Foundation, either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see [https://www.gnu.org/licenses/].
+
+ If you have Internet connection, the latest version of the AGPL should be
+ available at: [https://www.gnu.org/licenses/agpl-3.0.html]
+
+ - Spine, Doc Reform (related to SiSU) uses standard:
+ - docReform markup syntax (based on SiSU markup)
+ - standard SiSU markup syntax with modified headers and minor
+ modifications
+ - docReform object numbering (based on SiSU object citation numbering)
+ - standard SiSU document object numbering
+
+ - Homepages:
+ [https://www.sisudoc.org]
diff --git a/gleam/src/ocda/dump.gleam b/gleam/src/ocda/dump.gleam
new file mode 100644
index 0000000..a75da67
--- /dev/null
+++ b/gleam/src/ocda/dump.gleam
@@ -0,0 +1,247 @@
+//// The model, written out canonically.
+////
+//// Not an output format: an *oracle*. Two libraries in two languages
+//// implement the same model from the same specification, and the cheap way
+//// to ask whether they agree is to have each write the whole loaded
+//// document as text and diff the two. Byte for byte identical to the D
+//// library's `ocda/dump.d`, which is the whole point of it.
+////
+//// The rules, which both implementations follow:
+////
+//// - every line is `<key> <value>`, the value running to end of line,
+//// and only one free-form value per line;
+//// - a field at its default is *omitted*, so the output says what the
+//// document has rather than what the type declares;
+//// - dictionaries come in sorted key order, because neither language's
+//// map has an order worth relying on;
+//// - sections come in the fixed order of the format.
+////
+//// Anything that reads this to reconstruct a document has misunderstood
+//// it. Load the `.ssp` or the `.ocda.db`.
+
+import gleam/dict.{type Dict}
+import gleam/int
+import gleam/list
+import gleam/string
+import ocda/model.{type Document, type Object}
+
+pub fn document(doc: Document) -> String {
+ let lines =
+ []
+ |> kv("version", doc.format_version)
+ |> kv("source_filename", doc.source_filename)
+ |> entries("source", doc.source)
+ |> entries("meta", doc.metadata)
+ |> entries("make", doc.make)
+ |> entries("doc_has", doc.doc_has)
+ |> counts(doc)
+ |> objects(doc)
+ lines |> list.reverse |> join
+}
+
+/// What the loader met and did not understand, sorted.
+///
+/// Separate from the document dump: it describes the loader, not the
+/// document, and the `.ocda.db` loader has nothing to report where the
+/// `.ssp` loader does.
+pub fn unknown(doc: Document) -> String {
+ []
+ |> numbered("property", doc.unknown.property)
+ |> numbered("block", doc.unknown.block)
+ |> numbered("line", doc.unknown.line)
+ |> list.reverse
+ |> join
+}
+
+fn join(lines: List(String)) -> String {
+ case lines {
+ [] -> ""
+ _ -> string.join(lines, "\n") <> "\n"
+ }
+}
+
+/// Accumulated newest-first and reversed once, so that nothing appends to
+/// the tail of a list that grows to a few hundred thousand lines.
+fn kv(acc: List(String), key: String, value: String) -> List(String) {
+ case value {
+ "" -> acc
+ v -> [key <> " " <> v, ..acc]
+ }
+}
+
+fn kn(acc: List(String), key: String, value: Int) -> List(String) {
+ case value {
+ 0 -> acc
+ n -> [key <> " " <> int.to_string(n), ..acc]
+ }
+}
+
+fn kb(acc: List(String), key: String, value: Bool) -> List(String) {
+ case value {
+ False -> acc
+ True -> [key <> " 1", ..acc]
+ }
+}
+
+fn entries(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, String),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines]
+ })
+}
+
+fn numbered(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, Int),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines]
+ })
+}
+
+fn counts(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ case model.section(doc, name) {
+ [] -> lines
+ objects ->
+ [
+ "section " <> name <> " " <> int.to_string(list.length(objects)),
+ ..lines
+ ]
+ }
+ })
+}
+
+fn objects(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ model.section(doc, name)
+ |> list.index_fold(lines, fn(l, obj, i) {
+ object(l, name, i, obj)
+ })
+ })
+}
+
+fn object(
+ acc: List(String),
+ section: String,
+ index: Int,
+ obj: Object,
+) -> List(String) {
+ let acc = ["object " <> section <> " " <> int.to_string(index), ..acc]
+ acc
+ |> kn("ocn", obj.ocn)
+ |> kv("is_a", obj.is_a)
+ |> kv("own_section", obj.own_section)
+ |> kv("level", obj.heading_level)
+ |> kv("identifier", obj.identifier)
+ |> kv("part", obj.part)
+ |> kv("is_of_type", obj.is_of_type)
+ |> kv("segment_html_is", obj.segment_html_is)
+ |> kv("segment_epub", obj.segment_epub)
+ |> kv("segment_epub_is", obj.segment_epub_is)
+ |> kn("parent", obj.parent_ocn)
+ |> kn("last_descendant", obj.last_descendant_ocn)
+ |> kv("children", ints(obj.children))
+ |> kv("ancestors", ints(obj.ancestors))
+ |> kb("dummy", obj.dummy_heading)
+ |> kb("ocn_off", obj.ocn_off)
+ |> kn("indent_base", obj.indent_base)
+ |> kn("indent_hang", obj.indent_hang)
+ |> kb("bullet", obj.bullet)
+ |> kv("lang", obj.lang)
+ |> kb("has_links", obj.has_links)
+ |> kb("has_notes_reg", obj.has_notes_reg)
+ |> kb("has_notes_star", obj.has_notes_star)
+ |> kb("has_images", obj.has_images)
+ |> kv("segment", obj.segment)
+ |> kv("segment_prev", obj.segment_prev)
+ |> kv("segment_next", obj.segment_next)
+ |> kv("anchor", obj.anchor)
+ |> kv("heading_lev_anchor", obj.heading_lev_anchor)
+ |> kn("table_cols", obj.table_cols)
+ |> kv("table_widths", obj.table_widths)
+ |> kv("table_aligns", obj.table_aligns)
+ |> kb("table_header", obj.table_header)
+ |> kv("code_syntax", obj.code_syntax)
+ |> kb("code_linenumbers", obj.code_linenumbers)
+ |> kv("heading_ancestors_text", obj.heading_ancestors_text)
+ |> kv("sha256", obj.sha256)
+ |> links(obj)
+ |> anchor_tags(obj)
+ |> images(obj)
+ |> text(obj)
+}
+
+fn links(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) })
+}
+
+fn anchor_tags(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) })
+}
+
+fn images(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.images, acc, fn(lines, img) {
+ // the name last, because a filename may hold a space where none of the
+ // other fields can
+ let lines =
+ kv(
+ lines,
+ "image",
+ img.sha256
+ <> " "
+ <> int.to_string(img.bytes)
+ <> " "
+ <> int.to_string(img.width)
+ <> "x"
+ <> int.to_string(img.height)
+ <> " "
+ <> case img.missing {
+ True -> "1"
+ False -> "0"
+ }
+ <> " "
+ <> img.name,
+ )
+ img.fields
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(lines, fn(l, pair) {
+ kv(l, "image_field", pair.0 <> " " <> pair.1)
+ })
+ })
+}
+
+/// Text, when there is any.
+///
+/// An object whose only text line is blank holds "" here, indistinguishable
+/// from no text at all, because the model joins the lines into one string.
+/// Both libraries lose the same distinction in the same place, which is
+/// what makes them comparable; it is recorded rather than hidden.
+fn text(acc: List(String), obj: Object) -> List(String) {
+ case obj.text {
+ "" -> acc
+ t -> {
+ let lines = string.split(t, on: "\n")
+ let acc = kv(acc, "text", int.to_string(list.length(lines)))
+ list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] })
+ }
+ }
+}
+
+fn ints(xs: List(Int)) -> String {
+ case xs {
+ [] -> ""
+ _ -> xs |> list.map(int.to_string) |> string.join(",")
+ }
+}
diff --git a/gleam/src/ocda/images.gleam b/gleam/src/ocda/images.gleam
new file mode 100644
index 0000000..2f3a56e
--- /dev/null
+++ b/gleam/src/ocda/images.gleam
@@ -0,0 +1,135 @@
+//// An image's bytes, from whichever source the document came from.
+////
+//// The model *names* images, with digest, byte size and pixel size; it does
+//// not hold them. That is right - an abstraction of a document is not a
+//// bundle of its assets - but a writer of any format that shows images needs
+//// the bytes, so something has to know where they are. This is that
+//// something, and it is in the library rather than in a writer because the
+//// answer depends on the *input form* and not on the output.
+////
+//// | `.ocda.db` | the `files` table, role 'image', by name |
+//// | `.zip` | the archive entry whose path ends in `/image/<name>` |
+//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot |
+//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` |
+
+import gleam/bit_array
+import gleam/dynamic/decode
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/zip
+import simplifile
+import sqlight
+
+/// The bytes, or an error saying why not.
+pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) {
+ case load.source_of(source) {
+ load.OcdaDb -> from_db(source, name)
+ load.Pod -> from_archive(source, name)
+ load.Ssp -> from_beside(source, name)
+ load.Unknown -> Error(source <> ": not a source this library reads")
+ }
+}
+
+/// Can this source give images at all?
+///
+/// A writer asks before it decides whether to emit them, because an `<img>`
+/// pointing at a file it did not include is worse than no image.
+pub fn can_give_images(source: String) -> Bool {
+ case load.source_of(source) {
+ load.OcdaDb -> True
+ load.Pod -> zip_readable(source)
+ load.Ssp -> image_dir_beside(source) != ""
+ load.Unknown -> False
+ }
+}
+
+fn zip_readable(path: String) -> Bool {
+ case simplifile.read_bits(path) {
+ Error(_) -> False
+ Ok(bytes) ->
+ case zip.entries(bytes) {
+ Ok(_) -> True
+ Error(_) -> False
+ }
+ }
+}
+
+fn from_db(path: String, name: String) -> Result(BitArray, String) {
+ case sqlight.open("file:" <> path <> "?mode=ro") {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let out =
+ sqlight.query(
+ "select data from files where role = 'image' and name = ?1",
+ on: conn,
+ with: [sqlight.text(name)],
+ expecting: blob(),
+ )
+ |> result.replace_error(path <> ": could not read " <> name)
+ let _ = sqlight.close(conn)
+ case out {
+ Ok([data, ..]) -> Ok(data)
+ Ok([]) -> Error(name <> ": not in this database")
+ Error(e) -> Error(e)
+ }
+ }
+ }
+}
+
+/// A blob column, read as bytes rather than as text: an image read as text
+/// stops at the first zero byte, and a png's ninth byte is zero.
+fn blob() -> decode.Decoder(BitArray) {
+ use v <- decode.field(0, decode.bit_array)
+ decode.success(v)
+}
+
+fn from_archive(path: String, name: String) -> Result(BitArray, String) {
+ use bytes <- result.try(
+ simplifile.read_bits(path)
+ |> result.replace_error(path <> ": cannot read"),
+ )
+ use entries <- result.try(zip.entries(bytes))
+ case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) {
+ Error(_) -> Error(name <> ": not in this archive")
+ Ok(entry) -> zip.read(bytes, entry)
+ }
+}
+
+fn from_beside(path: String, name: String) -> Result(BitArray, String) {
+ case image_dir_beside(path) {
+ "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it")
+ dir ->
+ simplifile.read_bits(dir <> "/" <> name)
+ |> result.replace_error(dir <> "/" <> name <> ": cannot read")
+ }
+}
+
+/// `media/abstraction/<x>.ssp` -> `media/image/`
+fn image_dir_beside(path: String) -> String {
+ let parts = string.split(path, on: "/")
+ let dir = case list.reverse(parts) {
+ [_file, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ case string.ends_with(dir, "abstraction") {
+ False -> ""
+ True -> {
+ let parent = case list.reverse(string.split(dir, on: "/")) {
+ [_, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ let image_dir = parent <> "/image"
+ case simplifile.is_directory(image_dir) {
+ Ok(True) -> image_dir
+ _ -> ""
+ }
+ }
+ }
+}
+
+/// Unused here, kept so that a caller can size a read before making it.
+pub fn byte_size(b: BitArray) -> Int {
+ bit_array.byte_size(b)
+}
diff --git a/gleam/src/ocda/load.gleam b/gleam/src/ocda/load.gleam
new file mode 100644
index 0000000..2a82ac9
--- /dev/null
+++ b/gleam/src/ocda/load.gleam
@@ -0,0 +1,79 @@
+//// What a path is, and loading it.
+////
+//// One place that decides, so that every reader and every writer accepts
+//// the same inputs and a new input form is added once. This mirrors
+//// spine's own `ocda/abstraction/load.d`, which names the sources it can
+//// enter the pipeline from and tells them apart by suffix.
+////
+//// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means
+//// implementing the parser, which is spine's work and a far larger thing
+//// than a reader or a sample writer needs. This library reads
+//// abstractions.
+
+import gleam/list
+import gleam/string
+import ocda/load_db
+import ocda/load_pod
+import ocda/load_ssp
+import ocda/model.{type Document}
+
+pub type Source {
+ /// `.ssp`, the abstraction as text
+ Ssp
+ /// `.ocda.db`, the abstraction as sqlite
+ OcdaDb
+ /// `.sisupod` or `.zip`, an archive carrying a `.ssp`
+ Pod
+ Unknown
+}
+
+/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote
+/// before, and every pod already published carries that name, so both are
+/// recognised. What is inside decides how it is opened, not the name.
+const suffixes = [
+ #(".ocda.db", OcdaDb),
+ #(".ssp", Ssp),
+ #(".sisupod", Pod),
+ #(".zip", Pod),
+ #(".db", OcdaDb),
+]
+
+pub fn source_of(path: String) -> Source {
+ case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) {
+ Ok(#(_, source)) -> source
+ Error(_) -> Unknown
+ }
+}
+
+pub fn source_name(source: Source) -> String {
+ case source {
+ Ssp -> ".ssp (the abstraction as text)"
+ OcdaDb -> ".ocda.db (the abstraction as sqlite)"
+ Pod -> "a pod archive carrying a .ssp"
+ Unknown -> "not an abstraction this library reads"
+ }
+}
+
+/// Load whatever the path is into the model.
+///
+/// `lang` names which language to take from a `.ocda.db`, which holds
+/// every language of its document, and from a pod, which may carry more than
+/// one `.ssp`. A `.ssp` is one language by construction and ignores it.
+pub fn load(
+ path: String,
+ lang: String,
+ any_version: Bool,
+) -> Result(Document, String) {
+ case source_of(path) {
+ Ssp -> load_ssp.load_file(path)
+ OcdaDb -> load_db.load_file(path, lang, any_version)
+ Pod -> load_pod.load_file(path, lang)
+ Unknown ->
+ Error(
+ path
+ <> ": "
+ <> source_name(Unknown)
+ <> " (expected .ssp, .ocda.db, .sisupod or .zip)",
+ )
+ }
+}
diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam
new file mode 100644
index 0000000..a5f044b
--- /dev/null
+++ b/gleam/src/ocda/load_db.gleam
@@ -0,0 +1,532 @@
+//// Loading a `.ocda.db` into the model.
+////
+//// The point of this module is that it returns *the same value*
+//// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is
+//// then a function from the model to bytes and cannot tell which
+//// serialisation it was given, exactly as spine's own writers cannot.
+////
+//// Written against `ocda-db-format-specification.md`, whose section 4
+//// gives the column by column correspondence this module implements.
+
+import gleam/dict
+import gleam/dynamic/decode
+import gleam/float
+import gleam/int
+import gleam/json
+import gleam/list
+import gleam/option
+import gleam/result
+import gleam/string
+import ocda/model.{type Document, Image, Object}
+import sqlight
+
+/// The schema major version this loader knows.
+pub const schema_major_read = "2"
+
+/// One language of the database, as the manifest gives it.
+pub type Language {
+ Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String)
+}
+
+/// Load one language of a database as the model. `lang` == "" takes the
+/// first language the manifest gives.
+pub fn load_file(
+ path: String,
+ lang: String,
+ any_version: Bool,
+) -> Result(Document, String) {
+ // read only: a loader has no business writing, and a read-write open
+ // *creates* an empty database where a path is mistyped, which then reads
+ // as a document with nothing in it, a failure that looks like an answer
+ use conn <- with_connection("file:" <> path <> "?mode=ro", path)
+
+ use _ <- result.try(case
+ has(conn, "documents") && has(conn, "objects") && has(conn, "metadata")
+ {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ path
+ <> ": no documents/objects/metadata tables; it does not look like a .ocda.db",
+ )
+ })
+
+ let schema_version =
+ one(conn, "select value from metadata where doc_id is null
+ and key = 'schema.version'")
+ use _ <- result.try(case schema_version, any_version {
+ "", _ -> Ok(Nil)
+ _, True -> Ok(Nil)
+ v, False ->
+ case major_of(v) == schema_major_read {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ path
+ <> " is schema v"
+ <> v
+ <> ", this loader knows v"
+ <> schema_major_read
+ <> ".x",
+ )
+ }
+ })
+
+ let languages = languages(conn)
+ use chosen <- result.try(case languages, lang {
+ [], _ -> Error(path <> ": no documents rows")
+ [first, ..], "" -> Ok(first)
+ _, wanted ->
+ case list.find(languages, fn(l) { l.lang == wanted }) {
+ Ok(found) -> Ok(found)
+ Error(_) ->
+ Error(
+ path
+ <> ": no language '"
+ <> wanted
+ <> "'; it holds: "
+ <> string.join(list.map(languages, fn(l) { l.lang }), " "),
+ )
+ }
+ })
+
+ let source =
+ dict.new()
+ |> dict.insert("language", chosen.lang)
+ |> dict.insert(
+ "languages",
+ string.join(list.map(languages, fn(l) { l.lang }), " "),
+ )
+ let digest = meta_value(conn, chosen.doc_id, "source.digest")
+ let source = case digest {
+ "" -> source
+ d -> dict.insert(source, "digest", d)
+ }
+
+ let document =
+ model.Document(
+ ..model.new_document(),
+ format_version: schema_version,
+ source_filename: chosen.source_filename,
+ source: source,
+ )
+ let document = metadata(conn, chosen.doc_id, document)
+ let document = objects(conn, chosen.doc_id, document)
+ Ok(document)
+}
+
+fn with_connection(
+ uri: String,
+ path: String,
+ then: fn(sqlight.Connection) -> Result(a, String),
+) -> Result(a, String) {
+ case sqlight.open(uri) {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let outcome = then(conn)
+ let _ = sqlight.close(conn)
+ outcome
+ }
+ }
+}
+
+/// Every language the database holds, in manifest order.
+pub fn languages(conn: sqlight.Connection) -> List(Language) {
+ let d = {
+ use id <- decode.field(0, decode.int)
+ use lang <- decode.field(1, decode.string)
+ use source <- decode.field(2, decode.string)
+ use digest <- decode.field(3, decode.string)
+ decode.success(Language(id, lang, source, digest))
+ }
+ sqlight.query(
+ "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'')
+ from documents order by id",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+/// The three preamble blocks of the text form, out of the one metadata
+/// table the database keeps them in.
+fn metadata(
+ conn: sqlight.Connection,
+ doc_id: Int,
+ document: Document,
+) -> Document {
+ let d = {
+ use key <- decode.field(0, decode.string)
+ use value <- decode.field(1, decode.string)
+ decode.success(#(key, value))
+ }
+ let rows =
+ sqlight.query(
+ "select key, value from metadata where doc_id = "
+ <> int.to_string(doc_id)
+ <> " order by key",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+
+ list.fold(rows, document, fn(doc, row) {
+ let #(key, raw) = row
+ let value = string.trim(raw)
+ case key {
+ // source.* describes the artefact rather than the document, and is
+ // @source in the text form
+ "source." <> _ -> doc
+ // translation.ocn_aligned says whether the languages of this document
+ // share one numbering. A .ocda.db can know it because it holds every
+ // language; a .ssp is one language and cannot. It is a fact about the
+ // file, not the document, and carrying it here would let a writer
+ // tell which serialisation it was handed.
+ "translation." <> _ -> doc
+ // the database tells the three blocks apart by a key prefix.
+ // Unprefixing them is what makes the two loaders give one model: a
+ // writer asks make["footer"], never metadata["make.footer"].
+ "make." <> rest ->
+ model.Document(..doc, make: dict.insert(doc.make, rest, value))
+ "doc_has." <> rest ->
+ model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value))
+ _ ->
+ model.Document(..doc, metadata: dict.insert(doc.metadata, key, value))
+ }
+ })
+}
+
+/// The objects, in document order within each section, the sections in the
+/// order the format gives rather than the alphabetical one sqlite would.
+fn objects(
+ conn: sqlight.Connection,
+ doc_id: Int,
+ document: Document,
+) -> Document {
+ let d = {
+ use section <- decode.field(0, decode.string)
+ use ocn <- decode.field(2, decode.int)
+ use is_a <- decode.field(3, decode.string)
+ use part <- decode.field(4, decode.string)
+ use is_of_section <- decode.field(5, decode.string)
+ use is_of_type <- decode.field(6, decode.string)
+ use heading_level <- decode.field(7, decode.int)
+ use identifier <- decode.field(8, decode.string)
+ use parent <- decode.field(9, decode.int)
+ use last_descendant <- decode.field(10, decode.int)
+ use children <- decode.field(11, decode.string)
+ use ancestors <- decode.field(12, decode.string)
+ use dummy <- decode.field(13, decode.int)
+ use ocn_off <- decode.field(14, decode.int)
+ use indent_base <- decode.field(15, decode.int)
+ use indent_hang <- decode.field(16, decode.int)
+ use bullet <- decode.field(17, decode.int)
+ use lang <- decode.field(18, decode.string)
+ use has_links <- decode.field(19, decode.int)
+ use has_notes_reg <- decode.field(20, decode.int)
+ use has_notes_star <- decode.field(21, decode.int)
+ use has_images <- decode.field(22, decode.int)
+ use segment <- decode.field(23, decode.string)
+ use segment_prev <- decode.field(24, decode.string)
+ use segment_next <- decode.field(25, decode.string)
+ use anchor <- decode.field(26, decode.string)
+ use segment_html_is <- decode.field(27, decode.string)
+ use table_cols <- decode.field(28, decode.int)
+ use table_widths <- decode.field(29, decode.string)
+ use table_aligns <- decode.field(30, decode.string)
+ use table_header <- decode.field(31, decode.int)
+ use code_linenumbers <- decode.field(32, decode.int)
+ use hat <- decode.field(33, decode.string)
+ use sha256 <- decode.field(34, decode.string)
+ use text <- decode.field(35, decode.string)
+ use id <- decode.field(36, decode.int)
+ use hla <- decode.field(37, decode.string)
+ use seg_epub <- decode.field(38, decode.string)
+ use seg_epub_is <- decode.field(39, decode.string)
+ decode.success(#(
+ id,
+ Object(
+ ocn: ocn,
+ is_a: is_a,
+ section: section,
+ heading_level: level_name(heading_level),
+ identifier: identifier,
+ part: part,
+ is_of_type: is_of_type,
+ own_section: case is_of_section == section {
+ True -> ""
+ False -> is_of_section
+ },
+ segment_html_is: segment_html_is,
+ table_aligns: json_words(table_aligns),
+ stow_link: [],
+ anchor_tags: [],
+ parent_ocn: parent,
+ last_descendant_ocn: last_descendant,
+ children: json_ints(children),
+ ancestors: json_ints(ancestors),
+ dummy_heading: dummy != 0,
+ ocn_off: ocn_off != 0,
+ indent_base: indent_base,
+ indent_hang: indent_hang,
+ bullet: bullet != 0,
+ lang: lang,
+ has_links: has_links != 0,
+ has_notes_reg: has_notes_reg != 0,
+ has_notes_star: has_notes_star != 0,
+ has_images: has_images != 0,
+ segment: segment,
+ segment_prev: segment_prev,
+ segment_next: segment_next,
+ anchor: anchor,
+ heading_lev_anchor: hla,
+ segment_epub: seg_epub,
+ segment_epub_is: seg_epub_is,
+ table_cols: table_cols,
+ table_widths: json_words(table_widths),
+ table_header: table_header != 0,
+ code_syntax: "",
+ code_linenumbers: code_linenumbers != 0,
+ heading_ancestors_text: json_joined(hat),
+ sha256: sha256,
+ images: [],
+ text: text,
+ ),
+ ))
+ }
+ let rows =
+ sqlight.query(object_sql(doc_id), on: conn, with: [], expecting: d)
+ |> result.unwrap([])
+
+ // Built newest-first per section and reversed once at the end. Appending
+ // to the tail of each section instead would be quadratic, and the largest
+ // document in the reference collection has some thousands of objects in
+ // one section.
+ let sections =
+ list.fold(rows, dict.new(), fn(acc, row) {
+ let #(id, object) = row
+ // the four repeatable properties of the text form are child tables
+ // here, one query each
+ let object =
+ Object(
+ ..object,
+ images: images(conn, id),
+ stow_link: links(conn, id),
+ anchor_tags: anchors(conn, id),
+ )
+ case model.is_section(object.section) {
+ False -> acc
+ True ->
+ dict.upsert(acc, object.section, fn(existing) {
+ case existing {
+ option.Some(previous) -> [object, ..previous]
+ option.None -> [object]
+ }
+ })
+ }
+ })
+ |> dict.map_values(fn(_, objects) { list.reverse(objects) })
+ model.Document(..document, sections: sections)
+}
+
+fn object_sql(doc_id: Int) -> String {
+ "select section, seq, ocn, is_a, coalesce(is_of_part,''),
+ coalesce(is_of_section,''), coalesce(is_of_type,''),
+ coalesce(heading_level,-1), coalesce(identifier,''),
+ coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0),
+ coalesce(children,''), coalesce(ancestors,''),
+ coalesce(dummy_heading,0), coalesce(object_number_off,0),
+ coalesce(indent_base,0), coalesce(indent_hang,0),
+ coalesce(bullet,0), coalesce(lang,''),
+ coalesce(has_links,0), coalesce(has_notes_reg,0),
+ coalesce(has_notes_star,0), coalesce(has_images,0),
+ coalesce(segment,''), coalesce(segment_prev,''),
+ coalesce(segment_next,''), coalesce(anchor,''),
+ coalesce(segment_html_is,''),
+ coalesce(table_cols,0), coalesce(table_widths,''),
+ coalesce(table_aligns,''), coalesce(table_header,0),
+ coalesce(code_linenumbers,0),
+ coalesce(heading_ancestors_text,''), coalesce(sha256,''),
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,'')
+ from objects where doc_id = " <> int.to_string(doc_id) <> "
+ order by case section
+ when 'head' then 1 when 'toc' then 2 when 'body' then 3
+ when 'endnotes' then 4 when 'glossary' then 5
+ when 'bibliography' then 6 when 'bookindex' then 7
+ when 'blurb' then 8 when 'tail' then 9 else 10 end, seq"
+}
+
+fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) {
+ let d = {
+ use name <- decode.field(0, decode.string)
+ use sha256 <- decode.field(1, decode.string)
+ use bytes <- decode.field(2, decode.int)
+ use width <- decode.field(3, decode.int)
+ use height <- decode.field(4, decode.int)
+ use missing <- decode.field(5, decode.int)
+ decode.success(Image(
+ name: name,
+ sha256: sha256,
+ bytes: bytes,
+ width: width,
+ height: height,
+ missing: missing != 0,
+ fields: dict.new(),
+ ))
+ }
+ sqlight.query(
+ "select name, coalesce(sha256,''), coalesce(bytes,0),
+ coalesce(width,0), coalesce(height,0), coalesce(missing,0)
+ from object_images where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn links(conn: sqlight.Connection, object_id: Int) -> List(String) {
+ let d = {
+ use url <- decode.field(0, decode.string)
+ decode.success(url)
+ }
+ sqlight.query(
+ "select url from object_links where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn anchors(conn: sqlight.Connection, object_id: Int) -> List(String) {
+ let d = {
+ use anchor <- decode.field(0, decode.string)
+ decode.success(anchor)
+ }
+ sqlight.query(
+ "select anchor from object_anchors where object_id = "
+ <> int.to_string(object_id)
+ <> " order by seq",
+ on: conn,
+ with: [],
+ expecting: d,
+ )
+ |> result.unwrap([])
+}
+
+fn one(conn: sqlight.Connection, sql: String) -> String {
+ let d = {
+ use v <- decode.field(0, decode.string)
+ decode.success(v)
+ }
+ case sqlight.query(sql, on: conn, with: [], expecting: d) {
+ Ok([v, ..]) -> v
+ _ -> ""
+ }
+}
+
+fn has(conn: sqlight.Connection, name: String) -> Bool {
+ one(
+ conn,
+ "select cast(count(*) as text) from sqlite_master where name = '"
+ <> name
+ <> "'",
+ )
+ != "0"
+}
+
+fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String {
+ one(
+ conn,
+ "select value from metadata where doc_id = "
+ <> int.to_string(doc_id)
+ <> " and key = '"
+ <> key
+ <> "'",
+ )
+}
+
+fn major_of(version: String) -> String {
+ case string.split_once(version, ".") {
+ Ok(#(major, _)) -> major
+ Error(_) -> version
+ }
+}
+
+/// The eight heading levels and their indices.
+fn level_name(index: Int) -> String {
+ case index {
+ 0 -> "A"
+ 1 -> "B"
+ 2 -> "C"
+ 3 -> "D"
+ 4 -> "1"
+ 5 -> "2"
+ 6 -> "3"
+ 7 -> "4"
+ _ -> ""
+ }
+}
+
+/// The JSON columns, read as JSON.
+///
+/// Not as a character scan that strips brackets and quotes: that is wrong
+/// the moment a value holds an escape, and spine's encoder writes "/" as
+/// "\/", so every heading containing a slash comes back with a backslash in
+/// it. The D library had exactly that bug and all 36 document-languages
+/// differed on heading_ancestors_text because of it.
+fn json_strings(value: String) -> List(String) {
+ case value {
+ "" -> []
+ v ->
+ case json.parse(v, decode.list(of: loose_string())) {
+ Ok(items) -> items
+ Error(_) -> []
+ }
+ }
+}
+
+/// An element that may be a string or a number, which is what these columns
+/// hold: children and ancestors are int arrays, table_aligns and
+/// heading_ancestors_text string arrays, and table_widths an int array
+/// except where the columns divide evenly, when it is [33.3333,...]. One
+/// table in the reference collection does that, and without the float
+/// branch the widths came back as three empty strings: the fallback turned
+/// a value it could not read into a value that looked read.
+fn loose_string() -> decode.Decoder(String) {
+ decode.one_of(decode.string, [
+ decode.int |> decode.map(int.to_string),
+ decode.float |> decode.map(float.to_string),
+ decode.success(""),
+ ])
+}
+
+fn json_ints(value: String) -> List(Int) {
+ value
+ |> json_strings
+ |> list.filter(fn(s) { s != "" })
+ |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) })
+}
+
+/// Space separated in the text form: table_widths, table_aligns.
+fn json_words(value: String) -> String {
+ value |> json_strings |> string.join(" ")
+}
+
+/// heading_ancestors_text is eight slots, joined with "|" in the text form
+/// and a JSON array here. The empty slots are significant: they say which
+/// levels the heading sits under.
+fn json_joined(value: String) -> String {
+ case json_strings(value) {
+ [] -> ""
+ parts -> string.join(parts, "|")
+ }
+}
diff --git a/gleam/src/ocda/load_pod.gleam b/gleam/src/ocda/load_pod.gleam
new file mode 100644
index 0000000..35483c6
--- /dev/null
+++ b/gleam/src/ocda/load_pod.gleam
@@ -0,0 +1,138 @@
+//// The abstraction out of a pod archive.
+////
+//// A pod built with `--pod2` carries the abstraction inside it, at
+//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is
+//// a third input form and *not* a third model: the `.ssp` inside is parsed by
+//// `ocda/load_ssp` and comes back as the same value. A writer over the
+//// library gains pods without being touched, which is the whole reason the
+//// dispatch is in one place.
+////
+//// ## The one thing this cannot do
+////
+//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip
+//// and **not the frame**, because there is no zstd on the BEAM without a NIF,
+//// and the one that exists, `ezstd`, clones zstd from github in its build, so
+//// it cannot be built in a sandbox without patching it. The D library has
+//// both, through a declared `libzstd`.
+////
+//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says
+//// why rather than pretending the file is malformed. It is the only place the
+//// two libraries differ in what they accept, and it is a packaging limit
+//// rather than anything about the format.
+
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load_ssp
+import ocda/model.{type Document}
+import ocda/zip
+import simplifile
+
+/// The magic number of a zstd frame, little endian 0xFD2FB528.
+const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>>
+
+/// Which languages of the abstraction the pod carries, sorted.
+pub fn languages(path: String) -> Result(List(String), String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ zip.find_suffix(entries, ".ssp")
+ |> list.filter_map(fn(e) { lang_of(e.name) })
+ |> list.sort(string.compare)
+ |> Ok
+}
+
+/// Load one language of the abstraction a pod carries.
+///
+/// `lang` == "" takes the only one there is, and where there is a choice it
+/// says so rather than choosing, as the `.ocda.db` loader does.
+pub fn load_file(path: String, lang: String) -> Result(Document, String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ let ssps = zip.find_suffix(entries, ".ssp")
+ use chosen <- result.try(case ssps, lang {
+ [], _ ->
+ Error(
+ path
+ <> ": carries no .ssp. A pod carries the abstraction only when spine"
+ <> " was asked for it: spine --pod2 puts it in, spine --source does not.",
+ )
+ [only], "" -> Ok(only)
+ _, "" ->
+ Error(
+ path
+ <> ": holds more than one language ("
+ <> string.join(langs(ssps), " ")
+ <> "); name the one wanted",
+ )
+ _, wanted ->
+ case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) {
+ Ok(found) -> Ok(found)
+ Error(_) ->
+ Error(
+ path
+ <> ": no language '"
+ <> wanted
+ <> "'; it holds: "
+ <> string.join(langs(ssps), " "),
+ )
+ }
+ })
+ use content <- result.try(zip.read(bytes, chosen))
+ case bit_array_to_string(content) {
+ Ok(text) -> Ok(load_ssp.parse(text))
+ Error(_) -> Error(chosen.name <> ": not valid UTF-8")
+ }
+}
+
+fn langs(entries: List(zip.Entry)) -> List(String) {
+ list.filter_map(entries, fn(e) { lang_of(e.name) })
+}
+
+/// `<uid>.<lang>.ssp` -> `<lang>`
+fn lang_of(name: String) -> Result(String, Nil) {
+ case string.ends_with(name, ".ssp") {
+ False -> Error(Nil)
+ True -> {
+ let stem = string.drop_end(name, 4)
+ case string.split(stem, on: ".") |> list.last {
+ Ok(lang) if lang != stem -> Ok(lang)
+ _ -> Error(Nil)
+ }
+ }
+ }
+}
+
+/// The bytes decide, not the name: a file misnamed either way is treated by
+/// what it holds.
+fn archive_bytes(path: String) -> Result(BitArray, String) {
+ case simplifile.read_bits(path) {
+ Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
+ Ok(bytes) ->
+ case is_zstd_frame(bytes) {
+ False -> Ok(bytes)
+ True ->
+ Error(
+ path
+ <> " is a zstd frame wrapping the archive, and this library has no"
+ <> " zstd: the only one on the BEAM is a NIF that fetches zstd in"
+ <> " its own build, so it cannot be built sandboxed. Unwrap it"
+ <> " first (zstd -d) and read the archive, or use the D library,"
+ <> " which reads both.",
+ )
+ }
+ }
+}
+
+fn is_zstd_frame(bytes: BitArray) -> Bool {
+ case bytes {
+ <<head:bytes-size(4), _:bits>> -> head == zstd_magic
+ _ -> False
+ }
+}
+
+@external(erlang, "unicode", "characters_to_binary")
+fn bit_array_to_string_ffi(b: BitArray) -> String
+
+fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
+ Ok(bit_array_to_string_ffi(b))
+}
diff --git a/gleam/src/ocda/load_ssp.gleam b/gleam/src/ocda/load_ssp.gleam
new file mode 100644
index 0000000..b48e1c5
--- /dev/null
+++ b/gleam/src/ocda/load_ssp.gleam
@@ -0,0 +1,478 @@
+//// Loading a `.ssp` into the model.
+////
+//// The format is line oriented and every line is one of eight shapes told
+//// apart by its first character, so the loader is a fold over the lines
+//// carrying the state the shapes need.
+////
+//// Written against `ssp-format-specification.md`. Where it and the D
+//// library differ the D library is the bug, or this is: they are meant to
+//// give the same model, and `ocda-dump` says whether they do.
+
+import gleam/dict.{type Dict}
+import gleam/int
+import gleam/list
+import gleam/option
+import gleam/result
+import gleam/string
+import ocda/model.{type Document, type Image, type Object, Image, Object}
+import simplifile
+
+pub fn load_file(path: String) -> Result(Document, String) {
+ case simplifile.read(path) {
+ Ok(content) -> Ok(parse(content))
+ Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
+ }
+}
+
+type State {
+ State(
+ document: Document,
+ /// the block currently open, "" when none
+ block: String,
+ /// the section currently open, "" when in the preamble
+ section: String,
+ /// objects of the current section, newest first
+ objects: List(Object),
+ /// the object being read, if any
+ current: Result(Object, Nil),
+ /// text lines of the object being read, newest first
+ text: List(String),
+ /// whether a text line has been seen for the object being read.
+ /// Not "is the text non-empty": an object whose first text line is
+ /// blank, which the reference collection has seven of, would then lose
+ /// it.
+ seen_text: Bool,
+ )
+}
+
+pub fn parse(content: String) -> Document {
+ content
+ |> string.split(on: "\n")
+ |> list.fold(from: initial(), with: line)
+ |> finish
+}
+
+fn initial() -> State {
+ State(
+ document: model.new_document(),
+ block: "",
+ section: "",
+ objects: [],
+ current: Error(Nil),
+ text: [],
+ seen_text: False,
+ )
+}
+
+fn line(state: State, raw: String) -> State {
+ case raw {
+ "% SiSU Document Abstraction v" <> version ->
+ case state.document.format_version {
+ "" ->
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ format_version: string.trim(version),
+ ),
+ )
+ _ -> state
+ }
+ "% Source: " <> filename ->
+ case state.document.source_filename {
+ "" ->
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ source_filename: string.trim(filename),
+ ),
+ )
+ _ -> state
+ }
+ "%" <> _ -> state
+ "@" <> rest -> open_block(state, rest)
+ "}" <> _ -> close_block(state)
+ "[" <> rest -> declaration(state, rest)
+ "." <> rest -> property(state, rest)
+ "|" <> rest -> text_line(state, rest)
+ _ ->
+ case string.trim(raw) {
+ // a blank line ends the object being read
+ "" -> commit(state)
+ _ ->
+ case model.is_preamble_block(state.block) {
+ True -> preamble_pair(state, raw)
+ False -> unknown_line(state, raw)
+ }
+ }
+ }
+}
+
+fn open_block(state: State, rest: String) -> State {
+ let name = case string.split_once(rest, "{") {
+ Ok(#(name, _)) -> string.trim(name)
+ Error(_) -> string.trim(rest)
+ }
+ case model.is_preamble_block(name) {
+ True -> State(..flush(state), block: name)
+ False -> {
+ let state = flush(state)
+ let document = case model.is_section(name) {
+ True -> state.document
+ // a block that is neither preamble nor one of the nine sections:
+ // counted, so that a reader can say the file held something it did
+ // not expect
+ False ->
+ model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ block: bump(state.document.unknown.block, name),
+ ),
+ )
+ }
+ State(..state, block: name, section: name, document: document)
+ }
+ }
+}
+
+fn close_block(state: State) -> State {
+ State(..flush(state), block: "")
+}
+
+/// Commit the object being read, then put the section's objects into the
+/// document and close the section.
+fn flush(state: State) -> State {
+ let state = commit(state)
+ case state.section, state.objects {
+ "", _ -> state
+ _, [] -> State(..state, section: "", objects: [])
+ name, objects ->
+ State(
+ ..state,
+ section: "",
+ objects: [],
+ document: model.Document(
+ ..state.document,
+ sections: add_section(
+ state.document.sections,
+ name,
+ list.reverse(objects),
+ ),
+ ),
+ )
+ }
+}
+
+fn preamble_pair(state: State, raw: String) -> State {
+ case string.split_once(string.trim_start(raw), ":") {
+ Error(_) -> state
+ Ok(#(key, value)) -> {
+ let key = string.trim(key)
+ let value = string.trim(value)
+ let document = state.document
+ let document = case state.block {
+ "meta" ->
+ model.Document(
+ ..document,
+ metadata: dict.insert(document.metadata, key, value),
+ )
+ "make" ->
+ model.Document(..document, make: dict.insert(document.make, key, value))
+ "source" ->
+ model.Document(
+ ..document,
+ source: dict.insert(document.source, key, value),
+ )
+ _ ->
+ model.Document(
+ ..document,
+ doc_has: dict.insert(document.doc_has, key, value),
+ )
+ }
+ State(..state, document: document)
+ }
+ }
+}
+
+/// `[<ocn>] <kind>`, and for a heading `[<ocn>] heading :<level>
+/// [<identifier>]`.
+fn declaration(state: State, rest: String) -> State {
+ let state = commit(state)
+ case string.split_once(rest, "]") {
+ Error(_) -> state
+ Ok(#(ocn_text, tail)) -> {
+ let ocn = parse_int(string.trim(ocn_text))
+ let object =
+ Object(..model.new_object(), ocn: ocn, section: state.section)
+ let object = case string.split(string.trim(tail), on: " ") {
+ ["heading", level, ..identifier] ->
+ Object(
+ ..object,
+ is_a: "heading",
+ heading_level: string.drop_start(level, 1),
+ identifier: string.join(identifier, " "),
+ )
+ [""] -> object
+ [kind, ..] -> Object(..object, is_a: kind)
+ [] -> object
+ }
+ State(..state, current: Ok(object), text: [], seen_text: False)
+ }
+ }
+}
+
+fn property(state: State, rest: String) -> State {
+ case state.current, string.split_once(rest, ":") {
+ Ok(object), Ok(#(name, raw_value)) -> {
+ let name = string.trim(name)
+ let value = string.trim(raw_value)
+ let #(object, known) = apply_property(object, name, value)
+ let document = case known {
+ True -> state.document
+ False ->
+ model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ property: bump(state.document.unknown.property, name),
+ ),
+ )
+ }
+ State(..state, current: Ok(object), document: document)
+ }
+ _, _ -> state
+ }
+}
+
+/// The properties this library reads, and what each sets.
+///
+/// A property it has no use for is counted rather than passed over:
+/// ignoring one is conforming within a major version, doing so in silence
+/// is what stops a reader being a check on the specification.
+fn apply_property(
+ object: Object,
+ name: String,
+ value: String,
+) -> #(Object, Bool) {
+ case name {
+ "part" -> #(Object(..object, part: value), True)
+ "is_of_type" -> #(Object(..object, is_of_type: value), True)
+ "section" -> #(Object(..object, own_section: value), True)
+ "identifier" -> #(Object(..object, identifier: value), True)
+ "segment_html_is" -> #(Object(..object, segment_html_is: value), True)
+ "segment_epub" -> #(Object(..object, segment_epub: value), True)
+ "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True)
+ "table_aligns" -> #(Object(..object, table_aligns: value), True)
+ "stow_link" -> #(
+ Object(..object, stow_link: list.append(object.stow_link, [value])),
+ True,
+ )
+ "anchor_tag" -> #(
+ Object(..object, anchor_tags: list.append(object.anchor_tags, [value])),
+ True,
+ )
+ "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True)
+ "last_descendant" -> #(
+ Object(..object, last_descendant_ocn: parse_int(value)),
+ True,
+ )
+ "children" -> #(Object(..object, children: numbers(value)), True)
+ "ancestors" -> #(Object(..object, ancestors: numbers(value)), True)
+ "dummy" -> #(Object(..object, dummy_heading: value == "true"), True)
+ "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True)
+ "indent" -> {
+ let parts = string.split(value, on: " ")
+ let base = case parts {
+ [first, ..] -> parse_int(first)
+ [] -> 0
+ }
+ let hang = case parts {
+ [_, second, ..] -> parse_int(second)
+ _ -> 0
+ }
+ #(Object(..object, indent_base: base, indent_hang: hang), True)
+ }
+ "bullet" -> #(Object(..object, bullet: value == "true"), True)
+ "lang" -> #(Object(..object, lang: value), True)
+ "has" -> #(has_flags(object, value), True)
+ "segment" -> #(Object(..object, segment: value), True)
+ "segment_prev" -> #(Object(..object, segment_prev: value), True)
+ "segment_next" -> #(Object(..object, segment_next: value), True)
+ "anchor" -> #(Object(..object, anchor: value), True)
+ "heading_lev_anchor" -> #(
+ Object(..object, heading_lev_anchor: value),
+ True,
+ )
+ "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True)
+ "table_widths" -> #(Object(..object, table_widths: value), True)
+ "table_header" -> #(Object(..object, table_header: value == "true"), True)
+ "code_syntax" -> #(Object(..object, code_syntax: value), True)
+ "code_linenumbers" -> #(
+ Object(..object, code_linenumbers: value == "true"),
+ True,
+ )
+ "heading_ancestors_text" -> #(
+ Object(..object, heading_ancestors_text: value),
+ True,
+ )
+ "sha256" -> #(Object(..object, sha256: value), True)
+ "image" -> #(
+ Object(..object, images: list.append(object.images, [image(value)])),
+ True,
+ )
+ _ -> #(object, False)
+ }
+}
+
+fn has_flags(object: Object, value: String) -> Object {
+ value
+ |> string.split(on: " ")
+ |> list.fold(from: object, with: fn(obj, flag) {
+ case flag {
+ "links" -> Object(..obj, has_links: True)
+ "notes_reg" -> Object(..obj, has_notes_reg: True)
+ "notes_star" -> Object(..obj, has_notes_star: True)
+ "images" -> Object(..obj, has_images: True)
+ _ -> obj
+ }
+ })
+}
+
+/// `<filename> sha256:<HEX> bytes:<n> [px:<w>x<h>] [missing:true] [...]`
+///
+/// The filename comes first, the rest are order-independent, and a field
+/// this library does not recognise is kept rather than discarded.
+fn image(value: String) -> Image {
+ case string.split(value, on: " ") {
+ [] -> model.new_image()
+ [name, ..fields] ->
+ list.fold(fields, model.new_image(), fn(img, field) {
+ case string.split_once(field, ":") {
+ Error(_) -> img
+ Ok(#(key, val)) ->
+ case key {
+ "sha256" -> Image(..img, sha256: val)
+ "bytes" -> Image(..img, bytes: parse_int(val))
+ "px" ->
+ case string.split_once(val, "x") {
+ Ok(#(w, h)) ->
+ Image(..img, width: parse_int(w), height: parse_int(h))
+ Error(_) -> img
+ }
+ "missing" -> Image(..img, missing: val == "true")
+ _ -> Image(..img, fields: dict.insert(img.fields, key, val))
+ }
+ }
+ })
+ |> fn(img) { Image(..img, name: name) }
+ }
+}
+
+/// `| <text>`, and `| ` alone for a blank line. Never a bare `|`: the
+/// emitter writes the space.
+fn text_line(state: State, rest: String) -> State {
+ case state.current {
+ Error(_) -> state
+ Ok(_) -> {
+ let content = string.drop_start(rest, 1)
+ State(..state, text: [content, ..state.text], seen_text: True)
+ }
+ }
+}
+
+fn unknown_line(state: State, raw: String) -> State {
+ let first = string.slice(raw, 0, 1)
+ State(
+ ..state,
+ document: model.Document(
+ ..state.document,
+ unknown: model.Unknown(
+ ..state.document.unknown,
+ line: bump(state.document.unknown.line, first),
+ ),
+ ),
+ )
+}
+
+fn commit(state: State) -> State {
+ case state.current {
+ Error(_) -> state
+ Ok(object) -> {
+ let text = case state.seen_text {
+ True -> string.join(list.reverse(state.text), "\n")
+ False -> ""
+ }
+ let object = derive_omitted(Object(..object, text: text))
+ let objects = case model.is_section(state.section) {
+ True -> [object, ..state.objects]
+ False -> state.objects
+ }
+ State(
+ ..state,
+ current: Error(Nil),
+ text: [],
+ seen_text: False,
+ objects: objects,
+ )
+ }
+ }
+}
+
+/// Two values the text form leaves out because they can be worked out.
+///
+/// `.identifier` is absent when it is simply the ocn, `.anchor` when it is
+/// the segment name. The `.ocda.db`, being columns rather than lines,
+/// stores both unconditionally. So a loader that takes the text form
+/// literally gives a different model from the one the database gives, and
+/// over one document that is 1,314 identifiers and 69 anchors.
+fn derive_omitted(object: Object) -> Object {
+ let object = case object.identifier, object.ocn {
+ "", 0 -> object
+ "", ocn -> Object(..object, identifier: int.to_string(ocn))
+ _, _ -> object
+ }
+ case object.anchor, object.segment {
+ "", "" -> object
+ "", segment -> Object(..object, anchor: segment)
+ _, _ -> object
+ }
+}
+
+fn finish(state: State) -> Document {
+ flush(state).document
+}
+
+fn add_section(
+ sections: Dict(String, List(Object)),
+ name: String,
+ objects: List(Object),
+) -> Dict(String, List(Object)) {
+ dict.upsert(sections, name, fn(existing) {
+ case existing {
+ option.Some(previous) -> list.append(previous, objects)
+ option.None -> objects
+ }
+ })
+}
+
+fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) {
+ dict.upsert(counts, key, fn(existing) {
+ case existing {
+ option.Some(n) -> n + 1
+ option.None -> 1
+ }
+ })
+}
+
+fn parse_int(value: String) -> Int {
+ value |> string.trim |> int.parse |> result.unwrap(0)
+}
+
+fn numbers(value: String) -> List(Int) {
+ value
+ |> string.split(on: " ")
+ |> list.filter(fn(s) { s != "" })
+ |> list.map(parse_int)
+}
diff --git a/gleam/src/ocda/model.gleam b/gleam/src/ocda/model.gleam
new file mode 100644
index 0000000..050a97b
--- /dev/null
+++ b/gleam/src/ocda/model.gleam
@@ -0,0 +1,278 @@
+//// The object-centric document abstraction, as Gleam types.
+////
+//// A deliberate mirror of the D library's `ocda.model`, field for field.
+//// The two are meant to agree, and the cheapest way to make that testable
+//// is for them to hold the same things under the same names: the dump of a
+//// loaded document is then identical by construction rather than by
+//// negotiation.
+////
+//// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for
+//// the model, the two format specifications for what each serialisation
+//// says about it.
+
+import gleam/dict.{type Dict}
+import gleam/list
+import gleam/string
+
+/// The format major version this library knows.
+///
+/// The `.ssp` and the `.ocda.db` carry one version between them, declared
+/// in band. The minor part rises when a property is added and a reader of
+/// the newer format still reads the older, so only the major is checked.
+pub const format_major_read = "2"
+
+/// The nine sections, in the order the format gives them, which is not the
+/// order a sorted key list or a `group by` would give.
+pub const section_names = [
+ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex",
+ "blurb", "tail",
+]
+
+/// The four blocks that are document preamble rather than sections of
+/// objects.
+pub const preamble_blocks = ["source", "meta", "make", "doc_has"]
+
+/// A repeatable `.image` record: the file as the abstraction found it.
+pub type Image {
+ Image(
+ /// filename as the object refers to it
+ name: String,
+ /// hex, uppercase
+ sha256: String,
+ bytes: Int,
+ /// the file's own pixel dimensions, not any display width
+ width: Int,
+ height: Int,
+ /// absent or unreadable, so no digest and no size
+ missing: Bool,
+ /// any further key:value fields, kept rather than dropped
+ fields: Dict(String, String),
+ )
+}
+
+pub fn new_image() -> Image {
+ Image(
+ name: "",
+ sha256: "",
+ bytes: 0,
+ width: 0,
+ height: 0,
+ missing: False,
+ fields: dict.new(),
+ )
+}
+
+/// One object of the abstraction.
+///
+/// `text` is one string with the lines joined by newline, as in the D
+/// library and for the same reason: an object whose only text line is
+/// blank is then indistinguishable from an object with no text, in both
+/// libraries, in the same place. A difference that exists in one language
+/// and not the other would be worse than a loss both share.
+pub type Object {
+ Object(
+ ocn: Int,
+ /// one of the sixteen kinds
+ is_a: String,
+ /// the section this object was found in
+ section: String,
+ /// headings: A B C D 1 2 3 4
+ heading_level: String,
+ identifier: String,
+ part: String,
+ /// how the text behaves: para, block, comment
+ is_of_type: String,
+ /// the object's own section, when it differs from the block it sits in
+ own_section: String,
+ segment_html_is: String,
+ segment_epub: String,
+ segment_epub_is: String,
+ table_aligns: String,
+ stow_link: List(String),
+ anchor_tags: List(String),
+ parent_ocn: Int,
+ last_descendant_ocn: Int,
+ children: List(Int),
+ ancestors: List(Int),
+ dummy_heading: Bool,
+ ocn_off: Bool,
+ indent_base: Int,
+ indent_hang: Int,
+ bullet: Bool,
+ lang: String,
+ has_links: Bool,
+ has_notes_reg: Bool,
+ has_notes_star: Bool,
+ has_images: Bool,
+ segment: String,
+ segment_prev: String,
+ segment_next: String,
+ anchor: String,
+ heading_lev_anchor: String,
+ table_cols: Int,
+ table_widths: String,
+ table_header: Bool,
+ code_syntax: String,
+ code_linenumbers: Bool,
+ heading_ancestors_text: String,
+ sha256: String,
+ images: List(Image),
+ text: String,
+ )
+}
+
+pub fn new_object() -> Object {
+ Object(
+ ocn: 0,
+ is_a: "",
+ section: "",
+ heading_level: "",
+ identifier: "",
+ part: "",
+ is_of_type: "",
+ own_section: "",
+ segment_html_is: "",
+ segment_epub: "",
+ segment_epub_is: "",
+ table_aligns: "",
+ stow_link: [],
+ anchor_tags: [],
+ parent_ocn: 0,
+ last_descendant_ocn: 0,
+ children: [],
+ ancestors: [],
+ dummy_heading: False,
+ ocn_off: False,
+ indent_base: 0,
+ indent_hang: 0,
+ bullet: False,
+ lang: "",
+ has_links: False,
+ has_notes_reg: False,
+ has_notes_star: False,
+ has_images: False,
+ segment: "",
+ segment_prev: "",
+ segment_next: "",
+ anchor: "",
+ heading_lev_anchor: "",
+ table_cols: 0,
+ table_widths: "",
+ table_header: False,
+ code_syntax: "",
+ code_linenumbers: False,
+ heading_ancestors_text: "",
+ sha256: "",
+ images: [],
+ text: "",
+ )
+}
+
+/// Whether this object is rendered at all.
+///
+/// `is_of_type` "comment" is markup commentary: carried by the abstraction
+/// so that nothing is lost, and not put into output. An output that
+/// renders it leaks the author's notes into the document, which is what
+/// spine itself does not do.
+pub fn renders(object: Object) -> Bool {
+ object.is_of_type != "comment"
+}
+
+/// The section this object belongs to: the block it sits in, unless it
+/// says otherwise.
+pub fn section_of(object: Object) -> String {
+ case object.own_section {
+ "" -> object.section
+ own -> own
+ }
+}
+
+/// What the loader met and did not understand.
+///
+/// A loader that silently drops what it does not know cannot say whether
+/// it understood the document, and so cannot be used to check a
+/// specification against an implementation: silence and understanding look
+/// alike.
+pub type Unknown {
+ Unknown(
+ /// `.name` -> times seen
+ property: Dict(String, Int),
+ /// `@name` -> times seen
+ block: Dict(String, Int),
+ /// first character -> times seen, for lines of no known shape
+ line: Dict(String, Int),
+ )
+}
+
+pub fn new_unknown() -> Unknown {
+ Unknown(property: dict.new(), block: dict.new(), line: dict.new())
+}
+
+pub fn any_unknown(unknown: Unknown) -> Bool {
+ dict.size(unknown.property) > 0
+ || dict.size(unknown.block) > 0
+ || dict.size(unknown.line) > 0
+}
+
+pub type Document {
+ Document(
+ /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when
+ /// absent
+ format_version: String,
+ source_filename: String,
+ /// language, languages, digest
+ source: Dict(String, String),
+ unknown: Unknown,
+ metadata: Dict(String, String),
+ make: Dict(String, String),
+ doc_has: Dict(String, String),
+ /// section name -> objects, in document order
+ sections: Dict(String, List(Object)),
+ )
+}
+
+pub fn new_document() -> Document {
+ Document(
+ format_version: "",
+ source_filename: "",
+ source: dict.new(),
+ unknown: new_unknown(),
+ metadata: dict.new(),
+ make: dict.new(),
+ doc_has: dict.new(),
+ sections: dict.new(),
+ )
+}
+
+/// The major part of the declared version, "" when none was declared.
+pub fn version_major(document: Document) -> String {
+ case string.split_once(document.format_version, ".") {
+ Ok(#(major, _)) -> major
+ Error(_) -> document.format_version
+ }
+}
+
+pub fn meta(document: Document, key: String) -> String {
+ case dict.get(document.metadata, key) {
+ Ok(value) -> value
+ Error(_) -> ""
+ }
+}
+
+/// The objects of a named section, empty when the document has none.
+pub fn section(document: Document, name: String) -> List(Object) {
+ case dict.get(document.sections, name) {
+ Ok(objects) -> objects
+ Error(_) -> []
+ }
+}
+
+/// Is this one of the nine sections of the format?
+pub fn is_section(name: String) -> Bool {
+ list.contains(section_names, name)
+}
+
+/// Is this one of the four preamble blocks?
+pub fn is_preamble_block(name: String) -> Bool {
+ list.contains(preamble_blocks, name)
+}
diff --git a/gleam/src/ocda/zip.gleam b/gleam/src/ocda/zip.gleam
new file mode 100644
index 0000000..5ce6e09
--- /dev/null
+++ b/gleam/src/ocda/zip.gleam
@@ -0,0 +1,170 @@
+//// Enough of the zip format to find one file in a pod and read it.
+////
+//// Not a zip library. A pod archive holds a handful of entries, spine writes
+//// them **stored** (uncompressed) so that the outer zstd frame does the
+//// compressing once over the whole thing, and what is wanted from it here is
+//// one named entry.
+////
+//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could
+//// inflate it, but a library that says it read an entry it in fact skipped is
+//// worse than one that says it cannot, and spine's own archives do not need
+//// it.
+
+import gleam/bit_array
+import gleam/int
+import gleam/list
+import gleam/result
+import gleam/string
+
+pub type Entry {
+ Entry(
+ name: String,
+ /// 0 stored, 8 deflate
+ method: Int,
+ compressed: Int,
+ uncompressed: Int,
+ local_offset: Int,
+ )
+}
+
+const eocd_sig = 0x06054b50
+
+const cd_sig = 0x02014b50
+
+const local_sig = 0x04034b50
+
+/// The entries, from the central directory at the end of the file.
+pub fn entries(bytes: BitArray) -> Result(List(Entry), String) {
+ let size = bit_array.byte_size(bytes)
+ use _ <- result.try(case size >= 22 {
+ True -> Ok(Nil)
+ False -> Error("too short to be a zip archive")
+ })
+ use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size)))
+ use count <- result.try(u16(bytes, eocd + 10))
+ use offset <- result.try(u32(bytes, eocd + 16))
+ read_central(bytes, offset, count, [])
+}
+
+/// The end of central directory record is last, and may be followed by a
+/// comment of up to 65535 bytes, so it is searched for backwards.
+fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) {
+ case at < lowest {
+ True ->
+ Error(
+ "no end of central directory record; it does not look like a zip archive",
+ )
+ False ->
+ case u32(bytes, at) {
+ Ok(sig) if sig == eocd_sig -> Ok(at)
+ _ -> find_eocd(bytes, at - 1, lowest)
+ }
+ }
+}
+
+fn lowest(size: Int) -> Int {
+ case size > 22 + 65_535 {
+ True -> size - 22 - 65_535
+ False -> 0
+ }
+}
+
+fn read_central(
+ bytes: BitArray,
+ at: Int,
+ remaining: Int,
+ acc: List(Entry),
+) -> Result(List(Entry), String) {
+ case remaining {
+ 0 -> Ok(list.reverse(acc))
+ _ -> {
+ use sig <- result.try(u32(bytes, at))
+ use _ <- result.try(case sig == cd_sig {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ "central directory entry "
+ <> int.to_string(list.length(acc))
+ <> " is malformed",
+ )
+ })
+ use method <- result.try(u16(bytes, at + 10))
+ use compressed <- result.try(u32(bytes, at + 20))
+ use uncompressed <- result.try(u32(bytes, at + 24))
+ use name_len <- result.try(u16(bytes, at + 28))
+ use extra_len <- result.try(u16(bytes, at + 30))
+ use cmnt_len <- result.try(u16(bytes, at + 32))
+ use local_offset <- result.try(u32(bytes, at + 42))
+ use name <- result.try(text(bytes, at + 46, name_len))
+ let entry =
+ Entry(
+ name: name,
+ method: method,
+ compressed: compressed,
+ uncompressed: uncompressed,
+ local_offset: local_offset,
+ )
+ read_central(
+ bytes,
+ at + 46 + name_len + extra_len + cmnt_len,
+ remaining - 1,
+ [entry, ..acc],
+ )
+ }
+ }
+}
+
+/// One entry's bytes, from its local header.
+pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) {
+ use sig <- result.try(u32(bytes, entry.local_offset))
+ use _ <- result.try(case sig == local_sig {
+ True -> Ok(Nil)
+ False -> Error(entry.name <> ": local header is malformed")
+ })
+ use name_len <- result.try(u16(bytes, entry.local_offset + 26))
+ use extra_len <- result.try(u16(bytes, entry.local_offset + 28))
+ use _ <- result.try(case entry.method {
+ 0 -> Ok(Nil)
+ m ->
+ Error(
+ entry.name
+ <> ": compression method "
+ <> int.to_string(m)
+ <> "; this reader takes only stored entries, which is what spine writes",
+ )
+ })
+ let start = entry.local_offset + 30 + name_len + extra_len
+ case bit_array.slice(bytes, start, entry.compressed) {
+ Ok(slice) -> Ok(slice)
+ Error(_) -> Error(entry.name <> ": runs past the end of the archive")
+ }
+}
+
+/// The entries whose names end this way, in archive order. Used to find
+/// media/abstraction/<uid>.<lang>.ssp without knowing the uid.
+pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) {
+ list.filter(entries, fn(e) { string.ends_with(e.name, suffix) })
+}
+
+fn u16(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 2) {
+ Ok(<<v:little-size(16)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn u32(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 4) {
+ Ok(<<v:little-size(32)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
+ case bit_array.slice(bytes, at, len) {
+ Ok(slice) ->
+ bit_array.to_string(slice)
+ |> result.replace_error("an entry name is not valid UTF-8")
+ Error(_) -> Error("an entry name runs past the end of the archive")
+ }
+}
diff --git a/gleam/src/ocda_lib.gleam b/gleam/src/ocda_lib.gleam
new file mode 100644
index 0000000..1e8a5ae
--- /dev/null
+++ b/gleam/src/ocda_lib.gleam
@@ -0,0 +1,136 @@
+//// ocda-lib, in Gleam: the abstraction as a library, put to the question.
+////
+//// A library has no binary, so this is the two programs that test its
+//// claims.
+////
+//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]
+////
+//// `dump` writes the whole loaded model out canonically, byte for byte as
+//// the D library's `ocda-dump` writes it, so that the two implementations
+//// can be diffed. `agree` asks the same question the D library's
+//// `ocda-agree` asks, within this language: does loading a document's
+//// `.ssp` give what loading its `.ocda.db` gives?
+
+import argv
+import gleam/int
+import gleam/io
+import gleam/list
+import gleam/string
+import ocda/dump
+import ocda/load
+import ocda/model.{type Document}
+
+pub fn main() {
+ case argv.load().arguments {
+ ["dump", "--unknown", path] -> dump_file(path, "", True)
+ ["dump", "--unknown", path, lang] -> dump_file(path, lang, True)
+ ["dump", path] -> dump_file(path, "", False)
+ ["dump", path, lang] -> dump_file(path, lang, False)
+ ["agree", ssp, db] -> agree(ssp, db, "")
+ ["agree", ssp, db, lang] -> agree(ssp, db, lang)
+ _ -> usage()
+ }
+}
+
+fn usage() -> Nil {
+ io.println_error(
+ "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang]
+ ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]",
+ )
+ halt(2)
+}
+
+fn dump_file(path: String, lang: String, unknown: Bool) -> Nil {
+ case load.load(path, lang, False) {
+ Error(message) -> fail(message)
+ Ok(doc) ->
+ case unknown {
+ True -> io.print(dump.unknown(doc))
+ False -> io.print(dump.document(doc))
+ }
+ }
+}
+
+/// The library's whole purpose is that a writer cannot tell which
+/// serialisation it was handed. That is only true if the two loaders give
+/// the same model, so this asks, and says where they part company rather
+/// than only that they did.
+fn agree(ssp_path: String, db_path: String, lang: String) -> Nil {
+ case load.load(ssp_path, "", False), load.load(db_path, lang, False) {
+ Error(message), _ -> fail(message)
+ _, Error(message) -> fail(message)
+ Ok(from_ssp), Ok(from_db) -> {
+ let differences = compare(from_ssp, from_db)
+ case differences {
+ [] -> io.println("agree: the two loaders give the same model")
+ _ -> {
+ list.each(list.take(differences, 12), io.println)
+ case list.length(differences) > 12 {
+ True -> io.println(" ... stopping after 12")
+ False -> Nil
+ }
+ io.println(int.to_string(list.length(differences)) <> " difference(s)")
+ halt(1)
+ }
+ }
+ }
+ }
+}
+
+/// Compared through the dump rather than field by field, which is both
+/// shorter and stricter: the dump carries every field, so nothing can
+/// differ without showing here.
+fn compare(from_ssp: Document, from_db: Document) -> List(String) {
+ let a = dump.document(from_ssp)
+ let b = dump.document(from_db)
+ case a == b {
+ True -> []
+ False -> diff_lines(a, b)
+ }
+}
+
+fn diff_lines(a: String, b: String) -> List(String) {
+ let la = split_lines(a)
+ let lb = split_lines(b)
+ case list.length(la) == list.length(lb) {
+ False -> [
+ " differ: line count",
+ " .ssp: " <> int.to_string(list.length(la)),
+ " .ocda.db: " <> int.to_string(list.length(lb)),
+ ..pairwise(la, lb)
+ ]
+ True -> pairwise(la, lb)
+ }
+}
+
+fn pairwise(la: List(String), lb: List(String)) -> List(String) {
+ list.zip(la, lb)
+ |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) })
+ |> list.filter(fn(t) { t.1 != t.2 })
+ |> list.map(fn(t) {
+ " differ: line "
+ <> int.to_string(t.0 + 1)
+ <> "\n .ssp: "
+ <> t.1
+ <> "\n .ocda.db: "
+ <> t.2
+ })
+}
+
+fn split_lines(text: String) -> List(String) {
+ case text {
+ "" -> []
+ _ -> string.split(text, on: "\n")
+ }
+}
+
+fn fail(message: String) -> Nil {
+ io.println_error("error: " <> message)
+ halt(2)
+}
+
+/// Gleam has no standard exit, and the exit status is what a sweep script
+/// reads.
+@external(erlang, "erlang", "halt")
+fn halt(code: Int) -> Nil