-*- mode: org -*- #+TITLE: gleam lib #+DESCRIPTION: gleam lib #+FILETAGS: :spine:build:tools: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: show2levels hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code #+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX * ocda ** ocda lib #+NAME: ocda_lib #+HEADER: :no-expand #+BEGIN_SRC gleam //// ocda-lib, in Gleam: the abstraction as a library, put to the question. //// //// A library has no binary, so this is the programs over it: the reader, //// and the two that test its claims. //// //// ocda-lib dump [--unknown] .ssp|.ocda.db [lang] //// ocda-lib agree .ssp .ocda.db [lang] //// ocda-lib read [options] [lang] (also installed as ocda-read) //// //// `dump` writes the whole loaded model out canonically, byte for byte as //// the D library's `ocda-dump` writes it, so that the two implementations //// can be diffed. `agree` asks the same question the D library's //// `ocda-agree` asks, within this language: does loading a document's //// `.ssp` give what loading its `.ocda.db` gives? import argv import gleam/int import gleam/io import gleam/list import gleam/string import ocda/dump import ocda/load import ocda/model.{type Document} import read/main as reader pub fn main() { case argv.load().arguments { ["dump", "--unknown", path] -> dump_file(path, "", True) ["dump", "--unknown", path, lang] -> dump_file(path, lang, True) ["dump", path] -> dump_file(path, "", False) ["dump", path, lang] -> dump_file(path, lang, False) ["agree", ssp, db] -> agree(ssp, db, "") ["agree", ssp, db, lang] -> agree(ssp, db, lang) ["read", ..rest] -> case reader.run(rest) { 0 -> Nil code -> halt(code) } _ -> usage() } } fn usage() -> Nil { io.println_error( "usage: ocda-lib dump [--unknown] .ssp|.ocda.db [lang] ocda-lib agree .ssp .ocda.db [lang] ocda-lib read [--help] [options] [lang]", ) halt(2) } fn dump_file(path: String, lang: String, unknown: Bool) -> Nil { case load.load(path, lang, False) { Error(message) -> fail(message) Ok(doc) -> case unknown { True -> io.print(dump.unknown(doc)) False -> io.print(dump.document(doc)) } } } /// The library's whole purpose is that a writer cannot tell which /// serialisation it was handed. That is only true if the two loaders give /// the same model, so this asks, and says where they part company rather /// than only that they did. fn agree(ssp_path: String, db_path: String, lang: String) -> Nil { case load.load(ssp_path, "", False), load.load(db_path, lang, False) { Error(message), _ -> fail(message) _, Error(message) -> fail(message) Ok(from_ssp), Ok(from_db) -> { let differences = compare(from_ssp, from_db) case differences { [] -> io.println("agree: the two loaders give the same model") _ -> { list.each(list.take(differences, 12), io.println) case list.length(differences) > 12 { True -> io.println(" ... stopping after 12") False -> Nil } io.println(int.to_string(list.length(differences)) <> " difference(s)") halt(1) } } } } } /// Compared through the dump rather than field by field, which is both /// shorter and stricter: the dump carries every field, so nothing can /// differ without showing here. fn compare(from_ssp: Document, from_db: Document) -> List(String) { let a = dump.document(from_ssp) let b = dump.document(from_db) case a == b { True -> [] False -> diff_lines(a, b) } } fn diff_lines(a: String, b: String) -> List(String) { let la = split_lines(a) let lb = split_lines(b) case list.length(la) == list.length(lb) { False -> [ " differ: line count", " .ssp: " <> int.to_string(list.length(la)), " .ocda.db: " <> int.to_string(list.length(lb)), ..pairwise(la, lb) ] True -> pairwise(la, lb) } } fn pairwise(la: List(String), lb: List(String)) -> List(String) { list.zip(la, lb) |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) }) |> list.filter(fn(t) { t.1 != t.2 }) |> list.map(fn(t) { " differ: line " <> int.to_string(t.0 + 1) <> "\n .ssp: " <> t.1 <> "\n .ocda.db: " <> t.2 }) } fn split_lines(text: String) -> List(String) { case text { "" -> [] _ -> string.split(text, on: "\n") } } fn fail(message: String) -> Nil { io.println_error("error: " <> message) halt(2) } /// Gleam has no standard exit, and the exit status is what a sweep script /// reads. @external(erlang, "erlang", "halt") fn halt(code: Int) -> Nil #+END_SRC ** ocda *** model #+NAME: model #+HEADER: :no-expand #+BEGIN_SRC gleam //// The object-centric document abstraction, as Gleam types. //// //// A deliberate mirror of the D library's `ocda.model`, field for field. //// The two are meant to agree, and the cheapest way to make that testable //// is for them to hold the same things under the same names: the dump of a //// loaded document is then identical by construction rather than by //// negotiation. //// //// Written against specs/sisudoc-abstraction-format: `ocda-model.md` for //// the model, the two format specifications for what each serialisation //// says about it. import gleam/dict.{type Dict} import gleam/list import gleam/string /// The format major version this library knows. /// /// The `.ssp` and the `.ocda.db` carry one version between them, declared /// in band. The minor part rises when a property is added and a reader of /// the newer format still reads the older, so only the major is checked. pub const format_major_read = "2" /// The nine sections, in the order the format gives them, which is not the /// order a sorted key list or a `group by` would give. pub const section_names = [ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", "tail", ] /// The four blocks that are document preamble rather than sections of /// objects. pub const preamble_blocks = ["source", "meta", "make", "doc_has"] /// A repeatable `.image` record: the file as the abstraction found it. pub type Image { Image( /// filename as the object refers to it name: String, /// hex, uppercase sha256: String, bytes: Int, /// the file's own pixel dimensions, not any display width width: Int, height: Int, /// absent or unreadable, so no digest and no size missing: Bool, /// any further key:value fields, kept rather than dropped fields: Dict(String, String), ) } pub fn new_image() -> Image { Image( name: "", sha256: "", bytes: 0, width: 0, height: 0, missing: False, fields: dict.new(), ) } /// One object of the abstraction. /// /// `text` is one string with the lines joined by newline, as in the D /// library and for the same reason: an object whose only text line is /// blank is then indistinguishable from an object with no text, in both /// libraries, in the same place. A difference that exists in one language /// and not the other would be worse than a loss both share. pub type Object { Object( ocn: Int, /// one of the sixteen kinds is_a: String, /// the section this object was found in section: String, /// headings: A B C D 1 2 3 4 heading_level: String, identifier: String, part: String, /// how the text behaves: para, block, comment is_of_type: String, /// the object's own section, when it differs from the block it sits in own_section: String, segment_html_is: String, segment_epub: String, segment_epub_is: String, table_aligns: String, stow_link: List(String), anchor_tags: List(String), parent_ocn: Int, last_descendant_ocn: Int, ocn_range_first: Int, ocn_range_last: Int, children: List(Int), ancestors: List(Int), dummy_heading: Bool, ocn_off: Bool, indent_base: Int, indent_hang: Int, bullet: Bool, lang: String, has_links: Bool, has_notes_reg: Bool, has_notes_star: Bool, has_images: Bool, segment: String, segment_prev: String, segment_next: String, anchor: String, heading_lev_anchor: String, table_cols: Int, table_widths: String, table_header: Bool, code_syntax: String, code_linenumbers: Bool, heading_ancestors_text: String, sha256: String, images: List(Image), text: String, ) } pub fn new_object() -> Object { Object( ocn: 0, is_a: "", section: "", heading_level: "", identifier: "", part: "", is_of_type: "", own_section: "", segment_html_is: "", segment_epub: "", segment_epub_is: "", table_aligns: "", stow_link: [], anchor_tags: [], parent_ocn: 0, last_descendant_ocn: 0, ocn_range_first: 0, ocn_range_last: 0, children: [], ancestors: [], dummy_heading: False, ocn_off: False, indent_base: 0, indent_hang: 0, bullet: False, lang: "", has_links: False, has_notes_reg: False, has_notes_star: False, has_images: False, segment: "", segment_prev: "", segment_next: "", anchor: "", heading_lev_anchor: "", table_cols: 0, table_widths: "", table_header: False, code_syntax: "", code_linenumbers: False, heading_ancestors_text: "", sha256: "", images: [], text: "", ) } /// Whether this object is rendered at all. /// /// `is_of_type` "comment" is markup commentary: carried by the abstraction /// so that nothing is lost, and not put into output. An output that /// renders it leaks the author's notes into the document, which is what /// spine itself does not do. pub fn renders(object: Object) -> Bool { object.is_of_type != "comment" } /// The section this object belongs to: the block it sits in, unless it /// says otherwise. pub fn section_of(object: Object) -> String { case object.own_section { "" -> object.section own -> own } } /// What the loader met and did not understand. /// /// A loader that silently drops what it does not know cannot say whether /// it understood the document, and so cannot be used to check a /// specification against an implementation: silence and understanding look /// alike. pub type Unknown { Unknown( /// `.name` -> times seen property: Dict(String, Int), /// `@name` -> times seen block: Dict(String, Int), /// first character -> times seen, for lines of no known shape line: Dict(String, Int), ) } pub fn new_unknown() -> Unknown { Unknown(property: dict.new(), block: dict.new(), line: dict.new()) } pub fn any_unknown(unknown: Unknown) -> Bool { dict.size(unknown.property) > 0 || dict.size(unknown.block) > 0 || dict.size(unknown.line) > 0 } pub type Document { Document( /// as declared on line 1 of a `.ssp` or in `schema.version`, "" when /// absent format_version: String, source_filename: String, /// language, languages, digest source: Dict(String, String), unknown: Unknown, metadata: Dict(String, String), make: Dict(String, String), doc_has: Dict(String, String), /// section name -> objects, in document order sections: Dict(String, List(Object)), ) } pub fn new_document() -> Document { Document( format_version: "", source_filename: "", source: dict.new(), unknown: new_unknown(), metadata: dict.new(), make: dict.new(), doc_has: dict.new(), sections: dict.new(), ) } /// The major part of the declared version, "" when none was declared. pub fn version_major(document: Document) -> String { case string.split_once(document.format_version, ".") { Ok(#(major, _)) -> major Error(_) -> document.format_version } } pub fn meta(document: Document, key: String) -> String { case dict.get(document.metadata, key) { Ok(value) -> value Error(_) -> "" } } /// The objects of a named section, empty when the document has none. pub fn section(document: Document, name: String) -> List(Object) { case dict.get(document.sections, name) { Ok(objects) -> objects Error(_) -> [] } } /// Is this one of the nine sections of the format? pub fn is_section(name: String) -> Bool { list.contains(section_names, name) } /// Is this one of the four preamble blocks? pub fn is_preamble_block(name: String) -> Bool { list.contains(preamble_blocks, name) } #+END_SRC *** what to load **** inventory #+NAME: ocda_inventory #+HEADER: :no-expand #+BEGIN_SRC gleam //// What a source holds, beyond the one document the model is loaded from. //// //// The model is one language of one document. Two questions a reader asks //// are about the file instead: which languages it holds, and, for a //// database, what else it carries. They are here rather than in a reader //// because the answer depends on the input form, which is the library's to //// know, and so that a reader asking them does not learn which form it was //// handed. A mirror of the D library's `ocda.inventory`. //// //// `languages` answers for every form, and answers the same: the languages //// in the order the manifest gave them (which every `.ssp` records, in //// `@source languages`), each with the markup file it was made from and the //// sha256 of its `.ssp`. The database stores that digest; for a `.ssp` and a //// pod it is taken from the bytes, which are the same bytes. //// //// `carried` is the database's: a `.ocda.db` holds the markup, images, //// configuration and catalogues it was built from, a `.ssp` holds none of //// them, and a pod's are its archive entries rather than anything the //// abstraction says. import gleam/bit_array import gleam/dict import gleam/list import gleam/result import gleam/string import ocda/load import ocda/load_db import ocda/load_pod import ocda/load_ssp import ocda/zip import simplifile pub type Language { Language(lang: String, source_filename: String, ssp_digest: String) } pub type Carried { Carried( role: String, compression: String, count: String, bytes: String, stored: String, ) } pub fn languages(path: String) -> Result(List(Language), String) { case load.source_of(path) { load.OcdaDb -> { use held <- result.try(load_db.languages_of_file(path)) Ok( list.map(held, fn(l) { Language(l.lang, l.source_filename, l.ssp_digest) }), ) } load.Ssp -> { use bytes <- result.try( simplifile.read_bits(path) |> result.map_error(fn(e) { path <> ": " <> simplifile.describe_error(e) }), ) use text <- result.try( bit_array.to_string(bytes) |> result.replace_error(path <> ": not valid UTF-8"), ) let doc = load_ssp.parse(text) Ok([ Language( source_value(doc.source, "language"), doc.source_filename, sha256_hex(bytes), ), ]) } load.Pod -> { use archive <- result.try(load_pod.archive_bytes(path)) use entries <- result.try(zip.entries(archive)) use found <- result.try( zip.find_suffix(entries, ".ssp") |> list.try_map(fn(e) { use bytes <- result.try(zip.read(archive, e)) use text <- result.try( bit_array.to_string(bytes) |> result.replace_error(e.name <> ": not valid UTF-8"), ) let doc = load_ssp.parse(text) let lang = result.unwrap(load_pod.lang_of(e.name), "") Ok(#( Language(lang, doc.source_filename, sha256_hex(bytes)), string.split(source_value(doc.source, "languages"), " "), )) }), ) let order = case found { [#(_, o), ..] -> list.filter(o, fn(l) { l != "" }) [] -> [] } let by_lang = list.map(found, fn(f) { #({ f.0 }.lang, f.0) }) |> dict.from_list // the manifest's order, and anything it did not name after it let named = list.filter_map(order, fn(l) { dict.get(by_lang, l) }) let rest = list.filter(found, fn(f) { !list.contains(order, { f.0 }.lang) }) |> list.map(fn(f) { f.0 }) Ok(list.append(named, rest)) } load.Unknown -> Error(path <> ": " <> load.source_name(load.Unknown)) } } /// Only a `.ocda.db` carries files. pub fn is_carrier(path: String) -> Bool { load.source_of(path) == load.OcdaDb } pub fn carried(path: String) -> Result(List(Carried), String) { case is_carrier(path) { False -> Ok([]) True -> { use rows <- result.try(load_db.carried_of_file(path)) Ok( list.filter_map(rows, fn(r) { case r { [role, compression, n, bytes, stored] -> Ok(Carried(role, compression, n, bytes, stored)) _ -> Error(Nil) } }), ) } } } fn source_value(source: dict.Dict(String, String), key: String) -> String { result.unwrap(dict.get(source, key), "") } @external(erlang, "ocda_ffi", "sha256_hex") fn sha256_hex(bytes: BitArray) -> String #+END_SRC **** load ***** load #+NAME: ocda_load #+HEADER: :no-expand #+BEGIN_SRC gleam //// What a path is, and loading it. //// //// One place that decides, so that every reader and every writer accepts //// the same inputs and a new input form is added once. This mirrors //// spine's own `ocda/abstraction/load.d`, which names the sources it can //// enter the pipeline from and tells them apart by suffix. //// //// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means //// implementing the parser, which is spine's work and a far larger thing //// than a reader or a sample writer needs. This library reads //// abstractions. import gleam/list import gleam/string import ocda/load_db import ocda/load_pod import ocda/load_ssp import ocda/model.{type Document} pub type Source { /// `.ssp`, the abstraction as text Ssp /// `.ocda.db`, the abstraction as sqlite OcdaDb /// `.sisupod` or `.zip`, an archive carrying a `.ssp` Pod Unknown } /// `.sisupod` is the archive spine writes now; `.zip` is what it wrote /// before, and every pod already published carries that name, so both are /// recognised. What is inside decides how it is opened, not the name. const suffixes = [ #(".ocda.db", OcdaDb), #(".ssp", Ssp), #(".sisupod", Pod), #(".zip", Pod), #(".db", OcdaDb), ] pub fn source_of(path: String) -> Source { case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) { Ok(#(_, source)) -> source Error(_) -> Unknown } } pub fn source_name(source: Source) -> String { case source { Ssp -> ".ssp (the abstraction as text)" OcdaDb -> ".ocda.db (the abstraction as sqlite)" Pod -> "a pod archive carrying a .ssp" Unknown -> "not an abstraction this library reads" } } /// Load whatever the path is into the model. /// /// `lang` names which language to take from a `.ocda.db`, which holds /// every language of its document, and from a pod, which may carry more than /// one `.ssp`. A `.ssp` is one language by construction and ignores it. pub fn load( path: String, lang: String, any_version: Bool, ) -> Result(Document, String) { case source_of(path) { Ssp -> load_ssp.load_file(path) OcdaDb -> load_db.load_file(path, lang, any_version) Pod -> load_pod.load_file(path, lang) Unknown -> Error( path <> ": " <> source_name(Unknown) <> " (expected .ssp, .ocda.db, .sisupod or .zip)", ) } } #+END_SRC ***** load pod #+NAME: ocda_load_pod #+HEADER: :no-expand #+BEGIN_SRC gleam //// The abstraction out of a pod archive. //// //// A pod built with `--pod2` carries the abstraction inside it, at //// `media/abstraction/..ssp`, and that is what is read. So this is //// a third input form and *not* a third model: the `.ssp` inside is parsed by //// `ocda/load_ssp` and comes back as the same value. A writer over the //// library gains pods without being touched, which is the whole reason the //// dispatch is in one place. //// //// ## The one thing this cannot do //// //// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip //// and **not the frame**, because there is no zstd on the BEAM without a NIF, //// and the one that exists, `ezstd`, clones zstd from github in its build, so //// it cannot be built in a sandbox without patching it. The D library has //// both, through a declared `libzstd`. //// //// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says //// why rather than pretending the file is malformed. It is the only place the //// two libraries differ in what they accept, and it is a packaging limit //// rather than anything about the format. import gleam/list import gleam/result import gleam/string import ocda/load_ssp import ocda/model.{type Document} import ocda/zip import simplifile /// The magic number of a zstd frame, little endian 0xFD2FB528. const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>> /// Which languages of the abstraction the pod carries, sorted. pub fn languages(path: String) -> Result(List(String), String) { use bytes <- result.try(archive_bytes(path)) use entries <- result.try(zip.entries(bytes)) zip.find_suffix(entries, ".ssp") |> list.filter_map(fn(e) { lang_of(e.name) }) |> list.sort(string.compare) |> Ok } /// Load one language of the abstraction a pod carries. /// /// `lang` == "" takes the only one there is, and where there is a choice it /// says so rather than choosing, as the `.ocda.db` loader does. pub fn load_file(path: String, lang: String) -> Result(Document, String) { use bytes <- result.try(archive_bytes(path)) use entries <- result.try(zip.entries(bytes)) let ssps = zip.find_suffix(entries, ".ssp") use chosen <- result.try(case ssps, lang { [], _ -> Error( path <> ": carries no .ssp. A pod carries the abstraction only when spine" <> " was asked for it: spine --pod2 puts it in, spine --source does not.", ) [only], "" -> Ok(only) _, "" -> Error( path <> ": holds more than one language (" <> string.join(langs(ssps), " ") <> "); name the one wanted", ) _, wanted -> case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) { Ok(found) -> Ok(found) Error(_) -> Error( path <> ": no language '" <> wanted <> "'; it holds: " <> string.join(langs(ssps), " "), ) } }) use content <- result.try(zip.read(bytes, chosen)) case bit_array_to_string(content) { Ok(text) -> Ok(load_ssp.parse(text)) Error(_) -> Error(chosen.name <> ": not valid UTF-8") } } fn langs(entries: List(zip.Entry)) -> List(String) { list.filter_map(entries, fn(e) { lang_of(e.name) }) } /// `..ssp` -> `` pub fn lang_of(name: String) -> Result(String, Nil) { case string.ends_with(name, ".ssp") { False -> Error(Nil) True -> { let stem = string.drop_end(name, 4) case string.split(stem, on: ".") |> list.last { Ok(lang) if lang != stem -> Ok(lang) _ -> Error(Nil) } } } } /// The bytes decide, not the name: a file misnamed either way is treated by /// what it holds. pub fn archive_bytes(path: String) -> Result(BitArray, String) { case simplifile.read_bits(path) { Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) Ok(bytes) -> case is_zstd_frame(bytes) { False -> Ok(bytes) True -> Error( path <> " is a zstd frame wrapping the archive, and this library has no" <> " zstd: the only one on the BEAM is a NIF that fetches zstd in" <> " its own build, so it cannot be built sandboxed. Unwrap it" <> " first (zstd -d) and read the archive, or use the D library," <> " which reads both.", ) } } } fn is_zstd_frame(bytes: BitArray) -> Bool { case bytes { <> -> head == zstd_magic _ -> False } } @external(erlang, "unicode", "characters_to_binary") fn bit_array_to_string_ffi(b: BitArray) -> String pub fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { Ok(bit_array_to_string_ffi(b)) } #+END_SRC ***** load ssp #+NAME: ocda_load_ssp #+HEADER: :no-expand #+BEGIN_SRC gleam //// Loading a `.ssp` into the model. //// //// The format is line oriented and every line is one of eight shapes told //// apart by its first character, so the loader is a fold over the lines //// carrying the state the shapes need. //// //// Written against `ssp-format-specification.org`. Where it and the D //// library differ the D library is the bug, or this is: they are meant to //// give the same model, and `ocda-dump` says whether they do. import gleam/dict.{type Dict} import gleam/int import gleam/list import gleam/option import gleam/result import gleam/string import ocda/model.{type Document, type Image, type Object, Image, Object} import simplifile pub fn load_file(path: String) -> Result(Document, String) { case simplifile.read(path) { Ok(content) -> Ok(parse(content)) Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) } } type State { State( document: Document, /// the block currently open, "" when none block: String, /// the section currently open, "" when in the preamble section: String, /// objects of the current section, newest first objects: List(Object), /// the object being read, if any current: Result(Object, Nil), /// text lines of the object being read, newest first text: List(String), /// whether a text line has been seen for the object being read. /// Not "is the text non-empty": an object whose first text line is /// blank, which the reference collection has seven of, would then lose /// it. seen_text: Bool, ) } pub fn parse(content: String) -> Document { content |> string.split(on: "\n") |> list.fold(from: initial(), with: line) |> finish } fn initial() -> State { State( document: model.new_document(), block: "", section: "", objects: [], current: Error(Nil), text: [], seen_text: False, ) } fn line(state: State, raw: String) -> State { case raw { "% SiSU Document Abstraction v" <> version -> case state.document.format_version { "" -> State( ..state, document: model.Document( ..state.document, format_version: string.trim(version), ), ) _ -> state } "% Source: " <> filename -> case state.document.source_filename { "" -> State( ..state, document: model.Document( ..state.document, source_filename: string.trim(filename), ), ) _ -> state } "%" <> _ -> state "@" <> rest -> open_block(state, rest) "}" <> _ -> close_block(state) "[" <> rest -> declaration(state, rest) "." <> rest -> property(state, rest) "|" <> rest -> text_line(state, rest) _ -> case string.trim(raw) { // a blank line ends the object being read "" -> commit(state) _ -> case model.is_preamble_block(state.block) { True -> preamble_pair(state, raw) False -> unknown_line(state, raw) } } } } fn open_block(state: State, rest: String) -> State { let name = case string.split_once(rest, "{") { Ok(#(name, _)) -> string.trim(name) Error(_) -> string.trim(rest) } case model.is_preamble_block(name) { True -> State(..flush(state), block: name) False -> { let state = flush(state) let document = case model.is_section(name) { True -> state.document // a block that is neither preamble nor one of the nine sections: // counted, so that a reader can say the file held something it did // not expect False -> model.Document( ..state.document, unknown: model.Unknown( ..state.document.unknown, block: bump(state.document.unknown.block, name), ), ) } State(..state, block: name, section: name, document: document) } } } fn close_block(state: State) -> State { State(..flush(state), block: "") } /// Commit the object being read, then put the section's objects into the /// document and close the section. fn flush(state: State) -> State { let state = commit(state) case state.section, state.objects { "", _ -> state _, [] -> State(..state, section: "", objects: []) name, objects -> State( ..state, section: "", objects: [], document: model.Document( ..state.document, sections: add_section( state.document.sections, name, list.reverse(objects), ), ), ) } } fn preamble_pair(state: State, raw: String) -> State { case string.split_once(string.trim_start(raw), ":") { Error(_) -> state Ok(#(key, value)) -> { let key = string.trim(key) let value = string.trim(value) let document = state.document let document = case state.block { "meta" -> model.Document( ..document, metadata: dict.insert(document.metadata, key, value), ) "make" -> model.Document(..document, make: dict.insert(document.make, key, value)) "source" -> model.Document( ..document, source: dict.insert(document.source, key, value), ) _ -> model.Document( ..document, doc_has: dict.insert(document.doc_has, key, value), ) } State(..state, document: document) } } } /// `[] `, and for a heading `[] heading : /// []`. fn declaration(state: State, rest: String) -> State { let state = commit(state) case string.split_once(rest, "]") { Error(_) -> state Ok(#(ocn_text, tail)) -> { let ocn = parse_int(string.trim(ocn_text)) let object = Object(..model.new_object(), ocn: ocn, section: state.section) let object = case string.split(string.trim(tail), on: " ") { ["heading", level, ..identifier] -> Object( ..object, is_a: "heading", heading_level: string.drop_start(level, 1), identifier: string.join(identifier, " "), ) [""] -> object [kind, ..] -> Object(..object, is_a: kind) [] -> object } State(..state, current: Ok(object), text: [], seen_text: False) } } } fn property(state: State, rest: String) -> State { case state.current, string.split_once(rest, ":") { Ok(object), Ok(#(name, raw_value)) -> { let name = string.trim(name) let value = string.trim(raw_value) let #(object, known) = apply_property(object, name, value) let document = case known { True -> state.document False -> model.Document( ..state.document, unknown: model.Unknown( ..state.document.unknown, property: bump(state.document.unknown.property, name), ), ) } State(..state, current: Ok(object), document: document) } _, _ -> state } } /// The properties this library reads, and what each sets. /// /// A property it has no use for is counted rather than passed over: /// ignoring one is conforming within a major version, doing so in silence /// is what stops a reader being a check on the specification. fn apply_property( object: Object, name: String, value: String, ) -> #(Object, Bool) { case name { "part" -> #(Object(..object, part: value), True) "is_of_type" -> #(Object(..object, is_of_type: value), True) "section" -> #(Object(..object, own_section: value), True) "identifier" -> #(Object(..object, identifier: value), True) "segment_html_is" -> #(Object(..object, segment_html_is: value), True) "segment_epub" -> #(Object(..object, segment_epub: value), True) "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True) "table_aligns" -> #(Object(..object, table_aligns: value), True) "stow_link" -> #( Object(..object, stow_link: list.append(object.stow_link, [value])), True, ) "anchor_tag" -> #( Object(..object, anchor_tags: list.append(object.anchor_tags, [value])), True, ) "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True) "last_descendant" -> #( Object(..object, last_descendant_ocn: parse_int(value)), True, ) "ocn_range" -> case numbers(value) { [first, last] -> #( Object(..object, ocn_range_first: first, ocn_range_last: last), True, ) _ -> #(object, True) } "children" -> #(Object(..object, children: numbers(value)), True) "ancestors" -> #(Object(..object, ancestors: numbers(value)), True) "dummy" -> #(Object(..object, dummy_heading: value == "true"), True) "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True) "indent" -> { let parts = string.split(value, on: " ") let base = case parts { [first, ..] -> parse_int(first) [] -> 0 } let hang = case parts { [_, second, ..] -> parse_int(second) _ -> 0 } #(Object(..object, indent_base: base, indent_hang: hang), True) } "bullet" -> #(Object(..object, bullet: value == "true"), True) "lang" -> #(Object(..object, lang: value), True) "has" -> #(has_flags(object, value), True) "segment" -> #(Object(..object, segment: value), True) "segment_prev" -> #(Object(..object, segment_prev: value), True) "segment_next" -> #(Object(..object, segment_next: value), True) "anchor" -> #(Object(..object, anchor: value), True) "heading_lev_anchor" -> #( Object(..object, heading_lev_anchor: value), True, ) "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True) "table_widths" -> #(Object(..object, table_widths: value), True) "table_header" -> #(Object(..object, table_header: value == "true"), True) "code_syntax" -> #(Object(..object, code_syntax: value), True) "code_linenumbers" -> #( Object(..object, code_linenumbers: value == "true"), True, ) "heading_ancestors_text" -> #( Object(..object, heading_ancestors_text: value), True, ) "sha256" -> #(Object(..object, sha256: value), True) "image" -> #( Object(..object, images: list.append(object.images, [image(value)])), True, ) _ -> #(object, False) } } fn has_flags(object: Object, value: String) -> Object { value |> string.split(on: " ") |> list.fold(from: object, with: fn(obj, flag) { case flag { "links" -> Object(..obj, has_links: True) "notes_reg" -> Object(..obj, has_notes_reg: True) "notes_star" -> Object(..obj, has_notes_star: True) "images" -> Object(..obj, has_images: True) _ -> obj } }) } /// ` sha256: bytes: [px:x] [missing:true] [...]` /// /// The filename comes first, the rest are order-independent, and a field /// this library does not recognise is kept rather than discarded. fn image(value: String) -> Image { case string.split(value, on: " ") { [] -> model.new_image() [name, ..fields] -> list.fold(fields, model.new_image(), fn(img, field) { case string.split_once(field, ":") { Error(_) -> img Ok(#(key, val)) -> case key { "sha256" -> Image(..img, sha256: val) "bytes" -> Image(..img, bytes: parse_int(val)) "px" -> case string.split_once(val, "x") { Ok(#(w, h)) -> Image(..img, width: parse_int(w), height: parse_int(h)) Error(_) -> img } "missing" -> Image(..img, missing: val == "true") _ -> Image(..img, fields: dict.insert(img.fields, key, val)) } } }) |> fn(img) { Image(..img, name: name) } } } /// `| `, and `| ` alone for a blank line. Never a bare `|`: the /// emitter writes the space. fn text_line(state: State, rest: String) -> State { case state.current { Error(_) -> state Ok(_) -> { let content = string.drop_start(rest, 1) State(..state, text: [content, ..state.text], seen_text: True) } } } fn unknown_line(state: State, raw: String) -> State { let first = string.slice(raw, 0, 1) State( ..state, document: model.Document( ..state.document, unknown: model.Unknown( ..state.document.unknown, line: bump(state.document.unknown.line, first), ), ), ) } fn commit(state: State) -> State { case state.current { Error(_) -> state Ok(object) -> { let text = case state.seen_text { True -> string.join(list.reverse(state.text), "\n") False -> "" } let object = derive_omitted(Object(..object, text: text)) let objects = case model.is_section(state.section) { True -> [object, ..state.objects] False -> state.objects } State( ..state, current: Error(Nil), text: [], seen_text: False, objects: objects, ) } } } /// Two values the text form leaves out because they can be worked out. /// /// `.identifier` is absent when it is simply the ocn, `.anchor` when it is /// the segment name. The `.ocda.db`, being columns rather than lines, /// stores both unconditionally. So a loader that takes the text form /// literally gives a different model from the one the database gives, and /// over one document that is 1,314 identifiers and 69 anchors. fn derive_omitted(object: Object) -> Object { let object = case object.identifier, object.ocn { "", 0 -> object "", ocn -> Object(..object, identifier: int.to_string(ocn)) _, _ -> object } case object.anchor, object.segment { "", "" -> object "", segment -> Object(..object, anchor: segment) _, _ -> object } } fn finish(state: State) -> Document { flush(state).document } fn add_section( sections: Dict(String, List(Object)), name: String, objects: List(Object), ) -> Dict(String, List(Object)) { dict.upsert(sections, name, fn(existing) { case existing { option.Some(previous) -> list.append(previous, objects) option.None -> objects } }) } fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) { dict.upsert(counts, key, fn(existing) { case existing { option.Some(n) -> n + 1 option.None -> 1 } }) } fn parse_int(value: String) -> Int { value |> string.trim |> int.parse |> result.unwrap(0) } fn numbers(value: String) -> List(Int) { value |> string.split(on: " ") |> list.filter(fn(s) { s != "" }) |> list.map(parse_int) } #+END_SRC ***** load db #+NAME: ocda_load_db #+HEADER: :no-expand #+BEGIN_SRC gleam //// Loading a `.ocda.db` into the model. //// //// The point of this module is that it returns *the same value* //// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is //// then a function from the model to bytes and cannot tell which //// serialisation it was given, exactly as spine's own writers cannot. //// //// Written against `ocda-db-format-specification.org`, whose section 4 //// gives the column by column correspondence this module implements. import gleam/dict import gleam/dynamic/decode import gleam/float import gleam/int import gleam/json import gleam/list import gleam/option import gleam/result import gleam/string import ocda/model.{type Document, Image, Object} import sqlight /// The schema major version this loader knows. pub const schema_major_read = "2" /// One language of the database, as the manifest gives it. pub type Language { Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String) } /// Load one language of a database as the model. `lang` == "" takes the /// first language the manifest gives. pub fn load_file( path: String, lang: String, any_version: Bool, ) -> Result(Document, String) { // read only: a loader has no business writing, and a read-write open // *creates* an empty database where a path is mistyped, which then reads // as a document with nothing in it, a failure that looks like an answer use conn <- with_connection("file:" <> path <> "?mode=ro", path) use _ <- result.try(case has(conn, "documents") && has(conn, "objects") && has(conn, "metadata") { True -> Ok(Nil) False -> Error( path <> ": no documents/objects/metadata tables; it does not look like a .ocda.db", ) }) let schema_version = one(conn, "select value from metadata where doc_id is null and key = 'schema.version'") use _ <- result.try(case schema_version, any_version { "", _ -> Ok(Nil) _, True -> Ok(Nil) v, False -> case major_of(v) == schema_major_read { True -> Ok(Nil) False -> Error( path <> " is schema v" <> v <> ", this loader knows v" <> schema_major_read <> ".x", ) } }) let languages = languages(conn) use chosen <- result.try(case languages, lang { [], _ -> Error(path <> ": no documents rows") [first, ..], "" -> Ok(first) _, wanted -> case list.find(languages, fn(l) { l.lang == wanted }) { Ok(found) -> Ok(found) Error(_) -> Error( path <> ": no language '" <> wanted <> "'; it holds: " <> string.join(list.map(languages, fn(l) { l.lang }), " "), ) } }) let source = dict.new() |> dict.insert("language", chosen.lang) |> dict.insert( "languages", string.join(list.map(languages, fn(l) { l.lang }), " "), ) let digest = meta_value(conn, chosen.doc_id, "source.digest") let source = case digest { "" -> source d -> dict.insert(source, "digest", d) } let document = model.Document( ..model.new_document(), format_version: schema_version, source_filename: chosen.source_filename, source: source, ) let document = metadata(conn, chosen.doc_id, document) let document = objects(conn, chosen.doc_id, document) let document = count_unread(conn, chosen.doc_id, document) Ok(document) } fn with_connection( uri: String, path: String, then: fn(sqlight.Connection) -> Result(a, String), ) -> Result(a, String) { case sqlight.open(uri) { Error(e) -> Error(path <> ": " <> e.message) Ok(conn) -> { let outcome = then(conn) let _ = sqlight.close(conn) outcome } } } /// Every language the database holds, in manifest order. pub fn languages(conn: sqlight.Connection) -> List(Language) { let d = { use id <- decode.field(0, decode.int) use lang <- decode.field(1, decode.string) use source <- decode.field(2, decode.string) use digest <- decode.field(3, decode.string) decode.success(Language(id, lang, source, digest)) } sqlight.query( "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') from documents order by id", on: conn, with: [], expecting: d, ) |> result.unwrap([]) } /// The three preamble blocks of the text form, out of the one metadata /// table the database keeps them in. fn metadata( conn: sqlight.Connection, doc_id: Int, document: Document, ) -> Document { let d = { use key <- decode.field(0, decode.string) use value <- decode.field(1, decode.string) decode.success(#(key, value)) } let rows = sqlight.query( "select key, value from metadata where doc_id = " <> int.to_string(doc_id) <> " order by key", on: conn, with: [], expecting: d, ) |> result.unwrap([]) list.fold(rows, document, fn(doc, row) { let #(key, raw) = row let value = string.trim(raw) case key { // source.* describes the artefact rather than the document, and is // @source in the text form "source." <> _ -> doc // translation.ocn_aligned says whether the languages of this document // share one numbering. A .ocda.db can know it because it holds every // language; a .ssp is one language and cannot. It is a fact about the // file, not the document, and carrying it here would let a writer // tell which serialisation it was handed. "translation." <> _ -> doc // the database tells the three blocks apart by a key prefix. // Unprefixing them is what makes the two loaders give one model: a // writer asks make["footer"], never metadata["make.footer"]. "make." <> rest -> model.Document(..doc, make: dict.insert(doc.make, rest, value)) "doc_has." <> rest -> model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value)) _ -> model.Document(..doc, metadata: dict.insert(doc.metadata, key, value)) } }) } /// The objects, in document order within each section, the sections in the /// order the format gives rather than the alphabetical one sqlite would. fn objects( conn: sqlight.Connection, doc_id: Int, document: Document, ) -> Document { let d = { use section <- decode.field(0, decode.string) use ocn <- decode.field(2, decode.int) use is_a <- decode.field(3, decode.string) use part <- decode.field(4, decode.string) use is_of_section <- decode.field(5, decode.string) use is_of_type <- decode.field(6, decode.string) use heading_level <- decode.field(7, decode.int) use identifier <- decode.field(8, decode.string) use parent <- decode.field(9, decode.int) use last_descendant <- decode.field(10, decode.int) use children <- decode.field(11, decode.string) use ancestors <- decode.field(12, decode.string) use dummy <- decode.field(13, decode.int) use ocn_off <- decode.field(14, decode.int) use indent_base <- decode.field(15, decode.int) use indent_hang <- decode.field(16, decode.int) use bullet <- decode.field(17, decode.int) use lang <- decode.field(18, decode.string) use has_links <- decode.field(19, decode.int) use has_notes_reg <- decode.field(20, decode.int) use has_notes_star <- decode.field(21, decode.int) use has_images <- decode.field(22, decode.int) use segment <- decode.field(23, decode.string) use segment_prev <- decode.field(24, decode.string) use segment_next <- decode.field(25, decode.string) use anchor <- decode.field(26, decode.string) use segment_html_is <- decode.field(27, decode.string) use table_cols <- decode.field(28, decode.int) use table_widths <- decode.field(29, decode.string) use table_aligns <- decode.field(30, decode.string) use table_header <- decode.field(31, decode.int) use code_linenumbers <- decode.field(32, decode.int) use hat <- decode.field(33, decode.string) use sha256 <- decode.field(34, decode.string) use text <- decode.field(35, decode.string) use id <- decode.field(36, decode.int) use hla <- decode.field(37, decode.string) use seg_epub <- decode.field(38, decode.string) use seg_epub_is <- decode.field(39, decode.string) use range_first <- decode.field(40, decode.int) use range_last <- decode.field(41, decode.int) decode.success(#( id, Object( ocn: ocn, is_a: is_a, section: section, heading_level: level_name(heading_level), identifier: identifier, part: part, is_of_type: is_of_type, own_section: case is_of_section == section { True -> "" False -> is_of_section }, segment_html_is: segment_html_is, table_aligns: json_words(table_aligns), stow_link: [], anchor_tags: [], parent_ocn: parent, last_descendant_ocn: last_descendant, ocn_range_first: range_first, ocn_range_last: range_last, children: json_ints(children), ancestors: json_ints(ancestors), dummy_heading: dummy != 0, ocn_off: ocn_off != 0, indent_base: indent_base, indent_hang: indent_hang, bullet: bullet != 0, lang: lang, has_links: has_links != 0, has_notes_reg: has_notes_reg != 0, has_notes_star: has_notes_star != 0, has_images: has_images != 0, segment: segment, segment_prev: segment_prev, segment_next: segment_next, anchor: anchor, heading_lev_anchor: hla, segment_epub: seg_epub, segment_epub_is: seg_epub_is, table_cols: table_cols, table_widths: json_words(table_widths), table_header: table_header != 0, code_syntax: "", code_linenumbers: code_linenumbers != 0, heading_ancestors_text: json_joined(hat), sha256: sha256, images: [], text: text, ), )) } let rows = sqlight.query( object_sql(doc_id, has_range_columns(conn)), on: conn, with: [], expecting: d, ) |> result.unwrap([]) // Built newest-first per section and reversed once at the end. Appending // to the tail of each section instead would be quadratic, and the largest // document in the reference collection has some thousands of objects in // one section. let sections = list.fold(rows, dict.new(), fn(acc, row) { let #(id, object) = row // the four repeatable properties of the text form are child tables // here, one query each let object = Object( ..object, images: images(conn, id), stow_link: links(conn, id), anchor_tags: anchors(conn, id), ) case model.is_section(object.section) { False -> acc True -> dict.upsert(acc, object.section, fn(existing) { case existing { option.Some(previous) -> [object, ..previous] option.None -> [object] } }) } }) |> dict.map_values(fn(_, objects) { list.reverse(objects) }) model.Document(..document, sections: sections) } /// 2.1 added the container range as two columns. A 2.0 database has /// neither, and reads as having no range, which is what 2.0 meant. fn has_range_columns(conn: sqlight.Connection) -> Bool { let d = decode.field(0, decode.int, decode.success) case sqlight.query( "select count(*) from pragma_table_info('objects') where name = 'ocn_range_first'", on: conn, with: [], expecting: d, ) { Ok([n]) -> n > 0 _ -> False } } fn object_sql(doc_id: Int, has_range: Bool) -> String { let range_cols = case has_range { True -> "coalesce(ocn_range_first,0), coalesce(ocn_range_last,0)" False -> "0, 0" } "select section, seq, ocn, is_a, coalesce(is_of_part,''), coalesce(is_of_section,''), coalesce(is_of_type,''), coalesce(heading_level,-1), coalesce(identifier,''), coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), coalesce(children,''), coalesce(ancestors,''), coalesce(dummy_heading,0), coalesce(object_number_off,0), coalesce(indent_base,0), coalesce(indent_hang,0), coalesce(bullet,0), coalesce(lang,''), coalesce(has_links,0), coalesce(has_notes_reg,0), coalesce(has_notes_star,0), coalesce(has_images,0), coalesce(segment,''), coalesce(segment_prev,''), coalesce(segment_next,''), coalesce(anchor,''), coalesce(segment_html_is,''), coalesce(table_cols,0), coalesce(table_widths,''), coalesce(table_aligns,''), coalesce(table_header,0), coalesce(code_linenumbers,0), coalesce(heading_ancestors_text,''), coalesce(sha256,''), coalesce(text,''), id, coalesce(heading_lev_anchor,''), coalesce(segment_epub,''), coalesce(segment_epub_is,''), " <> range_cols <> " from objects where doc_id = " <> int.to_string(doc_id) <> " order by case section when 'head' then 1 when 'toc' then 2 when 'body' then 3 when 'endnotes' then 4 when 'glossary' then 5 when 'bibliography' then 6 when 'bookindex' then 7 when 'blurb' then 8 when 'tail' then 9 else 10 end, seq" } fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) { let d = { use name <- decode.field(0, decode.string) use sha256 <- decode.field(1, decode.string) use bytes <- decode.field(2, decode.int) use width <- decode.field(3, decode.int) use height <- decode.field(4, decode.int) use missing <- decode.field(5, decode.int) decode.success(Image( name: name, sha256: sha256, bytes: bytes, width: width, height: height, missing: missing != 0, fields: dict.new(), )) } sqlight.query( "select name, coalesce(sha256,''), coalesce(bytes,0), coalesce(width,0), coalesce(height,0), coalesce(missing,0) from object_images where object_id = " <> int.to_string(object_id) <> " order by seq", on: conn, with: [], expecting: d, ) |> result.unwrap([]) } fn links(conn: sqlight.Connection, object_id: Int) -> List(String) { let d = { use url <- decode.field(0, decode.string) decode.success(url) } sqlight.query( "select url from object_links where object_id = " <> int.to_string(object_id) <> " order by seq", on: conn, with: [], expecting: d, ) |> result.unwrap([]) } fn anchors(conn: sqlight.Connection, object_id: Int) -> List(String) { let d = { use anchor <- decode.field(0, decode.string) decode.success(anchor) } sqlight.query( "select anchor from object_anchors where object_id = " <> int.to_string(object_id) <> " order by seq", on: conn, with: [], expecting: d, ) |> result.unwrap([]) } fn one(conn: sqlight.Connection, sql: String) -> String { let d = { use v <- decode.field(0, decode.string) decode.success(v) } case sqlight.query(sql, on: conn, with: [], expecting: d) { Ok([v, ..]) -> v _ -> "" } } fn has(conn: sqlight.Connection, name: String) -> Bool { one( conn, "select cast(count(*) as text) from sqlite_master where name = '" <> name <> "'", ) != "0" } fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String { one( conn, "select value from metadata where doc_id = " <> int.to_string(doc_id) <> " and key = '" <> key <> "'", ) } fn major_of(version: String) -> String { case string.split_once(version, ".") { Ok(#(major, _)) -> major Error(_) -> version } } /// The eight heading levels and their indices. fn level_name(index: Int) -> String { case index { 0 -> "A" 1 -> "B" 2 -> "C" 3 -> "D" 4 -> "1" 5 -> "2" 6 -> "3" 7 -> "4" _ -> "" } } /// The JSON columns, read as JSON. /// /// Not as a character scan that strips brackets and quotes: that is wrong /// the moment a value holds an escape, and spine's encoder writes "/" as /// "\/", so every heading containing a slash comes back with a backslash in /// it. The D library had exactly that bug and all 36 document-languages /// differed on heading_ancestors_text because of it. fn json_strings(value: String) -> List(String) { case value { "" -> [] v -> case json.parse(v, decode.list(of: loose_string())) { Ok(items) -> items Error(_) -> [] } } } /// An element that may be a string or a number, which is what these columns /// hold: children and ancestors are int arrays, table_aligns and /// heading_ancestors_text string arrays, and table_widths an int array /// except where the columns divide evenly, when it is [33.3333,...]. One /// table in the reference collection does that, and without the float /// branch the widths came back as three empty strings: the fallback turned /// a value it could not read into a value that looked read. fn loose_string() -> decode.Decoder(String) { decode.one_of(decode.string, [ decode.int |> decode.map(int.to_string), decode.float |> decode.map(float.to_string), decode.success(""), ]) } fn json_ints(value: String) -> List(Int) { value |> json_strings |> list.filter(fn(s) { s != "" }) |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) }) } /// Space separated in the text form: table_widths, table_aligns. fn json_words(value: String) -> String { value |> json_strings |> string.join(" ") } /// heading_ancestors_text is eight slots, joined with "|" in the text form /// and a JSON array here. The empty slots are significant: they say which /// levels the heading sits under. fn json_joined(value: String) -> String { case json_strings(value) { [] -> "" parts -> string.join(parts, "|") } } /// What the database holds and the model does not. /// /// The text loader meets every property line and counts those it does not /// keep, so a reader can say what was read and not understood. This loader /// asks for columns by name, and would never see the ones it did not ask /// for: it would report that it had understood everything, which is the /// silence the count exists to prevent. So it counts them, on the /// conditions under which the `.ssp` writes the same property /// (specification section 6.3), and the report is the same from either /// form, as the D loader's is. fn count_unread( conn: sqlight.Connection, doc_id: Int, document: Document, ) -> Document { let id = int.to_string(doc_id) let counts = [ #("ancestors_collapsed", "coalesce(ancestors_collapsed,'') != ''"), #("dom_status", "coalesce(dom_status,'') != ''"), #("dom_status_collapsed", "coalesce(dom_status_collapsed,'') != ''"), #("heading_lev_collapsed", "heading_lev_collapsed is not null"), #("parent_lev", "coalesce(parent_lev,0) != 0"), #("segment_lv4_is", "coalesce(segment_lv4_is,'') != ''"), #("attrib", "coalesce(attrib,'') != ''"), ] let property = list.fold(counts, document.unknown.property, fn(acc, c) { let n = count( conn, "select count(*) from objects where doc_id = " <> id <> " and " <> c.1, ) case n > 0 { True -> dict.insert(acc, c.0, n) False -> acc } }) let property = case has(conn, "object_subtoc") { False -> property True -> { let n = count( conn, "select count(*) from object_subtoc s join objects o on o.id = s.object_id where o.doc_id = " <> id, ) case n > 0 { True -> dict.insert(property, "lev4_subtoc", n) False -> property } } } model.Document( ..document, unknown: model.Unknown(..document.unknown, property: property), ) } fn count(conn: sqlight.Connection, sql: String) -> Int { let d = decode.field(0, decode.int, decode.success) case sqlight.query(sql, on: conn, with: [], expecting: d) { Ok([n, ..]) -> n _ -> 0 } } /// Every language the database at `path` holds, in manifest order. pub fn languages_of_file(path: String) -> Result(List(Language), String) { use conn <- with_connection("file:" <> path <> "?mode=ro", path) Ok(languages(conn)) } /// What the database at `path` carries besides the abstraction, by role /// and storage: role, compression, count, original bytes, stored bytes. pub fn carried_of_file(path: String) -> Result(List(List(String)), String) { use conn <- with_connection("file:" <> path <> "?mode=ro", path) case has(conn, "files") { False -> Ok([]) True -> { let d = { use role <- decode.field(0, decode.string) use compression <- decode.field(1, decode.string) use n <- decode.field(2, decode.int) use bytes <- decode.field(3, decode.int) use stored <- decode.field(4, decode.int) decode.success([ role, compression, int.to_string(n), int.to_string(bytes), int.to_string(stored), ]) } sqlight.query( "select role, compression, count(*), sum(bytes), sum(length(data)) from files group by role, compression order by role", on: conn, with: [], expecting: d, ) |> result.map_error(fn(e) { path <> ": " <> e.message }) } } } #+END_SRC **** images #+NAME: ocda_images #+HEADER: :no-expand #+BEGIN_SRC gleam //// An image's bytes, from whichever source the document came from. //// //// The model *names* images, with digest, byte size and pixel size; it does //// not hold them. That is right - an abstraction of a document is not a //// bundle of its assets - but a writer of any format that shows images needs //// the bytes, so something has to know where they are. This is that //// something, and it is in the library rather than in a writer because the //// answer depends on the *input form* and not on the output. //// //// | `.ocda.db` | the `files` table, role 'image', by name | //// | `.zip` | the archive entry whose path ends in `/image/` | //// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot | //// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` | import gleam/bit_array import gleam/dynamic/decode import gleam/list import gleam/result import gleam/string import ocda/load import ocda/zip import simplifile import sqlight /// The bytes, or an error saying why not. pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) { case load.source_of(source) { load.OcdaDb -> from_db(source, name) load.Pod -> from_archive(source, name) load.Ssp -> from_beside(source, name) load.Unknown -> Error(source <> ": not a source this library reads") } } /// Can this source give images at all? /// /// A writer asks before it decides whether to emit them, because an `` /// pointing at a file it did not include is worse than no image. pub fn can_give_images(source: String) -> Bool { case load.source_of(source) { load.OcdaDb -> True load.Pod -> zip_readable(source) load.Ssp -> image_dir_beside(source) != "" load.Unknown -> False } } fn zip_readable(path: String) -> Bool { case simplifile.read_bits(path) { Error(_) -> False Ok(bytes) -> case zip.entries(bytes) { Ok(_) -> True Error(_) -> False } } } fn from_db(path: String, name: String) -> Result(BitArray, String) { case sqlight.open("file:" <> path <> "?mode=ro") { Error(e) -> Error(path <> ": " <> e.message) Ok(conn) -> { let out = sqlight.query( "select data from files where role = 'image' and name = ?1", on: conn, with: [sqlight.text(name)], expecting: blob(), ) |> result.replace_error(path <> ": could not read " <> name) let _ = sqlight.close(conn) case out { Ok([data, ..]) -> Ok(data) Ok([]) -> Error(name <> ": not in this database") Error(e) -> Error(e) } } } } /// A blob column, read as bytes rather than as text: an image read as text /// stops at the first zero byte, and a png's ninth byte is zero. fn blob() -> decode.Decoder(BitArray) { use v <- decode.field(0, decode.bit_array) decode.success(v) } fn from_archive(path: String, name: String) -> Result(BitArray, String) { use bytes <- result.try( simplifile.read_bits(path) |> result.replace_error(path <> ": cannot read"), ) use entries <- result.try(zip.entries(bytes)) case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) { Error(_) -> Error(name <> ": not in this archive") Ok(entry) -> zip.read(bytes, entry) } } fn from_beside(path: String, name: String) -> Result(BitArray, String) { case image_dir_beside(path) { "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it") dir -> simplifile.read_bits(dir <> "/" <> name) |> result.replace_error(dir <> "/" <> name <> ": cannot read") } } /// `media/abstraction/.ssp` -> `media/image/` fn image_dir_beside(path: String) -> String { let parts = string.split(path, on: "/") let dir = case list.reverse(parts) { [_file, ..rest] -> string.join(list.reverse(rest), "/") [] -> "" } case string.ends_with(dir, "abstraction") { False -> "" True -> { let parent = case list.reverse(string.split(dir, on: "/")) { [_, ..rest] -> string.join(list.reverse(rest), "/") [] -> "" } let image_dir = parent <> "/image" case simplifile.is_directory(image_dir) { Ok(True) -> image_dir _ -> "" } } } } /// Unused here, kept so that a caller can size a read before making it. pub fn byte_size(b: BitArray) -> Int { bit_array.byte_size(b) } #+END_SRC *** compression - zip #+NAME: ocda_zip #+HEADER: :no-expand #+BEGIN_SRC gleam //// Enough of the zip format to find one file in a pod and read it. //// //// Not a zip library. A pod archive holds a handful of entries, spine writes //// them **stored** (uncompressed) so that the outer zstd frame does the //// compressing once over the whole thing, and what is wanted from it here is //// one named entry. //// //// Deflate is *refused rather than mishandled*. Erlang's `zlib` could //// inflate it, but a library that says it read an entry it in fact skipped is //// worse than one that says it cannot, and spine's own archives do not need //// it. import gleam/bit_array import gleam/int import gleam/list import gleam/result import gleam/string pub type Entry { Entry( name: String, /// 0 stored, 8 deflate method: Int, compressed: Int, uncompressed: Int, local_offset: Int, ) } const eocd_sig = 0x06054b50 const cd_sig = 0x02014b50 const local_sig = 0x04034b50 /// The entries, from the central directory at the end of the file. pub fn entries(bytes: BitArray) -> Result(List(Entry), String) { let size = bit_array.byte_size(bytes) use _ <- result.try(case size >= 22 { True -> Ok(Nil) False -> Error("too short to be a zip archive") }) use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size))) use count <- result.try(u16(bytes, eocd + 10)) use offset <- result.try(u32(bytes, eocd + 16)) read_central(bytes, offset, count, []) } /// The end of central directory record is last, and may be followed by a /// comment of up to 65535 bytes, so it is searched for backwards. fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) { case at < lowest { True -> Error( "no end of central directory record; it does not look like a zip archive", ) False -> case u32(bytes, at) { Ok(sig) if sig == eocd_sig -> Ok(at) _ -> find_eocd(bytes, at - 1, lowest) } } } fn lowest(size: Int) -> Int { case size > 22 + 65_535 { True -> size - 22 - 65_535 False -> 0 } } fn read_central( bytes: BitArray, at: Int, remaining: Int, acc: List(Entry), ) -> Result(List(Entry), String) { case remaining { 0 -> Ok(list.reverse(acc)) _ -> { use sig <- result.try(u32(bytes, at)) use _ <- result.try(case sig == cd_sig { True -> Ok(Nil) False -> Error( "central directory entry " <> int.to_string(list.length(acc)) <> " is malformed", ) }) use method <- result.try(u16(bytes, at + 10)) use compressed <- result.try(u32(bytes, at + 20)) use uncompressed <- result.try(u32(bytes, at + 24)) use name_len <- result.try(u16(bytes, at + 28)) use extra_len <- result.try(u16(bytes, at + 30)) use cmnt_len <- result.try(u16(bytes, at + 32)) use local_offset <- result.try(u32(bytes, at + 42)) use name <- result.try(text(bytes, at + 46, name_len)) let entry = Entry( name: name, method: method, compressed: compressed, uncompressed: uncompressed, local_offset: local_offset, ) read_central( bytes, at + 46 + name_len + extra_len + cmnt_len, remaining - 1, [entry, ..acc], ) } } } /// One entry's bytes, from its local header. pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) { use sig <- result.try(u32(bytes, entry.local_offset)) use _ <- result.try(case sig == local_sig { True -> Ok(Nil) False -> Error(entry.name <> ": local header is malformed") }) use name_len <- result.try(u16(bytes, entry.local_offset + 26)) use extra_len <- result.try(u16(bytes, entry.local_offset + 28)) use _ <- result.try(case entry.method { 0 -> Ok(Nil) m -> Error( entry.name <> ": compression method " <> int.to_string(m) <> "; this reader takes only stored entries, which is what spine writes", ) }) let start = entry.local_offset + 30 + name_len + extra_len case bit_array.slice(bytes, start, entry.compressed) { Ok(slice) -> Ok(slice) Error(_) -> Error(entry.name <> ": runs past the end of the archive") } } /// The entries whose names end this way, in archive order. Used to find /// media/abstraction/..ssp without knowing the uid. pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) { list.filter(entries, fn(e) { string.ends_with(e.name, suffix) }) } fn u16(bytes: BitArray, at: Int) -> Result(Int, String) { case bit_array.slice(bytes, at, 2) { Ok(<>) -> Ok(v) _ -> Error("read past the end of the archive at " <> int.to_string(at)) } } fn u32(bytes: BitArray, at: Int) -> Result(Int, String) { case bit_array.slice(bytes, at, 4) { Ok(<>) -> Ok(v) _ -> Error("read past the end of the archive at " <> int.to_string(at)) } } fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { case bit_array.slice(bytes, at, len) { Ok(slice) -> bit_array.to_string(slice) |> result.replace_error("an entry name is not valid UTF-8") Error(_) -> Error("an entry name runs past the end of the archive") } } #+END_SRC *** dump #+NAME: ocda_dump #+HEADER: :no-expand #+BEGIN_SRC gleam //// The model, written out canonically. //// //// Not an output format: an *oracle*. Two libraries in two languages //// implement the same model from the same specification, and the cheap way //// to ask whether they agree is to have each write the whole loaded //// document as text and diff the two. Byte for byte identical to the D //// library's `ocda/dump.d`, which is the whole point of it. //// //// The rules, which both implementations follow: //// //// - every line is ` `, the value running to end of line, //// and only one free-form value per line; //// - a field at its default is *omitted*, so the output says what the //// document has rather than what the type declares; //// - dictionaries come in sorted key order, because neither language's //// map has an order worth relying on; //// - sections come in the fixed order of the format. //// //// Anything that reads this to reconstruct a document has misunderstood //// it. Load the `.ssp` or the `.ocda.db`. import gleam/dict.{type Dict} import gleam/int import gleam/list import gleam/string import ocda/model.{type Document, type Object} pub fn document(doc: Document) -> String { let lines = [] |> kv("version", doc.format_version) |> kv("source_filename", doc.source_filename) |> entries("source", doc.source) |> entries("meta", doc.metadata) |> entries("make", doc.make) |> entries("doc_has", doc.doc_has) |> counts(doc) |> objects(doc) lines |> list.reverse |> join } /// What the loader met and did not understand, sorted. /// /// Separate from the document dump: it describes the loader, not the /// document, and the `.ocda.db` loader has nothing to report where the /// `.ssp` loader does. pub fn unknown(doc: Document) -> String { [] |> numbered("property", doc.unknown.property) |> numbered("block", doc.unknown.block) |> numbered("line", doc.unknown.line) |> list.reverse |> join } fn join(lines: List(String)) -> String { case lines { [] -> "" _ -> string.join(lines, "\n") <> "\n" } } /// Accumulated newest-first and reversed once, so that nothing appends to /// the tail of a list that grows to a few hundred thousand lines. fn kv(acc: List(String), key: String, value: String) -> List(String) { case value { "" -> acc v -> [key <> " " <> v, ..acc] } } fn kn(acc: List(String), key: String, value: Int) -> List(String) { case value { 0 -> acc n -> [key <> " " <> int.to_string(n), ..acc] } } fn kb(acc: List(String), key: String, value: Bool) -> List(String) { case value { False -> acc True -> [key <> " 1", ..acc] } } fn entries( acc: List(String), prefix: String, d: Dict(String, String), ) -> List(String) { d |> dict.to_list |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) |> list.fold(acc, fn(lines, pair) { [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines] }) } fn numbered( acc: List(String), prefix: String, d: Dict(String, Int), ) -> List(String) { d |> dict.to_list |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) |> list.fold(acc, fn(lines, pair) { [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines] }) } fn counts(acc: List(String), doc: Document) -> List(String) { list.fold(model.section_names, acc, fn(lines, name) { case model.section(doc, name) { [] -> lines objects -> [ "section " <> name <> " " <> int.to_string(list.length(objects)), ..lines ] } }) } fn objects(acc: List(String), doc: Document) -> List(String) { list.fold(model.section_names, acc, fn(lines, name) { model.section(doc, name) |> list.index_fold(lines, fn(l, obj, i) { object(l, name, i, obj) }) }) } fn object( acc: List(String), section: String, index: Int, obj: Object, ) -> List(String) { let acc = ["object " <> section <> " " <> int.to_string(index), ..acc] acc |> kn("ocn", obj.ocn) |> kv("is_a", obj.is_a) |> kv("own_section", obj.own_section) |> kv("level", obj.heading_level) |> kv("identifier", obj.identifier) |> kv("part", obj.part) |> kv("is_of_type", obj.is_of_type) |> kv("segment_html_is", obj.segment_html_is) |> kv("segment_epub", obj.segment_epub) |> kv("segment_epub_is", obj.segment_epub_is) |> kn("parent", obj.parent_ocn) |> kn("last_descendant", obj.last_descendant_ocn) |> kn("ocn_range_first", obj.ocn_range_first) |> kn("ocn_range_last", obj.ocn_range_last) |> kv("children", ints(obj.children)) |> kv("ancestors", ints(obj.ancestors)) |> kb("dummy", obj.dummy_heading) |> kb("ocn_off", obj.ocn_off) |> kn("indent_base", obj.indent_base) |> kn("indent_hang", obj.indent_hang) |> kb("bullet", obj.bullet) |> kv("lang", obj.lang) |> kb("has_links", obj.has_links) |> kb("has_notes_reg", obj.has_notes_reg) |> kb("has_notes_star", obj.has_notes_star) |> kb("has_images", obj.has_images) |> kv("segment", obj.segment) |> kv("segment_prev", obj.segment_prev) |> kv("segment_next", obj.segment_next) |> kv("anchor", obj.anchor) |> kv("heading_lev_anchor", obj.heading_lev_anchor) |> kn("table_cols", obj.table_cols) |> kv("table_widths", obj.table_widths) |> kv("table_aligns", obj.table_aligns) |> kb("table_header", obj.table_header) |> kv("code_syntax", obj.code_syntax) |> kb("code_linenumbers", obj.code_linenumbers) |> kv("heading_ancestors_text", obj.heading_ancestors_text) |> kv("sha256", obj.sha256) |> links(obj) |> anchor_tags(obj) |> images(obj) |> text(obj) } fn links(acc: List(String), obj: Object) -> List(String) { list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) }) } fn anchor_tags(acc: List(String), obj: Object) -> List(String) { list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) }) } fn images(acc: List(String), obj: Object) -> List(String) { list.fold(obj.images, acc, fn(lines, img) { // the name last, because a filename may hold a space where none of the // other fields can let lines = kv( lines, "image", img.sha256 <> " " <> int.to_string(img.bytes) <> " " <> int.to_string(img.width) <> "x" <> int.to_string(img.height) <> " " <> case img.missing { True -> "1" False -> "0" } <> " " <> img.name, ) img.fields |> dict.to_list |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) |> list.fold(lines, fn(l, pair) { kv(l, "image_field", pair.0 <> " " <> pair.1) }) }) } /// Text, when there is any. /// /// An object whose only text line is blank holds "" here, indistinguishable /// from no text at all, because the model joins the lines into one string. /// Both libraries lose the same distinction in the same place, which is /// what makes them comparable; it is recorded rather than hidden. fn text(acc: List(String), obj: Object) -> List(String) { case obj.text { "" -> acc t -> { let lines = string.split(t, on: "\n") let acc = kv(acc, "text", int.to_string(list.length(lines))) list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] }) } } } fn ints(xs: List(Int)) -> String { case xs { [] -> "" _ -> xs |> list.map(int.to_string) |> string.join(",") } } #+END_SRC *** erlang #+NAME: ocda_erlang_ffi #+HEADER: :no-expand #+BEGIN_SRC erlang %% The one thing ocda-lib asks of Erlang directly: a sha256, which OTP's %% crypto has and the Gleam standard library does not. Upper case hex, as %% the database stores it and the D library writes it. -module(ocda_ffi). -export([sha256_hex/1]). sha256_hex(Bin) -> binary:encode_hex(crypto:hash(sha256, Bin)). #+END_SRC * read ** main #+NAME: read #+HEADER: :no-expand #+BEGIN_SRC gleam //// ocda-read, in Gleam: reads the abstraction, in any of the forms the //// library reads, and says what is in it. //// //// ocda-read [--tree|--outline|--languages|--files|--report-unknown] //// .ssp | .ocda.db | .zip [lang] //// //// The modes the Gleam readers had (the summary, the outline, the unknown //// report, and the database's languages and files), over the library, so //// that each reads every form. Their output is the D ocda-read's, byte for //// byte: two readers in unrelated languages, neither written from the other //// line by line, agreeing is what says each has read the model and not its //// own idea of it. A `.sisupod` is the one form the Gleam library cannot //// open (its zstd frame; see `ocda/load_pod`), and it says so. import gleam/dict import gleam/int import gleam/io import gleam/list import gleam/result import gleam/string import ocda/inventory.{type Language} import ocda/load import ocda/model.{type Document, type Object} /// The format major this reader knows. const major_read = "2" type Options { Options( tree: Bool, languages: Bool, files: Bool, report_unknown: Bool, any_version: Bool, lang: String, positional: List(String), ) } /// Run with the arguments after the program name; returns the exit status. pub fn run(args: List(String)) -> Int { case parse(args, Options(False, False, False, False, False, "", [])) { Error(message) -> { io.println_error("ocda-read: " <> message <> "; --help lists the options") 2 } Ok(Options(positional: [], ..)) -> { help() 0 } Ok(options) -> read(options) } } fn parse(args: List(String), o: Options) -> Result(Options, String) { case args { [] -> Ok(Options(..o, positional: list.reverse(o.positional))) ["--help", ..] | ["-h", ..] -> Ok(Options(..o, positional: [])) ["--tree", ..rest] | ["--outline", ..rest] -> parse(rest, Options(..o, tree: True)) ["--languages", ..rest] -> parse(rest, Options(..o, languages: True)) ["--files", ..rest] -> parse(rest, Options(..o, files: True)) ["--report-unknown", ..rest] -> parse(rest, Options(..o, report_unknown: True)) ["--any-version", ..rest] -> parse(rest, Options(..o, any_version: True)) ["--lang", code, ..rest] -> parse(rest, Options(..o, lang: code)) [flag, ..rest] -> case string.starts_with(flag, "--lang=") { True -> parse(rest, Options(..o, lang: string.drop_start(flag, 7))) False -> case string.starts_with(flag, "-") { True -> case list.contains( [ "--html", "--llm-text", "--llm-jsonl", "--images", "--verify", "--validate", ], flag, ) { True -> Error(flag <> " is a mode of the D ocda-read, not of this one") False -> Error("Unrecognized option " <> flag) } False -> parse(rest, Options(..o, positional: [flag, ..o.positional])) } } } } fn help() -> Nil { io.println("ocda-read - reads the document abstraction, in any of its forms") io.println("") io.println( "Usage: ocda-read [options] .ssp|.ocda.db|.zip [lang]", ) io.println("") io.println(" --tree the headings, as a tree") io.println(" --outline the same as --tree") io.println(" --languages the languages the file holds") io.println(" --files what a .ocda.db carries besides the abstraction") io.println(" --report-unknown what was read and not understood") io.println(" --lang= the language to read (default: the first)") io.println(" --any-version read a format major this reader does not know") } fn read(o: Options) -> Int { let assert [path, ..more] = o.positional let lang = case o.lang, more { "", [given, ..] -> given l, _ -> l } let lang_named = lang != "" let loaded = { use held <- result.try(inventory.languages(path)) let lang = case lang, held { "", [first, ..] -> first.lang l, _ -> l } use _ <- result.try(case held != [] && !list.any(held, fn(l) { l.lang == lang }) { True -> Error( "no language '" <> lang <> "' in " <> path <> "; it holds: " <> string.join(list.map(held, fn(l) { l.lang }), " "), ) False -> Ok(Nil) }) use doc <- result.try(load.load(path, lang, o.any_version)) Ok(#(held, lang, doc)) } case loaded { Error(message) -> { io.println_error("Error reading " <> path <> ": " <> message) 2 } Ok(#(held, lang, doc)) -> case gate(path, doc, o.any_version) { False -> 3 True -> { summary(path, doc, held, lang, lang_named) case o.languages { True -> print_languages(held) False -> Nil } case o.files { True -> print_files(path) False -> Nil } case o.tree { True -> tree(doc) False -> Nil } case o.report_unknown { True -> report_unknown(doc) False -> Nil } case o.tree || o.languages || o.files || o.report_unknown { True -> Nil False -> io.println( "\n(this is the summary: --help lists the modes, each adding its output)", ) } 0 } } } } /// The major part, and refuse what is not known. fn gate(path: String, doc: Document, any_version: Bool) -> Bool { case doc.format_version, model.version_major(doc) == major_read, any_version { "", _, _ -> { io.println_error("warning: " <> path <> " declares no format version") True } _, True, _ -> True v, False, True -> { io.println_error( "warning: " <> path <> " is format v" <> v <> ", this reader knows v" <> major_read <> ".x; reading anyway", ) True } v, False, False -> { io.println_error( "error: " <> path <> " is format v" <> v <> ", this reader knows v" <> major_read <> ".x", ) io.println_error( " a major version differs: a property may have changed meaning or gone away.", ) io.println_error(" --any-version reads it regardless.") False } } } const sections = [ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", "tail", ] /// The sections whose images are counted: every one but the tail, as the D /// reader counts them. const image_sections = [ "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb", ] fn summary( path: String, doc: Document, held: List(Language), lang: String, lang_named: Bool, ) -> Nil { io.println("Read: " <> path) case doc.format_version { "" -> Nil v -> io.println(" Format: v" <> v) } case dict.get(doc.source, "language") { Error(_) -> Nil Ok(language) -> { let of = case dict.get(doc.source, "languages") { Ok(ls) -> " of " <> ls Error(_) -> "" } let digest = case dict.get(doc.source, "digest") { Ok(d) -> ", markup digest " <> string.slice(d, 0, 16) <> "..." Error(_) -> "" } io.println(" Source: " <> language <> of <> digest) } } case list.length(held) > 1 && !lang_named { True -> io.println(" (reading " <> lang <> "; --lang= for another)") False -> Nil } io.println(" Title: " <> model.meta(doc, "title.main")) io.println(" Author: " <> model.meta(doc, "creator.author")) io.println( " Sections: " <> string.join( list.map(sections, fn(s) { s <> "=" <> int.to_string(list.length(model.section(doc, s))) }), " ", ), ) let records = list.fold(image_sections, 0, fn(n, s) { list.fold(model.section(doc, s), n, fn(n, o) { n + list.length(o.images) }) }) case records { 0 -> Nil 1 -> io.println(" Images: 1 record") n -> io.println(" Images: " <> int.to_string(n) <> " records") } } fn print_languages(held: List(Language)) -> Nil { io.println("") io.println(" lang source ssp digest") list.each(held, fn(l) { let digest = case string.length(l.ssp_digest) > 16 { True -> string.slice(l.ssp_digest, 0, 16) <> "..." False -> l.ssp_digest } io.println( " " <> string.pad_end(l.lang, 7, " ") <> " " <> string.pad_end(l.source_filename, 31, " ") <> " " <> digest, ) }) } fn print_files(path: String) -> Nil { io.println("") case inventory.is_carrier(path) { False -> io.println( " carried files: only a .ocda.db carries files; this is " <> load.source_name(load.source_of(path)), ) True -> case inventory.carried(path) { Error(message) -> io.println_error("error: " <> message) Ok([]) -> io.println(" carries no files") Ok(rows) -> { io.println(" carried files:") io.println(" role stored n original in the file") list.each(rows, fn(r) { io.println( " " <> string.pad_end(r.role, 9, " ") <> " " <> string.pad_end(r.compression, 7, " ") <> " " <> string.pad_end(r.count, 5, " ") <> " " <> string.pad_end(r.bytes, 13, " ") <> " " <> r.stored, ) }) } } } } /// The headings, as a tree: from the document's root heading, down the /// children each heading lists, as the D reader walks it. fn tree(doc: Document) -> Nil { io.println("Document: " <> model.meta(doc, "title.main")) case model.meta(doc, "title.subtitle") { "" -> Nil sub -> io.println(" " <> sub) } io.println("Author: " <> model.meta(doc, "creator.author")) io.println("") let headings = model.section(doc, "body") |> list.filter(fn(o) { o.is_a == "heading" && o.ocn > 0 }) |> list.map(fn(o) { #(o.ocn, o) }) |> dict.from_list case model.section(doc, "head") { [root, ..] if root.is_a == "heading" -> { io.println(heading_line(root, 0)) walk(root, headings, 1) } _ -> Nil } } fn walk(parent: Object, headings: dict.Dict(Int, Object), depth: Int) -> Nil { list.each(parent.children, fn(ocn) { case dict.get(headings, ocn) { Ok(child) -> { io.println(heading_line(child, depth)) walk(child, headings, depth + 1) } Error(_) -> Nil } }) } fn heading_line(o: Object, depth: Int) -> String { let level = case o.heading_level { "" -> "?" l -> l } let ocn = case o.ocn > 0 { True -> " [" <> int.to_string(o.ocn) <> "]" False -> "" } string.repeat(" ", depth) <> level <> "~" <> ocn <> " " <> strip_markup(o.text) } /// The internal markers a heading's text may carry, removed for display, as /// the D reader removes them. fn strip_markup(text: String) -> String { text |> string.replace("⑆*", "") |> string.replace("⑆!", "") |> string.replace("⑆/", "") |> string.replace("⑆_", "") |> string.replace("⑆^", "") |> string.replace("⑆,", "") |> string.replace("┨", "") |> string.replace("┣", "") |> string.replace("┙", " ") |> string.replace("┘", "\n") } /// Say what was read and not understood. fn report_unknown(doc: Document) -> Nil { let u = doc.unknown case model.any_unknown(u) { False -> io.println( "\nread and understood: every property, block and line was known", ) True -> { io.println("\nread and NOT understood:") section_of_unknown(u.property, " properties ignored (", fn(name, n) { " ." <> string.pad_end(name, 26, " ") <> " " <> string.pad_start(int.to_string(n), 8, " ") }) section_of_unknown(u.block, " blocks ignored (", fn(name, n) { " @" <> string.pad_end(name, 26, " ") <> " " <> string.pad_start(int.to_string(n), 8, " ") }) case dict.size(u.line) { 0 -> Nil _ -> { io.println(" lines of no known shape, by first character:") dict.to_list(u.line) |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) |> list.each(fn(e) { io.println( " " <> string.pad_end("\"" <> e.0 <> "\"", 27, " ") <> " " <> string.pad_start(int.to_string(e.1), 8, " "), ) }) } } io.println( " (ignoring these is conforming within a major version; the count is so that it is not silent)", ) } } } fn section_of_unknown( counts: dict.Dict(String, Int), heading: String, line: fn(String, Int) -> String, ) -> Nil { case dict.size(counts) { 0 -> Nil n -> { io.println(heading <> int.to_string(n) <> " distinct):") dict.to_list(counts) |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) |> list.each(fn(e) { io.println(line(e.0, e.1)) }) } } } #+END_SRC * test #+NAME: test #+HEADER: :no-expand #+BEGIN_SRC gleam import gleam/dict import gleam/list import simplifile import gleam/string import gleeunit import gleeunit/should import ocda/dump import ocda/load import ocda/load_ssp import ocda/model pub fn main() { gleeunit.main() } const minimal = "% SiSU Document Abstraction v2.0 % Source: example.sst @source { language: en languages: en digest: ABC } @meta { title.main: An Example } @body { [1] heading :A the_title .part: frontmatter .sha256: DEAD | An Example [2] para .part: body .parent: 1 | First line | | Third line, after a blank one } " pub fn version_test() { let doc = load_ssp.parse(minimal) doc.format_version |> should.equal("2.0") model.version_major(doc) |> should.equal("2") doc.source_filename |> should.equal("example.sst") } pub fn preamble_test() { let doc = load_ssp.parse(minimal) model.meta(doc, "title.main") |> should.equal("An Example") } pub fn objects_test() { let doc = load_ssp.parse(minimal) let body = model.section(doc, "body") body |> list.length |> should.equal(2) } /// The blank line in the middle of an object's text is kept, and so is the /// count: three lines, not two. pub fn blank_text_line_test() { let doc = load_ssp.parse(minimal) case model.section(doc, "body") { [_, second] -> second.text |> should.equal("First line\n\nThird line, after a blank one") _ -> should.fail() } } /// A heading's identifier comes from its declaration line; an object whose /// identifier is simply its ocn has it derived, because the text form does /// not write what it can work out. pub fn derived_identifier_test() { let doc = load_ssp.parse(minimal) case model.section(doc, "body") { [first, second] -> { first.identifier |> should.equal("the_title") second.identifier |> should.equal("2") } _ -> should.fail() } } pub fn source_of_test() { load.source_of("a/b.ssp") |> should.equal(load.Ssp) load.source_of("a/b.ocda.db") |> should.equal(load.OcdaDb) load.source_of("a/b.sst") |> should.equal(load.Unknown) } /// The dump omits a field at its default, so a minimal document gives a /// short dump and every line in it is a fact. pub fn dump_test() { let text = dump.document(load_ssp.parse(minimal)) text |> string.contains(contain: "version 2.0") |> should.be_true text |> string.contains(contain: "meta title.main An Example") |> should.be_true text |> string.contains(contain: "object body 0") |> should.be_true text |> string.contains(contain: "level A") |> should.be_true // a false boolean is not written at all text |> string.contains(contain: "bullet") |> should.be_false } /// A property is read, which the fixture above had to be corrected to show: /// a property line is `.name: value` at the start of the line, and an /// indented one is a line of no known shape. The fixture indented them, so /// every property in it was being counted as malformed and the tests around /// it passed for the wrong reason. pub fn property_test() { let doc = load_ssp.parse(minimal) case model.section(doc, "body") { [first, second] -> { first.part |> should.equal("frontmatter") first.sha256 |> should.equal("DEAD") second.parent_ocn |> should.equal(1) } _ -> should.fail() } // and nothing in the fixture is malformed any more dict_size(doc.unknown.line) |> should.equal(0) } @external(erlang, "maps", "size") fn dict_size(d: dict.Dict(a, b)) -> Int /// What a path is, including the two archive forms. pub fn pod_source_test() { load.source_of("a/b.sisupod") |> should.equal(load.Pod) load.source_of("a/b.zip") |> should.equal(load.Pod) } /// A zstd-wrapped archive is refused, and the refusal says why rather than /// claiming the file is malformed. This is the one place the D and Gleam /// libraries differ in what they accept, and it is a packaging limit: the /// only zstd on the BEAM is a NIF that fetches zstd in its own build. pub fn zstd_refusal_test() { let path = "test/fixture-zstd-frame.sisupod" let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>) let assert Error(message) = load.load(path, "", False) message |> string.contains(contain: "zstd") |> should.be_true message |> string.contains(contain: "zstd -d") |> should.be_true let _ = simplifile.delete(path) Nil } #+END_SRC * org-tangles ** ocda lib #+HEADER: :tangle ../gleam/src/ocda_lib.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC ** ocda *** model #+HEADER: :tangle ../gleam/src/ocda/model.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC *** what to load **** inventory #+HEADER: :tangle ../gleam/src/ocda/inventory.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC **** load ***** load #+HEADER: :tangle ../gleam/src/ocda/load.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC ***** load pod #+HEADER: :tangle ../gleam/src/ocda/load_pod.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC ***** load ssp #+HEADER: :tangle ../gleam/src/ocda/load_ssp.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC ***** load db #+HEADER: :tangle ../gleam/src/ocda/load_db.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC **** images #+HEADER: :tangle ../gleam/src/ocda/images.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC *** compression - zip #+HEADER: :tangle ../gleam/src/ocda/zip.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC *** dump #+HEADER: :tangle ../gleam/src/ocda/dump.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC *** erlang #+HEADER: :tangle ../gleam/src/ocda_ffi.erl #+HEADER: :noweb yes #+BEGIN_SRC org %%<> <> #+END_SRC ** read *** main #+HEADER: :tangle ../gleam/src/read/main.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC ** test #+HEADER: :tangle ../gleam/test/ocda_lib_test.gleam #+HEADER: :noweb yes #+BEGIN_SRC org ////<> <> #+END_SRC * config ** nix flake.nix :flake: #+HEADER: :tangle ../gleam/flake.nix #+HEADER: :noweb yes #+BEGIN_SRC nix { description = "ocda-lib: the object-centric document abstraction as a Gleam library"; inputs.nixpkgs.url = "<>"; outputs = { self, nixpkgs }: let system = "x86_64-linux"; # adjust for aarch64-linux / darwin pkgs = import nixpkgs { inherit system; }; name = "ocda_lib"; /* rebar3 with the port compiler already in it. esqlite builds a NIF, and its rebar3 configuration asks for the `pc` plugin, which rebar3 would otherwise fetch from Hex at build time: impossible in a sandbox, and the build fails with "missing_package pc". nixpkgs carries the plugin, so the answer is a rebar3 that has it before it starts. */ rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { plugins = [ pkgs.beamPackages.pc ]; }; /* The Hex dependencies, as manifest.toml pins them. . There is no Gleam builder in nixpkgs and `gleam build` fetches from Hex, so a sandboxed build cannot reach them. A fixed output derivation may, because its result is pinned by hash. . When a dependency changes, manifest.toml changes and this hash must change with it: build, take the hash nix reports, put it here. */ deps = pkgs.stdenv.mkDerivation { pname = "${name}-deps"; version = "0.1.0"; src = ./.; nativeBuildInputs = [ pkgs.gleam pkgs.cacert ]; buildPhase = '' export HOME=$TMPDIR gleam deps download ''; installPhase = '' mkdir -p $out cp -r build/packages/* $out/ ''; dontFixup = true; outputHashMode = "recursive"; outputHashAlgo = "sha256"; outputHash = "sha256-nWU753YRsC28gsaGyy+A3aZaWf15ctULYiUlvsfyB1c="; }; in { packages.${system} = { default = self.packages.${system}.${name}; /* Shipped as an Erlang shipment rather than as an escript, because sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into a shared object, and a NIF cannot be loaded from inside an escript archive, the runtime needing a real file on disk. The shipment keeps priv/ on disk, so it loads. */ ${name} = pkgs.stdenv.mkDerivation { pname = "ocda-lib"; version = "0.1.0"; src = ./.; nativeBuildInputs = [ pkgs.gleam pkgs.erlang rebar3-pc pkgs.makeWrapper ]; buildPhase = '' runHook preBuild export HOME=$TMPDIR # the dependencies, in place of the fetch gleam would do mkdir -p build/packages cp -r ${deps}/* build/packages/ chmod -R u+w build/packages gleam export erlang-shipment runHook postBuild ''; installPhase = '' runHook preInstall mkdir -p $out/lib $out/bin cp -r build/erlang-shipment $out/lib/${name} makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-lib \ --add-flags run \ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} # the reader under its own name, as the D flake installs it makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-read \ --add-flags "run read" \ --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.erlang pkgs.coreutils ]} runHook postInstall ''; meta = { description = "the abstraction as a Gleam library, with its reader, dump and agreement check"; mainProgram = "ocda-lib"; }; }; }; devShells.${system}.default = pkgs.mkShell { packages = [ pkgs.gleam pkgs.erlang pkgs.rebar3 pkgs.sqlite ]; }; }; } #+END_SRC ** README.md #+HEADER: :tangle ../gleam/README.md #+HEADER: :noweb yes #+BEGIN_SRC md # ocda-lib, in Gleam The abstraction's model and both its loaders, as a Gleam library, with its reader and the two programs that put it to the question. | | | |------------------|-----------------------------------------------------------------------------------------| | `ocda/model` | the document, the object, the image. Depends on nothing but the standard library | | `ocda/load_ssp` | text -> model | | `ocda/load_db` | sqlite -> model | | `ocda/load` | what a path is, and loading it | | `ocda/inventory` | what a file holds beyond the one document: its languages, and what a `.ocda.db` carries | | `ocda/dump` | the model written out canonically, as an oracle | | `read/main` | `ocda-read`, the reader | | `ocda_ffi.erl` | a sha256, from OTP's crypto, which the standard library lacks | ## Use ```gleam import ocda/load import ocda/model case load.load(path, "en", False) { Error(why) -> io.println_error(why) Ok(doc) -> { // and from here a writer neither knows nor cares which of the two // files it was handed list.each(model.section(doc, "body"), fn(obj) { ... }) } } ``` ## The reader ```sh nix build ./result/bin/ocda-read [--tree|--outline|--languages|--files|--report-unknown] .ssp|.ocda.db|.zip [lang] ``` `ocda-lib read`, installed also as `ocda-read`. The modes the Gleam readers had, over the library, so each reads every form this library reads; the others are the D reader's. **Its output is the D `ocda-read`'s, byte for byte**, in every mode both have. In `src/read/`, because Gleam compiles nothing outside `src/`; a program depending on the library compiles it too, and does not use it. A `.sisupod` it refuses, saying why: the zstd frame, below. ## The two programs ```sh nix build ./result/bin/ocda-lib agree .ssp .ocda.db [lang] ./result/bin/ocda-lib dump [--unknown] .ssp|.ocda.db [lang] ``` `agree` asks whether the two loaders give the same model. `dump` writes the whole model out canonically, and is the stronger check of the two, because it is **byte for byte what the D library's `ocda-dump` writes**. So the four questions can all be asked with `diff`: | | | |-----------------------------|-----------------------------| | D: `.ssp` == `.ocda.db` | 36 of 36 document-languages | | Gleam: `.ssp` == `.ocda.db` | 36 of 36 | | `.ssp`: D == Gleam | 36 of 36 | | `.ocda.db`: D == Gleam | 36 of 36 | ## Deliberately a mirror of the D library Field for field, name for name, including where that is not the most Gleamish choice: - `text` is one `String` with the lines joined by newline, not a `List(String)`. So an object whose only text line is blank is indistinguishable from an object with no text, *in both languages, in the same place*. A difference that existed in one and not the other would be worse than a loss both share, because it would show up in every dump and mean nothing. - the properties are typed fields rather than a list of pairs. A writer wants `obj.parent_ocn`, not a lookup that might miss. The point is that the two agree. Making them agree is much cheaper when they are shaped alike, and the shape is the specification's, not either language's. ## What the agreement found Four disagreements, none of which a comparison of section counts or of a handful of fields would have shown. All four are recorded in the D library's history; the short version: 1. `.identifier` and `.anchor` are omitted by the text form when they can be worked out (the identifier is the ocn, the anchor is the segment). Both loaders now derive them. Over one document that is 1,314 identifiers. 2. `make` and `doc_has` are one metadata table in the database, distinguished by a key prefix. Both loaders unprefix them. 3. The JSON columns must be *parsed*, not scanned: the encoder writes `/` as `\/` and every heading with a slash in it came back wrong. 4. `translation.ocn_aligned` is in the database and cannot be in a `.ssp`, which is one language. It is a fact about the file, so neither loader puts it in the document. ## Build ```sh nix build # sandboxed, no network nix develop -c gleam test ``` Shipped as an Erlang shipment rather than an escript, because sqlite on the BEAM is a NIF and a NIF cannot be loaded from inside an escript archive. The dependency hash in `flake.nix` pins what `manifest.toml` pins; change one and the other must change with it. ## Format Specified in **sisudoc-abstraction-format**: `ocda-model.org` for the model, `ssp-format-specification.org` and `ocda-db-format-specification.org` for the two serialisations. At format **2.x**. #+END_SRC ** gleam.toml #+HEADER: :tangle ../gleam/gleam.toml #+HEADER: :noweb yes #+BEGIN_SRC toml name = "ocda_lib" version = "0.1.0" description = "the object-centric document abstraction as a library: the model, and loaders for its two serialisations" licences = ["AGPL-3.0-or-later"] [dependencies] gleam_stdlib = ">= 1.0.0 and < 2.0.0" argv = ">= 1.1.0 and < 2.0.0" simplifile = ">= 2.7.0 and < 3.0.0" sqlight = ">= 1.0.0 and < 2.0.0" gleam_json = ">= 3.0.0 and < 4.0.0" [dev-dependencies] gleeunit = ">= 1.0.0 and < 2.0.0" #+END_SRC * org includes ** project version #+NAME: spine_version #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> #+END_SRC ** year #+NAME: year #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC ** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * nixpkgs source #+NAME: nixpkgs_unstable #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> #+END_SRC * __END__