diff options
Diffstat (limited to 'gleam/src')
| -rw-r--r-- | gleam/src/ocda/dump.gleam | 239 | ||||
| -rw-r--r-- | gleam/src/ocda/load.gleam | 69 | ||||
| -rw-r--r-- | gleam/src/ocda/load_db.gleam | 503 | ||||
| -rw-r--r-- | gleam/src/ocda/load_ssp.gleam | 468 | ||||
| -rw-r--r-- | gleam/src/ocda_lib.gleam | 136 |
5 files changed, 1415 insertions, 0 deletions
diff --git a/gleam/src/ocda/dump.gleam b/gleam/src/ocda/dump.gleam new file mode 100644 index 0000000..1ab37a6 --- /dev/null +++ b/gleam/src/ocda/dump.gleam @@ -0,0 +1,239 @@ +//// The model, written out canonically. +//// +//// Not an output format: an *oracle*. Two libraries in two languages +//// implement the same model from the same specification, and the cheap way +//// to ask whether they agree is to have each write the whole loaded +//// document as text and diff the two. Byte for byte identical to the D +//// library's `ocda/dump.d`, which is the whole point of it. +//// +//// The rules, which both implementations follow: +//// +//// - every line is `<key> <value>`, the value running to end of line, +//// and only one free-form value per line; +//// - a field at its default is *omitted*, so the output says what the +//// document has rather than what the type declares; +//// - dictionaries come in sorted key order, because neither language's +//// map has an order worth relying on; +//// - sections come in the fixed order of the format. +//// +//// Anything that reads this to reconstruct a document has misunderstood +//// it. Load the `.ssp` or the `.ocda.db`. + +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/string +import ocda/model.{type Document, type Object} + +pub fn document(doc: Document) -> String { + let lines = + [] + |> kv("version", doc.format_version) + |> kv("source_filename", doc.source_filename) + |> entries("source", doc.source) + |> entries("meta", doc.metadata) + |> entries("make", doc.make) + |> entries("doc_has", doc.doc_has) + |> counts(doc) + |> objects(doc) + lines |> list.reverse |> join +} + +/// What the loader met and did not understand, sorted. +/// +/// Separate from the document dump: it describes the loader, not the +/// document, and the `.ocda.db` loader has nothing to report where the +/// `.ssp` loader does. +pub fn unknown(doc: Document) -> String { + [] + |> numbered("property", doc.unknown.property) + |> numbered("block", doc.unknown.block) + |> numbered("line", doc.unknown.line) + |> list.reverse + |> join +} + +fn join(lines: List(String)) -> String { + case lines { + [] -> "" + _ -> string.join(lines, "\n") <> "\n" + } +} + +/// Accumulated newest-first and reversed once, so that nothing appends to +/// the tail of a list that grows to a few hundred thousand lines. +fn kv(acc: List(String), key: String, value: String) -> List(String) { + case value { + "" -> acc + v -> [key <> " " <> v, ..acc] + } +} + +fn kn(acc: List(String), key: String, value: Int) -> List(String) { + case value { + 0 -> acc + n -> [key <> " " <> int.to_string(n), ..acc] + } +} + +fn kb(acc: List(String), key: String, value: Bool) -> List(String) { + case value { + False -> acc + True -> [key <> " 1", ..acc] + } +} + +fn entries( + acc: List(String), + prefix: String, + d: Dict(String, String), +) -> List(String) { + d + |> dict.to_list + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.fold(acc, fn(lines, pair) { + [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines] + }) +} + +fn numbered( + acc: List(String), + prefix: String, + d: Dict(String, Int), +) -> List(String) { + d + |> dict.to_list + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.fold(acc, fn(lines, pair) { + [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines] + }) +} + +fn counts(acc: List(String), doc: Document) -> List(String) { + list.fold(model.section_names, acc, fn(lines, name) { + case model.section(doc, name) { + [] -> lines + objects -> + [ + "section " <> name <> " " <> int.to_string(list.length(objects)), + ..lines + ] + } + }) +} + +fn objects(acc: List(String), doc: Document) -> List(String) { + list.fold(model.section_names, acc, fn(lines, name) { + model.section(doc, name) + |> list.index_fold(lines, fn(l, obj, i) { + object(l, name, i, obj) + }) + }) +} + +fn object( + acc: List(String), + section: String, + index: Int, + obj: Object, +) -> List(String) { + let acc = ["object " <> section <> " " <> int.to_string(index), ..acc] + acc + |> kn("ocn", obj.ocn) + |> kv("is_a", obj.is_a) + |> kv("own_section", obj.own_section) + |> kv("level", obj.heading_level) + |> kv("identifier", obj.identifier) + |> kv("part", obj.part) + |> kv("is_of_type", obj.is_of_type) + |> kv("segment_html_is", obj.segment_html_is) + |> kn("parent", obj.parent_ocn) + |> kn("last_descendant", obj.last_descendant_ocn) + |> kv("children", ints(obj.children)) + |> kv("ancestors", ints(obj.ancestors)) + |> kb("dummy", obj.dummy_heading) + |> kb("ocn_off", obj.ocn_off) + |> kn("indent_base", obj.indent_base) + |> kn("indent_hang", obj.indent_hang) + |> kb("bullet", obj.bullet) + |> kv("lang", obj.lang) + |> kb("has_links", obj.has_links) + |> kb("has_notes_reg", obj.has_notes_reg) + |> kb("has_notes_star", obj.has_notes_star) + |> kb("has_images", obj.has_images) + |> kv("segment", obj.segment) + |> kv("segment_prev", obj.segment_prev) + |> kv("segment_next", obj.segment_next) + |> kv("anchor", obj.anchor) + |> kn("table_cols", obj.table_cols) + |> kv("table_widths", obj.table_widths) + |> kv("table_aligns", obj.table_aligns) + |> kb("table_header", obj.table_header) + |> kv("code_syntax", obj.code_syntax) + |> kb("code_linenumbers", obj.code_linenumbers) + |> kv("heading_ancestors_text", obj.heading_ancestors_text) + |> kv("sha256", obj.sha256) + |> links(obj) + |> images(obj) + |> text(obj) +} + +fn links(acc: List(String), obj: Object) -> List(String) { + list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) }) +} + +fn images(acc: List(String), obj: Object) -> List(String) { + list.fold(obj.images, acc, fn(lines, img) { + // the name last, because a filename may hold a space where none of the + // other fields can + let lines = + kv( + lines, + "image", + img.sha256 + <> " " + <> int.to_string(img.bytes) + <> " " + <> int.to_string(img.width) + <> "x" + <> int.to_string(img.height) + <> " " + <> case img.missing { + True -> "1" + False -> "0" + } + <> " " + <> img.name, + ) + img.fields + |> dict.to_list + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.fold(lines, fn(l, pair) { + kv(l, "image_field", pair.0 <> " " <> pair.1) + }) + }) +} + +/// Text, when there is any. +/// +/// An object whose only text line is blank holds "" here, indistinguishable +/// from no text at all, because the model joins the lines into one string. +/// Both libraries lose the same distinction in the same place, which is +/// what makes them comparable; it is recorded rather than hidden. +fn text(acc: List(String), obj: Object) -> List(String) { + case obj.text { + "" -> acc + t -> { + let lines = string.split(t, on: "\n") + let acc = kv(acc, "text", int.to_string(list.length(lines))) + list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] }) + } + } +} + +fn ints(xs: List(Int)) -> String { + case xs { + [] -> "" + _ -> xs |> list.map(int.to_string) |> string.join(",") + } +} diff --git a/gleam/src/ocda/load.gleam b/gleam/src/ocda/load.gleam new file mode 100644 index 0000000..1bbf869 --- /dev/null +++ b/gleam/src/ocda/load.gleam @@ -0,0 +1,69 @@ +//// What a path is, and loading it. +//// +//// One place that decides, so that every reader and every writer accepts +//// the same inputs and a new input form is added once. This mirrors +//// spine's own `ocda/abstraction/load.d`, which names the sources it can +//// enter the pipeline from and tells them apart by suffix. +//// +//// Deliberately *not* here: `.sst` and `.ssm` markup. Reading markup means +//// implementing the parser, which is spine's work and a far larger thing +//// than a reader or a sample writer needs. This library reads +//// abstractions. + +import gleam/string +import ocda/load_db +import ocda/load_ssp +import ocda/model.{type Document} + +pub type Source { + /// `.ssp`, the abstraction as text + Ssp + /// `.ocda.db`, the abstraction as sqlite + OcdaDb + Unknown +} + +pub fn source_of(path: String) -> Source { + case string.ends_with(path, ".ocda.db") { + True -> OcdaDb + False -> + case string.ends_with(path, ".ssp") { + True -> Ssp + False -> + case string.ends_with(path, ".db") { + True -> OcdaDb + False -> Unknown + } + } + } +} + +pub fn source_name(source: Source) -> String { + case source { + Ssp -> ".ssp (the abstraction as text)" + OcdaDb -> ".ocda.db (the abstraction as sqlite)" + Unknown -> "not an abstraction this library reads" + } +} + +/// Load whatever the path is into the model. +/// +/// `lang` applies only to a `.ocda.db`, which holds every language of its +/// document; a `.ssp` is one language and the argument is ignored. +pub fn load( + path: String, + lang: String, + any_version: Bool, +) -> Result(Document, String) { + case source_of(path) { + Ssp -> load_ssp.load_file(path) + OcdaDb -> load_db.load_file(path, lang, any_version) + Unknown -> + Error( + path + <> ": " + <> source_name(Unknown) + <> " (expected .ssp or .ocda.db)", + ) + } +} diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam new file mode 100644 index 0000000..bcd65d8 --- /dev/null +++ b/gleam/src/ocda/load_db.gleam @@ -0,0 +1,503 @@ +//// Loading a `.ocda.db` into the model. +//// +//// The point of this module is that it returns *the same value* +//// `ocda/load_ssp` returns from the same document's `.ssp`. A writer is +//// then a function from the model to bytes and cannot tell which +//// serialisation it was given, exactly as spine's own writers cannot. +//// +//// Written against `ocda-db-format-specification.md`, whose section 4 +//// gives the column by column correspondence this module implements. + +import gleam/dict +import gleam/dynamic/decode +import gleam/float +import gleam/int +import gleam/json +import gleam/list +import gleam/option +import gleam/result +import gleam/string +import ocda/model.{type Document, Image, Object} +import sqlight + +/// The schema major version this loader knows. +pub const schema_major_read = "2" + +/// One language of the database, as the manifest gives it. +pub type Language { + Language(doc_id: Int, lang: String, source_filename: String, ssp_digest: String) +} + +/// Load one language of a database as the model. `lang` == "" takes the +/// first language the manifest gives. +pub fn load_file( + path: String, + lang: String, + any_version: Bool, +) -> Result(Document, String) { + // read only: a loader has no business writing, and a read-write open + // *creates* an empty database where a path is mistyped, which then reads + // as a document with nothing in it, a failure that looks like an answer + use conn <- with_connection("file:" <> path <> "?mode=ro", path) + + use _ <- result.try(case + has(conn, "documents") && has(conn, "objects") && has(conn, "metadata") + { + True -> Ok(Nil) + False -> + Error( + path + <> ": no documents/objects/metadata tables; it does not look like a .ocda.db", + ) + }) + + let schema_version = + one(conn, "select value from metadata where doc_id is null + and key = 'schema.version'") + use _ <- result.try(case schema_version, any_version { + "", _ -> Ok(Nil) + _, True -> Ok(Nil) + v, False -> + case major_of(v) == schema_major_read { + True -> Ok(Nil) + False -> + Error( + path + <> " is schema v" + <> v + <> ", this loader knows v" + <> schema_major_read + <> ".x", + ) + } + }) + + let languages = languages(conn) + use chosen <- result.try(case languages, lang { + [], _ -> Error(path <> ": no documents rows") + [first, ..], "" -> Ok(first) + _, wanted -> + case list.find(languages, fn(l) { l.lang == wanted }) { + Ok(found) -> Ok(found) + Error(_) -> + Error( + path + <> ": no language '" + <> wanted + <> "'; it holds: " + <> string.join(list.map(languages, fn(l) { l.lang }), " "), + ) + } + }) + + let source = + dict.new() + |> dict.insert("language", chosen.lang) + |> dict.insert( + "languages", + string.join(list.map(languages, fn(l) { l.lang }), " "), + ) + let digest = meta_value(conn, chosen.doc_id, "source.digest") + let source = case digest { + "" -> source + d -> dict.insert(source, "digest", d) + } + + let document = + model.Document( + ..model.new_document(), + format_version: schema_version, + source_filename: chosen.source_filename, + source: source, + ) + let document = metadata(conn, chosen.doc_id, document) + let document = objects(conn, chosen.doc_id, document) + Ok(document) +} + +fn with_connection( + uri: String, + path: String, + then: fn(sqlight.Connection) -> Result(a, String), +) -> Result(a, String) { + case sqlight.open(uri) { + Error(e) -> Error(path <> ": " <> e.message) + Ok(conn) -> { + let outcome = then(conn) + let _ = sqlight.close(conn) + outcome + } + } +} + +/// Every language the database holds, in manifest order. +pub fn languages(conn: sqlight.Connection) -> List(Language) { + let d = { + use id <- decode.field(0, decode.int) + use lang <- decode.field(1, decode.string) + use source <- decode.field(2, decode.string) + use digest <- decode.field(3, decode.string) + decode.success(Language(id, lang, source, digest)) + } + sqlight.query( + "select id, lang, coalesce(source_filename,''), coalesce(ssp_digest,'') + from documents order by id", + on: conn, + with: [], + expecting: d, + ) + |> result.unwrap([]) +} + +/// The three preamble blocks of the text form, out of the one metadata +/// table the database keeps them in. +fn metadata( + conn: sqlight.Connection, + doc_id: Int, + document: Document, +) -> Document { + let d = { + use key <- decode.field(0, decode.string) + use value <- decode.field(1, decode.string) + decode.success(#(key, value)) + } + let rows = + sqlight.query( + "select key, value from metadata where doc_id = " + <> int.to_string(doc_id) + <> " order by key", + on: conn, + with: [], + expecting: d, + ) + |> result.unwrap([]) + + list.fold(rows, document, fn(doc, row) { + let #(key, raw) = row + let value = string.trim(raw) + case key { + // source.* describes the artefact rather than the document, and is + // @source in the text form + "source." <> _ -> doc + // translation.ocn_aligned says whether the languages of this document + // share one numbering. A .ocda.db can know it because it holds every + // language; a .ssp is one language and cannot. It is a fact about the + // file, not the document, and carrying it here would let a writer + // tell which serialisation it was handed. + "translation." <> _ -> doc + // the database tells the three blocks apart by a key prefix. + // Unprefixing them is what makes the two loaders give one model: a + // writer asks make["footer"], never metadata["make.footer"]. + "make." <> rest -> + model.Document(..doc, make: dict.insert(doc.make, rest, value)) + "doc_has." <> rest -> + model.Document(..doc, doc_has: dict.insert(doc.doc_has, rest, value)) + _ -> + model.Document(..doc, metadata: dict.insert(doc.metadata, key, value)) + } + }) +} + +/// The objects, in document order within each section, the sections in the +/// order the format gives rather than the alphabetical one sqlite would. +fn objects( + conn: sqlight.Connection, + doc_id: Int, + document: Document, +) -> Document { + let d = { + use section <- decode.field(0, decode.string) + use ocn <- decode.field(2, decode.int) + use is_a <- decode.field(3, decode.string) + use part <- decode.field(4, decode.string) + use is_of_section <- decode.field(5, decode.string) + use is_of_type <- decode.field(6, decode.string) + use heading_level <- decode.field(7, decode.int) + use identifier <- decode.field(8, decode.string) + use parent <- decode.field(9, decode.int) + use last_descendant <- decode.field(10, decode.int) + use children <- decode.field(11, decode.string) + use ancestors <- decode.field(12, decode.string) + use dummy <- decode.field(13, decode.int) + use ocn_off <- decode.field(14, decode.int) + use indent_base <- decode.field(15, decode.int) + use indent_hang <- decode.field(16, decode.int) + use bullet <- decode.field(17, decode.int) + use lang <- decode.field(18, decode.string) + use has_links <- decode.field(19, decode.int) + use has_notes_reg <- decode.field(20, decode.int) + use has_notes_star <- decode.field(21, decode.int) + use has_images <- decode.field(22, decode.int) + use segment <- decode.field(23, decode.string) + use segment_prev <- decode.field(24, decode.string) + use segment_next <- decode.field(25, decode.string) + use anchor <- decode.field(26, decode.string) + use segment_html_is <- decode.field(27, decode.string) + use table_cols <- decode.field(28, decode.int) + use table_widths <- decode.field(29, decode.string) + use table_aligns <- decode.field(30, decode.string) + use table_header <- decode.field(31, decode.int) + use code_linenumbers <- decode.field(32, decode.int) + use hat <- decode.field(33, decode.string) + use sha256 <- decode.field(34, decode.string) + use text <- decode.field(35, decode.string) + use id <- decode.field(36, decode.int) + decode.success(#( + id, + Object( + ocn: ocn, + is_a: is_a, + section: section, + heading_level: level_name(heading_level), + identifier: identifier, + part: part, + is_of_type: is_of_type, + own_section: case is_of_section == section { + True -> "" + False -> is_of_section + }, + segment_html_is: segment_html_is, + table_aligns: json_words(table_aligns), + stow_link: [], + parent_ocn: parent, + last_descendant_ocn: last_descendant, + children: json_ints(children), + ancestors: json_ints(ancestors), + dummy_heading: dummy != 0, + ocn_off: ocn_off != 0, + indent_base: indent_base, + indent_hang: indent_hang, + bullet: bullet != 0, + lang: lang, + has_links: has_links != 0, + has_notes_reg: has_notes_reg != 0, + has_notes_star: has_notes_star != 0, + has_images: has_images != 0, + segment: segment, + segment_prev: segment_prev, + segment_next: segment_next, + anchor: anchor, + table_cols: table_cols, + table_widths: json_words(table_widths), + table_header: table_header != 0, + code_syntax: "", + code_linenumbers: code_linenumbers != 0, + heading_ancestors_text: json_joined(hat), + sha256: sha256, + images: [], + text: text, + ), + )) + } + let rows = + sqlight.query(object_sql(doc_id), on: conn, with: [], expecting: d) + |> result.unwrap([]) + + // Built newest-first per section and reversed once at the end. Appending + // to the tail of each section instead would be quadratic, and the largest + // document in the reference collection has some thousands of objects in + // one section. + let sections = + list.fold(rows, dict.new(), fn(acc, row) { + let #(id, object) = row + // the four repeatable properties of the text form are child tables + // here, one query each + let object = + Object(..object, images: images(conn, id), stow_link: links(conn, id)) + case model.is_section(object.section) { + False -> acc + True -> + dict.upsert(acc, object.section, fn(existing) { + case existing { + option.Some(previous) -> [object, ..previous] + option.None -> [object] + } + }) + } + }) + |> dict.map_values(fn(_, objects) { list.reverse(objects) }) + model.Document(..document, sections: sections) +} + +fn object_sql(doc_id: Int) -> String { + "select section, seq, ocn, is_a, coalesce(is_of_part,''), + coalesce(is_of_section,''), coalesce(is_of_type,''), + coalesce(heading_level,-1), coalesce(identifier,''), + coalesce(parent_ocn,0), coalesce(last_descendant_ocn,0), + coalesce(children,''), coalesce(ancestors,''), + coalesce(dummy_heading,0), coalesce(object_number_off,0), + coalesce(indent_base,0), coalesce(indent_hang,0), + coalesce(bullet,0), coalesce(lang,''), + coalesce(has_links,0), coalesce(has_notes_reg,0), + coalesce(has_notes_star,0), coalesce(has_images,0), + coalesce(segment,''), coalesce(segment_prev,''), + coalesce(segment_next,''), coalesce(anchor,''), + coalesce(segment_html_is,''), + coalesce(table_cols,0), coalesce(table_widths,''), + coalesce(table_aligns,''), coalesce(table_header,0), + coalesce(code_linenumbers,0), + coalesce(heading_ancestors_text,''), coalesce(sha256,''), + coalesce(text,''), id + from objects where doc_id = " <> int.to_string(doc_id) <> " + order by case section + when 'head' then 1 when 'toc' then 2 when 'body' then 3 + when 'endnotes' then 4 when 'glossary' then 5 + when 'bibliography' then 6 when 'bookindex' then 7 + when 'blurb' then 8 when 'tail' then 9 else 10 end, seq" +} + +fn images(conn: sqlight.Connection, object_id: Int) -> List(model.Image) { + let d = { + use name <- decode.field(0, decode.string) + use sha256 <- decode.field(1, decode.string) + use bytes <- decode.field(2, decode.int) + use width <- decode.field(3, decode.int) + use height <- decode.field(4, decode.int) + use missing <- decode.field(5, decode.int) + decode.success(Image( + name: name, + sha256: sha256, + bytes: bytes, + width: width, + height: height, + missing: missing != 0, + fields: dict.new(), + )) + } + sqlight.query( + "select name, coalesce(sha256,''), coalesce(bytes,0), + coalesce(width,0), coalesce(height,0), coalesce(missing,0) + from object_images where object_id = " + <> int.to_string(object_id) + <> " order by seq", + on: conn, + with: [], + expecting: d, + ) + |> result.unwrap([]) +} + +fn links(conn: sqlight.Connection, object_id: Int) -> List(String) { + let d = { + use url <- decode.field(0, decode.string) + decode.success(url) + } + sqlight.query( + "select url from object_links where object_id = " + <> int.to_string(object_id) + <> " order by seq", + on: conn, + with: [], + expecting: d, + ) + |> result.unwrap([]) +} + +fn one(conn: sqlight.Connection, sql: String) -> String { + let d = { + use v <- decode.field(0, decode.string) + decode.success(v) + } + case sqlight.query(sql, on: conn, with: [], expecting: d) { + Ok([v, ..]) -> v + _ -> "" + } +} + +fn has(conn: sqlight.Connection, name: String) -> Bool { + one( + conn, + "select cast(count(*) as text) from sqlite_master where name = '" + <> name + <> "'", + ) + != "0" +} + +fn meta_value(conn: sqlight.Connection, doc_id: Int, key: String) -> String { + one( + conn, + "select value from metadata where doc_id = " + <> int.to_string(doc_id) + <> " and key = '" + <> key + <> "'", + ) +} + +fn major_of(version: String) -> String { + case string.split_once(version, ".") { + Ok(#(major, _)) -> major + Error(_) -> version + } +} + +/// The eight heading levels and their indices. +fn level_name(index: Int) -> String { + case index { + 0 -> "A" + 1 -> "B" + 2 -> "C" + 3 -> "D" + 4 -> "1" + 5 -> "2" + 6 -> "3" + 7 -> "4" + _ -> "" + } +} + +/// The JSON columns, read as JSON. +/// +/// Not as a character scan that strips brackets and quotes: that is wrong +/// the moment a value holds an escape, and spine's encoder writes "/" as +/// "\/", so every heading containing a slash comes back with a backslash in +/// it. The D library had exactly that bug and all 36 document-languages +/// differed on heading_ancestors_text because of it. +fn json_strings(value: String) -> List(String) { + case value { + "" -> [] + v -> + case json.parse(v, decode.list(of: loose_string())) { + Ok(items) -> items + Error(_) -> [] + } + } +} + +/// An element that may be a string or a number, which is what these columns +/// hold: children and ancestors are int arrays, table_aligns and +/// heading_ancestors_text string arrays, and table_widths an int array +/// except where the columns divide evenly, when it is [33.3333,...]. One +/// table in the reference collection does that, and without the float +/// branch the widths came back as three empty strings: the fallback turned +/// a value it could not read into a value that looked read. +fn loose_string() -> decode.Decoder(String) { + decode.one_of(decode.string, [ + decode.int |> decode.map(int.to_string), + decode.float |> decode.map(float.to_string), + decode.success(""), + ]) +} + +fn json_ints(value: String) -> List(Int) { + value + |> json_strings + |> list.filter(fn(s) { s != "" }) + |> list.map(fn(s) { s |> int.parse |> result.unwrap(0) }) +} + +/// Space separated in the text form: table_widths, table_aligns. +fn json_words(value: String) -> String { + value |> json_strings |> string.join(" ") +} + +/// heading_ancestors_text is eight slots, joined with "|" in the text form +/// and a JSON array here. The empty slots are significant: they say which +/// levels the heading sits under. +fn json_joined(value: String) -> String { + case json_strings(value) { + [] -> "" + parts -> string.join(parts, "|") + } +} diff --git a/gleam/src/ocda/load_ssp.gleam b/gleam/src/ocda/load_ssp.gleam new file mode 100644 index 0000000..6dcc38a --- /dev/null +++ b/gleam/src/ocda/load_ssp.gleam @@ -0,0 +1,468 @@ +//// Loading a `.ssp` into the model. +//// +//// The format is line oriented and every line is one of eight shapes told +//// apart by its first character, so the loader is a fold over the lines +//// carrying the state the shapes need. +//// +//// Written against `ssp-format-specification.md`. Where it and the D +//// library differ the D library is the bug, or this is: they are meant to +//// give the same model, and `ocda-dump` says whether they do. + +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/option +import gleam/result +import gleam/string +import ocda/model.{type Document, type Image, type Object, Image, Object} +import simplifile + +pub fn load_file(path: String) -> Result(Document, String) { + case simplifile.read(path) { + Ok(content) -> Ok(parse(content)) + Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) + } +} + +type State { + State( + document: Document, + /// the block currently open, "" when none + block: String, + /// the section currently open, "" when in the preamble + section: String, + /// objects of the current section, newest first + objects: List(Object), + /// the object being read, if any + current: Result(Object, Nil), + /// text lines of the object being read, newest first + text: List(String), + /// whether a text line has been seen for the object being read. + /// Not "is the text non-empty": an object whose first text line is + /// blank, which the reference collection has seven of, would then lose + /// it. + seen_text: Bool, + ) +} + +pub fn parse(content: String) -> Document { + content + |> string.split(on: "\n") + |> list.fold(from: initial(), with: line) + |> finish +} + +fn initial() -> State { + State( + document: model.new_document(), + block: "", + section: "", + objects: [], + current: Error(Nil), + text: [], + seen_text: False, + ) +} + +fn line(state: State, raw: String) -> State { + case raw { + "% SiSU Document Abstraction v" <> version -> + case state.document.format_version { + "" -> + State( + ..state, + document: model.Document( + ..state.document, + format_version: string.trim(version), + ), + ) + _ -> state + } + "% Source: " <> filename -> + case state.document.source_filename { + "" -> + State( + ..state, + document: model.Document( + ..state.document, + source_filename: string.trim(filename), + ), + ) + _ -> state + } + "%" <> _ -> state + "@" <> rest -> open_block(state, rest) + "}" <> _ -> close_block(state) + "[" <> rest -> declaration(state, rest) + "." <> rest -> property(state, rest) + "|" <> rest -> text_line(state, rest) + _ -> + case string.trim(raw) { + // a blank line ends the object being read + "" -> commit(state) + _ -> + case model.is_preamble_block(state.block) { + True -> preamble_pair(state, raw) + False -> unknown_line(state, raw) + } + } + } +} + +fn open_block(state: State, rest: String) -> State { + let name = case string.split_once(rest, "{") { + Ok(#(name, _)) -> string.trim(name) + Error(_) -> string.trim(rest) + } + case model.is_preamble_block(name) { + True -> State(..flush(state), block: name) + False -> { + let state = flush(state) + let document = case model.is_section(name) { + True -> state.document + // a block that is neither preamble nor one of the nine sections: + // counted, so that a reader can say the file held something it did + // not expect + False -> + model.Document( + ..state.document, + unknown: model.Unknown( + ..state.document.unknown, + block: bump(state.document.unknown.block, name), + ), + ) + } + State(..state, block: name, section: name, document: document) + } + } +} + +fn close_block(state: State) -> State { + State(..flush(state), block: "") +} + +/// Commit the object being read, then put the section's objects into the +/// document and close the section. +fn flush(state: State) -> State { + let state = commit(state) + case state.section, state.objects { + "", _ -> state + _, [] -> State(..state, section: "", objects: []) + name, objects -> + State( + ..state, + section: "", + objects: [], + document: model.Document( + ..state.document, + sections: add_section( + state.document.sections, + name, + list.reverse(objects), + ), + ), + ) + } +} + +fn preamble_pair(state: State, raw: String) -> State { + case string.split_once(string.trim_start(raw), ":") { + Error(_) -> state + Ok(#(key, value)) -> { + let key = string.trim(key) + let value = string.trim(value) + let document = state.document + let document = case state.block { + "meta" -> + model.Document( + ..document, + metadata: dict.insert(document.metadata, key, value), + ) + "make" -> + model.Document(..document, make: dict.insert(document.make, key, value)) + "source" -> + model.Document( + ..document, + source: dict.insert(document.source, key, value), + ) + _ -> + model.Document( + ..document, + doc_has: dict.insert(document.doc_has, key, value), + ) + } + State(..state, document: document) + } + } +} + +/// `[<ocn>] <kind>`, and for a heading `[<ocn>] heading :<level> +/// [<identifier>]`. +fn declaration(state: State, rest: String) -> State { + let state = commit(state) + case string.split_once(rest, "]") { + Error(_) -> state + Ok(#(ocn_text, tail)) -> { + let ocn = parse_int(string.trim(ocn_text)) + let object = + Object(..model.new_object(), ocn: ocn, section: state.section) + let object = case string.split(string.trim(tail), on: " ") { + ["heading", level, ..identifier] -> + Object( + ..object, + is_a: "heading", + heading_level: string.drop_start(level, 1), + identifier: string.join(identifier, " "), + ) + [""] -> object + [kind, ..] -> Object(..object, is_a: kind) + [] -> object + } + State(..state, current: Ok(object), text: [], seen_text: False) + } + } +} + +fn property(state: State, rest: String) -> State { + case state.current, string.split_once(rest, ":") { + Ok(object), Ok(#(name, raw_value)) -> { + let name = string.trim(name) + let value = string.trim(raw_value) + let #(object, known) = apply_property(object, name, value) + let document = case known { + True -> state.document + False -> + model.Document( + ..state.document, + unknown: model.Unknown( + ..state.document.unknown, + property: bump(state.document.unknown.property, name), + ), + ) + } + State(..state, current: Ok(object), document: document) + } + _, _ -> state + } +} + +/// The properties this library reads, and what each sets. +/// +/// A property it has no use for is counted rather than passed over: +/// ignoring one is conforming within a major version, doing so in silence +/// is what stops a reader being a check on the specification. +fn apply_property( + object: Object, + name: String, + value: String, +) -> #(Object, Bool) { + case name { + "part" -> #(Object(..object, part: value), True) + "is_of_type" -> #(Object(..object, is_of_type: value), True) + "section" -> #(Object(..object, own_section: value), True) + "identifier" -> #(Object(..object, identifier: value), True) + "segment_html_is" -> #(Object(..object, segment_html_is: value), True) + "table_aligns" -> #(Object(..object, table_aligns: value), True) + "stow_link" -> #( + Object(..object, stow_link: list.append(object.stow_link, [value])), + True, + ) + "parent" -> #(Object(..object, parent_ocn: parse_int(value)), True) + "last_descendant" -> #( + Object(..object, last_descendant_ocn: parse_int(value)), + True, + ) + "children" -> #(Object(..object, children: numbers(value)), True) + "ancestors" -> #(Object(..object, ancestors: numbers(value)), True) + "dummy" -> #(Object(..object, dummy_heading: value == "true"), True) + "ocn_off" -> #(Object(..object, ocn_off: value == "true"), True) + "indent" -> { + let parts = string.split(value, on: " ") + let base = case parts { + [first, ..] -> parse_int(first) + [] -> 0 + } + let hang = case parts { + [_, second, ..] -> parse_int(second) + _ -> 0 + } + #(Object(..object, indent_base: base, indent_hang: hang), True) + } + "bullet" -> #(Object(..object, bullet: value == "true"), True) + "lang" -> #(Object(..object, lang: value), True) + "has" -> #(has_flags(object, value), True) + "segment" -> #(Object(..object, segment: value), True) + "segment_prev" -> #(Object(..object, segment_prev: value), True) + "segment_next" -> #(Object(..object, segment_next: value), True) + "anchor" -> #(Object(..object, anchor: value), True) + "table_cols" -> #(Object(..object, table_cols: parse_int(value)), True) + "table_widths" -> #(Object(..object, table_widths: value), True) + "table_header" -> #(Object(..object, table_header: value == "true"), True) + "code_syntax" -> #(Object(..object, code_syntax: value), True) + "code_linenumbers" -> #( + Object(..object, code_linenumbers: value == "true"), + True, + ) + "heading_ancestors_text" -> #( + Object(..object, heading_ancestors_text: value), + True, + ) + "sha256" -> #(Object(..object, sha256: value), True) + "image" -> #( + Object(..object, images: list.append(object.images, [image(value)])), + True, + ) + _ -> #(object, False) + } +} + +fn has_flags(object: Object, value: String) -> Object { + value + |> string.split(on: " ") + |> list.fold(from: object, with: fn(obj, flag) { + case flag { + "links" -> Object(..obj, has_links: True) + "notes_reg" -> Object(..obj, has_notes_reg: True) + "notes_star" -> Object(..obj, has_notes_star: True) + "images" -> Object(..obj, has_images: True) + _ -> obj + } + }) +} + +/// `<filename> sha256:<HEX> bytes:<n> [px:<w>x<h>] [missing:true] [...]` +/// +/// The filename comes first, the rest are order-independent, and a field +/// this library does not recognise is kept rather than discarded. +fn image(value: String) -> Image { + case string.split(value, on: " ") { + [] -> model.new_image() + [name, ..fields] -> + list.fold(fields, model.new_image(), fn(img, field) { + case string.split_once(field, ":") { + Error(_) -> img + Ok(#(key, val)) -> + case key { + "sha256" -> Image(..img, sha256: val) + "bytes" -> Image(..img, bytes: parse_int(val)) + "px" -> + case string.split_once(val, "x") { + Ok(#(w, h)) -> + Image(..img, width: parse_int(w), height: parse_int(h)) + Error(_) -> img + } + "missing" -> Image(..img, missing: val == "true") + _ -> Image(..img, fields: dict.insert(img.fields, key, val)) + } + } + }) + |> fn(img) { Image(..img, name: name) } + } +} + +/// `| <text>`, and `| ` alone for a blank line. Never a bare `|`: the +/// emitter writes the space. +fn text_line(state: State, rest: String) -> State { + case state.current { + Error(_) -> state + Ok(_) -> { + let content = string.drop_start(rest, 1) + State(..state, text: [content, ..state.text], seen_text: True) + } + } +} + +fn unknown_line(state: State, raw: String) -> State { + let first = string.slice(raw, 0, 1) + State( + ..state, + document: model.Document( + ..state.document, + unknown: model.Unknown( + ..state.document.unknown, + line: bump(state.document.unknown.line, first), + ), + ), + ) +} + +fn commit(state: State) -> State { + case state.current { + Error(_) -> state + Ok(object) -> { + let text = case state.seen_text { + True -> string.join(list.reverse(state.text), "\n") + False -> "" + } + let object = derive_omitted(Object(..object, text: text)) + let objects = case model.is_section(state.section) { + True -> [object, ..state.objects] + False -> state.objects + } + State( + ..state, + current: Error(Nil), + text: [], + seen_text: False, + objects: objects, + ) + } + } +} + +/// Two values the text form leaves out because they can be worked out. +/// +/// `.identifier` is absent when it is simply the ocn, `.anchor` when it is +/// the segment name. The `.ocda.db`, being columns rather than lines, +/// stores both unconditionally. So a loader that takes the text form +/// literally gives a different model from the one the database gives, and +/// over one document that is 1,314 identifiers and 69 anchors. +fn derive_omitted(object: Object) -> Object { + let object = case object.identifier, object.ocn { + "", 0 -> object + "", ocn -> Object(..object, identifier: int.to_string(ocn)) + _, _ -> object + } + case object.anchor, object.segment { + "", "" -> object + "", segment -> Object(..object, anchor: segment) + _, _ -> object + } +} + +fn finish(state: State) -> Document { + flush(state).document +} + +fn add_section( + sections: Dict(String, List(Object)), + name: String, + objects: List(Object), +) -> Dict(String, List(Object)) { + dict.upsert(sections, name, fn(existing) { + case existing { + option.Some(previous) -> list.append(previous, objects) + option.None -> objects + } + }) +} + +fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) { + dict.upsert(counts, key, fn(existing) { + case existing { + option.Some(n) -> n + 1 + option.None -> 1 + } + }) +} + +fn parse_int(value: String) -> Int { + value |> string.trim |> int.parse |> result.unwrap(0) +} + +fn numbers(value: String) -> List(Int) { + value + |> string.split(on: " ") + |> list.filter(fn(s) { s != "" }) + |> list.map(parse_int) +} diff --git a/gleam/src/ocda_lib.gleam b/gleam/src/ocda_lib.gleam new file mode 100644 index 0000000..1e8a5ae --- /dev/null +++ b/gleam/src/ocda_lib.gleam @@ -0,0 +1,136 @@ +//// ocda-lib, in Gleam: the abstraction as a library, put to the question. +//// +//// A library has no binary, so this is the two programs that test its +//// claims. +//// +//// ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] +//// ocda-lib agree <doc>.ssp <doc>.ocda.db [lang] +//// +//// `dump` writes the whole loaded model out canonically, byte for byte as +//// the D library's `ocda-dump` writes it, so that the two implementations +//// can be diffed. `agree` asks the same question the D library's +//// `ocda-agree` asks, within this language: does loading a document's +//// `.ssp` give what loading its `.ocda.db` gives? + +import argv +import gleam/int +import gleam/io +import gleam/list +import gleam/string +import ocda/dump +import ocda/load +import ocda/model.{type Document} + +pub fn main() { + case argv.load().arguments { + ["dump", "--unknown", path] -> dump_file(path, "", True) + ["dump", "--unknown", path, lang] -> dump_file(path, lang, True) + ["dump", path] -> dump_file(path, "", False) + ["dump", path, lang] -> dump_file(path, lang, False) + ["agree", ssp, db] -> agree(ssp, db, "") + ["agree", ssp, db, lang] -> agree(ssp, db, lang) + _ -> usage() + } +} + +fn usage() -> Nil { + io.println_error( + "usage: ocda-lib dump [--unknown] <file>.ssp|<file>.ocda.db [lang] + ocda-lib agree <doc>.ssp <doc>.ocda.db [lang]", + ) + halt(2) +} + +fn dump_file(path: String, lang: String, unknown: Bool) -> Nil { + case load.load(path, lang, False) { + Error(message) -> fail(message) + Ok(doc) -> + case unknown { + True -> io.print(dump.unknown(doc)) + False -> io.print(dump.document(doc)) + } + } +} + +/// The library's whole purpose is that a writer cannot tell which +/// serialisation it was handed. That is only true if the two loaders give +/// the same model, so this asks, and says where they part company rather +/// than only that they did. +fn agree(ssp_path: String, db_path: String, lang: String) -> Nil { + case load.load(ssp_path, "", False), load.load(db_path, lang, False) { + Error(message), _ -> fail(message) + _, Error(message) -> fail(message) + Ok(from_ssp), Ok(from_db) -> { + let differences = compare(from_ssp, from_db) + case differences { + [] -> io.println("agree: the two loaders give the same model") + _ -> { + list.each(list.take(differences, 12), io.println) + case list.length(differences) > 12 { + True -> io.println(" ... stopping after 12") + False -> Nil + } + io.println(int.to_string(list.length(differences)) <> " difference(s)") + halt(1) + } + } + } + } +} + +/// Compared through the dump rather than field by field, which is both +/// shorter and stricter: the dump carries every field, so nothing can +/// differ without showing here. +fn compare(from_ssp: Document, from_db: Document) -> List(String) { + let a = dump.document(from_ssp) + let b = dump.document(from_db) + case a == b { + True -> [] + False -> diff_lines(a, b) + } +} + +fn diff_lines(a: String, b: String) -> List(String) { + let la = split_lines(a) + let lb = split_lines(b) + case list.length(la) == list.length(lb) { + False -> [ + " differ: line count", + " .ssp: " <> int.to_string(list.length(la)), + " .ocda.db: " <> int.to_string(list.length(lb)), + ..pairwise(la, lb) + ] + True -> pairwise(la, lb) + } +} + +fn pairwise(la: List(String), lb: List(String)) -> List(String) { + list.zip(la, lb) + |> list.index_map(fn(pair, i) { #(i, pair.0, pair.1) }) + |> list.filter(fn(t) { t.1 != t.2 }) + |> list.map(fn(t) { + " differ: line " + <> int.to_string(t.0 + 1) + <> "\n .ssp: " + <> t.1 + <> "\n .ocda.db: " + <> t.2 + }) +} + +fn split_lines(text: String) -> List(String) { + case text { + "" -> [] + _ -> string.split(text, on: "\n") + } +} + +fn fail(message: String) -> Nil { + io.println_error("error: " <> message) + halt(2) +} + +/// Gleam has no standard exit, and the exit status is what a sweep script +/// reads. +@external(erlang, "erlang", "halt") +fn halt(code: Int) -> Nil |
