summaryrefslogtreecommitdiffhomepage
path: root/org/gleam-ocda-misc.org
diff options
context:
space:
mode:
Diffstat (limited to 'org/gleam-ocda-misc.org')
-rw-r--r--org/gleam-ocda-misc.org649
1 files changed, 649 insertions, 0 deletions
diff --git a/org/gleam-ocda-misc.org b/org/gleam-ocda-misc.org
new file mode 100644
index 0000000..cc32dcb
--- /dev/null
+++ b/org/gleam-ocda-misc.org
@@ -0,0 +1,649 @@
+-*- mode: org -*-
+#+TITLE: gleam lib
+#+DESCRIPTION: gleam lib
+#+FILETAGS: :spine:build:tools:
+#+AUTHOR: Ralph Amissah
+#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
+#+LANGUAGE: en
+#+STARTUP: content hideblocks hidestars noindent entitiespretty
+#+PROPERTY: header-args+ :eval never-export :exports code
+#+PROPERTY: header-args+ :tangle-mode :noweb no :padline no :tangle-mode :no-expand
+#+PROPERTY: header-args+ :results silent :cache no
+#+PROPERTY: header-args+ :mkdirp yes
+#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
+- magic single double-quote → " ← FIX changes hilighting behavior (occuring
+ after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
+
+* gleam :gleam:
+** zip gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/zip.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<zip>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: zip
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// Enough of the zip format to find one file in a pod and read it.
+////
+//// Not a zip library. A pod archive holds a handful of entries, spine writes
+//// them **stored** (uncompressed) so that the outer zstd frame does the
+//// compressing once over the whole thing, and what is wanted from it here is
+//// one named entry.
+////
+//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could
+//// inflate it, but a library that says it read an entry it in fact skipped is
+//// worse than one that says it cannot, and spine's own archives do not need
+//// it.
+
+import gleam/bit_array
+import gleam/int
+import gleam/list
+import gleam/result
+import gleam/string
+
+pub type Entry {
+ Entry(
+ name: String,
+ /// 0 stored, 8 deflate
+ method: Int,
+ compressed: Int,
+ uncompressed: Int,
+ local_offset: Int,
+ )
+}
+
+const eocd_sig = 0x06054b50
+
+const cd_sig = 0x02014b50
+
+const local_sig = 0x04034b50
+
+/// The entries, from the central directory at the end of the file.
+pub fn entries(bytes: BitArray) -> Result(List(Entry), String) {
+ let size = bit_array.byte_size(bytes)
+ use _ <- result.try(case size >= 22 {
+ True -> Ok(Nil)
+ False -> Error("too short to be a zip archive")
+ })
+ use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size)))
+ use count <- result.try(u16(bytes, eocd + 10))
+ use offset <- result.try(u32(bytes, eocd + 16))
+ read_central(bytes, offset, count, [])
+}
+
+/// The end of central directory record is last, and may be followed by a
+/// comment of up to 65535 bytes, so it is searched for backwards.
+fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) {
+ case at < lowest {
+ True ->
+ Error(
+ "no end of central directory record; it does not look like a zip archive",
+ )
+ False ->
+ case u32(bytes, at) {
+ Ok(sig) if sig == eocd_sig -> Ok(at)
+ _ -> find_eocd(bytes, at - 1, lowest)
+ }
+ }
+}
+
+fn lowest(size: Int) -> Int {
+ case size > 22 + 65_535 {
+ True -> size - 22 - 65_535
+ False -> 0
+ }
+}
+
+fn read_central(
+ bytes: BitArray,
+ at: Int,
+ remaining: Int,
+ acc: List(Entry),
+) -> Result(List(Entry), String) {
+ case remaining {
+ 0 -> Ok(list.reverse(acc))
+ _ -> {
+ use sig <- result.try(u32(bytes, at))
+ use _ <- result.try(case sig == cd_sig {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ "central directory entry "
+ <> int.to_string(list.length(acc))
+ <> " is malformed",
+ )
+ })
+ use method <- result.try(u16(bytes, at + 10))
+ use compressed <- result.try(u32(bytes, at + 20))
+ use uncompressed <- result.try(u32(bytes, at + 24))
+ use name_len <- result.try(u16(bytes, at + 28))
+ use extra_len <- result.try(u16(bytes, at + 30))
+ use cmnt_len <- result.try(u16(bytes, at + 32))
+ use local_offset <- result.try(u32(bytes, at + 42))
+ use name <- result.try(text(bytes, at + 46, name_len))
+ let entry =
+ Entry(
+ name: name,
+ method: method,
+ compressed: compressed,
+ uncompressed: uncompressed,
+ local_offset: local_offset,
+ )
+ read_central(
+ bytes,
+ at + 46 + name_len + extra_len + cmnt_len,
+ remaining - 1,
+ [entry, ..acc],
+ )
+ }
+ }
+}
+
+/// One entry's bytes, from its local header.
+pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) {
+ use sig <- result.try(u32(bytes, entry.local_offset))
+ use _ <- result.try(case sig == local_sig {
+ True -> Ok(Nil)
+ False -> Error(entry.name <> ": local header is malformed")
+ })
+ use name_len <- result.try(u16(bytes, entry.local_offset + 26))
+ use extra_len <- result.try(u16(bytes, entry.local_offset + 28))
+ use _ <- result.try(case entry.method {
+ 0 -> Ok(Nil)
+ m ->
+ Error(
+ entry.name
+ <> ": compression method "
+ <> int.to_string(m)
+ <> "; this reader takes only stored entries, which is what spine writes",
+ )
+ })
+ let start = entry.local_offset + 30 + name_len + extra_len
+ case bit_array.slice(bytes, start, entry.compressed) {
+ Ok(slice) -> Ok(slice)
+ Error(_) -> Error(entry.name <> ": runs past the end of the archive")
+ }
+}
+
+/// The entries whose names end this way, in archive order. Used to find
+/// media/abstraction/<uid>.<lang>.ssp without knowing the uid.
+pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) {
+ list.filter(entries, fn(e) { string.ends_with(e.name, suffix) })
+}
+
+fn u16(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 2) {
+ Ok(<<v:little-size(16)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn u32(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 4) {
+ Ok(<<v:little-size(32)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
+ case bit_array.slice(bytes, at, len) {
+ Ok(slice) ->
+ bit_array.to_string(slice)
+ |> result.replace_error("an entry name is not valid UTF-8")
+ Error(_) -> Error("an entry name runs past the end of the archive")
+ }
+}
+#+END_SRC
+
+** imapges gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/images.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<images>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: images
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// An image's bytes, from whichever source the document came from.
+////
+//// The model *names* images, with digest, byte size and pixel size; it does
+//// not hold them. That is right - an abstraction of a document is not a
+//// bundle of its assets - but a writer of any format that shows images needs
+//// the bytes, so something has to know where they are. This is that
+//// something, and it is in the library rather than in a writer because the
+//// answer depends on the *input form* and not on the output.
+////
+//// | `.ocda.db` | the `files` table, role 'image', by name |
+//// | `.zip` | the archive entry whose path ends in `/image/<name>` |
+//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot |
+//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` |
+
+import gleam/bit_array
+import gleam/dynamic/decode
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/zip
+import simplifile
+import sqlight
+
+/// The bytes, or an error saying why not.
+pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) {
+ case load.source_of(source) {
+ load.OcdaDb -> from_db(source, name)
+ load.Pod -> from_archive(source, name)
+ load.Ssp -> from_beside(source, name)
+ load.Unknown -> Error(source <> ": not a source this library reads")
+ }
+}
+
+/// Can this source give images at all?
+///
+/// A writer asks before it decides whether to emit them, because an `<img>`
+/// pointing at a file it did not include is worse than no image.
+pub fn can_give_images(source: String) -> Bool {
+ case load.source_of(source) {
+ load.OcdaDb -> True
+ load.Pod -> zip_readable(source)
+ load.Ssp -> image_dir_beside(source) != ""
+ load.Unknown -> False
+ }
+}
+
+fn zip_readable(path: String) -> Bool {
+ case simplifile.read_bits(path) {
+ Error(_) -> False
+ Ok(bytes) ->
+ case zip.entries(bytes) {
+ Ok(_) -> True
+ Error(_) -> False
+ }
+ }
+}
+
+fn from_db(path: String, name: String) -> Result(BitArray, String) {
+ case sqlight.open("file:" <> path <> "?mode=ro") {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let out =
+ sqlight.query(
+ "select data from files where role = 'image' and name = ?1",
+ on: conn,
+ with: [sqlight.text(name)],
+ expecting: blob(),
+ )
+ |> result.replace_error(path <> ": could not read " <> name)
+ let _ = sqlight.close(conn)
+ case out {
+ Ok([data, ..]) -> Ok(data)
+ Ok([]) -> Error(name <> ": not in this database")
+ Error(e) -> Error(e)
+ }
+ }
+ }
+}
+
+/// A blob column, read as bytes rather than as text: an image read as text
+/// stops at the first zero byte, and a png's ninth byte is zero.
+fn blob() -> decode.Decoder(BitArray) {
+ use v <- decode.field(0, decode.bit_array)
+ decode.success(v)
+}
+
+fn from_archive(path: String, name: String) -> Result(BitArray, String) {
+ use bytes <- result.try(
+ simplifile.read_bits(path)
+ |> result.replace_error(path <> ": cannot read"),
+ )
+ use entries <- result.try(zip.entries(bytes))
+ case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) {
+ Error(_) -> Error(name <> ": not in this archive")
+ Ok(entry) -> zip.read(bytes, entry)
+ }
+}
+
+fn from_beside(path: String, name: String) -> Result(BitArray, String) {
+ case image_dir_beside(path) {
+ "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it")
+ dir ->
+ simplifile.read_bits(dir <> "/" <> name)
+ |> result.replace_error(dir <> "/" <> name <> ": cannot read")
+ }
+}
+
+/// `media/abstraction/<x>.ssp` -> `media/image/`
+fn image_dir_beside(path: String) -> String {
+ let parts = string.split(path, on: "/")
+ let dir = case list.reverse(parts) {
+ [_file, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ case string.ends_with(dir, "abstraction") {
+ False -> ""
+ True -> {
+ let parent = case list.reverse(string.split(dir, on: "/")) {
+ [_, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ let image_dir = parent <> "/image"
+ case simplifile.is_directory(image_dir) {
+ Ok(True) -> image_dir
+ _ -> ""
+ }
+ }
+ }
+}
+
+/// Unused here, kept so that a caller can size a read before making it.
+pub fn byte_size(b: BitArray) -> Int {
+ bit_array.byte_size(b)
+}
+#+END_SRC
+
+** dump gleam
+*** tangle
+
+#+HEADER: :tangle ../gleam/src/ocda/dump.gleam
+#+HEADER: :noweb yes
+#+BEGIN_SRC org
+////<<doc_header_including_copyright_and_license>>
+<<dump>>
+#+END_SRC
+
+*** gleam source
+
+#+NAME: dump
+#+HEADER: :no-expand
+#+BEGIN_SRC gleam
+//// The model, written out canonically.
+////
+//// Not an output format: an *oracle*. Two libraries in two languages
+//// implement the same model from the same specification, and the cheap way
+//// to ask whether they agree is to have each write the whole loaded
+//// document as text and diff the two. Byte for byte identical to the D
+//// library's `ocda/dump.d`, which is the whole point of it.
+////
+//// The rules, which both implementations follow:
+////
+//// - every line is `<key> <value>`, the value running to end of line,
+//// and only one free-form value per line;
+//// - a field at its default is *omitted*, so the output says what the
+//// document has rather than what the type declares;
+//// - dictionaries come in sorted key order, because neither language's
+//// map has an order worth relying on;
+//// - sections come in the fixed order of the format.
+////
+//// Anything that reads this to reconstruct a document has misunderstood
+//// it. Load the `.ssp` or the `.ocda.db`.
+
+import gleam/dict.{type Dict}
+import gleam/int
+import gleam/list
+import gleam/string
+import ocda/model.{type Document, type Object}
+
+pub fn document(doc: Document) -> String {
+ let lines =
+ []
+ |> kv("version", doc.format_version)
+ |> kv("source_filename", doc.source_filename)
+ |> entries("source", doc.source)
+ |> entries("meta", doc.metadata)
+ |> entries("make", doc.make)
+ |> entries("doc_has", doc.doc_has)
+ |> counts(doc)
+ |> objects(doc)
+ lines |> list.reverse |> join
+}
+
+/// What the loader met and did not understand, sorted.
+///
+/// Separate from the document dump: it describes the loader, not the
+/// document, and the `.ocda.db` loader has nothing to report where the
+/// `.ssp` loader does.
+pub fn unknown(doc: Document) -> String {
+ []
+ |> numbered("property", doc.unknown.property)
+ |> numbered("block", doc.unknown.block)
+ |> numbered("line", doc.unknown.line)
+ |> list.reverse
+ |> join
+}
+
+fn join(lines: List(String)) -> String {
+ case lines {
+ [] -> ""
+ _ -> string.join(lines, "\n") <> "\n"
+ }
+}
+
+/// Accumulated newest-first and reversed once, so that nothing appends to
+/// the tail of a list that grows to a few hundred thousand lines.
+fn kv(acc: List(String), key: String, value: String) -> List(String) {
+ case value {
+ "" -> acc
+ v -> [key <> " " <> v, ..acc]
+ }
+}
+
+fn kn(acc: List(String), key: String, value: Int) -> List(String) {
+ case value {
+ 0 -> acc
+ n -> [key <> " " <> int.to_string(n), ..acc]
+ }
+}
+
+fn kb(acc: List(String), key: String, value: Bool) -> List(String) {
+ case value {
+ False -> acc
+ True -> [key <> " 1", ..acc]
+ }
+}
+
+fn entries(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, String),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> pair.1, ..lines]
+ })
+}
+
+fn numbered(
+ acc: List(String),
+ prefix: String,
+ d: Dict(String, Int),
+) -> List(String) {
+ d
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(acc, fn(lines, pair) {
+ [prefix <> " " <> pair.0 <> " " <> int.to_string(pair.1), ..lines]
+ })
+}
+
+fn counts(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ case model.section(doc, name) {
+ [] -> lines
+ objects ->
+ [
+ "section " <> name <> " " <> int.to_string(list.length(objects)),
+ ..lines
+ ]
+ }
+ })
+}
+
+fn objects(acc: List(String), doc: Document) -> List(String) {
+ list.fold(model.section_names, acc, fn(lines, name) {
+ model.section(doc, name)
+ |> list.index_fold(lines, fn(l, obj, i) {
+ object(l, name, i, obj)
+ })
+ })
+}
+
+fn object(
+ acc: List(String),
+ section: String,
+ index: Int,
+ obj: Object,
+) -> List(String) {
+ let acc = ["object " <> section <> " " <> int.to_string(index), ..acc]
+ acc
+ |> kn("ocn", obj.ocn)
+ |> kv("is_a", obj.is_a)
+ |> kv("own_section", obj.own_section)
+ |> kv("level", obj.heading_level)
+ |> kv("identifier", obj.identifier)
+ |> kv("part", obj.part)
+ |> kv("is_of_type", obj.is_of_type)
+ |> kv("segment_html_is", obj.segment_html_is)
+ |> kv("segment_epub", obj.segment_epub)
+ |> kv("segment_epub_is", obj.segment_epub_is)
+ |> kn("parent", obj.parent_ocn)
+ |> kn("last_descendant", obj.last_descendant_ocn)
+ |> kn("ocn_range_first", obj.ocn_range_first)
+ |> kn("ocn_range_last", obj.ocn_range_last)
+ |> kv("children", ints(obj.children))
+ |> kv("ancestors", ints(obj.ancestors))
+ |> kb("dummy", obj.dummy_heading)
+ |> kb("ocn_off", obj.ocn_off)
+ |> kn("indent_base", obj.indent_base)
+ |> kn("indent_hang", obj.indent_hang)
+ |> kb("bullet", obj.bullet)
+ |> kv("lang", obj.lang)
+ |> kb("has_links", obj.has_links)
+ |> kb("has_notes_reg", obj.has_notes_reg)
+ |> kb("has_notes_star", obj.has_notes_star)
+ |> kb("has_images", obj.has_images)
+ |> kv("segment", obj.segment)
+ |> kv("segment_prev", obj.segment_prev)
+ |> kv("segment_next", obj.segment_next)
+ |> kv("anchor", obj.anchor)
+ |> kv("heading_lev_anchor", obj.heading_lev_anchor)
+ |> kn("table_cols", obj.table_cols)
+ |> kv("table_widths", obj.table_widths)
+ |> kv("table_aligns", obj.table_aligns)
+ |> kb("table_header", obj.table_header)
+ |> kv("code_syntax", obj.code_syntax)
+ |> kb("code_linenumbers", obj.code_linenumbers)
+ |> kv("heading_ancestors_text", obj.heading_ancestors_text)
+ |> kv("sha256", obj.sha256)
+ |> links(obj)
+ |> anchor_tags(obj)
+ |> images(obj)
+ |> text(obj)
+}
+
+fn links(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.stow_link, acc, fn(lines, url) { kv(lines, "stow_link", url) })
+}
+
+fn anchor_tags(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.anchor_tags, acc, fn(lines, a) { kv(lines, "anchor_tag", a) })
+}
+
+fn images(acc: List(String), obj: Object) -> List(String) {
+ list.fold(obj.images, acc, fn(lines, img) {
+ // the name last, because a filename may hold a space where none of the
+ // other fields can
+ let lines =
+ kv(
+ lines,
+ "image",
+ img.sha256
+ <> " "
+ <> int.to_string(img.bytes)
+ <> " "
+ <> int.to_string(img.width)
+ <> "x"
+ <> int.to_string(img.height)
+ <> " "
+ <> case img.missing {
+ True -> "1"
+ False -> "0"
+ }
+ <> " "
+ <> img.name,
+ )
+ img.fields
+ |> dict.to_list
+ |> list.sort(fn(a, b) { string.compare(a.0, b.0) })
+ |> list.fold(lines, fn(l, pair) {
+ kv(l, "image_field", pair.0 <> " " <> pair.1)
+ })
+ })
+}
+
+/// Text, when there is any.
+///
+/// An object whose only text line is blank holds "" here, indistinguishable
+/// from no text at all, because the model joins the lines into one string.
+/// Both libraries lose the same distinction in the same place, which is
+/// what makes them comparable; it is recorded rather than hidden.
+fn text(acc: List(String), obj: Object) -> List(String) {
+ case obj.text {
+ "" -> acc
+ t -> {
+ let lines = string.split(t, on: "\n")
+ let acc = kv(acc, "text", int.to_string(list.length(lines)))
+ list.fold(lines, acc, fn(l, line) { ["| " <> line, ..l] })
+ }
+ }
+}
+
+fn ints(xs: List(Int)) -> String {
+ case xs {
+ [] -> ""
+ _ -> xs |> list.map(int.to_string) |> string.join(",")
+ }
+}
+#+END_SRC
+
+* org includes
+** project version
+
+#+NAME: spine_version
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
+#+END_SRC
+
+** year
+
+#+NAME: year
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
+#+END_SRC
+
+** document header including copyright & license
+
+#+NAME: doc_header_including_copyright_and_license
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
+#+END_SRC
+
+* __END__