From fda48e417435e31a3122bfa7ca563333628aed99 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Sat, 26 Sep 2026 09:47:07 -0400 Subject: format 2.0: abstraction as library, in D and Gleam The object-centric document abstraction that sisudoc-spine builds from SiSU markup, read as a library: the model, and a loader for each form it is written in - the .ssp (text), the .ocda.db (sqlite), and a pod archive carrying a .ssp (.sisupod, .zip). Begun 2026-09-25; this is the library as it stood at format 2.0. The readers (sisudoc-ssp-reader, sisudoc-ocda-db-reader) each read one form and print. A writer needs neither: it needs the document. So the model and the loaders live here, ocda.model imports nothing, and a program built on the library cannot tell which form it was handed. ocda.load is the one place that decides what a path is, so a new input form is added once and every program gains it. The model and the .ssp loader began as the .ssp reader's code, moved and renamed to the level they belong to; the .ocda.db loader is new. The Gleam library is a deliberate field-for-field mirror of the D one, including where that is not the most idiomatic Gleam: a difference in one language and not the other would show in every comparison and mean nothing. The claim holds only if the loaders agree, so the library asks: ocda-agree .ssp .ocda.db [lang] ten fields, per object ocda-dump [lang] the whole model, canonically The dump is an oracle, not an output format, and the D and Gleam dumps are byte identical, so four questions are a diff: D and Gleam, each from the .ssp and from the .ocda.db. Over the markup document samples all four hold, every document-language; a .sisupod gives the same model as the .ssp beside it. None of that held at first, and what the checks found is the record of how the specification became enough: - an object whose first text line is blank lost it (the loader asked whether the text was empty, not whether a line had been seen); seven sample documents open verse that way. Found by agree; fixed here and in the .ssp reader, which carried the same code. - found by the dump: the identifier and the anchor are omitted by the text form when they can be derived and stored by the database always, so the loaders derive them, and the rule went into the specification (derived values); make and doc_has share one table in the database under a key prefix, which the loaders strip; the JSON columns were scanned, not parsed, and Phobos writes "/" as "\/"; and translation.ocn_aligned is a fact about the file, not the document, so neither loader puts it in the model. - in Gleam, a JSON element decoder fell through to a fallback giving "", so a float column read as empty strings: a value that could not be read turned into one that looked read. - found by the writers: anchor_tags, heading_lev_anchor, segment_epub and segment_epub_is were in the specification and read by nothing. ocda.images gives an image's bytes from whichever form the document came from (the database's files table, the archive entry, or beside a .ssp at media/image/), and says beforehand whether it can. The Gleam library reads a .zip pod but not a .sisupod: the zstd frame around it has no implementation on the BEAM that builds sandboxed. It says so, and says zstd -d, rather than calling the file malformed. gleam/ builds with nix-gleam's builder, kept in gleam/nix-gleam/ (MIT) Use: dlang/ nix build ocda-dump, ocda-agree; or a dub path dependency on dlang/ gleam/ nix build ocda-lib dump | agree See README.md, and dlang/README.md and gleam/README.md. (assisted by Claude-Code) --- dlang/src/ocda/load_ssp.d | 337 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 dlang/src/ocda/load_ssp.d (limited to 'dlang/src/ocda/load_ssp.d') diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d new file mode 100644 index 0000000..34d156c --- /dev/null +++ b/dlang/src/ocda/load_ssp.d @@ -0,0 +1,337 @@ +module ocda.load_ssp; + +import std.algorithm : splitter, startsWith; +import std.array : array, split; +import std.conv : to, ConvException; +import std.file : readText; +import std.stdio : writeln; +import std.string : strip, stripLeft, indexOf; +import ocda.model; + +OcdaDocument parseSspFile(string filename) { + string content = readText(filename); + return parseSsp(content); +} + +OcdaDocument parseSsp(string content) { + OcdaDocument doc; + OcdaObject current_obj; + bool in_object = false; + /+ ↓ whether a text line has been seen for the object being read. + Not "is the text non-empty": an object whose first text line is blank + (a verse opening on an empty line, which the sample set has seven of) + would then lose it, the next line overwriting rather than appending. + +/ + bool seen_text = false; + string current_section = ""; + string current_block = ""; // meta, make, doc_has, or section name + + foreach (line; content.splitter("\n")) { + /+ ↓ comment. The first line of a .ssp declares the format version, and + it is the one comment that carries meaning: a reader checks the + major part and refuses what it does not know rather than loading + an unfamiliar artefact half populated in silence. + +/ + if (line.length > 0 && line[0] == '%') { + enum _banner = "% SiSU Document Abstraction v"; + enum _source = "% Source: "; + if (doc.format_version.length == 0 && line.startsWith(_banner)) { + doc.format_version = line[_banner.length .. $].strip; + } else if (doc.source_filename.length == 0 && line.startsWith(_source)) { + doc.source_filename = line[_source.length .. $].strip; + } + continue; + } + + /+ ↓ blank line - finalize current object +/ + if (line.length == 0 || line.strip.length == 0) { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + continue; + } + + /+ ↓ section/block open +/ + if (line[0] == '@') { + auto brace_pos = line.indexOf('{'); + if (brace_pos > 0) { + string name = line[1 .. brace_pos].strip; + current_block = name; + if (!isPreambleBlock(name)) { + current_section = name; + /+ ↓ a block that is neither preamble nor a section this reader + knows: counted, so that --report-unknown can say so +/ + if (doc.sectionByName(name) is null) { + doc.unknown.block[name]++; + } + } + } + continue; + } + + /+ ↓ section/block close +/ + if (line[0] == '}') { + if (in_object) { + commitObject(doc, current_section, current_obj); + current_obj = OcdaObject.init; + in_object = false; + seen_text = false; + } + current_block = ""; + continue; + } + + /+ ↓ inside meta/make/doc_has blocks: key-value pairs +/ + if (isPreambleBlock(current_block)) { + string stripped = line.stripLeft; + auto colon_pos = stripped.indexOf(':'); + if (colon_pos > 0) { + string key = stripped[0 .. colon_pos].strip; + string value = (colon_pos + 1 < stripped.length) + ? stripped[colon_pos + 1 .. $].strip : ""; + if (current_block == "meta") { doc.metadata[key] = value; } + else if (current_block == "make") { doc.make[key] = value; } + else if (current_block == "source") { doc.source[key] = value; } + else { doc.doc_has[key] = value; } + } + continue; + } + + /+ ↓ object declaration: [N] type ... +/ + if (line[0] == '[') { + if (in_object) { + commitObject(doc, current_section, current_obj); + } + current_obj = OcdaObject.init; + current_obj.section = current_section; + in_object = true; + seen_text = false; + parseObjectDeclaration(line, current_obj); + continue; + } + + /+ ↓ property: .name: value +/ + if (line[0] == '.') { + if (in_object) { + parseProperty(line, current_obj, doc.unknown); + } + continue; + } + + /+ ↓ text content: | content +/ + if (line.length >= 1 && line[0] == '|') { + if (in_object) { + string text_content = (line.length > 2) ? line[2 .. $] : ""; + if (seen_text) { + current_obj.text ~= "\n" ~ text_content; + } else { + current_obj.text = text_content; + seen_text = true; + } + } + continue; + } + + /+ ↓ no known shape. Counted by its first character so that a reader + can be told the file held something this one did not expect + +/ + doc.unknown.line[line[0 .. 1].idup]++; + } + + /+ ↓ finalize last object +/ + if (in_object) { + commitObject(doc, current_section, current_obj); + } + + return doc; +} + +private void parseObjectDeclaration(string line, ref OcdaObject obj) { + /+ parse: [N] type :level identifier +/ + auto bracket_close = line.indexOf(']'); + if (bracket_close < 0) return; + try { + obj.ocn = line[1 .. bracket_close].strip.to!int; + } catch (ConvException) { + obj.ocn = 0; + } + string rest = (bracket_close + 1 < line.length) + ? line[bracket_close + 1 .. $].strip : ""; + if (rest.length == 0) return; + + auto parts = rest.split(" "); + if (parts.length == 0) return; + obj.is_a = parts[0]; + + if (obj.is_a == "heading" && parts.length >= 2) { + /+ parse :A, :B, :1, etc. +/ + string lev = parts[1]; + if (lev.length >= 2 && lev[0] == ':') { + obj.heading_level = lev[1 .. $]; + } + /+ optional identifier after level +/ + if (parts.length >= 3) { + obj.identifier = parts[2]; + } + } +} + +/+ ↓ the blocks that are document preamble rather than sections of objects +/ +private bool isPreambleBlock(string name) { + return name == "source" || name == "meta" || name == "make" || name == "doc_has"; +} + +private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unknown) { + /+ parse: .name: value +/ + auto colon_pos = line.indexOf(':'); + if (colon_pos < 1) return; + string name = line[1 .. colon_pos].strip; + string value = (colon_pos + 1 < line.length) + ? line[colon_pos + 1 .. $].strip : ""; + + switch (name) { + case "part": obj.part = value; break; + case "is_of_type": obj.is_of_type = value; break; + case "section": obj.own_section = value; break; + case "identifier": obj.identifier = value; break; + case "segment_html_is": obj.segment_html_is = value; break; + /+ ↓ the epub segments at heading levels 1 to 4 where the html + segments at level 4 alone, so the two are different files and + the abstraction records both +/ + case "segment_epub": obj.segment_epub = value; break; + case "segment_epub_is": obj.segment_epub_is = value; break; + case "table_aligns": obj.table_aligns = value; break; + case "stow_link": obj.stow_link ~= value; break; + case "anchor_tag": obj.anchor_tags ~= value; break; + case "parent": obj.parent_ocn = toIntSafe(value); break; + case "last_descendant": obj.last_descendant_ocn = toIntSafe(value); break; + case "children": obj.children = parseIntArray(value); break; + case "ancestors": obj.ancestors = parseIntArray(value); break; + case "dummy": obj.dummy_heading = (value == "true"); break; + case "ocn_off": obj.ocn_off = (value == "true"); break; + case "indent": + auto parts = value.split(" "); + if (parts.length >= 1) obj.indent_base = toIntSafe(parts[0]); + if (parts.length >= 2) obj.indent_hang = toIntSafe(parts[1]); + break; + case "bullet": obj.bullet = (value == "true"); break; + case "lang": obj.lang = value; break; + case "has": + foreach (flag; value.split(" ")) { + switch (flag) { + case "links": obj.has_links = true; break; + case "notes_reg": obj.has_notes_reg = true; break; + case "notes_star": obj.has_notes_star = true; break; + case "images": obj.has_images = true; break; + default: break; + } + } + break; + case "segment": obj.segment = value; break; + case "segment_prev": obj.segment_prev = value; break; + case "segment_next": obj.segment_next = value; break; + case "anchor": obj.anchor = value; break; + case "heading_lev_anchor": obj.heading_lev_anchor = value; break; + case "table_cols": obj.table_cols = toIntSafe(value); break; + case "table_widths": obj.table_widths = value; break; + case "table_header": obj.table_header = (value == "true"); break; + case "code_syntax": obj.code_syntax = value; break; + case "code_linenumbers": obj.code_linenumbers = (value == "true"); break; + case "heading_ancestors_text": obj.heading_ancestors_text = value; break; + case "sha256": obj.sha256 = value; break; + case "image": obj.images ~= parseImageRecord(value); break; + /+ ↓ a property this reader has no use for, or has never heard of. + Ignored, which is the conforming behaviour within a major + version, but counted rather than passed over in silence + +/ + default: unknown.property[name]++; break; + } +} + +private OcdaImage parseImageRecord(string value) { + /+ parse: sha256: bytes: [key:value ...] + filename first, remaining fields order-independent; unrecognised + fields are kept rather than discarded +/ + OcdaImage img; + bool first = true; + foreach (field; value.split(" ")) { + if (field.length == 0) continue; + if (first) { + img.name = field; + first = false; + continue; + } + auto colon_pos = field.indexOf(':'); + if (colon_pos < 1) continue; + string key = field[0 .. colon_pos]; + string val = (colon_pos + 1 < field.length) ? field[colon_pos + 1 .. $] : ""; + switch (key) { + case "sha256": img.sha256 = val; break; + case "bytes": img.bytes = toUlongSafe(val); break; + case "px": + auto dims = val.split("x"); + if (dims.length == 2) { + img.width = toIntSafe(dims[0]); + img.height = toIntSafe(dims[1]); + } + break; + case "missing": img.missing = (val == "true"); break; + default: img.fields[key] = val; break; + } + } + return img; +} + +/+ ↓ two values the text form leaves out because they can be worked out. + . + The .ssp emits a property only when it carries information, so + .identifier is absent when it is simply the ocn, and .anchor is absent + when it is the segment name. The .ocda.db, being columns rather than + lines, stores both unconditionally. So a loader that takes the text + form literally gives a *different model* from the one the database + gives, and the difference is not small: in the reference collection + the identifier is omitted on all but a handful of objects. + . + The model is the thing both serialisations describe, so the loader + fills them in. Found by dumping both models and diffing them, which no + amount of comparing section counts would have shown. ++/ +private void deriveOmitted(ref OcdaObject obj) { + import std.conv : to; + if (obj.identifier.length == 0 && obj.ocn != 0) { + obj.identifier = obj.ocn.to!string; + } + if (obj.anchor.length == 0 && obj.segment.length > 0) { + obj.anchor = obj.segment; + } +} + +private void commitObject(ref OcdaDocument doc, string section, ref OcdaObject obj) { + deriveOmitted(obj); + auto sec = doc.sectionByName(section); + if (sec !is null) { + *sec ~= obj; + } +} + +private int toIntSafe(string s) { + try { return s.strip.to!int; } + catch (ConvException) { return 0; } +} + +private ulong toUlongSafe(string s) { + try { return s.strip.to!ulong; } + catch (ConvException) { return 0; } +} + +private int[] parseIntArray(string s) { + int[] result; + foreach (part; s.split(" ")) { + auto v = toIntSafe(part); + result ~= v; + } + return result; +} -- cgit v1.2.3