diff options
| -rw-r--r-- | README.md | 202 | ||||
| -rw-r--r-- | dlang/README.md | 162 | ||||
| -rw-r--r-- | dlang/dub.json (renamed from dub.json) | 0 | ||||
| -rw-r--r-- | dlang/flake.lock | 27 | ||||
| -rw-r--r-- | dlang/flake.nix | 67 | ||||
| -rw-r--r-- | dlang/src/main.d (renamed from src/main.d) | 16 | ||||
| -rw-r--r-- | dlang/src/output_html.d (renamed from src/output_html.d) | 0 | ||||
| -rw-r--r-- | dlang/src/output_images.d (renamed from src/output_images.d) | 0 | ||||
| -rw-r--r-- | dlang/src/output_llm_breadcrumbs.d (renamed from src/output_llm_breadcrumbs.d) | 0 | ||||
| -rw-r--r-- | dlang/src/output_llm_jsonl.d (renamed from src/output_llm_jsonl.d) | 0 | ||||
| -rw-r--r-- | dlang/src/output_tree.d (renamed from src/output_tree.d) | 0 | ||||
| -rw-r--r-- | dlang/src/report_unknown.d (renamed from src/report_unknown.d) | 0 | ||||
| -rw-r--r-- | dlang/src/ssp_parser.d (renamed from src/ssp_parser.d) | 0 | ||||
| -rw-r--r-- | dlang/src/ssp_types.d (renamed from src/ssp_types.d) | 0 | ||||
| -rw-r--r-- | dlang/src/validate.d | 264 | ||||
| -rw-r--r-- | dlang/src/verify_images.d (renamed from src/verify_images.d) | 0 | ||||
| -rw-r--r-- | gleam/.envrc | 1 | ||||
| -rw-r--r-- | gleam/README.md | 82 | ||||
| -rw-r--r-- | gleam/flake.lock | 27 | ||||
| -rw-r--r-- | gleam/flake.nix | 61 | ||||
| -rw-r--r-- | gleam/gleam.toml | 21 | ||||
| -rw-r--r-- | gleam/manifest.toml | 21 | ||||
| -rw-r--r-- | gleam/nix-gleam/LICENSE | 21 | ||||
| -rw-r--r-- | gleam/nix-gleam/README.md | 32 | ||||
| -rw-r--r-- | gleam/nix-gleam/default.nix | 280 | ||||
| -rw-r--r-- | gleam/src/ssp/parse.gleam | 284 | ||||
| -rw-r--r-- | gleam/src/ssp/types.gleam | 109 | ||||
| -rw-r--r-- | gleam/src/ssp_reader.gleam | 199 | ||||
| -rw-r--r-- | gleam/test/ssp_reader_test.gleam | 142 |
29 files changed, 1869 insertions, 149 deletions
@@ -1,151 +1,61 @@ # ssp-reader -Reader for the **`.ssp` document-abstraction text format** emitted -by [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine). +Readers for the **`.ssp`** document abstraction, the object-centric +document abstraction written as text, which `sisudoc-spine` emits and +reads. -Reads a `.ssp` file (spine's PEG-parsable serialisation of its -internal document abstraction) and renders it into a chosen output: -an outline tree, scroll HTML, plain text with heading breadcrumbs, -JSONL chunks suitable for LLM / embedding pipelines, or an image -manifest for verifying the images a document embeds. - -## What this is for - -- Inspecting and debugging the abstraction stage of spine without - rebuilding spine itself. -- Round-trip experiments: markup -> spine -> `.ssp` -> ssp-reader - -> downstream artefact. -- Re-using spine's parsed document structure outside the spine - binary: feeding it to an indexer, an LLM pipeline, or a custom - renderer. -- A worked example of consuming `.ssp` for anyone implementing - another reader in another language. - -## Audience - -Developers building tools that consume spine's output: search -indexers, LLM ingestion pipelines, custom renderers, alternative -output formats, and anyone studying the abstraction stage of -spine itself. - -End-user document authors do **not** need this tool to publish a -document. Run spine directly for that. - -## Relationship to spine - -Spine's processing pipeline has three stages: - -1. **Input:** parse `.sst` / `.ssm` markup. -2. **Abstraction:** build a numbered object model (the "spine - abstraction"; one of spine's named outputs is `.ssp`, a - PEG-parsable text representation of this model). -3. **Output:** render to HTML / EPUB / PDF / SQLite / etc. - -`ssp-reader` picks up the abstraction at the end of stage 2 (the -`.ssp` file) and runs alternative stage-3 renderings on top of it. -The `.ssp` format itself is documented in -[`doc-abstraction-format`](https://git.sisudoc.org/tools/doc-abstraction-format). - -## Quickstart - -### Build - -Inside the project's nix dev environment (the same one spine uses): - -```sh -nix develop ".#dsh-nixpkgs-ldc" --print-build-logs -c $SHELL -dub build --compiler=ldmd2 -``` - -The binary lands at `./bin/ssp-reader`. - -### Run - -```sh -# generate a .ssp from a sample document via spine, then: -ssp-reader --tree doc.ssp # outline view -ssp-reader --html doc.ssp # HTML scroll output -ssp-reader --llm-text doc.ssp # plain text with breadcrumbs -ssp-reader --llm-jsonl doc.ssp # JSONL chunks for embeddings -ssp-reader --images doc.ssp # image manifest (digest, size, pixels) -ssp-reader --verify --image-dir=pod/media/image doc.ssp # check images against the records -ssp-reader --report-unknown doc.ssp # what it read and did not understand -ssp-reader --tree --output=out.txt doc.ssp -ssp-reader --help -``` - -## Format version - -The reader knows abstraction format **2.x**. Every `.ssp` declares its -version on its first line, and the reader checks the **major** part and -refuses what it does not know: an unknown major means a property may have -changed meaning or gone away, and loading such a file half populated in -silence is the failure the check exists to prevent. `--any-version` reads -it regardless, for looking at a file rather than trusting one. - -## What it does not understand - -Within a major version a reader **should** ignore properties it does not -know, since the minor part is raised precisely by adding them. This reader -does, and `--report-unknown` says what it ignored and how often: - -``` -read and NOT understood: - properties ignored (15 distinct): - .is_of_type 328 - .segment_html_is 138 - ... -``` - -### What it ignores, and why - -Twelve properties are ignored across the reference collection, and each is -ignored for a reason rather than by oversight: - -| ignored | why | -|---------|-----| -| `.ancestors_collapsed`, `.parent_lev`, `.heading_lev_collapsed`, `.dom_status`, `.dom_status_collapsed` | level bookkeeping; this reader works from `.ancestors` and the heading level | -| `.segment_epub`, `.segment_epub_is`, `.segment_lv4_is`, `.heading_lev_anchor`, `.anchor_tag`, `.lev4_subtoc` | naming for segmented HTML and EPUB output, which this reader does not produce | -| `.attrib` | one occurrence in the whole collection, and no output uses it | - -The list was longer. `--report-unknown` found that `.is_of_type` was -ignored, and **that was a defect**: objects of type `comment` are markup -commentary, carried by the abstraction and not put into output, and three -of the four outputs were rendering them. A markup comment was appearing in -the HTML, in the plain text, and as an embedding chunk in the JSONL. -Fixed, and the reason the report exists. - -That is deliberate and is what makes the reader usable as a check on the -[format specification](https://git.sisudoc.org/specs/doc-abstraction-format): -a reader that drops what it does not know is conforming, but one that -cannot say what it dropped cannot tell you whether the specification it was -written from was complete. Silence and understanding look alike otherwise. +**One reader per language, in a directory named for the language.** -## Documentation +| | | +|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| [`dlang/`](dlang/) | D. Outline, HTML scroll, breadcrumbed text, JSONL chunks for embedding pipelines, an image manifest, image verification, and `--validate` | +| [`gleam/`](gleam/) | Gleam. Summary, outline, and what it did not understand | +| `ruby/` | to come | -- The `.ssp` format itself: see - [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format). -- Source: [`src/`](src/) - one module per output format - (`output_tree.d`, `output_html.d`, `output_llm_breadcrumbs.d`, - `output_llm_jsonl.d`, `output_images.d`, `verify_images.d`), plus - `ssp_parser.d` and `ssp_types.d`. - -## License - -AGPL-3.0-or-later. See [`dub.json`](dub.json) and the parent -[sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine) -project for full licence text. - -## Repository - -- **Canonical URL:** https://git.sisudoc.org/tools/ssp-reader -- **Clone:** `git clone --depth=1 git://git.sisudoc.org/tools/ssp-reader` - -## Related projects - -- [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine) - - produces the `.ssp` files this tool consumes. -- [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format) - - specification of the `.ssp` text format. -- [tree-sitter-sisu](https://git.sisudoc.org/tools/tree-sitter-sisu) - - tree-sitter grammar for the upstream `.sst` / `.ssm` markup. +## Why more than one + +These are not three tools that happen to share a name. They are **the same +specification implemented more than once**, and the interesting thing about +them is that they agree. + +Each was written from the format specification, `doc-abstraction-format`, +rather than from spine's emitter or from each other. Run over the 36 +document reference collection, **the D and Gleam readers agree on the +section counts of all 36**. + +That is the point of the exercise. A specification that one implementation +follows is a description of that implementation; a specification that two +independent implementations follow, in languages with nothing in common, is +a description of the format. + +## What a reader must do + +From the specification's conformance section, and both readers do it: + +- **check the major version** declared on the first line and refuse a major + it does not know, rather than loading an unfamiliar artefact half + populated in silence; +- **tolerate unknown properties** within a major version, since the minor + part is raised precisely by adding them; +- **say what it ignored**, on request. A reader that drops what it does not + know is conforming; one that cannot say what it dropped cannot show a + specification to be complete, silence and understanding being alike from + outside. + +The last is why both carry `--report-unknown`, and it has earned its place: +it found that the D reader was rendering markup comments into its HTML, +text and JSONL output, because it was ignoring the property that says an +object is commentary. + +## Building + +Each language directory builds in its own way and says how in its own +README. Neither depends on the other. + +## The format + +Specified in **doc-abstraction-format**: `ocda-model.md` for the model both +serialisations share, `ssp-format-specification.md` for this one, and +`ocda-db-format-specification.md` for the sqlite form. + +The readers here are at format **2.x**. diff --git a/dlang/README.md b/dlang/README.md new file mode 100644 index 0000000..fa39eab --- /dev/null +++ b/dlang/README.md @@ -0,0 +1,162 @@ +# ssp-reader + +Reader for the **`.ssp` document-abstraction text format** emitted +by [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine). + +Reads a `.ssp` file (spine's PEG-parsable serialisation of its +internal document abstraction) and renders it into a chosen output: +an outline tree, scroll HTML, plain text with heading breadcrumbs, +JSONL chunks suitable for LLM / embedding pipelines, or an image +manifest for verifying the images a document embeds. + +## What this is for + +- Inspecting and debugging the abstraction stage of spine without + rebuilding spine itself. +- Round-trip experiments: markup -> spine -> `.ssp` -> ssp-reader + -> downstream artefact. +- Re-using spine's parsed document structure outside the spine + binary: feeding it to an indexer, an LLM pipeline, or a custom + renderer. +- A worked example of consuming `.ssp` for anyone implementing + another reader in another language. + +## Audience + +Developers building tools that consume spine's output: search +indexers, LLM ingestion pipelines, custom renderers, alternative +output formats, and anyone studying the abstraction stage of +spine itself. + +End-user document authors do **not** need this tool to publish a +document. Run spine directly for that. + +## Relationship to spine + +Spine's processing pipeline has three stages: + +1. **Input:** parse `.sst` / `.ssm` markup. +2. **Abstraction:** build a numbered object model (the "spine + abstraction"; one of spine's named outputs is `.ssp`, a + PEG-parsable text representation of this model). +3. **Output:** render to HTML / EPUB / PDF / SQLite / etc. + +`ssp-reader` picks up the abstraction at the end of stage 2 (the +`.ssp` file) and runs alternative stage-3 renderings on top of it. +The `.ssp` format itself is documented in +[`doc-abstraction-format`](https://git.sisudoc.org/tools/doc-abstraction-format). + +## Quickstart + +### Build + +```sh +nix build # the reader, with ldc +./result/bin/ssp-reader <file.ssp> +``` + +The reader depends on nothing beyond phobos, so `nix build` needs no +network and works in a sandbox. `nix build .#ssp-reader-dmd` builds it with +dmd instead, which is given gcc14 for the same reason spine gives it: GCC +15 introduced `nullptr` in its headers and dmd's ImportC parser does not +yet handle it. + +For development, or to build without nix: + +```sh +nix develop # ldc, dub, dtools +dub build --compiler=ldmd2 +``` + +The binary then lands at `./bin/ssp-reader`. + +### Run + +```sh +# generate a .ssp from a sample document via spine, then: +ssp-reader --tree doc.ssp # outline view +ssp-reader --html doc.ssp # HTML scroll output +ssp-reader --llm-text doc.ssp # plain text with breadcrumbs +ssp-reader --llm-jsonl doc.ssp # JSONL chunks for embeddings +ssp-reader --images doc.ssp # image manifest (digest, size, pixels) +ssp-reader --verify --image-dir=pod/media/image doc.ssp # check images against the records +ssp-reader --report-unknown doc.ssp # what it read and did not understand +ssp-reader --tree --output=out.txt doc.ssp +ssp-reader --help +``` + +## Format version + +The reader knows abstraction format **2.x**. Every `.ssp` declares its +version on its first line, and the reader checks the **major** part and +refuses what it does not know: an unknown major means a property may have +changed meaning or gone away, and loading such a file half populated in +silence is the failure the check exists to prevent. `--any-version` reads +it regardless, for looking at a file rather than trusting one. + +## What it does not understand + +Within a major version a reader **should** ignore properties it does not +know, since the minor part is raised precisely by adding them. This reader +does, and `--report-unknown` says what it ignored and how often: + +``` +read and NOT understood: + properties ignored (15 distinct): + .is_of_type 328 + .segment_html_is 138 + ... +``` + +### What it ignores, and why + +Twelve properties are ignored across the reference collection, and each is +ignored for a reason rather than by oversight: + +| ignored | why | +|--------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------| +| `.ancestors_collapsed`, `.parent_lev`, `.heading_lev_collapsed`, `.dom_status`, `.dom_status_collapsed` | level bookkeeping; this reader works from `.ancestors` and the heading level | +| `.segment_epub`, `.segment_epub_is`, `.segment_lv4_is`, `.heading_lev_anchor`, `.anchor_tag`, `.lev4_subtoc` | naming for segmented HTML and EPUB output, which this reader does not produce | +| `.attrib` | one occurrence in the whole collection, and no output uses it | + +The list was longer. `--report-unknown` found that `.is_of_type` was +ignored, and **that was a defect**: objects of type `comment` are markup +commentary, carried by the abstraction and not put into output, and three +of the four outputs were rendering them. A markup comment was appearing in +the HTML, in the plain text, and as an embedding chunk in the JSONL. +Fixed, and the reason the report exists. + +That is deliberate and is what makes the reader usable as a check on the +[format specification](https://git.sisudoc.org/specs/doc-abstraction-format): +a reader that drops what it does not know is conforming, but one that +cannot say what it dropped cannot tell you whether the specification it was +written from was complete. Silence and understanding look alike otherwise. + +## Documentation + +- The `.ssp` format itself: see + [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format). +- Source: [`src/`](src/) - one module per output format + (`output_tree.d`, `output_html.d`, `output_llm_breadcrumbs.d`, + `output_llm_jsonl.d`, `output_images.d`, `verify_images.d`), plus + `ssp_parser.d` and `ssp_types.d`. + +## License + +AGPL-3.0-or-later. See [`dub.json`](dub.json) and the parent +[sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine) +project for full licence text. + +## Repository + +- **Canonical URL:** https://git.sisudoc.org/tools/ssp-reader +- **Clone:** `git clone --depth=1 git://git.sisudoc.org/tools/ssp-reader` + +## Related projects + +- [sisudoc-spine](https://git.sisudoc.org/projects/sisudoc-spine) - + produces the `.ssp` files this tool consumes. +- [doc-abstraction-format](https://git.sisudoc.org/tools/doc-abstraction-format) - + specification of the `.ssp` text format. +- [tree-sitter-sisu](https://git.sisudoc.org/tools/tree-sitter-sisu) - + tree-sitter grammar for the upstream `.sst` / `.ssm` markup. diff --git a/dub.json b/dlang/dub.json index 51f7132..51f7132 100644 --- a/dub.json +++ b/dlang/dub.json diff --git a/dlang/flake.lock b/dlang/flake.lock new file mode 100644 index 0000000..5f236ab --- /dev/null +++ b/dlang/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1790185690, + "narHash": "sha256-xJ+X4hBtOcAFGBOe5nAMyMUeF9foJBmIOu3NjBqBycU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/dlang/flake.nix b/dlang/flake.nix new file mode 100644 index 0000000..554b8e0 --- /dev/null +++ b/dlang/flake.nix @@ -0,0 +1,67 @@ +{ + description = "ssp-reader: a reader for the .ssp document abstraction, in D"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + + name = "ssp-reader"; + + /* The reader has no dependencies beyond phobos, so there is nothing + to vendor and nothing to fetch: --skip-registry=all keeps dub from + reaching for a registry it does not need, which is also what lets + this build in a sandbox without network. + */ + mkReader = { compiler, compilerBin }: pkgs.stdenv.mkDerivation { + pname = name; + version = "2.0.0"; + src = self; + nativeBuildInputs = [ pkgs.dub compiler ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + dub build \ + --cache=local \ + --compiler=$(type -P ${compilerBin}) \ + --build=release \ + --skip-registry=all + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 bin/${name} $out/bin/${name} + runHook postInstall + ''; + meta = { + description = "reads a .ssp document abstraction and says what is in it"; + mainProgram = name; + }; + }; + in { + packages.${system} = { + default = self.packages.${system}.ssp-reader-ldc; + + ssp-reader-ldc = mkReader { + compiler = pkgs.ldc; + compilerBin = "ldmd2"; + }; + + /* dmd is given gcc14: GCC 15 introduced nullptr in its headers and + dmd's ImportC parser does not yet handle it. The same override + sisudoc-spine carries, for the same reason. + */ + ssp-reader-dmd = mkReader { + compiler = pkgs.dmd.override { stdenv = pkgs.gcc14Stdenv; }; + compilerBin = "dmd"; + }; + }; + + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.ldc pkgs.dub pkgs.dtools ]; + }; + }; +} diff --git a/src/main.d b/dlang/src/main.d index 78ceb69..a7a6537 100644 --- a/src/main.d +++ b/dlang/src/main.d @@ -9,6 +9,7 @@ import output_llm_jsonl; import output_images; import verify_images; import report_unknown; +import validate; int main(string[] args) { bool do_tree = false; @@ -18,6 +19,7 @@ int main(string[] args) { bool do_images = false; bool do_verify = false; bool do_report_unknown = false; + bool do_validate = false; bool any_version = false; string image_dir = ""; string output_file = ""; @@ -32,6 +34,7 @@ int main(string[] args) { "image-dir", "image directory to verify against", &image_dir, "output", "output file (default: stdout)", &output_file, "report-unknown", "list what was read and not understood", &do_report_unknown, + "validate", "check the file against the .ssp specification", &do_validate, "any-version", "read a format major version this reader does not know", &any_version, ); @@ -137,12 +140,19 @@ int main(string[] args) { reportUnknown(doc); } + int _validation_findings = 0; + if (do_validate) { + auto _found = validateSspFile(input_file, ssp_format_major_read); + reportValidation(input_file, _found); + _validation_findings = cast(int) _found.length; + } + if (!do_tree && !do_html && !do_llm_text && !do_llm_jsonl && !do_images - && !do_verify && !do_report_unknown) { + && !do_verify && !do_report_unknown && !do_validate) { writeln("\nUse --tree, --html, --llm-text, --llm-jsonl, --images," ~ " --verify --image-dir=<dir>" - ~ " or --report-unknown to produce output."); + ~ " --report-unknown or --validate to produce output."); } - return (_verify_failures > 0) ? 1 : 0; + return (_verify_failures > 0 || _validation_findings > 0) ? 1 : 0; } diff --git a/src/output_html.d b/dlang/src/output_html.d index b1c711c..b1c711c 100644 --- a/src/output_html.d +++ b/dlang/src/output_html.d diff --git a/src/output_images.d b/dlang/src/output_images.d index 69fb458..69fb458 100644 --- a/src/output_images.d +++ b/dlang/src/output_images.d diff --git a/src/output_llm_breadcrumbs.d b/dlang/src/output_llm_breadcrumbs.d index 5b240d2..5b240d2 100644 --- a/src/output_llm_breadcrumbs.d +++ b/dlang/src/output_llm_breadcrumbs.d diff --git a/src/output_llm_jsonl.d b/dlang/src/output_llm_jsonl.d index 3d34089..3d34089 100644 --- a/src/output_llm_jsonl.d +++ b/dlang/src/output_llm_jsonl.d diff --git a/src/output_tree.d b/dlang/src/output_tree.d index 88c1b4c..88c1b4c 100644 --- a/src/output_tree.d +++ b/dlang/src/output_tree.d diff --git a/src/report_unknown.d b/dlang/src/report_unknown.d index 1d5238e..1d5238e 100644 --- a/src/report_unknown.d +++ b/dlang/src/report_unknown.d diff --git a/src/ssp_parser.d b/dlang/src/ssp_parser.d index 1a1efe2..1a1efe2 100644 --- a/src/ssp_parser.d +++ b/dlang/src/ssp_parser.d diff --git a/src/ssp_types.d b/dlang/src/ssp_types.d index 3960df0..3960df0 100644 --- a/src/ssp_types.d +++ b/dlang/src/ssp_types.d diff --git a/dlang/src/validate.d b/dlang/src/validate.d new file mode 100644 index 0000000..7b48cd8 --- /dev/null +++ b/dlang/src/validate.d @@ -0,0 +1,264 @@ +module validate; + +/+ ↓ check a .ssp against the format specification. + . + Written from specs/doc-abstraction-format/ssp-format-specification.md + and from nothing else: the point of it is to be what the specification + says, executed, so that a file can be held against the document rather + than against an implementation. Each check names the section it comes + from, so a disagreement can be taken back to the text. ++/ + +import std.algorithm : canFind, startsWith; +import std.array : array, split; +import std.conv : to; +import std.file : readText; +import std.stdio : writeln, writefln; +import std.string : indexOf, strip; + +struct SspFinding { + size_t line; // 1-based + string section; // the specification section the rule is from + string detail; +} + +/+ ↓ specification section 6.3: the properties, in the order they are + emitted. A reader may rely on the order, so a validator can check it +/ +private immutable string[] property_order = [ + "identifier", "part", "section", "parent", "last_descendant", "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", + "bullet", "lang", "has", "table_cols", "table_widths", "table_aligns", + "table_header", "code_linenumbers", "stow_link", "segment", "anchor", + "segment_prev", "segment_next", "heading_lev_anchor", "segment_epub", + "segment_html_is", "segment_epub_is", "segment_lv4_is", + "heading_ancestors_text", "lev4_subtoc", "anchor_tag", +]; + +/+ ↓ section 6.1: the sixteen kinds +/ +private immutable string[] kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", + "blurb", "group", +]; + +/+ ↓ section 4: the nine sections, in the order they appear +/ +private immutable string[] section_order = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", + "bookindex", "blurb", "tail", +]; + +/+ ↓ section 4: the four preamble blocks, in the order they appear +/ +private immutable string[] preamble_order = ["source", "meta", "make", "doc_has"]; + +/+ ↓ section 6.1: the heading levels +/ +private immutable string[] levels = ["A", "B", "C", "D", "1", "2", "3", "4"]; + +private int propertyRank(string name) { + foreach (i, p; property_order) { if (p == name) { return cast(int) i; } } + return -1; +} + +SspFinding[] validateSspFile(string filename, string major_known) { + SspFinding[] found; + auto lines = readText(filename).split("\n"); + + bool seen_banner = false; + bool seen_source_line = false; + string block = ""; // the block currently open, "" when none + int preamble_at = -1; // how far through the preamble order + int section_at = -1; // how far through the section order + bool in_object = false; + int prev_rank = -1; + size_t doc_has_keys = 0; + bool in_text = false; // text lines come after properties + + foreach (i, line; lines) { + size_t no = i + 1; + + /+ ↓ section 2: the version, on the first line +/ + if (no == 1) { + enum banner = "% SiSU Document Abstraction v"; + if (!line.startsWith(banner)) { + found ~= SspFinding(no, "2", "first line is not the version banner"); + } else { + seen_banner = true; + string v = line[banner.length .. $].strip; + auto dot = v.indexOf('.'); + string major = (dot > 0) ? v[0 .. dot] : v; + if (major != major_known) { + found ~= SspFinding(no, "2.1", + "format major v" ~ major ~ ", this validator knows v" ~ major_known); + } + } + continue; + } + if (no == 2) { + if (!line.startsWith("% Source: ")) { + found ~= SspFinding(no, "4", "second line is not \"% Source: \""); + } else { seen_source_line = true; } + continue; + } + + if (line.length == 0) { in_object = false; in_text = false; prev_rank = -1; continue; } + + /+ ↓ section 3: a comment is the file header only +/ + if (line[0] == '%') { + found ~= SspFinding(no, "3", "a comment outside the two header lines"); + continue; + } + + /+ ↓ block open +/ + if (line[0] == '@') { + auto brace = line.indexOf(" {"); + if (brace < 1 || line[$ - 1] != '{') { + found ~= SspFinding(no, "3", "block open is not \"@name {\""); + continue; + } + string name = line[1 .. brace]; + if (preamble_order.canFind(name)) { + int at = 0; + foreach (k, p; preamble_order) { if (p == name) { at = cast(int) k; } } + if (at <= preamble_at) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " out of order"); + } + preamble_at = at; + if (section_at >= 0) { + found ~= SspFinding(no, "4", "preamble block @" ~ name ~ " after a section"); + } + } else if (section_order.canFind(name)) { + int at = 0; + foreach (k, sname; section_order) { if (sname == name) { at = cast(int) k; } } + if (at <= section_at) { + found ~= SspFinding(no, "4", "section @" ~ name ~ " out of order"); + } + section_at = at; + } else { + found ~= SspFinding(no, "4", "@" ~ name ~ " is neither a preamble block nor a section"); + } + block = name; + doc_has_keys = 0; + continue; + } + + if (line[0] == '}') { + if (line.length != 1) { + found ~= SspFinding(no, "3", "block close carries more than \"}\""); + } + /+ ↓ section 5.4: ten keys, always +/ + if (block == "doc_has" && doc_has_keys != 10) { + found ~= SspFinding(no, "5.4", + "@doc_has has " ~ doc_has_keys.to!string ~ " keys, the specification says ten"); + } + block = ""; + in_object = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 5: inside a preamble block, " key: value" +/ + if (preamble_order.canFind(block)) { + if (!line.startsWith(" ") || line.indexOf(": ") < 3) { + found ~= SspFinding(no, "5", "not \" key: value\" inside @" ~ block); + } else if (block == "doc_has") { + doc_has_keys++; + } + continue; + } + + /+ ↓ section 6.1: the object declaration line +/ + if (line[0] == '[') { + auto close = line.indexOf("] "); + if (close < 2) { + found ~= SspFinding(no, "6.1", "declaration is not \"[ocn] ...\""); + continue; + } + string ocn = line[1 .. close]; + foreach (c; ocn) { + if (c < '0' || c > '9') { + found ~= SspFinding(no, "6.1", "ocn \"" ~ ocn ~ "\" is not a number"); + break; + } + } + auto rest = line[close + 2 .. $].split(" "); + if (rest.length == 0 || !kinds.canFind(rest[0])) { + found ~= SspFinding(no, "6.1", + "\"" ~ ((rest.length > 0) ? rest[0] : "") ~ "\" is not one of the sixteen kinds"); + } else if (rest[0] == "heading") { + if (rest.length < 2 || rest[1].length < 2 || rest[1][0] != ':') { + found ~= SspFinding(no, "6.1", "a heading declaration carries no \":level\""); + } else if (!levels.canFind(rest[1][1 .. $])) { + found ~= SspFinding(no, "6.1", "level \"" ~ rest[1][1 .. $] ~ "\" is not one of the eight"); + } + } + in_object = true; + in_text = false; + prev_rank = -1; + continue; + } + + /+ ↓ section 6.3: properties, known and in order +/ + if (line[0] == '.') { + if (!in_object) { + found ~= SspFinding(no, "6", "a property outside an object record"); + continue; + } + if (in_text) { + found ~= SspFinding(no, "6", "a property after the object text began"); + } + auto colon = line.indexOf(": "); + if (colon < 2) { + found ~= SspFinding(no, "3", "property is not \".name: value\""); + continue; + } + string name = line[1 .. colon]; + int rank = propertyRank(name); + if (rank < 0) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is not one of the 42 properties"); + } else { + /+ ↓ the four repeatable properties may recur at the same rank +/ + if (rank < prev_rank) { + found ~= SspFinding(no, "6.3", "." ~ name ~ " is out of the emitted order"); + } + prev_rank = rank; + } + continue; + } + + /+ ↓ section 6.2: text lines +/ + if (line[0] == '|') { + if (!in_object) { + found ~= SspFinding(no, "6.2", "a text line outside an object record"); + } + if (line.length < 2 || line[1] != ' ') { + found ~= SspFinding(no, "6.2", "a text line is not \"| \" then the text"); + } + in_text = true; + continue; + } + + found ~= SspFinding(no, "3", "line of no shape the specification defines"); + } + + if (!seen_banner) { found ~= SspFinding(0, "2", "no version banner"); } + if (!seen_source_line) { found ~= SspFinding(0, "4", "no \"% Source:\" line"); } + return found; +} + +void reportValidation(string filename, SspFinding[] found) { + if (found.length == 0) { + writeln("\nvalid: ", filename, + " conforms to the .ssp specification as this validator reads it"); + return; + } + writefln("\n%d finding%s against the specification:", + found.length, (found.length == 1) ? "" : "s"); + foreach (f; found) { + if (f.line > 0) { + writefln(" line %-7d [%-4s] %s", f.line, f.section, f.detail); + } else { + writefln(" %-13s [%-4s] %s", "(file)", f.section, f.detail); + } + } +} diff --git a/src/verify_images.d b/dlang/src/verify_images.d index 216c02a..216c02a 100644 --- a/src/verify_images.d +++ b/dlang/src/verify_images.d diff --git a/gleam/.envrc b/gleam/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/gleam/.envrc @@ -0,0 +1 @@ +use flake diff --git a/gleam/README.md b/gleam/README.md new file mode 100644 index 0000000..32d4cf7 --- /dev/null +++ b/gleam/README.md @@ -0,0 +1,82 @@ +# ssp-reader, in Gleam + +Reads a `.ssp`, the object-centric document abstraction written as text, +and says what it found and what it did not understand. + +**Written from the specification alone**, as an experiment in whether that +specification suffices: + +- `ssp-format-specification.md`, from the **doc-abstraction-format** + specification +- `ocda-model.md`, from the same + +No reference was made to spine's emitter or to the D reader while writing +it. The constants it carries (the sixteen kinds, the nine sections, the +four preamble blocks, the 42 properties in their emitted order) were taken +from the specification's own tables. + +## Building and running + +```sh +nix build # the reader, as a single-file escript +./result/bin/ssp_reader <file.ssp> +``` + +`nix build` works **sandboxed and without network**. There is no Gleam +builder in nixpkgs and `gleam build` fetches from Hex, so the build uses +nix-gleam's builder, kept in `nix-gleam/` (MIT, see its README), which +fetches each Hex package on its own by the checksum `manifest.toml` already +carries. There is no hash in `flake.nix` to keep up to date. The result +needs only what it carries: run it with an empty environment and it works. + +For development: + +```sh +nix develop # gleam, erlang, rebar3 +gleam run -- <file.ssp> +gleam run -- --outline <file.ssp> +gleam run -- --report-unknown <file.ssp> +gleam run -- --any-version <file.ssp> +``` + +## What it does + +- Reads the version banner and **checks the major part**, refusing a major + it does not know (specification section 2.1). `--any-version` overrides. +- Parses the four preamble blocks, the nine sections, and the object + records: declaration line, properties, text. +- `--outline` prints the headings indented by level. +- `--report-unknown` lists every property it did not use, with counts. + +## Agreement with the D reader + +Both implementations were run over the 36 document reference collection +and **agree on the section counts of all 36**. + +## Tests + +```sh +gleam test +``` + +Ten, against small literals rather than files, since the parser is a pure +function from text to a document. Each names the specification section it +holds the parser to: the version banner, the `% Source:` line, `@source` +being preamble and not a section, the object declaration, text line order, +ancestors as numbers, that a `comment` object does not render, that an +unused property is counted rather than dropped, and that a line of no known +shape is counted. + +## What it does not do + +- The `.ocda.db`, which would want an Erlang sqlite binding. +- HTML, JSONL or plain-text output. +- Packaging as an escript, so it needs `gleam run`. + +## Layout + +``` +src/ssp_reader.gleam the program: arguments, version gate, output +src/ssp/types.gleam the document, the object, the version rule +src/ssp/parse.gleam the parser, a fold over the lines +``` diff --git a/gleam/flake.lock b/gleam/flake.lock new file mode 100644 index 0000000..5f236ab --- /dev/null +++ b/gleam/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1790185690, + "narHash": "sha256-xJ+X4hBtOcAFGBOe5nAMyMUeF9foJBmIOu3NjBqBycU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/gleam/flake.nix b/gleam/flake.nix new file mode 100644 index 0000000..57c92e3 --- /dev/null +++ b/gleam/flake.nix @@ -0,0 +1,61 @@ +{ + description = "ssp-reader: a reader for the .ssp document abstraction, in Gleam"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + + name = "ssp_reader"; + + /* The builder, nix-gleam's, kept here in nix-gleam/ (see the README + there). It fetches each Hex dependency on its own, with the + checksum gleam has already written into manifest.toml, so there is + no hash in this file to keep up to date, and nothing a run of gleam + could make come out differently. + */ + inherit (pkgs.callPackage ./nix-gleam { }) buildGleamApplication; + in { + packages.${system} = { + default = self.packages.${system}.${name}; + + /* The reader as an escript: precompiled Erlang in a single file, + which needs only erlang to run, no gleam and no dependencies. + Verified reproducible: a second export is byte identical. + nix-gleam builds an erlang-shipment by default; the build phase + is replaced, its configure phase having already put the + dependencies in place. + */ + ${name} = buildGleamApplication { + pname = name; + version = "2.0.0"; + src = ./.; + nativeBuildInputs = [ pkgs.makeWrapper ]; + buildPhase = '' + runHook preBuild + export HOME=$TMPDIR + gleam export escript + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ${name} $out/bin/${name} + wrapProgram $out/bin/${name} \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.beamPackages.erlang pkgs.coreutils ]} + runHook postInstall + ''; + meta = { + description = "reads a .ssp document abstraction and says what is in it"; + mainProgram = name; + }; + }; + }; + + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.gleam pkgs.beamPackages.erlang pkgs.rebar3 ]; + }; + }; +} diff --git a/gleam/gleam.toml b/gleam/gleam.toml new file mode 100644 index 0000000..8a56dbb --- /dev/null +++ b/gleam/gleam.toml @@ -0,0 +1,21 @@ +name = "ssp_reader" +version = "2.0.0" + +# Fill out these fields if you intend to generate HTML documentation or publish +# your project to the Hex package manager. +# +# description = "" +# licences = ["Apache-2.0"] +# repository = { type = "github", user = "", repo = "" } +# links = [{ title = "Website", href = "" }] +# +# For a full reference of all the available options, you can have a look at +# https://gleam.run/writing-gleam/gleam-toml/. + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +argv = ">= 1.1.0 and < 2.0.0" + +[dev_dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/gleam/manifest.toml b/gleam/manifest.toml new file mode 100644 index 0000000..1f256ca --- /dev/null +++ b/gleam/manifest.toml @@ -0,0 +1,21 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "argv", version = "1.1.0", build_tools = ["gleam"], requirements = [], otp_app = "argv", source = "hex", outer_checksum = "3277D100448BDB4A29B6D58C0F36F631CBC349E8BDD09766C6309DF202831140" }, + { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, + { name = "gleam_stdlib", version = "1.0.5", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "CEE5B6C076A85B45F60C585F4316C63EC8B7127C119D5738C3958A9C4D50404E" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, + { name = "simplifile", version = "2.7.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "A2727627B063E87351934C7F7F008F2D1FDB16F6DE0B8C79F9E46459CFC9C164" }, +] + +[requirements] +argv = { version = ">= 1.1.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +simplifile = { version = ">= 2.7.0 and < 3.0.0" } diff --git a/gleam/nix-gleam/LICENSE b/gleam/nix-gleam/LICENSE new file mode 100644 index 0000000..c35ad09 --- /dev/null +++ b/gleam/nix-gleam/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Arnar Gauti Ingason + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gleam/nix-gleam/README.md b/gleam/nix-gleam/README.md new file mode 100644 index 0000000..39bd1b6 --- /dev/null +++ b/gleam/nix-gleam/README.md @@ -0,0 +1,32 @@ +# nix-gleam, kept here + +`default.nix` is the builder of **nix-gleam**, by Arnar Gauti Ingason, +copied unchanged from + +- https://github.com/arnarg/nix-gleam +- `builder/default.nix` at commit `1f38154` (2026-10-06) + +under the MIT licence, which is in `LICENSE` beside it. + +It is kept here rather than taken as a flake input so that the build does +not depend on a third repository being reachable, and so that what builds +this package is in the package. + +## Why it is used + +There is no Gleam builder in nixpkgs, and a sandboxed build cannot fetch +from Hex. The usual answer, one fixed output derivation that runs +`gleam deps download` and is pinned by a hash of everything it fetched, has +two faults: the hash has to be updated by hand whenever a dependency +changes, and gleam writes `packages.toml` in a different order on each run, +so the same dependencies do not always give the same hash. + +nix-gleam fetches each Hex package on its own with `fetchHex`, using the +checksum gleam has already written for it into `manifest.toml`, and writes +`packages.toml` itself. So there is no hash to maintain, and nothing a run +of gleam could make come out differently. + +## Updating it + +Copy `builder/default.nix` from a newer commit over `default.nix`, update +the commit above, and build. diff --git a/gleam/nix-gleam/default.nix b/gleam/nix-gleam/default.nix new file mode 100644 index 0000000..d074263 --- /dev/null +++ b/gleam/nix-gleam/default.nix @@ -0,0 +1,280 @@ +{ + stdenv, + lib, + git, + xxhash, + fetchHex, + gleam, + beamPackages, + rsync, + nodejs, +}: let + inherit (builtins) fromTOML readFile; +in { + buildGleamApplication = { + src, + nativeBuildInputs ? [], + localPackages ? [], + erlangPackage ? beamPackages.erlang, + rebar3Package ? beamPackages.rebar3, + ... + } @ attrs: let + # gleam.toml contains an application name and version. + gleamToml = lib.importTOML (src + "/gleam.toml"); + + # manifest.toml contains a list of required packages including a sha256 checksum + # that can be used by nix fetchHex fetcher. + manifestToml = lib.importTOML (src + "/manifest.toml"); + + # Specify which target to build for. + buildTarget = attrs.target or gleamToml.target or "erlang"; + + # Generates a packages.toml expected by gleam compiler. + packagesTOML = with lib; + concatStringsSep "\n" ( + ["[packages]"] + ++ (map + (p: "${p.name} = \"${p.version}\"") + manifestToml.packages) + ++ (lib.optional (gitRecordsTOML != "") "") + ++ (lib.optional (gitRecordsTOML != "") gitRecordsTOML) + ); + + # Gleam records a git dependency twice: once in `[packages]`, + # and once in a `[git.<name>]` table with the commit. + # Without the second, Gleam will try to download it. + gitRecordsTOML = with lib; + concatMapStringsSep "\n" ( + p: "[git.${p.name}]\ncommit = \"${p.commit}\"" + ) (filterPackagesBySource "git" manifestToml.packages); + + # Helper function to filter manifest.toml packages + filterPackagesBySource = type: packages: lib.lists.filter (p: p.source == type) packages; + + gitDerivs = + map + (p: { + name = p.name; + derivation = fetchGit { + url = p.repo; + rev = p.commit; + }; + }) + (filterPackagesBySource "git" manifestToml.packages); + + # Fetch all dependencies + depsDerivs = + map + (p: { + name = p.name; + derivation = fetchHex { + inherit (p) version; + pkg = p.name; + sha256 = p.outer_checksum; + }; + }) + (filterPackagesBySource "hex" manifestToml.packages); + + # Find replacement paths for `local` package dependencies + # from `localPackages` list. + localDeps = let + # Build a lookup attrset for local packages. + localDerivs = lib.mergeAttrsList (map ( + p: let + name = (fromTOML (readFile (p + "/gleam.toml"))).name; + in { + "${name}" = p; + } + ) + localPackages); + in + map ( + p: { + inherit (p) name path; + localSrc = + if localDerivs ? "${p.name}" + then localDerivs.${p.name} + else throw "Local dependency \"${p.name}\" not found in `localPackages`."; + # Keep local packages in a writable location during build. + newPath = p.path; + } + ) (filterPackagesBySource "local" manifestToml.packages); + + # Check if elixir is needed in nativeBuildInputs by checking if "mix" is in + # required build_tools. + isElixirProject = with lib; p: any (t: t == "mix") p.build_tools; + needsElixir = with lib; any isElixirProject manifestToml.packages; + + # nativeBuildInputs needed for both targets. + defaultNativeBuildInputs = [gleam beamPackages.hex rsync git xxhash]; + in + # Base common mkDerivation attributes + stdenv.mkDerivation (attrs + // { + pname = attrs.pname or gleamToml.name; + version = attrs.version or gleamToml.version; + + src = lib.cleanSource attrs.src; + + # Here we must copy the dependencies into the right spot and + # create a packages.toml file so the gleam compiler does not + # attempt to pull the dependencies from the internet. + configurePhase = + attrs.configurePhase + or '' + runHook preConfigure + + mkdir -p build/packages + + # Write the packages.toml file + cat <<EOF > build/packages/packages.toml + ${packagesTOML} + EOF + + ${ + # Gleam writes build output into the local dependency's own source + # directory, so it cannot be the read-only store path. + lib.concatStringsSep "\n" ( + lib.forEach localDeps ( + d: '' + mkdir -p "${d.newPath}" + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.localSrc}/* "${d.newPath}/" + '' + ) + ) + } + + ${ + lib.concatStringsSep "\n" ( + lib.forEach gitDerivs ( + d: '' + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.derivation}/* build/packages/${d.name}/ + '' + ) + ) + } + + # Copy all the dependencies into place + ${lib.concatStringsSep "\n" ( + lib.forEach depsDerivs ( + # gleam outputs files inside the dependency's source directory + # and therefor it needs to have permissive permissions. + d: '' + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.derivation}/* build/packages/${d.name}/ + '' + ) + )} + + # To prevent dependency resolution in Gleam 1.15+, local packages + # need to have their fingerprint up-to-date. + ${lib.concatStringsSep "\n" ( + lib.forEach localDeps (d: '' + printf "%u" 0x$(xxhsum -H3 ${d.newPath}/gleam.toml | cut -d' ' -f1 | cut -d '_' -f2) > build/packages/${d.name}.config_fingerprint + '') + )} + + ${ + # Prime local package dependency caches so they do not try to fetch. + # + # Gleam compiles a local dependency inside that dependency's own + # directory, using the package cache found there. + lib.concatStringsSep "\n" ( + lib.forEach localDeps ( + d: '' + mkdir -p "${d.newPath}/build/packages" + cp build/packages/packages.toml "${d.newPath}/build/packages/packages.toml" + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r build/packages/* "${d.newPath}/build/packages/" + '' + ) + ) + } + + runHook postConfigure + ''; + } + # When the build target is erlang + // lib.optionalAttrs (buildTarget == "erlang") { + nativeBuildInputs = + defaultNativeBuildInputs + ++ [erlangPackage rebar3Package] + ++ (lib.optional needsElixir beamPackages.elixir) + ++ nativeBuildInputs; + + # The gleam compiler has a nice export function for erlang shipment. + buildPhase = + attrs.buildPhase + or '' + runHook preBuild + + export REBAR_CACHE_DIR="$TMP/.rebar-cache" + + gleam export erlang-shipment + + runHook postBuild + ''; + + # Install all built packages into lib and create an entrypoint script + # that starts the application. + installPhase = + attrs.installPhase + or '' + runHook preInstall + + mkdir -p $out/{bin,lib} + + rsync --exclude=entrypoint.sh -r build/erlang-shipment/* $out/lib/ + + cat <<EOF > $out/bin/${gleamToml.name} + #!/usr/bin/env sh + ${erlangPackage}/bin/erl \ + -pa $out/lib/*/ebin \ + -eval "${gleamToml.name}@@main:run(${gleamToml.name})" \ + -noshell \ + -extra "\$@" + EOF + chmod +x $out/bin/${gleamToml.name} + + runHook postInstall + ''; + } + # When the build target is javascript + // lib.optionalAttrs (buildTarget == "javascript") { + nativeBuildInputs = defaultNativeBuildInputs ++ nativeBuildInputs; + + # The gleam compiler doesn't provide an export mechanism for javascript target. + buildPhase = + attrs.buildPhase + or '' + runHook prebuild + + gleam build --target javascript + + runHook postBuild + ''; + + # Install all built packages into lib and create an entrypoint script + # that starts the application. + installPhase = + attrs.installPhase + or '' + runHook preInstall + + mkdir -p $out/{bin,lib} + + rsync --exclude=gleam.lock --exclude=gleam_version -r build/dev/javascript/* $out/lib/ + + cat <<EOF > $out/lib/${gleamToml.name}/main.mjs + import { main } from "./${gleamToml.name}.mjs"; + main(); + EOF + + cat <<EOF > $out/bin/${gleamToml.name} + #!/usr/bin/env sh + ${nodejs}/bin/node $out/lib/${gleamToml.name}/main.mjs "\$@" + EOF + chmod +x $out/bin/${gleamToml.name} + + runHook postInstall + ''; + }); +} diff --git a/gleam/src/ssp/parse.gleam b/gleam/src/ssp/parse.gleam new file mode 100644 index 0000000..9458317 --- /dev/null +++ b/gleam/src/ssp/parse.gleam @@ -0,0 +1,284 @@ +//// Parsing a .ssp. +//// +//// Specification section 3: the format is line oriented, and every line +//// is one of eight shapes told apart by its first character. So the +//// parser is a fold over the lines carrying the state the shapes need. + +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/option +import gleam/string +import ssp/types.{type Document, type Object, type Section, Object, Section} + +/// specification section 6.1: the sixteen kinds +pub const kinds = [ + "para", "heading", "toc", "bookindex", "endnote", "code", "bibliography", + "comment", "verse", "glossary", "quote", "poem", "block", "table", "blurb", + "group", +] + +/// specification section 4: the four preamble blocks +pub const preamble_blocks = ["source", "meta", "make", "doc_has"] + +/// specification section 4: the nine sections, in the order they appear +pub const section_names = [ + "head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", + "blurb", "tail", +] + +/// specification section 6.3: the 42 properties, in the order emitted +pub const properties = [ + "identifier", "part", "section", "parent", "last_descendant", "children", + "ancestors", "ancestors_collapsed", "dom_status", "dom_status_collapsed", + "heading_lev_collapsed", "parent_lev", "dummy", "ocn_off", "is_of_type", + "attrib", "meta_lang", "meta_syntax", "sha256", "image", "indent", "bullet", + "lang", "has", "table_cols", "table_widths", "table_aligns", "table_header", + "code_linenumbers", "stow_link", "segment", "anchor", "segment_prev", + "segment_next", "heading_lev_anchor", "segment_epub", "segment_html_is", + "segment_epub_is", "segment_lv4_is", "heading_ancestors_text", "lev4_subtoc", + "anchor_tag", +] + +/// The properties this reader makes use of. The rest are counted and +/// ignored, which is conforming behaviour within a major version +/// (specification section 8.1) so long as it is not done in silence. +const properties_used = [ + "part", "parent", "ancestors", "is_of_type", "identifier", +] + +type State { + State( + document: Document, + /// the block currently open, "" when none + block: String, + /// the section currently open, "" when in the preamble + section: String, + /// objects of the current section, newest first + objects: List(Object), + /// the object being read, if any + current: Result(Object, Nil), + /// sections completed, newest first + done: List(Section), + ) +} + +pub fn parse(content: String) -> Document { + let state = + content + |> string.split(on: "\n") + |> list.fold(from: initial_state(), with: line) + finish(state) +} + +fn initial_state() -> State { + State( + document: types.new_document(), + block: "", + section: "", + objects: [], + current: Error(Nil), + done: [], + ) +} + +fn line(state: State, raw: String) -> State { + case raw { + "" -> commit_object(state) + "% SiSU Document Abstraction v" <> version -> + State( + ..state, + document: types.Document( + ..state.document, + version: string.trim(version), + ), + ) + "% Source: " <> filename -> + State( + ..state, + document: types.Document( + ..state.document, + source_filename: string.trim(filename), + ), + ) + "%" <> _ -> state + "@" <> rest -> open_block(state, rest) + "}" -> close_block(state) + " " <> rest -> preamble_pair(state, rest) + "[" <> rest -> declaration(state, rest) + "." <> rest -> property(state, rest) + "| " <> rest -> text_line(state, rest) + _ -> + State( + ..state, + document: types.Document( + ..state.document, + malformed_lines: state.document.malformed_lines + 1, + ), + ) + } +} + +fn open_block(state: State, rest: String) -> State { + let name = case string.split_once(rest, " {") { + Ok(#(name, _)) -> name + Error(_) -> rest + } + case list.contains(preamble_blocks, name), list.contains(section_names, name) { + True, _ -> State(..state, block: name) + _, True -> State(..state, block: name, section: name, objects: []) + _, _ -> + State( + ..state, + block: name, + document: types.Document( + ..state.document, + unknown_blocks: bump(state.document.unknown_blocks, name), + ), + ) + } +} + +fn close_block(state: State) -> State { + let state = commit_object(state) + case state.section { + "" -> State(..state, block: "") + name -> + State( + ..state, + block: "", + section: "", + objects: [], + done: [Section(name: name, objects: list.reverse(state.objects)), ..state.done], + ) + } +} + +fn preamble_pair(state: State, rest: String) -> State { + case string.split_once(rest, ": ") { + Error(_) -> state + Ok(#(key, value)) -> { + let document = state.document + let document = case state.block { + "source" -> + types.Document(..document, source: dict.insert(document.source, key, value)) + "meta" -> + types.Document(..document, meta: dict.insert(document.meta, key, value)) + "make" -> + types.Document(..document, make: dict.insert(document.make, key, value)) + "doc_has" -> + types.Document( + ..document, + doc_has: dict.insert(document.doc_has, key, value), + ) + _ -> document + } + State(..state, document: document) + } + } +} + +/// specification section 6.1 +/// [<ocn>] <kind> +/// [<ocn>] heading :<level> +/// [<ocn>] heading :<level> <identifier> +fn declaration(state: State, rest: String) -> State { + let state = commit_object(state) + case string.split_once(rest, "] ") { + Error(_) -> state + Ok(#(ocn_text, tail)) -> { + let ocn = case int.parse(ocn_text) { + Ok(n) -> n + Error(_) -> 0 + } + let object = case string.split(tail, on: " ") { + ["heading", level, ..identifier] -> + Object( + ..types.new_object(), + ocn: ocn, + kind: "heading", + level: string.drop_start(level, 1), + identifier: string.join(identifier, " "), + ) + [kind, ..] -> Object(..types.new_object(), ocn: ocn, kind: kind) + [] -> types.new_object() + } + State(..state, current: Ok(object)) + } + } +} + +fn property(state: State, rest: String) -> State { + case state.current, string.split_once(rest, ": ") { + Ok(object), Ok(#(name, value)) -> { + let object = + Object(..object, properties: [#(name, value), ..object.properties]) + let object = case name { + "part" -> Object(..object, part: value) + "is_of_type" -> Object(..object, is_of_type: value) + "identifier" -> Object(..object, identifier: value) + "parent" -> + Object(..object, parent: case int.parse(value) { + Ok(n) -> n + Error(_) -> 0 + }) + "ancestors" -> Object(..object, ancestors: numbers(value)) + _ -> object + } + // a property this reader does not use, whether or not the + // specification names it, is counted rather than passed over: within + // a major version ignoring it is conforming, doing so in silence is + // what stops a reader being a check on the specification + let document = case list.contains(properties_used, name) { + True -> state.document + False -> + types.Document( + ..state.document, + unknown_properties: bump(state.document.unknown_properties, name), + ) + } + State(..state, current: Ok(object), document: document) + } + _, _ -> state + } +} + +fn text_line(state: State, rest: String) -> State { + case state.current { + Ok(object) -> State(..state, current: Ok(Object(..object, text: [rest, ..object.text]))) + Error(_) -> state + } +} + +fn commit_object(state: State) -> State { + case state.current { + Error(_) -> state + Ok(object) -> + State( + ..state, + current: Error(Nil), + objects: [Object(..object, text: list.reverse(object.text), + properties: list.reverse(object.properties)), ..state.objects], + ) + } +} + +fn finish(state: State) -> Document { + let state = close_block(commit_object(state)) + types.Document(..state.document, sections: list.reverse(state.done)) +} + +fn bump(counts: Dict(String, Int), key: String) -> Dict(String, Int) { + dict.upsert(counts, key, fn(existing) { + case existing { + option.Some(n) -> n + 1 + option.None -> 1 + } + }) +} + +fn numbers(value: String) -> List(Int) { + value + |> string.split(on: " ") + |> list.filter_map(int.parse) +} diff --git a/gleam/src/ssp/types.gleam b/gleam/src/ssp/types.gleam new file mode 100644 index 0000000..1e98886 --- /dev/null +++ b/gleam/src/ssp/types.gleam @@ -0,0 +1,109 @@ +//// The .ssp document abstraction, as Gleam types. +//// +//// Written from the doc-abstraction-format specification, +//// ssp-format-specification.md and ocda-model.md, and from nothing else: the exercise is to find out +//// whether the specification suffices to write a reader from. + +import gleam/dict.{type Dict} +import gleam/string + +/// The format major version this reader knows. +/// +/// Specification section 2.1: a reader checks the major part and refuses +/// what it does not know, rather than loading an unfamiliar artefact half +/// populated in silence. +pub const format_major_read = "2" + +/// One object of the abstraction. +/// +/// Model section 2: an object carries a citation number, a kind and a +/// type, its place in the structure, its text and a digest of the text. +pub type Object { + Object( + ocn: Int, + /// section 6.1: one of the sixteen kinds + kind: String, + /// headings only: A B C D 1 2 3 4 + level: String, + identifier: String, + /// section 6.3: how the text behaves, para / block / comment + is_of_type: String, + part: String, + parent: Int, + ancestors: List(Int), + /// every property as read, in the order read, including the ones + /// this reader has no use for + properties: List(#(String, String)), + /// section 6.2, in order, without the "| " prefix + text: List(String), + ) +} + +pub fn new_object() -> Object { + Object( + ocn: 0, + kind: "", + level: "", + identifier: "", + is_of_type: "", + part: "", + parent: 0, + ancestors: [], + properties: [], + text: [], + ) +} + +/// Model section 2.1: the type says how the text behaves, and "comment" +/// is markup commentary that no output carries. +pub fn renders(object: Object) -> Bool { + object.is_of_type != "comment" +} + +/// One section of a document, in the order of section 4. +pub type Section { + Section(name: String, objects: List(Object)) +} + +pub type Document { + Document( + /// as declared on the first line, "" when absent + version: String, + source_filename: String, + /// the four preamble blocks of section 5 + source: Dict(String, String), + meta: Dict(String, String), + make: Dict(String, String), + doc_has: Dict(String, String), + sections: List(Section), + /// what was met and not understood, counted rather than dropped in + /// silence: a reader that cannot say what it ignored cannot show that + /// a specification is complete + unknown_properties: Dict(String, Int), + unknown_blocks: Dict(String, Int), + malformed_lines: Int, + ) +} + +pub fn new_document() -> Document { + Document( + version: "", + source_filename: "", + source: dict.new(), + meta: dict.new(), + make: dict.new(), + doc_has: dict.new(), + sections: [], + unknown_properties: dict.new(), + unknown_blocks: dict.new(), + malformed_lines: 0, + ) +} + +/// The major part of the declared version, "" when none was declared. +pub fn version_major(document: Document) -> String { + case string.split_once(document.version, ".") { + Ok(#(major, _)) -> major + Error(_) -> document.version + } +} diff --git a/gleam/src/ssp_reader.gleam b/gleam/src/ssp_reader.gleam new file mode 100644 index 0000000..d8c4c78 --- /dev/null +++ b/gleam/src/ssp_reader.gleam @@ -0,0 +1,199 @@ +//// ssp-reader, in Gleam. +//// +//// Reads a .ssp, the object-centric document abstraction written as text, +//// and says what it found and what it did not understand. +//// +//// Written from the specification alone: +//// ssp-format-specification.md, of the doc-abstraction-format spec +//// ocda-model.md, of the same + +import argv +import gleam/dict +import gleam/int +import gleam/io +import gleam/list +import gleam/result +import gleam/string +import simplifile +import ssp/parse +import ssp/types.{type Document, type Object} + +pub fn main() -> Nil { + case argv.load().arguments { + [] -> usage() + args -> { + let any_version = list.contains(args, "--any-version") + let outline = list.contains(args, "--outline") + let report = list.contains(args, "--report-unknown") + case list.filter(args, fn(a) { !string.starts_with(a, "--") }) { + [path, ..] -> run(path, any_version, outline, report) + [] -> usage() + } + } + } +} + +fn usage() -> Nil { + io.println("ssp-reader - reads a .ssp and says what is in it") + io.println("") + io.println(" ssp_reader [options] <file.ssp>") + io.println("") + io.println(" --outline the headings, as a tree") + io.println(" --report-unknown what was read and not understood") + io.println(" --any-version read a format major this reader does not know") +} + +fn run(path: String, any_version: Bool, outline: Bool, report: Bool) -> Nil { + case simplifile.read(from: path) { + Error(_) -> io.println("error: cannot read " <> path) + Ok(content) -> { + let document = parse.parse(content) + case version_ok(document, any_version) { + False -> Nil + True -> { + summary(document, path) + case outline { + True -> print_outline(document) + False -> Nil + } + case report { + True -> print_unknown(document) + False -> Nil + } + } + } + } + } +} + +/// Specification section 2.1: check the major part and refuse what is not +/// known, rather than loading an unfamiliar artefact half populated. +fn version_ok(document: Document, any_version: Bool) -> Bool { + let major = types.version_major(document) + case document.version, major == types.format_major_read, any_version { + "", _, _ -> { + io.println("warning: no version banner, reading anyway") + True + } + _, True, _ -> True + _, False, True -> { + io.println( + "warning: format v" + <> document.version + <> ", this reader knows v" + <> types.format_major_read + <> ".x; reading anyway", + ) + True + } + _, False, False -> { + io.println( + "error: format v" + <> document.version + <> ", this reader knows v" + <> types.format_major_read + <> ".x", + ) + io.println( + " a major version differs: a property may have changed meaning", + ) + io.println(" --any-version reads it regardless.") + False + } + } +} + +fn summary(document: Document, path: String) -> Nil { + io.println("Parsed: " <> path) + io.println(" Format: v" <> document.version) + io.println(" Source: " <> document.source_filename) + case dict.get(document.source, "language") { + Ok(language) -> io.println(" Language: " <> language) + Error(_) -> Nil + } + io.println(" Title: " <> meta(document, "title.main")) + io.println(" Author: " <> meta(document, "creator.author")) + let counts = + document.sections + |> list.map(fn(s) { s.name <> "=" <> int.to_string(list.length(s.objects)) }) + |> string.join(" ") + io.println(" Sections: " <> counts) +} + +fn meta(document: Document, key: String) -> String { + document.meta |> dict.get(key) |> result.unwrap("") +} + +/// The headings, indented by level. Model section 4: the eight levels are +/// A to D for the title matter and 1 to 4 for the body. +fn print_outline(document: Document) -> Nil { + io.println("") + document.sections + |> list.each(fn(section) { + section.objects + |> list.filter(fn(o) { o.kind == "heading" && types.renders(o) }) + |> list.each(fn(o) { io.println(heading_line(o)) }) + }) +} + +fn heading_line(object: Object) -> String { + let depth = case object.level { + "A" -> 0 + "B" -> 1 + "C" -> 2 + "D" -> 3 + "1" -> 4 + "2" -> 5 + "3" -> 6 + "4" -> 7 + _ -> 8 + } + string.repeat(" ", depth) + <> object.level + <> "~ [" + <> int.to_string(object.ocn) + <> "] " + <> first_line(object.text) +} + +fn first_line(text: List(String)) -> String { + case text { + [line, ..] -> line + [] -> "" + } +} + +/// A reader that drops what it does not know is conforming; one that +/// cannot say what it dropped cannot show a specification to be complete. +fn print_unknown(document: Document) -> Nil { + io.println("") + case dict.size(document.unknown_properties) { + 0 -> io.println("read and understood: every property was used") + n -> { + io.println("read and not used (" <> int.to_string(n) <> " distinct):") + document.unknown_properties + |> dict.to_list + |> list.sort(fn(a, b) { string.compare(a.0, b.0) }) + |> list.each(fn(pair) { + io.println( + " ." <> pair.0 <> " " <> int.to_string(pair.1), + ) + }) + } + } + case dict.size(document.unknown_blocks) { + 0 -> Nil + _ -> { + io.println("blocks not understood:") + document.unknown_blocks + |> dict.to_list + |> list.each(fn(pair) { + io.println(" @" <> pair.0 <> " " <> int.to_string(pair.1)) + }) + } + } + case document.malformed_lines { + 0 -> Nil + n -> io.println("lines of no known shape: " <> int.to_string(n)) + } +} diff --git a/gleam/test/ssp_reader_test.gleam b/gleam/test/ssp_reader_test.gleam new file mode 100644 index 0000000..86f1cce --- /dev/null +++ b/gleam/test/ssp_reader_test.gleam @@ -0,0 +1,142 @@ +//// Tests for the .ssp parser. +//// +//// The parser is a pure function from text to a document, so the tests +//// are small literals rather than files. Each names the specification +//// section it holds the parser to. + +import gleam/dict +import gleam/list +import gleeunit +import ssp/parse +import ssp/types + +pub fn main() -> Nil { + gleeunit.main() +} + +/// A minimal but well-formed document: the two header lines, the four +/// preamble blocks, one section, one object. +fn sample() -> String { + "% SiSU Document Abstraction v2.0 +% Source: example.sst + +@source { + language: en + digest: ABCD +} + +@meta { + title.main: An Example + creator.author: A Writer +} + +@make { + doc_type: article +} + +@doc_has { + inline_links: 2 +} + +@body { + +[1] heading :A +.part: body +.is_of_type: para +| The Title + +[2] para +.part: body +.parent: 1 +.ancestors: 1 0 0 0 0 0 0 0 +.is_of_type: para +.sha256: BEEF +| Some text. +| A second line. + +[0] comment +.part: comment +.is_of_type: comment +| % a markup comment + +} +" +} + +// specification section 2: the version is declared on the first line +pub fn version_is_read_test() { + let document = parse.parse(sample()) + assert document.version == "2.0" + assert types.version_major(document) == "2" +} + +// specification section 4: the "% Source:" line names the markup +pub fn source_filename_is_read_test() { + assert parse.parse(sample()).source_filename == "example.sst" +} + +// specification section 5.1: the @source block, which is preamble and not +// a section +pub fn source_block_is_preamble_test() { + let document = parse.parse(sample()) + assert dict.get(document.source, "language") == Ok("en") + assert dict.get(document.source, "digest") == Ok("ABCD") + // and it did not become a section of objects + assert list.length(document.sections) == 1 +} + +// specification section 5.2 +pub fn meta_is_read_test() { + let document = parse.parse(sample()) + assert dict.get(document.meta, "title.main") == Ok("An Example") + assert dict.get(document.meta, "creator.author") == Ok("A Writer") +} + +// specification section 6.1: [ocn] heading :level +pub fn heading_declaration_test() { + let document = parse.parse(sample()) + let assert [section] = document.sections + let assert [heading, ..] = section.objects + assert heading.kind == "heading" + assert heading.level == "A" + assert heading.ocn == 1 +} + +// specification section 6.2: text lines, in order, without the prefix +pub fn text_lines_are_in_order_test() { + let document = parse.parse(sample()) + let assert [section] = document.sections + let assert [_, para, ..] = section.objects + assert para.text == ["Some text.", "A second line."] +} + +// model section 4: ancestors is eight slots indexed by level +pub fn ancestors_are_numbers_test() { + let document = parse.parse(sample()) + let assert [section] = document.sections + let assert [_, para, ..] = section.objects + assert para.ancestors == [1, 0, 0, 0, 0, 0, 0, 0] + assert para.parent == 1 +} + +// model section 2.1: is_of_type "comment" is markup commentary, which no +// output carries +pub fn comments_do_not_render_test() { + let document = parse.parse(sample()) + let assert [section] = document.sections + let rendered = list.filter(section.objects, types.renders) + assert list.length(section.objects) == 3 + assert list.length(rendered) == 2 +} + +// a property the reader does not use is counted, not dropped in silence +pub fn unused_properties_are_counted_test() { + let document = parse.parse(sample()) + assert dict.get(document.unknown_properties, "sha256") == Ok(1) +} + +// a line of no shape the specification defines is counted +pub fn malformed_lines_are_counted_test() { + let document = parse.parse("% SiSU Document Abstraction v2.0\n% Source: x.sst\n\nnonsense line\n") + assert document.malformed_lines == 1 +} |
