diff options
Diffstat (limited to 'gleam')
| -rw-r--r-- | gleam/COPYRIGHT | 48 | ||||
| -rw-r--r-- | gleam/README.md | 113 | ||||
| -rw-r--r-- | gleam/flake.lock | 44 | ||||
| -rw-r--r-- | gleam/flake.nix | 88 | ||||
| -rw-r--r-- | gleam/gleam.toml | 16 | ||||
| -rw-r--r-- | gleam/manifest.toml | 30 | ||||
| -rw-r--r-- | gleam/nix-gleam/LICENSE | 21 | ||||
| -rw-r--r-- | gleam/nix-gleam/README.md | 32 | ||||
| -rw-r--r-- | gleam/nix-gleam/default.nix | 280 | ||||
| -rw-r--r-- | gleam/src/COPYRIGHT | 45 | ||||
| -rw-r--r-- | gleam/src/ocda_write.gleam | 149 | ||||
| -rw-r--r-- | gleam/src/write/epub.gleam | 527 | ||||
| -rw-r--r-- | gleam/src/write/html.gleam | 999 | ||||
| -rw-r--r-- | gleam/src/write/markdown.gleam | 667 | ||||
| -rw-r--r-- | gleam/src/write/rgx.gleam | 282 | ||||
| -rw-r--r-- | gleam/src/write/text.gleam | 399 | ||||
| -rw-r--r-- | gleam/src/write/typst.gleam | 909 | ||||
| -rw-r--r-- | gleam/src/write/zip.gleam | 132 | ||||
| -rw-r--r-- | gleam/test/ocda_write_test.gleam | 233 |
19 files changed, 5014 insertions, 0 deletions
diff --git a/gleam/COPYRIGHT b/gleam/COPYRIGHT new file mode 100644 index 0000000..0da8543 --- /dev/null +++ b/gleam/COPYRIGHT @@ -0,0 +1,48 @@ +- Name: sisudoc-ocda-writer + - Description: part of SiSU spine (Doc Reform): documents, structuring, + processing, publishing and search + - [https://www.sisudoc.org] + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2026 Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later, for everything in this repository unless a file or + a directory specifies otherwise.: + + sisudoc-ocda-writer + + part of SiSU spine (Doc Reform): documents, structuring, + processing, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License (AGPLv3) as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at: [https://www.gnu.org/licenses/agpl-3.0.html] + + - Spine, Doc Reform (related to SiSU) uses standard: + - docReform markup syntax (based on SiSU markup) + - standard SiSU markup syntax with modified headers and minor + modifications + - docReform object numbering (based on SiSU object citation numbering) + - standard SiSU document object numbering + + - Homepages: + [https://www.sisudoc.org] + + - Copyright for ./nix-gleam/ 2023 Arnar Gauti Ingason + - License for ./nix-gleam/ MIT License diff --git a/gleam/README.md b/gleam/README.md new file mode 100644 index 0000000..8a2bea9 --- /dev/null +++ b/gleam/README.md @@ -0,0 +1,113 @@ +# ocda-write, in Gleam + +Sample writers over the abstraction: + + ocda-write --text|--md|--html|--epub|--typst <file> [lang] + ocda-write --md <file> [lang] [--image-prefix <path>] + +writes a document out, from any serialisation of its abstraction. + +## Modules + +| | | +|---|---| +| `write/text` | the model -> spine's plain text, byte for byte | +| `write/html` | the model -> one self-contained, valid html5 file | +| `write/epub` | the model -> an epub3, one file per segment | +| `write/markdown` | the model -> CommonMark, object numbers and all | +| `write/typst` | the model -> typst markup, for a pdf | +| `write/zip` | writing a zip, stored only, with no timestamps | +| `write/rgx` | the internal markup, as compiled patterns | +| `ocda_write` | the program: which format, which file, which language | + +`write.text.write` takes an `ocda/model.Document` and returns a `String`. It +is given no file, no path and no option: a writer is a function from the +model to bytes. + +## Build and run + +```sh +nix build --override-input ocda-lib path:../../sisudoc-ocda-lib +./result/bin/ocda-write --text <doc>.ssp +./result/bin/ocda-write --text <doc>.ocda.db de +./result/bin/ocda-write --md <doc>.ssp +./result/bin/ocda-write --epub <doc>.ocda.db en --out <doc>.epub +./result/bin/ocda-write --typst <doc>.ocda.db en --out out/<doc>.typ +``` + +The override applies to `nix develop` too, until the library is published: + +```sh +nix develop --override-input ocda-lib path:../../sisudoc-ocda-lib -c gleam build +``` + +## The check + +```sh +./result/bin/ocda-write --text <doc>.ssp | cmp - <the D writer's> +``` + +**36 of 36 document-languages, from both serialisations, byte identical to the +D writer** - which is the check that matters now, since byte identity with +`spine --text` is no longer the target. + +And for `--md`, `--html`, `--epub` and `--typst`, **byte identical to the D +writer on all 36**, which is the agreement that says the model is enough. None +of them is held to spine's bytes: the epub's judge is epubcheck, which read the +specification so that the writer need not be trusted to have, and the pdf's is +typst, where a link to a label that does not exist is a compile error. See the +repository README for what they do differently and why. + +## The library, as a path dependency inside the package + +`gleam.toml` names it `./ocda-lib`, and that is a **symlink** to +`../../sisudoc-ocda-lib/gleam`, kept out of git. + +It has to be inside the package: a path dependency reaching outside cannot +be seen by a sandboxed build, and a store path in `gleam.toml` would change +with every change to the library. So the path is stable, and what sits at +the end of it is arranged by whoever is building: a symlink in a working +tree, a copy from the store in the nix build, where nix-gleam puts the +library in place as a local package. + +Create it after cloning: + +```sh +ln -s ../../sisudoc-ocda-lib/gleam ocda-lib +``` + +## Places this is not idiomatic Gleam, on purpose + +- **A column's width is the widest cell in bytes, and the padding is counted + in codepoints.** Not a typo. Spine takes the width from `cell.length`, + which for a D string is a byte count, and pads with `format("%-*s")`, which + counts codepoints. So a cell holding `©` is padded one place further than + one holding `c`. Measured rather than guessed, and reproduced rather than + corrected, because the output is compared byte for byte. +- **The escape in `write/typst` walks codepoints and folds a list**, where + everything else here is a regular expression. It has to: the escaping skips + protected runs, the protection nests, and the D writer walks codepoints, so + a grapheme walk would part company with it over a combining mark. `label_of` + walks codepoints for the same reason, and once did not: it used graphemes + while the D one used bytes, and five documents with typographic quotation + marks in a book index entry disagreed. +- **`gleam/regexp` has no replace-with-function**, so `replace_all` here is + `scan` plus a walk over the subject. It is the one piece of scaffolding + this writer needs that the D one gets from Phobos. +- **The claimed-id set is threaded across segments, not restarted in each.** + An id has to be unique in a *file* to be valid, and unique in the + *publication* to be useful: a link carries a file and a fragment, and an + anchor with two destinations has none. Restarting the set per segment gave + twelve documents that epubcheck passed and the D writer disagreed with. +- **Every replacement goes through the function form, never the string + form.** Erlang's `re:replace` reads `&` in a replacement *string* as the + whole match, so replacing `&` with `"&"` yields `"quot;` where the + match was a quotation mark. Byte identity with the D writer is what caught + it, on one document's copyright notice, and nothing else would have: the + file still parsed and still read correctly everywhere else. + +## Shipping + +An Erlang shipment rather than an escript, because the library reaches +sqlite through a NIF and a NIF cannot be loaded from inside an escript +archive. diff --git a/gleam/flake.lock b/gleam/flake.lock new file mode 100644 index 0000000..ab89304 --- /dev/null +++ b/gleam/flake.lock @@ -0,0 +1,44 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1790185690, + "narHash": "sha256-xJ+X4hBtOcAFGBOe5nAMyMUeF9foJBmIOu3NjBqBycU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "ocda-lib": { + "flake": false, + "locked": { + "lastModified": 1790430427, + "narHash": "sha256-GSfZ1mJstp+rpvCvQJzPlXLPlrziV6uDa3Tvs/9b9p0=", + "ref": "refs/heads/main", + "rev": "fda48e417435e31a3122bfa7ca563333628aed99", + "revCount": 1, + "type": "git", + "url": "git://git.sisudoc.org/tools/sisudoc-ocda-lib" + }, + "original": { + "type": "git", + "url": "git://git.sisudoc.org/tools/sisudoc-ocda-lib" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "ocda-lib": "ocda-lib" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/gleam/flake.nix b/gleam/flake.nix new file mode 100644 index 0000000..49cd3f1 --- /dev/null +++ b/gleam/flake.nix @@ -0,0 +1,88 @@ +{ + description = "ocda-write: sample writers over the object-centric document abstraction, in Gleam"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + /* The library this is built on, as a source tree rather than a flake. + gleam.toml names it as the path dependency ./ocda-lib, which in a + working tree is a symlink to the library's gleam/ and is not carried + in git; the build is given the library from this input instead, and + nix-gleam puts it in place as a local package. + + flake.lock pins the commit of the library this writer was built and + checked against. To build against a working copy of it instead: + + nix build --override-input ocda-lib path:../../sisudoc-ocda-lib + */ + inputs.ocda-lib = { + url = "git+git://git.sisudoc.org/tools/sisudoc-ocda-lib"; + flake = false; + }; + + outputs = { self, nixpkgs, ocda-lib }: + let + system = "x86_64-linux"; # adjust for aarch64-linux / darwin + pkgs = import nixpkgs { inherit system; }; + + name = "ocda_write"; + + /* The builder, nix-gleam's, kept here in nix-gleam/ (see the README + there). It fetches each Hex dependency on its own, with the + checksum gleam has already written into manifest.toml, so there is + no hash in this file to keep up to date, and nothing a run of gleam + could make come out differently. + */ + inherit (pkgs.callPackage ./nix-gleam { }) buildGleamApplication; + + /* rebar3 with the port compiler already in it. + + esqlite builds a NIF, and its rebar3 configuration asks for the + `pc` plugin, which rebar3 would otherwise fetch from Hex at build + time: impossible in a sandbox, and the build fails with + "missing_package pc". nixpkgs carries the plugin, so the answer is + a rebar3 that has it before it starts. + */ + rebar3-pc = pkgs.beamPackages.rebar3WithPlugins { + plugins = [ pkgs.beamPackages.pc ]; + }; + in { + packages.${system} = { + default = self.packages.${system}.${name}; + + /* Shipped as an Erlang shipment rather than as an escript, because + sqlite on the BEAM is a NIF: esqlite compiles its own sqlite into + a shared object, and a NIF cannot be loaded from inside an + escript archive, the runtime needing a real file on disk. The + shipment keeps priv/ on disk, so it loads. + */ + ${name} = buildGleamApplication { + pname = "ocda-write"; + version = "2.0.0"; + src = ./.; + localPackages = [ "${ocda-lib}/gleam" ]; + rebar3Package = rebar3-pc; + nativeBuildInputs = [ pkgs.makeWrapper ]; + preBuild = '' + export HOME=$TMPDIR + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/lib $out/bin + cp -r build/erlang-shipment $out/lib/${name} + makeWrapper $out/lib/${name}/entrypoint.sh $out/bin/ocda-write \ + --add-flags run \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.beamPackages.erlang pkgs.coreutils ]} + runHook postInstall + ''; + meta = { + description = "writes a document out from either serialisation of its abstraction"; + mainProgram = "ocda-write"; + }; + }; + }; + + devShells.${system}.default = pkgs.mkShell { + packages = [ pkgs.gleam pkgs.beamPackages.erlang pkgs.rebar3 pkgs.sqlite ]; + }; + }; +} diff --git a/gleam/gleam.toml b/gleam/gleam.toml new file mode 100644 index 0000000..93880ca --- /dev/null +++ b/gleam/gleam.toml @@ -0,0 +1,16 @@ +name = "ocda_write" +version = "2.0.0" +description = "sample writers over the object-centric document abstraction" +licences = ["AGPL-3.0-or-later"] + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +gleam_regexp = ">= 1.0.0 and < 2.0.0" +argv = ">= 1.1.0 and < 2.0.0" +simplifile = ">= 2.7.0 and < 3.0.0" +sqlight = ">= 1.0.0 and < 2.0.0" +gleam_json = ">= 3.0.0 and < 4.0.0" +ocda_lib = { path = "./ocda-lib" } + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/gleam/manifest.toml b/gleam/manifest.toml new file mode 100644 index 0000000..8f22b12 --- /dev/null +++ b/gleam/manifest.toml @@ -0,0 +1,30 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "argv", version = "1.1.0", build_tools = ["gleam"], requirements = [], otp_app = "argv", source = "hex", outer_checksum = "3277D100448BDB4A29B6D58C0F36F631CBC349E8BDD09766C6309DF202831140" }, + { name = "esqlite", version = "0.9.0", build_tools = ["rebar3"], requirements = [], otp_app = "esqlite", source = "hex", outer_checksum = "CCF72258A4EE152EC7AD92AA9A03552EB6CA1B06B65C93AD5B6E55C302E05855" }, + { name = "filepath", version = "1.1.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "B06A9AF0BF10E51401D64B98E4B627F1D2E48C154967DA7AF4D0914780A6D40A" }, + { name = "gleam_json", version = "3.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "44FDAA8847BE8FC48CA7A1C089706BD54BADCC4C45B237A992EDDF9F2CDB2836" }, + { name = "gleam_regexp", version = "1.1.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_regexp", source = "hex", outer_checksum = "9C215C6CA84A5B35BB934A9B61A9A306EC743153BE2B0425A0D032E477B062A9" }, + { name = "gleam_stdlib", version = "1.0.5", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "CEE5B6C076A85B45F60C585F4316C63EC8B7127C119D5738C3958A9C4D50404E" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, + { name = "ocda_lib", version = "2.0.0", build_tools = ["gleam"], requirements = ["argv", "gleam_json", "gleam_stdlib", "simplifile", "sqlight"], source = "local", path = "./ocda-lib" }, + { name = "simplifile", version = "2.7.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "A2727627B063E87351934C7F7F008F2D1FDB16F6DE0B8C79F9E46459CFC9C164" }, + { name = "sqlight", version = "1.2.0", build_tools = ["gleam"], requirements = ["esqlite", "gleam_stdlib"], otp_app = "sqlight", source = "hex", outer_checksum = "841768D0A81107EE2DB46B949354F284A68B2E8098315C28A1FE3C07D206C17A" }, +] + +[requirements] +argv = { version = ">= 1.1.0 and < 2.0.0" } +gleam_json = { version = ">= 3.0.0 and < 4.0.0" } +gleam_regexp = { version = ">= 1.0.0 and < 2.0.0" } +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } +ocda_lib = { path = "./ocda-lib" } +simplifile = { version = ">= 2.7.0 and < 3.0.0" } +sqlight = { version = ">= 1.0.0 and < 2.0.0" } diff --git a/gleam/nix-gleam/LICENSE b/gleam/nix-gleam/LICENSE new file mode 100644 index 0000000..c35ad09 --- /dev/null +++ b/gleam/nix-gleam/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Arnar Gauti Ingason + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gleam/nix-gleam/README.md b/gleam/nix-gleam/README.md new file mode 100644 index 0000000..39bd1b6 --- /dev/null +++ b/gleam/nix-gleam/README.md @@ -0,0 +1,32 @@ +# nix-gleam, kept here + +`default.nix` is the builder of **nix-gleam**, by Arnar Gauti Ingason, +copied unchanged from + +- https://github.com/arnarg/nix-gleam +- `builder/default.nix` at commit `1f38154` (2026-10-06) + +under the MIT licence, which is in `LICENSE` beside it. + +It is kept here rather than taken as a flake input so that the build does +not depend on a third repository being reachable, and so that what builds +this package is in the package. + +## Why it is used + +There is no Gleam builder in nixpkgs, and a sandboxed build cannot fetch +from Hex. The usual answer, one fixed output derivation that runs +`gleam deps download` and is pinned by a hash of everything it fetched, has +two faults: the hash has to be updated by hand whenever a dependency +changes, and gleam writes `packages.toml` in a different order on each run, +so the same dependencies do not always give the same hash. + +nix-gleam fetches each Hex package on its own with `fetchHex`, using the +checksum gleam has already written for it into `manifest.toml`, and writes +`packages.toml` itself. So there is no hash to maintain, and nothing a run +of gleam could make come out differently. + +## Updating it + +Copy `builder/default.nix` from a newer commit over `default.nix`, update +the commit above, and build. diff --git a/gleam/nix-gleam/default.nix b/gleam/nix-gleam/default.nix new file mode 100644 index 0000000..d074263 --- /dev/null +++ b/gleam/nix-gleam/default.nix @@ -0,0 +1,280 @@ +{ + stdenv, + lib, + git, + xxhash, + fetchHex, + gleam, + beamPackages, + rsync, + nodejs, +}: let + inherit (builtins) fromTOML readFile; +in { + buildGleamApplication = { + src, + nativeBuildInputs ? [], + localPackages ? [], + erlangPackage ? beamPackages.erlang, + rebar3Package ? beamPackages.rebar3, + ... + } @ attrs: let + # gleam.toml contains an application name and version. + gleamToml = lib.importTOML (src + "/gleam.toml"); + + # manifest.toml contains a list of required packages including a sha256 checksum + # that can be used by nix fetchHex fetcher. + manifestToml = lib.importTOML (src + "/manifest.toml"); + + # Specify which target to build for. + buildTarget = attrs.target or gleamToml.target or "erlang"; + + # Generates a packages.toml expected by gleam compiler. + packagesTOML = with lib; + concatStringsSep "\n" ( + ["[packages]"] + ++ (map + (p: "${p.name} = \"${p.version}\"") + manifestToml.packages) + ++ (lib.optional (gitRecordsTOML != "") "") + ++ (lib.optional (gitRecordsTOML != "") gitRecordsTOML) + ); + + # Gleam records a git dependency twice: once in `[packages]`, + # and once in a `[git.<name>]` table with the commit. + # Without the second, Gleam will try to download it. + gitRecordsTOML = with lib; + concatMapStringsSep "\n" ( + p: "[git.${p.name}]\ncommit = \"${p.commit}\"" + ) (filterPackagesBySource "git" manifestToml.packages); + + # Helper function to filter manifest.toml packages + filterPackagesBySource = type: packages: lib.lists.filter (p: p.source == type) packages; + + gitDerivs = + map + (p: { + name = p.name; + derivation = fetchGit { + url = p.repo; + rev = p.commit; + }; + }) + (filterPackagesBySource "git" manifestToml.packages); + + # Fetch all dependencies + depsDerivs = + map + (p: { + name = p.name; + derivation = fetchHex { + inherit (p) version; + pkg = p.name; + sha256 = p.outer_checksum; + }; + }) + (filterPackagesBySource "hex" manifestToml.packages); + + # Find replacement paths for `local` package dependencies + # from `localPackages` list. + localDeps = let + # Build a lookup attrset for local packages. + localDerivs = lib.mergeAttrsList (map ( + p: let + name = (fromTOML (readFile (p + "/gleam.toml"))).name; + in { + "${name}" = p; + } + ) + localPackages); + in + map ( + p: { + inherit (p) name path; + localSrc = + if localDerivs ? "${p.name}" + then localDerivs.${p.name} + else throw "Local dependency \"${p.name}\" not found in `localPackages`."; + # Keep local packages in a writable location during build. + newPath = p.path; + } + ) (filterPackagesBySource "local" manifestToml.packages); + + # Check if elixir is needed in nativeBuildInputs by checking if "mix" is in + # required build_tools. + isElixirProject = with lib; p: any (t: t == "mix") p.build_tools; + needsElixir = with lib; any isElixirProject manifestToml.packages; + + # nativeBuildInputs needed for both targets. + defaultNativeBuildInputs = [gleam beamPackages.hex rsync git xxhash]; + in + # Base common mkDerivation attributes + stdenv.mkDerivation (attrs + // { + pname = attrs.pname or gleamToml.name; + version = attrs.version or gleamToml.version; + + src = lib.cleanSource attrs.src; + + # Here we must copy the dependencies into the right spot and + # create a packages.toml file so the gleam compiler does not + # attempt to pull the dependencies from the internet. + configurePhase = + attrs.configurePhase + or '' + runHook preConfigure + + mkdir -p build/packages + + # Write the packages.toml file + cat <<EOF > build/packages/packages.toml + ${packagesTOML} + EOF + + ${ + # Gleam writes build output into the local dependency's own source + # directory, so it cannot be the read-only store path. + lib.concatStringsSep "\n" ( + lib.forEach localDeps ( + d: '' + mkdir -p "${d.newPath}" + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.localSrc}/* "${d.newPath}/" + '' + ) + ) + } + + ${ + lib.concatStringsSep "\n" ( + lib.forEach gitDerivs ( + d: '' + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.derivation}/* build/packages/${d.name}/ + '' + ) + ) + } + + # Copy all the dependencies into place + ${lib.concatStringsSep "\n" ( + lib.forEach depsDerivs ( + # gleam outputs files inside the dependency's source directory + # and therefor it needs to have permissive permissions. + d: '' + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r ${d.derivation}/* build/packages/${d.name}/ + '' + ) + )} + + # To prevent dependency resolution in Gleam 1.15+, local packages + # need to have their fingerprint up-to-date. + ${lib.concatStringsSep "\n" ( + lib.forEach localDeps (d: '' + printf "%u" 0x$(xxhsum -H3 ${d.newPath}/gleam.toml | cut -d' ' -f1 | cut -d '_' -f2) > build/packages/${d.name}.config_fingerprint + '') + )} + + ${ + # Prime local package dependency caches so they do not try to fetch. + # + # Gleam compiles a local dependency inside that dependency's own + # directory, using the package cache found there. + lib.concatStringsSep "\n" ( + lib.forEach localDeps ( + d: '' + mkdir -p "${d.newPath}/build/packages" + cp build/packages/packages.toml "${d.newPath}/build/packages/packages.toml" + rsync --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r -r build/packages/* "${d.newPath}/build/packages/" + '' + ) + ) + } + + runHook postConfigure + ''; + } + # When the build target is erlang + // lib.optionalAttrs (buildTarget == "erlang") { + nativeBuildInputs = + defaultNativeBuildInputs + ++ [erlangPackage rebar3Package] + ++ (lib.optional needsElixir beamPackages.elixir) + ++ nativeBuildInputs; + + # The gleam compiler has a nice export function for erlang shipment. + buildPhase = + attrs.buildPhase + or '' + runHook preBuild + + export REBAR_CACHE_DIR="$TMP/.rebar-cache" + + gleam export erlang-shipment + + runHook postBuild + ''; + + # Install all built packages into lib and create an entrypoint script + # that starts the application. + installPhase = + attrs.installPhase + or '' + runHook preInstall + + mkdir -p $out/{bin,lib} + + rsync --exclude=entrypoint.sh -r build/erlang-shipment/* $out/lib/ + + cat <<EOF > $out/bin/${gleamToml.name} + #!/usr/bin/env sh + ${erlangPackage}/bin/erl \ + -pa $out/lib/*/ebin \ + -eval "${gleamToml.name}@@main:run(${gleamToml.name})" \ + -noshell \ + -extra "\$@" + EOF + chmod +x $out/bin/${gleamToml.name} + + runHook postInstall + ''; + } + # When the build target is javascript + // lib.optionalAttrs (buildTarget == "javascript") { + nativeBuildInputs = defaultNativeBuildInputs ++ nativeBuildInputs; + + # The gleam compiler doesn't provide an export mechanism for javascript target. + buildPhase = + attrs.buildPhase + or '' + runHook prebuild + + gleam build --target javascript + + runHook postBuild + ''; + + # Install all built packages into lib and create an entrypoint script + # that starts the application. + installPhase = + attrs.installPhase + or '' + runHook preInstall + + mkdir -p $out/{bin,lib} + + rsync --exclude=gleam.lock --exclude=gleam_version -r build/dev/javascript/* $out/lib/ + + cat <<EOF > $out/lib/${gleamToml.name}/main.mjs + import { main } from "./${gleamToml.name}.mjs"; + main(); + EOF + + cat <<EOF > $out/bin/${gleamToml.name} + #!/usr/bin/env sh + ${nodejs}/bin/node $out/lib/${gleamToml.name}/main.mjs "\$@" + EOF + chmod +x $out/bin/${gleamToml.name} + + runHook postInstall + ''; + }); +} diff --git a/gleam/src/COPYRIGHT b/gleam/src/COPYRIGHT new file mode 100644 index 0000000..a8ee2b4 --- /dev/null +++ b/gleam/src/COPYRIGHT @@ -0,0 +1,45 @@ +- Name: sisudoc-ocda-writer + - Description: part of SiSU spine (Doc Reform): documents, structuring, + processing, publishing and search + - [https://www.sisudoc.org] + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2026 Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later, for everything in this repository unless a file or + a directory specifies otherwise.: + + sisudoc-ocda-writer + + part of SiSU spine (Doc Reform): documents, structuring, + processing, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License (AGPLv3) as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at: [https://www.gnu.org/licenses/agpl-3.0.html] + + - Spine, Doc Reform (related to SiSU) uses standard: + - docReform markup syntax (based on SiSU markup) + - standard SiSU markup syntax with modified headers and minor + modifications + - docReform object numbering (based on SiSU object citation numbering) + - standard SiSU document object numbering + + - Homepages: + [https://www.sisudoc.org] diff --git a/gleam/src/ocda_write.gleam b/gleam/src/ocda_write.gleam new file mode 100644 index 0000000..3c224fa --- /dev/null +++ b/gleam/src/ocda_write.gleam @@ -0,0 +1,149 @@ +//// ocda-write: a sample writer over the abstraction. +//// +//// ocda-write --text|--html <file>.ssp|<file>.ocda.db|<file>.zip [lang] +//// +//// The input may be either serialisation, and the writer is not told which. +//// That is the claim being demonstrated: a writer is a function from the +//// model to bytes, so a document renders the same whichever of its two +//// abstraction files it is given. +//// +//// Output goes to stdout, so that the comparison against spine's own output +//// is a shell pipeline and not a directory of files to clean up. + +import argv +import gleam/list +import gleam/string +import ocda/images +import gleam/io +import ocda/load +import simplifile +import write/epub +import write/html +import write/markdown +import write/text +import write/typst + +pub fn main() { + case argv.load().arguments { + ["--text", path] -> render(path, "", "text") + ["--text", path, lang] -> render(path, lang, "text") + ["--html", path] -> render(path, "", "html") + ["--html", path, lang] -> render(path, lang, "html") + ["--epub", path, "--out", out] -> write_epub(path, "", out) + ["--epub", path, lang, "--out", out] -> write_epub(path, lang, out) + ["--md", path] -> render_markdown(path, "", "image") + ["--md", path, lang] -> render_markdown(path, lang, "image") + ["--md", path, "--image-prefix", pth] -> render_markdown(path, "", pth) + ["--md", path, lang, "--image-prefix", pth] -> + render_markdown(path, lang, pth) + ["--typ", path] -> render(path, "", "typst") + ["--typ", path, lang] -> render(path, lang, "typst") + ["--typst", path] -> render(path, "", "typst") + ["--typst", path, lang] -> render(path, lang, "typst") + ["--typst", path, "--out", out] -> write_typst(path, "", out) + ["--typst", path, lang, "--out", out] -> write_typst(path, lang, out) + _ -> usage() + } +} + +fn usage() -> Nil { + io.println_error( + "usage: ocda-write --text|--html|--typst <file> [lang]\n" + <> " ocda-write --md <file> [lang] [--image-prefix <path>]\n" + <> " ocda-write --epub <file> [lang] --out <file>.epub", + ) + halt(2) +} + +fn render(path: String, lang: String, format: String) -> Nil { + case load.load(path, lang, False) { + Error(message) -> { + io.println_error("error: " <> message) + halt(2) + } + Ok(doc) -> + io.print(case format { + "html" -> html.write(doc) + "typst" -> typst.write(doc) + _ -> text.write(doc) + }) + } +} + +/// Markdown takes one thing the other formats do not: where the images are, +/// relative to wherever the .md is put. It has its own entry for that, as the +/// epub and the typst do for --out. +fn render_markdown(path: String, lang: String, image_pth: String) -> Nil { + case load.load(path, lang, False) { + Error(message) -> { + io.println_error("error: " <> message) + halt(2) + } + Ok(doc) -> io.print(markdown.write(doc, image_pth)) + } +} + +/// An epub is a zip, so it goes to a file rather than to stdout. +fn write_epub(path: String, lang: String, out: String) -> Nil { + case load.load(path, lang, False) { + Error(message) -> { + io.println_error("error: " <> message) + halt(2) + } + Ok(doc) -> + case simplifile.write_bits(out, epub.write(doc, path)) { + Ok(_) -> Nil + Error(e) -> { + io.println_error("error: " <> out <> ": " <> simplifile.describe_error(e)) + halt(2) + } + } + } +} + +/// Gleam has no standard exit, and the exit status is what a sweep script +/// reads. +@external(erlang, "erlang", "halt") +fn halt(code: Int) -> Nil + +/// A .typ names its images by path rather than carrying them, so when it goes +/// to a file the images are written beside it. +fn write_typst(path: String, lang: String, out: String) -> Nil { + case load.load(path, lang, False) { + Error(message) -> { + io.println_error("error: " <> message) + halt(2) + } + Ok(doc) -> { + case simplifile.write(out, typst.write(doc)) { + Ok(_) -> Nil + Error(e) -> { + io.println_error( + "error: " <> out <> ": " <> simplifile.describe_error(e), + ) + halt(2) + } + } + let dir = case string.split(out, "/") { + [_] -> "image" + parts -> + string.join(list.take(parts, list.length(parts) - 1), "/") <> "/image" + } + case images.can_give_images(path) { + False -> Nil + True -> + epub.images_named(doc) + |> list.each(fn(name) { + case images.image_bytes(path, name) { + Ok(bits) -> { + let _ = simplifile.create_directory_all(dir) + let _ = simplifile.write_bits(dir <> "/" <> name, bits) + Nil + } + Error(_) -> Nil + } + }) + } + } + } +} diff --git a/gleam/src/write/epub.gleam b/gleam/src/write/epub.gleam new file mode 100644 index 0000000..77f4f42 --- /dev/null +++ b/gleam/src/write/epub.gleam @@ -0,0 +1,527 @@ +//// The epub3 writer: a zip of xhtml, one file per segment. +//// +//// This is the step the scroll html put off. A scroll is one file, so an +//// internal link is a fragment and a segment is a notion the writer can +//// ignore; an epub is *the segmented output plus a container*, so the writer +//// has to answer two questions the scroll writer did not: which file does +//// this object belong to, and what is the link from one file to another. +//// +//// The model answers both: `segment_epub` says which segment a heading opens, +//// `segment_epub_is` which segment an object sits in, and an internal link +//// names its own segment. +//// +//// The judge is *epubcheck*, which is better than a diff: it reads the +//// specification so that this writer does not have to be trusted to have read +//// it. And, as with the html, the two languages' writers must produce the +//// same bytes. + +import gleam/bit_array +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/regexp +import gleam/string +import ocda/images +import ocda/model.{type Document, type Object} +import write/html +import write/zip + +const generator_name = "spine" + +/// One segment: its file name and the objects in it, in document order. +pub type Segment { + Segment(file: String, title: String, objects: List(Object)) +} + +/// The epub, as bytes. `source` is the path the document was loaded from, +/// which is how the images are found: the model names them and does not hold +/// them. +pub fn write(doc: Document, source: String) -> BitArray { + let segs = segments(doc) + let targets = target_map(segs) + let files = list.map(segs, fn(s) { s.file }) + let image_names = case images.can_give_images(source) { + False -> [] + True -> images_named(doc) + } + let #(image_entries, images_kept) = + list.fold(image_names, #([], []), fn(state, name) { + let #(entries, kept) = state + case images.image_bytes(source, name) { + Ok(data) -> + case bit_array.byte_size(data) > 0 { + True -> #( + [zip.Entry("OEBPS/image/" <> name, data), ..entries], + [name, ..kept], + ) + False -> #(entries, kept) + } + Error(_) -> #(entries, kept) + } + }) + let image_entries = list.reverse(image_entries) + let images_kept = list.reverse(images_kept) + + // The claimed ids are threaded *across* segments, not restarted in each. + // + // An id must be unique in the publication, not merely in a file, because a + // link carries a file and a fragment and an anchor that appears in two files + // has two destinations. Restarting the set per segment emitted an anchor + // again in every segment that carried it, which the D writer does not, and + // twelve documents differed between the two writers until this threaded. + let #(segment_entries, _) = + list.fold(segs, #([], html.reserved(doc)), fn(state, seg) { + let #(entries, ids) = state + let ctx = html.Segmented(in_file: targets, files: files, current: seg.file) + let #(body, ids) = html.render_with(seg.objects, doc, ctx, ids) + #( + [ + zip.Entry("OEBPS/" <> seg.file <> ".xhtml", bytes(xhtml(doc, seg, body))), + ..entries + ], + ids, + ) + }) + let segment_entries = list.reverse(segment_entries) + + zip.write( + list.flatten([ + // "mimetype" first, stored, no extra field. That is the one ordering the + // format insists on, so that a reading system can identify the file from + // its first bytes. + [zip.Entry("mimetype", bytes("application/epub+zip"))], + [zip.Entry("META-INF/container.xml", bytes(container))], + segment_entries, + image_entries, + [zip.Entry("OEBPS/style.css", bytes(html.style_sheet()))], + [zip.Entry("OEBPS/nav.xhtml", bytes(nav(doc, segs)))], + [zip.Entry("OEBPS/content.opf", bytes(opf(doc, files, images_kept)))], + ]), + ) +} + +/// The segments, in document order. +/// +/// An object says which segment it is in, and the first object of a segment is +/// where that segment begins. Objects with no segment at all - the document +/// title, the table of contents - go into a segment named for the section they +/// sit in, because an epub has no place for an object outside the reading +/// order. +pub fn segments(doc: Document) -> List(Segment) { + // Grouped by name, in order of first appearance, *not* by run. + // + // A segment's objects are not always contiguous: a document may put an + // object with no segment of its own after a segment has begun, and the + // fallback then names it for its section, so "body" recurs. Grouping by run + // made two zip entries of one name, which is an epubcheck error (OPF-060) + // and was one in nine of the collection. + let pairs = + html.sections(doc) + |> list.flat_map(fn(name) { + model.section(doc, name) + |> list.filter(model.renders) + |> list.map(fn(obj) { #(segment_of(obj, name), obj) }) + }) + let #(order, by_name) = + list.fold(pairs, #([], dict.new()), fn(state, pair) { + let #(order, by_name) = state + let #(seg, obj) = pair + let title = heading_text(obj) + case dict.get(by_name, seg) { + Ok(Segment(file, have, objects)) -> #( + order, + dict.insert( + by_name, + seg, + Segment( + file, + case have, title { + "", t -> t + h, _ -> h + }, + [obj, ..objects], + ), + ), + ) + Error(_) -> #( + [seg, ..order], + dict.insert(by_name, seg, Segment(seg, title, [obj])), + ) + } + }) + order + |> list.reverse + |> list.filter_map(fn(seg) { + case dict.get(by_name, seg) { + Ok(s) -> Ok(Segment(s.file, s.title, list.reverse(s.objects))) + Error(_) -> Error(Nil) + } + }) +} + +fn segment_of(obj: Object, section: String) -> String { + case obj.segment_epub_is, obj.segment { + "", "" -> safe_name(section) + "", seg -> safe_name(seg) + seg, _ -> safe_name(seg) + } +} + +/// A file name a zip entry and an href can both carry. A segment name comes +/// from the markup, so it may hold anything; these are file names inside the +/// container and a reading system has to be able to resolve them. +fn safe_name(s: String) -> String { + let out = + s + |> string.to_graphemes + |> list.map(fn(c) { + case is_safe(c) { + True -> c + False -> "_" + } + }) + |> string.concat + case out { + "" -> "segment" + _ -> out + } +} + +fn is_safe(c: String) -> Bool { + string.contains( + does: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.", + contain: c, + ) +} + +fn heading_text(obj: Object) -> String { + case obj.is_a { + "heading" | "toc" -> plain(obj.text) + _ -> "" + } +} + +/// Every target an object carries, and the file it is in. This is what a +/// cross-segment link needs and what only the whole document can say. +pub fn target_map(segs: List(Segment)) -> Dict(String, String) { + list.fold(segs, dict.new(), fn(acc, seg) { + list.fold(seg.objects, acc, fn(acc, obj) { + let put = fn(d, key) { + case key { + "" -> d + k -> + case dict.has_key(d, k) { + True -> d + False -> dict.insert(d, k, seg.file) + } + } + } + let acc = case obj.ocn { + 0 -> acc + n -> put(acc, int.to_string(n)) + } + let acc = put(acc, obj.identifier) + let acc = put(acc, obj.heading_lev_anchor) + let acc = list.fold(obj.anchor_tags, acc, put) + // and the note references this object's own text will create: a note + // reference is an id the *writer* makes, and the gathered endnote links + // back to it from another segment + let acc = + list.fold(regexp.scan(note_mark(), obj.text), acc, fn(d, m) { + put(d, "noteref_" <> html.submatch(m, 0)) + }) + // and an anchor the markup placed *in the text*, ┃name┃. Those are ids + // too, and a link to one is a bare fragment rather than an internal + // link naming its segment, so only this map can say which file it is in. + list.fold(regexp.scan(inline_anchor(), obj.text), acc, fn(d, m) { + put(d, html.submatch(m, 0)) + }) + }) + }) +} + +fn inline_anchor() -> regexp.Regexp { + let assert Ok(r) = + regexp.compile( + "┃(\\S+?)┃", + regexp.Options(case_insensitive: False, multi_line: True), + ) + r +} + +fn note_mark() -> regexp.Regexp { + let assert Ok(r) = + regexp.compile( + "【(\\d+|(?:[*]|[+])+)\\s", + regexp.Options(case_insensitive: False, multi_line: True), + ) + r +} + +/// The images the document names, each once, in the order met. +pub fn images_named(doc: Document) -> List(String) { + html.sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.flat_map(fn(obj) { obj.images }) + |> list.filter(fn(img) { img.name != "" && !img.missing }) + |> list.map(fn(img) { img.name }) + |> list.fold([], fn(acc, name) { + case list.contains(acc, name) { + True -> acc + False -> [name, ..acc] + } + }) + |> list.reverse +} + +const container = "<?xml version=\"1.0\" encoding=\"utf-8\"?> +<container version=\"1.0\" xmlns=\"urn:oasis:names:tc:opendocument:xmlns:container\"> + <rootfiles> + <rootfile full-path=\"OEBPS/content.opf\" media-type=\"application/oebps-package+xml\" /> + </rootfiles> +</container> +" + +/// One segment as xhtml. xhtml rather than html: an epub's content documents +/// are xml, so the namespace is declared and every element is closed. +fn xhtml(doc: Document, seg: Segment, body: String) -> String { + let lang = language(doc) + let title = case seg.title { + "" -> model.meta(doc, "title.full") + t -> t + } + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + <> "<html xmlns=\"http://www.w3.org/1999/xhtml\"" + <> " xmlns:epub=\"http://www.idpf.org/2007/ops\"" + <> " lang=\"" + <> html.attribute(lang) + <> "\" xml:lang=\"" + <> html.attribute(lang) + <> "\">\n" + <> "<head>\n<title>" + <> html.escape_text(title) + <> "</title>\n" + <> "<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" />\n" + <> "</head>\n<body>\n" + <> body + <> "</body>\n</html>\n" +} + +/// The navigation document, which epub3 requires and which is the document's +/// own table of contents rather than a second one. +/// +/// Built from the segments, not from the toc objects: a toc object's link may +/// point inside a segment, and a navigation entry has to name the file. One +/// level, because the segment list is already flat. +fn nav(doc: Document, segs: List(Segment)) -> String { + let lang = language(doc) + let title = model.meta(doc, "title.full") + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + <> "<html xmlns=\"http://www.w3.org/1999/xhtml\"" + <> " xmlns:epub=\"http://www.idpf.org/2007/ops\"" + <> " lang=\"" + <> html.attribute(lang) + <> "\" xml:lang=\"" + <> html.attribute(lang) + <> "\">\n" + <> "<head>\n<title>" + <> html.escape_text(title) + <> "</title>\n" + <> "<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" />\n" + <> "</head>\n<body>\n" + <> "<nav epub:type=\"toc\" id=\"toc\" role=\"doc-toc\">\n" + <> "<h1>" + <> html.escape_text(title) + <> "</h1>\n<ol>\n" + <> { + segs + |> list.map(fn(seg) { + let label = case seg.title { + "" -> seg.file + t -> t + } + "<li><a href=\"" + <> html.attribute(seg.file) + <> ".xhtml\">" + <> html.escape_text(label) + <> "</a></li>\n" + }) + |> string.concat + } + <> "</ol>\n</nav>\n</body>\n</html>\n" +} + +/// The package document. +/// +/// dcterms:modified is required and must be to the second in UTC. It is taken +/// from the document's own publication date rather than from the clock, so +/// that the same document gives the same epub twice: a writer that stamps the +/// time of its run cannot be compared with itself. +fn opf(doc: Document, files: List(String), image_names: List(String)) -> String { + let lang = language(doc) + let m = fn(key) { model.meta(doc, key) } + let title = case m("title.full") { + "" -> m("title.main") + t -> t + } + let element = fn(name, value) { + case value { + "" -> "" + v -> "<" <> name <> ">" <> html.escape_text(v) <> "</" <> name <> ">\n" + } + } + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + <> "<package version=\"3.0\" xmlns=\"http://www.idpf.org/2007/opf\"" + <> " unique-identifier=\"bookid\" xml:lang=\"" + <> html.attribute(lang) + <> "\">\n" + <> "<metadata xmlns:dc=\"http://purl.org/dc/elements/1.1/\">\n" + <> "<dc:identifier id=\"bookid\">urn:uuid:" + <> uuid(doc) + <> "</dc:identifier>\n" + <> element("dc:title", title) + <> element("dc:creator", m("creator.author")) + <> element("dc:language", lang) + <> element("dc:date", m("date.published")) + <> element("dc:rights", m("rights.copyright")) + <> "<meta property=\"dcterms:modified\">" + <> modified(doc) + <> "</meta>\n" + <> "</metadata>\n<manifest>\n" + <> "<item id=\"nav\" href=\"nav.xhtml\" media-type=\"application/xhtml+xml\" properties=\"nav\" />\n" + <> "<item id=\"css\" href=\"style.css\" media-type=\"text/css\" />\n" + <> { + files + |> list.map(fn(f) { + "<item id=\"s_" + <> html.attribute(f) + <> "\" href=\"" + <> html.attribute(f) + <> ".xhtml\" media-type=\"application/xhtml+xml\" />\n" + }) + |> string.concat + } + <> { + image_names + |> list.index_map(fn(name, i) { + "<item id=\"i_" + <> int.to_string(i) + <> "\" href=\"image/" + <> html.attribute(name) + <> "\" media-type=\"" + <> media_type(name) + <> "\" />\n" + }) + |> string.concat + } + <> "</manifest>\n<spine>\n" + <> { + files + |> list.map(fn(f) { "<itemref idref=\"s_" <> html.attribute(f) <> "\" />\n" }) + |> string.concat + } + <> "</spine>\n</package>\n" +} + +/// The language, as a language tag rather than as a locale name. +/// +/// BCP 47 separates a language from its region with a hyphen, and one document +/// of the reference collection declares "pt_BR", which epubcheck refuses: +/// "Invalid subtag: pt_BR". The underscore is the POSIX locale spelling and +/// the abstraction carries it as the markup gave it, so the conversion belongs +/// here, in the writer that has a specification to satisfy. +fn language(doc: Document) -> String { + case dict.get(doc.source, "language") { + Ok(l) if l != "" -> string.replace(l, each: "_", with: "-") + _ -> "en" + } +} + +/// A uuid from the document's own digest, so that the same document has the +/// same identifier every time it is written. A random one would make two runs +/// of the same input two different publications. +fn uuid(doc: Document) -> String { + let seed = case dict.get(doc.source, "digest") { + Ok(d) if d != "" -> d + _ -> model.meta(doc, "title.full") <> doc.source_filename + } + let h = + sha1(<<seed:utf8>>) + |> bit_array.base16_encode + |> string.lowercase + // version 5 shape: the version nibble and the variant bits set, which is + // what says "this uuid is a hash of a name" + string.slice(h, 0, 8) + <> "-" + <> string.slice(h, 8, 4) + <> "-5" + <> string.slice(h, 13, 3) + <> "-a" + <> string.slice(h, 17, 3) + <> "-" + <> string.slice(h, 20, 12) +} + +fn modified(doc: Document) -> String { + let d = case model.meta(doc, "date.modified") { + "" -> model.meta(doc, "date.published") + v -> v + } + case string.length(d) >= 10 { + True -> string.slice(d, 0, 10) <> "T00:00:00Z" + False -> "1970-01-01T00:00:00Z" + } +} + +fn media_type(name: String) -> String { + let n = string.lowercase(name) + case string.ends_with(n, ".png") { + True -> "image/png" + False -> + case string.ends_with(n, ".jpg") || string.ends_with(n, ".jpeg") { + True -> "image/jpeg" + False -> + case string.ends_with(n, ".gif") { + True -> "image/gif" + False -> + case string.ends_with(n, ".svg") { + True -> "image/svg+xml" + False -> "application/octet-stream" + } + } + } + } +} + +/// sha1, from the runtime's own crypto rather than from a package. +@external(erlang, "crypto", "hash") +fn crypto_hash(algorithm: HashName, data: BitArray) -> BitArray + +type HashName + +@external(erlang, "erlang", "binary_to_atom") +fn atom(name: String) -> HashName + +fn sha1(data: BitArray) -> BitArray { + crypto_hash(atom("sha"), data) +} + +/// An object's text with the internal markup taken out, for a title or a +/// navigation label, where no markup may go. +fn plain(txt: String) -> String { + let assert Ok(markers) = + regexp.compile( + "⑆.┨|┣.|┥|┝|┤.*?├|【.+?】|░|┘|┙|┚|┃", + regexp.Options(case_insensitive: False, multi_line: True), + ) + html.escape_text(string.trim(regexp.replace(markers, txt, ""))) +} + +fn bytes(s: String) -> BitArray { + <<s:utf8>> +} + +/// Unused, kept so the generator can be named where a format wants it. +pub fn generator() -> String { + generator_name +} diff --git a/gleam/src/write/html.gleam b/gleam/src/write/html.gleam new file mode 100644 index 0000000..ca33bc3 --- /dev/null +++ b/gleam/src/write/html.gleam @@ -0,0 +1,999 @@ +//// The scroll html writer: one file, self-contained, valid html5. +//// +//// Byte for byte what the D writer produces, which is the agreement that +//// matters: two implementations from one model, and the model is enough. +//// +//// Not a reproduction of spine's html. The text format had nothing to +//// improve, so that writer reproduces spine exactly; html does, and the +//// reasons are in the D writer's own comments and in the repository README. +//// The short list: h1..h6 rather than <h0>, one charset declaration, no +//// placeholder metadata, span rather than label for the object number, +//// section with a dpub-aria role, a self-contained stylesheet, no site +//// furniture, an id for every anchor tag rather than only for an endnote, +//// and indents as data attributes rather than runs of . + +import gleam/dict +import gleam/int +import gleam/list +import gleam/regexp +import gleam/set.{type Set} +import gleam/string +import ocda/model.{type Document, type Object} +import write/rgx + +/// Set to spine's own values so that the comparison is of the writer and not +/// of the signature. +const generator_name = "spine" + +const generator_url = "https://sisudoc.org" + +/// The ids already taken. An id must be unique, so for each anchor tag the +/// *first* occurrence becomes the id and the rest become markers. +pub type Ids = + Set(String) + +/// Where a link to a target inside the document goes. +/// +/// In one file a "#x" is "#x". In a set of files it is +/// "<the file holding x>#x", and which file that is is a fact about the +/// document that only the whole document knows. So the object rendering takes +/// the answer rather than working it out, and the same code serves the scroll +/// html and the epub. +pub type LinkContext { + /// One file: nothing to resolve. + OneFile + /// Many files: the map from target to file, the files that exist, and the + /// one being written. + Segmented( + in_file: dict.Dict(String, String), + files: List(String), + current: String, + ) +} + +/// Render a list of objects with a link context, which is what the epub +/// writer needs: it renders one segment at a time. +pub fn render( + objects: List(Object), + doc: Document, + ctx: LinkContext, +) -> #(String, Ids) { + render_with(objects, doc, ctx, reserve_identifiers(doc)) +} + +/// The same, continuing from ids already claimed, so that a document rendered +/// segment by segment claims each id once across the whole of it. +pub fn render_with( + objects: List(Object), + doc: Document, + ctx: LinkContext, + ids: Ids, +) -> #(String, Ids) { + list.fold(objects, #("", ids), fn(state, obj) { + let #(acc, i) = state + let #(piece, i) = object_in(obj, doc, i, ctx) + #(acc <> piece, i) + }) +} + +/// The document-wide id reservation, exposed so a segmented writer can make +/// it once and thread it. +pub fn reserved(doc: Document) -> Ids { + reserve_identifiers(doc) +} + +/// The escaping, exposed for a writer building its own head and titles. +pub fn escape_text(txt: String) -> String { + escape(txt) +} + +pub fn attribute(txt: String) -> String { + attr(txt) +} + +/// Where a link to a target inside the document goes, given the context. +/// +/// An internal link carries its own segment: "¤<segment>.fnSuffix#<fragment>", +/// where ".fnSuffix" stands where an output's file suffix goes. The +/// abstraction *names the segment*, which is exactly what a segmented output +/// needs and what a single file has no use for. +fn resolve_target(target: String, ctx: LinkContext) -> String { + case string.starts_with(target, "¤") { + True -> + case regexp.scan(rgx.internal_target(), target) { + [m, ..] -> { + let seg = safe_name(rgx.group(m.submatches, 0)) + let frag = rgx.group(m.submatches, 1) + case ctx { + OneFile -> frag + Segmented(_, files, current) -> + case list.contains(files, seg) { + // a segment this writer did not write is one it cannot link + // to, so the fragment alone is better than a file that is + // not there + False -> frag + True -> + case seg == current { + True -> frag + False -> seg <> ".xhtml" <> frag + } + } + } + } + [] -> target + } + False -> + case ctx, string.starts_with(target, "#") { + Segmented(in_file, _, current), True -> { + let name = string.drop_start(target, 1) + case dict.get(in_file, name) { + Ok(file) if file != current -> file <> ".xhtml" <> target + _ -> target + } + } + _, _ -> target + } + } +} + +/// A name a zip entry, a file and an href can all carry. A segment name +/// comes from the markup, so it may hold anything. +pub fn safe_name(s: String) -> String { + let out = + s + |> string.to_graphemes + |> list.map(fn(c) { + case string.contains( + does: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.", + contain: c, + ) { + True -> c + False -> "_" + } + }) + |> string.concat + case out { + "" -> "segment" + _ -> out + } +} + +/// The stylesheet, exposed for a writer that packages it as a file. +pub fn style_sheet() -> String { + stylesheet +} + +/// A match's submatch, exposed so a writer can read a pattern's groups the +/// same way this module does. +pub fn submatch(m: regexp.Match, index: Int) -> String { + rgx.group(m.submatches, index) +} + +pub fn write(doc: Document) -> String { + let ids = reserve_identifiers(doc) + let #(body, _) = + sections(doc) + |> list.fold(#("", ids), fn(state, name) { + let #(acc, ids) = state + let objects = list.filter(model.section(doc, name), model.renders) + case objects { + [] -> #(acc, ids) + _ -> { + let #(inner, ids) = + list.fold(objects, #("", ids), fn(st, obj) { + let #(a, i) = st + let #(piece, i) = object_in(obj, doc, i, OneFile) + #(a <> piece, i) + }) + #(acc <> section_open(name) <> inner <> "</section>\n", ids) + } + } + }) + head(doc) <> body <> tail(doc) +} + +/// The same sequence the text writer uses, and for the same reason: it is the +/// document's own order, and a trailing section holding only its own heading +/// is not a section. +pub fn sections(doc: Document) -> List(String) { + let trailing = + ["endnotes", "glossary", "bibliography", "bookindex", "blurb"] + |> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 }) + list.append(["head", "toc", "body"], trailing) +} + +/// Every object's identifier is an id, so those names are taken before any +/// anchor can ask for one. +/// +/// They collide in practice: a heading's level anchor may be named "1", and +/// "1" is also the identifier of the object whose ocn is 1. Claiming the +/// identifiers first means the collision costs an anchor rather than making +/// the same id twice, and an id that appears twice is not a target. +fn reserve_identifiers(doc: Document) -> Ids { + sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.fold(set.new(), fn(ids, obj) { + let ids = case obj.identifier { + "" -> ids + id -> set.insert(ids, id) + } + case obj.ocn { + 0 -> ids + n -> set.insert(ids, int.to_string(n)) + } + }) +} + +fn claim(ids: Ids, tag: String) -> #(Bool, Ids) { + case set.contains(ids, tag) { + True -> #(False, ids) + False -> #(True, set.insert(ids, tag)) + } +} + +/// The dpub-aria role of a section, where it has one. A section with no role +/// of its own gets none rather than an invented one. +fn section_role(name: String) -> String { + case name { + "toc" -> "doc-toc" + "endnotes" -> "doc-endnotes" + "bibliography" -> "doc-bibliography" + "glossary" -> "doc-glossary" + "bookindex" -> "doc-index" + _ -> "" + } +} + +/// A section, with no id of its own. +/// +/// It needs none: the document already carries the targets its table of +/// contents links to, as anchor tags on the section headings, and this writer +/// emits those. Inventing a second id for the same place would put the same +/// name in the document twice. +fn section_open(name: String) -> String { + "<section class=\"" + <> name + <> "\"" + <> case section_role(name) { + "" -> "" + role -> " role=\"" <> role <> "\"" + } + <> ">\n" +} + +fn object_in( + obj: Object, + doc: Document, + ids: Ids, + ctx: LinkContext, +) -> #(String, Ids) { + case obj.is_a { + "heading" -> heading(obj, doc, ids, ctx) + "toc" -> para(obj, doc, ids, ctx, "toc") + "para" -> para(obj, doc, ids, ctx, "para") + "endnote" -> para(obj, doc, ids, ctx, "endnote") + "glossary" -> para(obj, doc, ids, ctx, "glossary") + "bibliography" -> para(obj, doc, ids, ctx, "bibliography") + "bookindex" -> para(obj, doc, ids, ctx, "bookindex") + "blurb" -> para(obj, doc, ids, ctx, "blurb") + "quote" -> blockish(obj, doc, ids, ctx, "blockquote", "quote") + "group" -> blockish(obj, doc, ids, ctx, "div", "group") + "block" -> blockish(obj, doc, ids, ctx, "div", "block") + "verse" -> blockish(obj, doc, ids, ctx, "div", "verse") + "code" -> #(code(obj), ids) + "table" -> table(obj, doc, ids, ctx) + _ -> #("", ids) + } +} + +/// Eight markup levels into six html ones. +/// +/// A B C D are the title levels and 1 2 3 4 the body levels beneath them, so +/// the outline is eight deep and html offers six. The first six take an +/// element each; deeper headings take h6 and say their real depth in +/// aria-level, which is what a reading system uses and what h7 would have +/// meant if html had one. +/// +/// Spine passes the markup level straight through, so a level A heading +/// becomes `<h0>`. There is no h0. +fn heading_depth(level: String) -> Int { + case level { + "A" -> 1 + "B" -> 2 + "C" -> 3 + "D" -> 4 + "1" -> 5 + "2" -> 6 + "3" -> 7 + "4" -> 8 + _ -> 1 + } +} + +fn heading( + obj: Object, + doc: Document, + ids: Ids, + ctx: LinkContext, +) -> #(String, Ids) { + // a dummy heading is furniture the abstraction generated so that a section + // has a heading, and its text repeats the real heading above it. Its text + // is not rendered - spine does the same - but its identifier and anchors + // *are*, because they are what the table of contents links to. Spine drops + // them, which is why its own toc entry for the endnotes points at a target + // that is not in the file. + case obj.dummy_heading { + True -> { + let #(tags, ids) = anchor_tags(obj, ids) + #( + case obj.identifier { + "" -> "" + id -> "<a id=\"" <> attr(id) <> "\"></a>" + } + <> tags, + ids, + ) + } + False -> { + let depth = heading_depth(obj.heading_level) + let element = int.to_string(int.min(depth, 6)) + let #(txt, ids) = inline_markup(obj, doc, ids, ctx) + case string.trim(txt) { + "" -> #("", ids) + _ -> { + let #(lev, ids) = level_anchor(obj, ids) + #( + "<div class=\"substance\">" + <> ocn_mark(obj) + <> "<h" + <> element + <> case obj.identifier { + "" -> "" + id -> " id=\"" <> attr(id) <> "\"" + } + <> case depth > 6 { + True -> " aria-level=\"" <> int.to_string(depth) <> "\"" + False -> "" + } + <> ">" + <> lev + <> txt + <> "</h" + <> element + <> ">" + <> "</div>\n", + ids, + ) + } + } + } + } +} + +/// The name the markup gave this heading, which is what a link written as +/// "#that-name" points at: 1,854 of them in the reference collection, and a +/// link to one is the ordinary way a document refers to its own sections. It +/// is a second name for the same place, so it is an empty anchor rather than +/// the heading's own id. +fn level_anchor(obj: Object, ids: Ids) -> #(String, Ids) { + case obj.heading_lev_anchor { + "" -> #("", ids) + name -> + case claim(ids, name) { + #(True, ids) -> #("<a id=\"" <> attr(name) <> "\"></a>", ids) + #(False, ids) -> #("", ids) + } + } +} + +fn para( + obj: Object, + doc: Document, + ids: Ids, + ctx: LinkContext, + cls: String, +) -> #(String, Ids) { + // a dummy table of contents entry names a heading the abstraction + // generated, "[Intro]" or "[Part III]", and it is furniture in the same way + // a dummy heading is. Its anchors are kept and its text is not. + // + // Spine drops it here and *keeps* it in the text output, which is an + // inconsistency between its own two writers rather than a rule. + case obj.dummy_heading { + True -> anchor_tags(obj, ids) + False -> { + let #(txt, ids) = inline_markup(obj, doc, ids, ctx) + case string.trim(txt) { + "" -> #("", ids) + _ -> #( + "<div class=\"substance\">" + <> ocn_mark(obj) + <> "<p class=\"" + <> cls + <> "\"" + <> case obj.identifier { + "" -> "" + id -> " id=\"" <> attr(id) <> "\"" + } + // the indent is the document's, so it is data rather than + // presentation; the stylesheet reads it + <> case obj.indent_base > 0 || obj.indent_hang > 0 { + True -> + " data-indent=\"" + <> int.to_string(obj.indent_base) + <> "\" data-hang=\"" + <> int.to_string(obj.indent_hang) + <> "\"" + False -> "" + } + // a gathered endnote is a note body whatever section it sits in, + // and the reference to it carries doc-noteref at the other end + <> case obj.is_a { + "endnote" -> " role=\"doc-footnote\"" + _ -> "" + } + <> ">" + <> case obj.bullet { + True -> "<span class=\"bullet\">\u{25cf}</span> " + False -> "" + } + <> txt + <> "</p></div>\n", + ids, + ) + } + } + } +} + +/// quote, group, block and verse: one object whose text holds its own line +/// structure, so the lines are kept and not re-flowed. +fn blockish( + obj: Object, + doc: Document, + ids: Ids, + ctx: LinkContext, + element: String, + cls: String, +) -> #(String, Ids) { + let #(txt, ids) = inline_markup(obj, doc, ids, ctx) + case string.trim(txt) { + "" -> #("", ids) + _ -> #( + "<div class=\"substance\">" + <> ocn_mark(obj) + <> "<" + <> element + <> " class=\"" + <> cls + <> "\"" + <> case obj.identifier { + "" -> "" + id -> " id=\"" <> attr(id) <> "\"" + } + <> ">" + <> txt + <> "</" + <> element + <> ">" + <> "</div>\n", + ids, + ) + } +} + +/// A code block: pre and code, and nothing inside it interpreted. The text is +/// escaped and otherwise untouched, which is the whole point of a code block. +fn code(obj: Object) -> String { + "<div class=\"substance\">" + <> ocn_mark(obj) + <> "<pre class=\"code\"" + <> case obj.identifier { + "" -> "" + id -> " id=\"" <> attr(id) <> "\"" + } + <> "><code" + <> case obj.code_syntax { + "" -> "" + s -> " class=\"language-" <> attr(s) <> "\"" + } + <> ">" + <> escape(obj.text) + <> "</code></pre>" + <> "</div>\n" +} + +/// A table. The object's text is the whole table: rows separated by newlines, +/// columns by ┊. A header row becomes th, and the column widths the markup +/// declared become a colgroup, which is where a width belongs. +fn table( + obj: Object, + _doc: Document, + ids: Ids, + _ctx: LinkContext, +) -> #(String, Ids) { + let rows = + regexp.split(rgx.table_delimiter_row(), obj.text) + |> list.filter(fn(r) { string.trim(r) != "" }) + let widths = string.split(obj.table_widths, on: " ") + let aligns = string.split(obj.table_aligns, on: " ") + let colgroup = case widths { + [first, ..] if first != "" -> + "<colgroup>" + <> { + widths + |> list.map(fn(w) { "<col style=\"width: " <> attr(w) <> "%\" />" }) + |> string.concat + } + <> "</colgroup>" + _ -> "" + } + let body = + rows + |> list.index_map(fn(row, row_idx) { + let header = row_idx == 0 && obj.table_header + let cell_element = case header { + True -> "th" + False -> "td" + } + let cells = + regexp.split(rgx.table_delimiter_col(), row) + |> list.index_map(fn(cell, i) { + let a = case list_at(aligns, i) { + Ok(a) -> a + Error(_) -> "l" + } + "<" + <> cell_element + <> " class=\"align-" + <> align_name(a) + <> "\">" + // a cell carries the same inline markup a paragraph does, line + // breaks among it: a table of features has cells of several lines + <> line_breaks(font_face(escape(cell)), obj) + <> "</" + <> cell_element + <> ">" + }) + |> string.concat + case header { + True -> "<thead><tr>" <> cells <> "</tr></thead>" + False -> "<tr>" <> cells <> "</tr>" + } + }) + |> string.concat + #( + "<div class=\"substance\">" + <> ocn_mark(obj) + <> "<table class=\"table\"" + <> case obj.identifier { + "" -> "" + id -> " id=\"" <> attr(id) <> "\"" + } + <> ">" + <> colgroup + <> body + <> "</table></div>\n", + ids, + ) +} + +fn align_name(a: String) -> String { + case a { + "c" -> "centre" + "r" -> "right" + _ -> "left" + } +} + +fn list_at(items: List(a), index: Int) -> Result(a, Nil) { + case items, index { + [], _ -> Error(Nil) + [first, ..], 0 -> Ok(first) + [_, ..rest], n -> list_at(rest, n - 1) + } +} + +/// The object number, which is also the anchor every citation of this object +/// points at. +/// +/// A span rather than a label: `<label>` is for form controls, and an object +/// number labels nothing that can be focused. +/// +/// The object's own `.ocn_off` property is deliberately *not* consulted. It +/// looks as though it should be, and it is not what suppresses a number: what +/// suppresses one in spine is the command line option `--ocn-off`. The object +/// property is set on objects that do carry an ocn - every bibliography entry +/// in one document has both - and neither of spine's writers reads it. +fn ocn_mark(obj: Object) -> String { + case obj.ocn { + 0 -> "" + n -> { + let s = int.to_string(n) + "<span class=\"ocn\"><a href=\"#" <> s <> "\" class=\"lnkocn\">" <> s <> "</a></span>" + } + } +} + +/// Everything inside an object's text, in the order the markers require: +/// images and links first, because their markers hold text that must not be +/// escaped as content, then notes, then font faces, then the line and indent +/// markers. +fn inline_markup( + obj: Object, + _doc: Document, + ids: Ids, + ctx: LinkContext, +) -> #(String, Ids) { + let txt = escape(obj.text) + let txt = images(txt) + let txt = links(txt, obj, ctx) + let txt = notes(txt, ctx) + let txt = font_face(txt) + let txt = anchors(txt) + let txt = group_indents(txt, obj) + let txt = line_breaks(txt, obj) + let #(tags, ids) = anchor_tags(obj, ids) + #(tags <> txt, ids) +} + +/// The object's own named targets, ahead of its text. A tag unique in the +/// document becomes an id and so a link target; one that repeats is recorded +/// in a data attribute, which a script or a stylesheet can find and a link +/// cannot mistake for a destination. +fn anchor_tags(obj: Object, ids: Ids) -> #(String, Ids) { + list.fold(obj.anchor_tags, #("", ids), fn(state, a) { + let #(acc, ids) = state + case claim(ids, a) { + #(True, ids) -> #(acc <> "<a id=\"" <> attr(a) <> "\"></a>", ids) + #(False, ids) -> #( + acc <> "<span class=\"marker\" data-anchor=\"" <> attr(a) <> "\"></span>", + ids, + ) + } + }) +} + +/// The four characters html reserves, and nothing else. Done first, over the +/// whole text, so that a "<" the author wrote is a "<" the reader sees and not +/// the start of an element. The markup's own markers are not html and survive +/// it. +/// The replacements go through the function form, not the string form. +/// +/// Erlang's re:replace reads "&" in a replacement string as *the whole +/// match*, so replacing `&` with the string "&" yields """ written as +/// `"quot;`: the ampersand became the quotation mark that had just been +/// matched. Byte for byte against the D writer is what caught it, on one +/// document's copyright notice. +fn escape(txt: String) -> String { + txt + |> sub(rgx.ampersand(), fn(_) { "&" }) + |> sub(rgx.less_than(), fn(_) { "<" }) + |> sub(rgx.greater_than(), fn(_) { ">" }) + |> replace(rgx.nbsp_char(), " ") +} + +/// The same, for a value that goes in an attribute. +fn attr(txt: String) -> String { + txt + |> sub(rgx.ampersand(), fn(_) { "&" }) + |> sub(rgx.quotation(), fn(_) { """ }) + |> sub(rgx.less_than(), fn(_) { "<" }) + |> sub(rgx.greater_than(), fn(_) { ">" }) +} + +fn replace(txt: String, pattern: regexp.Regexp, with: String) -> String { + regexp.replace(pattern, txt, with) +} + +/// Subject first, so it reads in a pipe. +fn sub( + txt: String, + pattern: regexp.Regexp, + build: fn(fn(Int) -> String) -> String, +) -> String { + rgx.replace_with(pattern, txt, build) +} + +fn font_face(txt: String) -> String { + let faces = [ + #(rgx.inline_emphasis(), "<em>", "</em>"), + #(rgx.inline_bold(), "<strong>", "</strong>"), + #(rgx.inline_italics(), "<i>", "</i>"), + #(rgx.inline_underscore(), "<u>", "</u>"), + #(rgx.inline_superscript(), "<sup>", "</sup>"), + #(rgx.inline_subscript(), "<sub>", "</sub>"), + #(rgx.inline_mono(), "<code>", "</code>"), + #(rgx.inline_strike(), "<del>", "</del>"), + #(rgx.inline_insert(), "<ins>", "</ins>"), + #(rgx.inline_cite(), "<cite>", "</cite>"), + ] + list.fold(faces, txt, fn(acc, face) { + let #(pattern, open, close) = face + rgx.replace_with(pattern, acc, fn(g) { open <> g(0) <> close }) + }) +} + +/// A note reference, 【<n> the note text】. +/// +/// The reference becomes a superscript link to the gathered note, and the +/// note's own text is dropped here because the endnotes section carries it: an +/// object and the endnotes section both holding it would put it in the +/// document twice. +fn notes(txt: String, ctx: LinkContext) -> String { + rgx.replace_with(rgx.inline_notes_any_mark(), txt, fn(g) { + "<sup class=\"noteref\"><a id=\"noteref_" + <> g(0) + <> "\" href=\"" + <> resolve_target("#note_" <> g(0), ctx) + <> "\" role=\"doc-noteref\">" + <> g(0) + <> "</a></sup>" + }) +} + +/// A link, ┥text┝┤target├. +/// +/// Before the writers see it the target may still be a number, which is an +/// index into the object's own `.stow_link` list rather than a url. That +/// indirection is the abstraction's, so undoing it is the writer's job. +fn links(txt: String, obj: Object, ctx: LinkContext) -> String { + let txt = + rgx.replace_with(rgx.inline_link_number_only(), txt, fn(g) { + let url = case int.parse(g(1)) { + Ok(n) -> + case list_at(obj.stow_link, n) { + Ok(u) -> u + Error(_) -> "" + } + Error(_) -> "" + } + // escaped here, because a url from the abstraction has not been through + // the text escaping: the text was escaped before the writers saw it and + // this comes from .stow_link, beside it. An unescaped "&" in an href is + // a fatal xml parse error in an epub, and one url in the collection has + // one. + g(0) <> "┤" <> attr(url) <> "├" + }) + // a link the markup left with no target at all keeps its text + let txt = rgx.replace_with(rgx.inline_link_empty(), txt, fn(g) { g(0) }) + let txt = + rgx.replace_with(rgx.inline_link(), txt, fn(g) { + "<a href=\"" <> resolve_target(g(1), ctx) <> "\">" <> g(0) <> "</a>" + }) + regexp.replace(rgx.mark_internal_site_lnk(), txt, "") +} + +/// An image, ┥☼<file>,w<W>h<H> "alt"┝┤target├. +/// +/// alt is required. An image with nothing to say for itself takes alt="", +/// which is how to say "decorative, skip me"; no alt at all is an omission, +/// and a reading system then announces the filename. +/// +/// The dimensions are dropped when they are zero, because width="0" is a valid +/// html integer meaning zero wide. +fn images(txt: String) -> String { + rgx.replace_with(rgx.inline_image(), txt, fn(g) { + let post = g(5) + let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) { + [m, ..] -> #( + rgx.group(m.submatches, 0), + case string.split_once(post, m.content) { + Ok(#(_, after)) -> after + Error(_) -> post + }, + ) + [] -> #("", post) + } + let dim = case g(3) == "0" && g(4) == "0" { + True -> "" + False -> " width=\"" <> g(3) <> "\" height=\"" <> g(4) <> "\"" + } + // the opening link marker is kept, because an image in the markup sits + // inside a link whose target may be empty or may be a url. Keeping it lets + // the link handling that follows either drop the wrapper or make the image + // a link, and dropping it here would leave the closing marker behind as + // text. + g(0) + <> "<img src=\"image/" + <> attr(g(2)) + <> "\" alt=\"" + <> attr(alt) + <> "\"" + <> dim + <> " /> " + <> rest + }) +} + +/// An anchor the markup placed in the text, ┃name┃. +fn anchors(txt: String) -> String { + rgx.replace_with(rgx.inline_link_anchor(), txt, fn(g) { + "<a id=\"" <> g(0) <> "\"></a>" + }) +} + +/// The line-leading markers of a grouped object. +/// +/// The indent is a number, so it is written as a number, in a data attribute +/// the stylesheet reads. Spine writes runs of instead, up to eighteen +/// of them for one line, which puts presentation in the content and cannot be +/// undone by a reader that wants its own. +fn group_indents(txt: String, obj: Object) -> String { + case obj.is_a { + "group" -> + txt + |> sub(rgx.grouped_para_indent_hang(), fn(g) { + "<span class=\"indent\" data-hang=\"" + <> g(0) + <> "\" data-indent=\"" + <> g(1) + <> "\"></span>" + }) + |> sub(rgx.grouped_para_bullet_at(), fn(g) { + "<span class=\"indent\" data-indent=\"" + <> g(0) + <> "\"></span><span class=\"bullet\">\u{25cf}</span> " + }) + |> sub(rgx.grouped_para_bullet(), fn(_) { + "<span class=\"bullet\">\u{25cf}</span> " + }) + |> sub(rgx.grouped_para_indent(), fn(g) { + "<span class=\"indent\" data-indent=\"" <> g(0) <> "\"></span>" + }) + _ -> txt + } +} + +/// The line breaks the markup asked for, and the ones a block preserves. Not +/// inside code, whose text is preserved as it stands. +fn line_breaks(txt: String, obj: Object) -> String { + case obj.is_a { + "code" -> txt + _ -> + txt + |> replace(rgx.br_line(), "<br />\n") + |> replace(rgx.br_line_inline(), "<br />\n") + |> replace(rgx.br_line_spaced(), "<br />\n<br />\n") + |> replace(rgx.line_break(), "<br />\n") + } +} + +/// The head. +/// +/// One charset declaration, and a Dublin Core meta only where there is a value +/// for it. Spine writes two conflicting Content-Type metas, the first of them +/// text/plain, and writes dc.publisher="Publisher" and dc.date.issued="year" +/// as literal strings when it has nothing. +fn head(doc: Document) -> String { + let m = fn(key) { model.meta(doc, key) } + let lang = case dict.get(doc.source, "language") { + Ok(l) if l != "" -> l + _ -> "en" + } + let title = case m("title.full") { + "" -> m("title.main") + t -> t + } + let author = m("creator.author") + let dc = fn(name, value) { + case value { + "" -> "" + v -> "<meta name=\"" <> name <> "\" content=\"" <> attr(v) <> "\" />\n" + } + } + "<!DOCTYPE html>\n<html lang=\"" + <> attr(lang) + <> "\">\n<head>\n" + <> "<meta charset=\"utf-8\" />\n" + <> "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n" + <> "<title>" + <> escape(title) + <> case author { + "" -> "" + a -> ", " <> escape(a) + } + <> "</title>\n" + <> dc("dc.title", title) + <> dc("dc.creator", author) + <> dc("dc.date", m("date.published")) + <> dc("dc.date.created", m("date.created")) + <> dc("dc.date.issued", m("date.issued")) + <> dc("dc.date.available", m("date.available")) + <> dc("dc.date.valid", m("date.valid")) + <> dc("dc.date.modified", m("date.modified")) + <> dc("dc.language", lang) + <> dc("dc.rights", m("rights.copyright")) + <> dc("dc.publisher", m("publisher")) + <> dc("dc.identifier", m("identifier.oclc")) + <> "<meta name=\"generator\" content=\"" + <> attr(generator_name) + <> "\" />\n" + <> "<style>\n" + <> stylesheet + <> "</style>\n" + <> "</head>\n<body>\n" +} + +fn tail(doc: Document) -> String { + let m = fn(key) { model.meta(doc, key) } + let line = fn(label, value) { + case value { + "" -> "" + v -> + "<p class=\"meta\"><span class=\"label\">" + <> label + <> "</span> " + <> escape(v) + <> "</p>\n" + } + } + "<footer>\n" + <> line("Title:", m("title.full")) + <> line("Author:", m("creator.author")) + <> line("Published:", m("date.published")) + <> line("Copyright:", m("rights.copyright")) + <> line("License:", m("rights.license")) + <> "<p class=\"generator\">" + <> escape(generator_name) + <> " <a href=\"" + <> attr(generator_url) + <> "\">" + <> escape(generator_url) + <> "</a></p>\n" + <> "</footer>\n</body>\n</html>\n" +} + +/// A stylesheet small enough to embed, so that the file renders on its own. It +/// styles what this writer emits and nothing else. Not spine's 4,747 line +/// sheet, which is a site's and not a document's. +const stylesheet = "html { font-size: 100%; } +body { + margin: 0 auto; padding: 1em 1em 1em 4em; max-width: 46em; + font-family: Georgia, \"Times New Roman\", serif; line-height: 1.5; + color: #111; background: #fff; +} +@media (prefers-color-scheme: dark) { + body { color: #ddd; background: #121212; } + a { color: #8ab4f8; } +} +.substance { position: relative; } +.ocn { + position: absolute; left: -3.6em; width: 3em; text-align: right; + font-size: 0.7em; line-height: 2.2; color: #888; +} +.ocn a { color: inherit; text-decoration: none; } +h1, h2, h3, h4, h5, h6 { line-height: 1.25; font-weight: 600; } +h1 { font-size: 1.7em; } h2 { font-size: 1.4em; } h3 { font-size: 1.2em; } +h4, h5, h6 { font-size: 1.05em; } +p { margin: 0.6em 0; } +p[data-indent=\"1\"] { margin-left: 1em; } p[data-indent=\"2\"] { margin-left: 2em; } +p[data-indent=\"3\"] { margin-left: 3em; } p[data-indent=\"4\"] { margin-left: 4em; } +p[data-indent=\"5\"] { margin-left: 5em; } p[data-indent=\"6\"] { margin-left: 6em; } +p[data-indent=\"7\"] { margin-left: 7em; } p[data-indent=\"8\"] { margin-left: 8em; } +p[data-indent=\"9\"] { margin-left: 9em; } +.indent { display: inline-block; } +.indent[data-indent=\"1\"] { width: 1em; } .indent[data-indent=\"2\"] { width: 2em; } +.indent[data-indent=\"3\"] { width: 3em; } .indent[data-indent=\"4\"] { width: 4em; } +.indent[data-indent=\"5\"] { width: 5em; } .indent[data-indent=\"6\"] { width: 6em; } +.indent[data-indent=\"7\"] { width: 7em; } .indent[data-indent=\"8\"] { width: 8em; } +.indent[data-indent=\"9\"] { width: 9em; } +blockquote.quote { margin: 0.8em 2em; font-style: normal; } +div.group, div.block, div.verse { margin: 0.8em 0; } +div.verse { white-space: pre-wrap; } +pre.code { + overflow-x: auto; padding: 0.6em; background: #f4f4f4; + font-family: ui-monospace, \"DejaVu Sans Mono\", monospace; font-size: 0.9em; +} +@media (prefers-color-scheme: dark) { pre.code { background: #1e1e1e; } } +table.table { border-collapse: collapse; margin: 0.8em 0; width: 100%; } +table.table th, table.table td { + border: 1px solid #ccc; padding: 0.2em 0.4em; vertical-align: top; +} +.align-right { text-align: right; } .align-centre { text-align: center; } +sup.noteref a { text-decoration: none; } +section[role=\"doc-endnotes\"] p { font-size: 0.9em; } +footer { margin-top: 3em; border-top: 1px solid #ccc; font-size: 0.85em; } +footer .label { color: #666; } +img { max-width: 100%; height: auto; } +" diff --git a/gleam/src/write/markdown.gleam b/gleam/src/write/markdown.gleam new file mode 100644 index 0000000..dc1b343 --- /dev/null +++ b/gleam/src/write/markdown.gleam @@ -0,0 +1,667 @@ +//// The markdown writer: a document as CommonMark, for reading anywhere. +//// +//// The fifth format, and the one asked for because markdown is what a +//// document is passed around as. The question was how much of spine's spirit +//// survives the translation, and the answer is: the object numbers do, which +//// is the part that matters. +//// +//// The dialect is CommonMark, plus pipe tables. Raw inline html is used in +//// exactly two places, both for the object number, because markdown has no +//// anchor syntax and nothing else will do: +//// +//// <a id="12"></a>The text of the paragraph.<sup>[12](#12)</sup> +//// +//// Markdown's own footnote syntax is not used: a renderer numbers footnotes +//// itself, in the order it meets them, and the document's note numbers are +//// part of its citation. So a note is written where it is referred to, with +//// the document's own number and a link back to the object - the same +//// decision the typst and the text writers made. +//// +//// Byte for byte the same as the D writer. + +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/regexp +import gleam/string +import ocda/model.{type Document, type Object} +import write/rgx + +const newline = "\n" + +const newlines = "\n\n" + +/// The document as markdown. +/// +/// image_pth is where the images are, relative to wherever this output is put. +/// Given rather than assumed: the model names an image and does not say where the +/// file naming it will sit. spine, whose markdown goes two directories down under +/// a language, passes "../../image". +pub fn write(doc: Document, image_pth: String) -> String { + let depths = toc_depths(doc) + { + front_matter(doc) + <> { + sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.filter(fn(obj) { + model.renders(obj) && !obj.dummy_heading && string.trim(obj.text) != "" + }) + |> list.map(fn(obj) { object(obj, depths) }) + |> string.concat + } + } + |> string.replace(cp(image_dir_mark), image_pth) +} + +/// head, toc, body, then each trailing section with more than one object. Not +/// endnotes: a note is written where it is referred to. +pub fn sections(doc: Document) -> List(String) { + let trailing = + ["glossary", "bibliography", "bookindex", "blurb"] + |> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 }) + list.append(["head", "toc", "body"], trailing) +} + +/// spine reserves indent_hang 1 to 3 for levels above the body, so a toc jumps +/// from 1 to 4; a markdown list nests by two spaces a level and cannot take a +/// jump, so the values the document uses are ranked. +fn toc_depths(doc: Document) -> Dict(Int, Int) { + model.section(doc, "toc") + |> list.filter(model.renders) + |> list.map(fn(obj) { obj.indent_hang }) + |> list.unique + |> list.sort(int.compare) + |> list.index_map(fn(level, i) { #(level, i) }) + |> dict.from_list +} + +/// The document's metadata as yaml front matter, which is how a markdown file +/// carries metadata and what every static site generator reads. +fn front_matter(doc: Document) -> String { + let language = case dict.get(doc.source, "language") { + Ok(l) -> l + Error(_) -> "" + } + "---" + <> newline + <> { + [ + #("title", model.meta(doc, "title.full")), + #("author", model.meta(doc, "creator.author")), + #("date", model.meta(doc, "date.published")), + #("language", language), + #("copyright", model.meta(doc, "rights.copyright")), + #("license", model.meta(doc, "rights.license")), + ] + |> list.map(fn(pair) { + let #(key, value) = pair + case value { + "" -> "" + _ -> key <> ": " <> yaml(value) <> newline + } + }) + |> string.concat + } + <> "---" + <> newlines +} + +/// A yaml scalar. Quoted always, so that a colon or a leading marker in a +/// title cannot change the shape of the document's own metadata. +fn yaml(s: String) -> String { + "\"" + <> { + s + |> string.replace("\\", "\\\\") + |> string.replace("\"", "\\\"") + |> string.replace("\n", " ") + } + <> "\"" +} + +/// One object, and then the notes it referred to. +/// +/// The notes are gathered here rather than inside each kind, because a +/// reference can be anywhere - the markup manual has one in a heading - and a +/// reference whose note was never written is a link to nothing. +fn object(obj: Object, depths: Dict(Int, Int)) -> String { + object_body(obj, depths) <> notes(obj) +} + +fn object_body(obj: Object, depths: Dict(Int, Int)) -> String { + case obj.is_a { + "heading" -> heading(obj) + "toc" -> toc_entry(obj, depths) + "table" -> table(obj) + "code" -> code(obj) + "quote" -> quote(obj) + "verse" | "poem" | "group" | "block" -> lines(obj) + "bookindex" -> index_entry(obj) + _ -> para(obj) + } +} + +/// Six levels of heading, which is all markdown has. A document may be eight +/// deep; the two deepest take the sixth, and the object number, the anchor and +/// the table of contents carry the true depth. +fn heading_level(level: String) -> Int { + case level { + "A" -> 1 + "B" -> 2 + "C" -> 3 + "D" -> 4 + "1" -> 5 + "2" | "3" | "4" -> 6 + _ -> 1 + } +} + +fn heading(obj: Object) -> String { + string.repeat("#", heading_level(obj.heading_level)) + <> " " + <> anchor(obj) + <> inline(obj) + <> ocn_mark(obj) + <> newlines +} + +/// A table of contents entry: a list item at its depth, the heading as a link, +/// and the object number as the locator. The number and not a page: a page is +/// a fact about one typesetting, and the object number is the same reference +/// in every output of the document. +fn toc_entry(obj: Object, depths: Dict(Int, Int)) -> String { + let indent = case dict.get(depths, obj.indent_hang) { + Ok(d) -> string.repeat(" ", d) + Error(_) -> "" + } + let locator = case internal_target(obj) { + "" -> "" + n -> " <sup>[" <> n <> "](#" <> n <> ")</sup>" + } + indent <> "- " <> inline(obj) <> locator <> newline +} + +fn para(obj: Object) -> String { + let bullet = case obj.bullet { + True -> "- " + False -> "" + } + string.repeat(" ", obj.indent_base) + <> bullet + <> anchor(obj) + <> inline(obj) + <> ocn_mark(obj) + <> newlines +} + +fn quote(obj: Object) -> String { + "> " <> anchor(obj) <> inline(obj) <> ocn_mark(obj) <> newlines +} + +/// Verse, a poem, a group or a block: every line kept, with CommonMark's own +/// hard line break - a backslash at the end of a line - rather than the two +/// trailing spaces an editor will strip. +fn lines(obj: Object) -> String { + anchor(obj) + <> { + string.split(inline(obj), newline) + |> string.join("\\" <> newline) + } + <> ocn_mark(obj) + <> newlines +} + +/// A book index entry: a list item, and every object it names a link. +fn index_entry(obj: Object) -> String { + let txt = regexp.replace(rgx.trailing_backslash(), obj.text, "") + "- " <> anchor(obj) <> inline_text(txt, obj) <> ocn_mark(obj) <> newlines +} + +/// A fenced code block, with the language where the document named one. +/// Nothing inside a fence is interpreted, which is the whole point. +fn code(obj: Object) -> String { + let txt = regexp.replace(rgx.nbsp_char(), obj.text, " ") + let fence = long_enough(txt, "```") + anchor(obj) + <> newline + <> fence + <> obj.code_syntax + <> newline + <> txt + <> newline + <> fence + <> newline + <> ocn_mark(obj) + <> newlines +} + +fn long_enough(txt: String, fence: String) -> String { + case string.contains(txt, fence) { + True -> long_enough(txt, fence <> "`") + False -> fence + } +} + +/// A pipe table, which needs a header row: markdown has no table without one, +/// so a table the document did not give a header gets an empty row and keeps +/// all of its own. +fn table(obj: Object) -> String { + let aligns = string.split(obj.table_aligns, on: " ") + let cols = int.max(obj.table_cols, 1) + let body = + regexp.split(rgx.table_delimiter_row(), obj.text) + |> list.filter(fn(r) { string.trim(r) != "" }) + |> list.map(fn(row) { + regexp.split(rgx.table_delimiter_col(), row) + |> list.map(fn(cell) { string.replace(inline_text(cell, obj), "|", "\\|") }) + }) + let rule = + "|" + <> { + list.repeat("", cols) + |> list.index_map(fn(_, i) { + case at(aligns, i, "l") { + "c" -> " :---: |" + "r" -> " ---: |" + _ -> " :--- |" + } + }) + |> string.concat + } + <> newline + let line = fn(cells: List(String)) { + "|" + <> { + list.repeat("", cols) + |> list.index_map(fn(_, i) { " " <> at(cells, i, "") <> " |" }) + |> string.concat + } + <> newline + } + let rows = case body, obj.table_header { + [first, ..rest], True -> + line(first) <> rule <> { rest |> list.map(line) |> string.concat } + _, _ -> line([]) <> rule <> { body |> list.map(line) |> string.concat } + } + anchor(obj) <> newline <> rows <> ocn_mark(obj) <> newlines +} + +/// The object's anchor, so that a citation can point at it. +fn anchor(obj: Object) -> String { + case obj.ocn { + 0 -> "" + n -> "<a id=\"" <> int.to_string(n) <> "\"></a>" + } +} + +/// The object's number, at the end of it, as a link to itself. +fn ocn_mark(obj: Object) -> String { + case obj.ocn { + 0 -> "" + n -> { + let s = int.to_string(n) + "<sup>[" <> s <> "](#" <> s <> ")</sup>" + } + } +} + +/// The notes the object referred to, each where it was referred to, with the +/// document's own number and a link back to the object. +fn notes(obj: Object) -> String { + let back = case obj.ocn { + 0 -> "" + n -> " [\u{21a9}](#" <> int.to_string(n) <> ")" + } + regexp.scan(rgx.inline_notes_any_mark(), obj.text) + |> list.map(fn(m) { + let mark = rgx.group(m.submatches, 0) + newlines + <> "<a id=\"note-" + <> mark + <> "\"></a><sup>" + <> inert(mark) + <> ".</sup> " + <> inline_text(rgx.group(m.submatches, 1), obj) + <> back + }) + |> string.concat + |> fn(s) { + // and a blank line after the last of them, or the next object begins on + // the same line and markdown reads the two as one paragraph + case s { + "" -> "" + _ -> s <> newlines + } + } +} + +fn inline(obj: Object) -> String { + inline_text(obj.text, obj) +} + +/// An object's text as markdown. +/// +/// The order is the one every writer here uses: images before links, because +/// an image sits inside a link; then the note references, the faces, the +/// anchors and the breaks. +fn inline_text(txt: String, obj: Object) -> String { + txt + |> images + |> links(obj) + |> note_refs + |> faces + |> regexp.replace(rgx.inline_link_anchor(), _, "") + |> breaks(obj) + |> escape + |> unprotect + |> guard_line_starts +} + +/// A "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a list +/// item, an enumeration item, a blockquote. The writer uses the first of them +/// itself, and a line of the document's that begins the same way would be read +/// as one - which is what a verse or a group can do. +/// +/// Guarded rather than escaped everywhere, because these are special only in +/// that position. +fn guard_line_starts(s: String) -> String { + string.split(s, "\n") + |> list.map(guard_one_line) + |> string.join("\n") +} + +fn guard_one_line(line: String) -> String { + let spaces = leading_spaces(line, 0) + let rest = string.drop_start(line, spaces) + let pad = string.repeat(" ", spaces) + case marker_at(rest) { + True -> pad <> "\\" <> rest + False -> + case enumerated_at(rest, 0) { + 0 -> line + n -> pad <> string.slice(rest, 0, n) <> "\\" <> string.drop_start(rest, n) + } + } +} + +fn leading_spaces(s: String, n: Int) -> Int { + case string.slice(s, n, 1) { + " " -> leading_spaces(s, n + 1) + _ -> n + } +} + +fn marker_at(s: String) -> Bool { + case string.slice(s, 0, 1), string.slice(s, 1, 1) { + "-", " " | "+", " " | ">", " " -> True + "-", "\t" | "+", "\t" | ">", "\t" -> True + _, _ -> False + } +} + +/// How many digits precede a "." or ")" and a space, or 0 for none. +fn enumerated_at(s: String, n: Int) -> Int { + case string.slice(s, n, 1) { + d if d == "0" || d == "1" || d == "2" || d == "3" || d == "4" -> enumerated_at(s, n + 1) + d if d == "5" || d == "6" || d == "7" || d == "8" || d == "9" -> enumerated_at(s, n + 1) + "." | ")" -> + case n > 0 && string.slice(s, n + 1, 1) == " " { + True -> n + False -> 0 + } + _ -> 0 + } +} + +// The markers that protect the markdown this writer emits from the escaping +// that follows. +// +// The escaping has to come *last*, and this is why: the abstraction's own +// markers are built from the characters markdown reserves - "⑆*┨" for emphasis, +// "⑆_┨" for an underscore face - so escaping first turns them into "⑆\*┨" and +// no face is ever recognised. +const keep_open = 0xe000 + +const keep_close = 0xe001 + +// A third marker of the same kind, standing for the directory the images are in. +// +// An image reference is written while an object is written, and where the file +// will be put is not something the model knows: it is told. So the reference +// carries a marker and write substitutes it once, at the end, out of the one +// argument that says. +const image_dir_mark = 0xe002 + +fn keep(markdown_source: String) -> String { + cp(keep_open) <> markdown_source <> cp(keep_close) +} + +fn unprotect(txt: String) -> String { + txt |> string.replace(cp(keep_open), "") |> string.replace(cp(keep_close), "") +} + +fn cp(n: Int) -> String { + let assert Ok(c) = string.utf_codepoint(n) + string.from_utf_codepoints([c]) +} + +/// The document's own text, escaped for a protected run: the same rule, run +/// early, and then flattened so that the result is inert. +fn inert(txt: String) -> String { + unprotect(escape(txt)) +} + +/// The characters markdown reads as syntax, escaped wherever they occur. +/// +/// The same rule the typst writer arrived at, and for the same reason: a closed +/// set is checkable and a rule about positions is not. The three html +/// characters become entities rather than backslash escapes, because markdown +/// passes raw html through and "<b" would open a tag. +fn escape(txt: String) -> String { + let #(out, _) = + string.to_utf_codepoints(txt) + |> list.fold(#([], 0), fn(state, c) { + let #(acc, depth) = state + let n = string.utf_codepoint_to_int(c) + case n == keep_open, n == keep_close, depth > 0 { + True, _, _ -> #(acc, depth + 1) + _, True, _ -> #(acc, int.max(depth - 1, 0)) + _, _, True -> #([string.from_utf_codepoints([c]), ..acc], depth) + _, _, False -> #([escaped(n, c), ..acc], depth) + } + }) + string.concat(list.reverse(out)) +} + +fn escaped(n: Int, c: UtfCodepoint) -> String { + case n { + 0x26 -> "&" + 0x3c -> "<" + 0x3e -> ">" + // \ ` * _ [ ] # | ~ + 0x5c | 0x60 | 0x2a | 0x5f | 0x5b | 0x5d | 0x23 | 0x7c | 0x7e -> + "\\" <> string.from_utf_codepoints([c]) + _ -> string.from_utf_codepoints([c]) + } +} + +/// An image, with its alt text where the document gave one. +fn images(txt: String) -> String { + rgx.replace_with(rgx.inline_image(), txt, fn(g) { + let post = g(5) + let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) { + [m, ..] -> #(rgx.group(m.submatches, 0), case + string.split_once(post, m.content) + { + Ok(#(_, after)) -> after + Error(_) -> post + }) + [] -> #("", post) + } + g(0) + <> keep("![" <> inert(alt) <> "](" <> cp(image_dir_mark) <> "/" <> g(2) <> ")") + <> rest + }) +} + +/// A link. An internal target becomes a fragment, which is the object number, +/// and so resolves against the anchor every object carries. +/// +/// And *only* an object number: those are the anchors this writer defines, so a +/// target that names something else - the endnotes section, an anchor tag the +/// markup placed - would be a link to nothing. The text stands on its own +/// instead, which is what a reader sees anyway. +fn links(txt: String, obj: Object) -> String { + rgx.replace_with(rgx.inline_link_number_only(), txt, fn(g) { + let url = case int.parse(g(1)) { + Ok(n) -> at(obj.stow_link, n, "") + Error(_) -> "" + } + g(0) <> "┤" <> url <> "├" + }) + |> rgx.replace_with(rgx.inline_link(), _, fn(g) { + let target = case string.starts_with(g(1), "¤") { + False -> g(1) + True -> + case regexp.scan(rgx.internal_target(), g(1)) { + [m, ..] -> rgx.group(m.submatches, 1) + [] -> g(1) + } + } + let shown = inert(faces(g(0))) + case target, string.starts_with(target, "#") && !is_object_number(target) { + "", _ -> keep(shown) + _, True -> keep(shown) + _, False -> keep("[" <> shown <> "](" <> target <> ")") + } + }) +} + +/// A link target is a url or a fragment and not prose, so the escaping the +/// text went through comes off it again. A "#" that stayed escaped made every +/// table of contents line point at nothing. +fn unescape_target(target: String) -> String { + target + |> string.replace("\\", "") + |> string.replace("&", "&") + |> string.replace("<", "<") + |> string.replace(">", ">") +} + +/// A fragment that names an object of this document. +fn is_object_number(clean: String) -> Bool { + case string.drop_start(clean, 1) { + "" -> False + "0" -> False + rest -> + case int.parse(rest) { + Ok(_) -> True + Error(_) -> False + } + } +} + +/// A note reference: a superscript link to the note, which sits under the +/// object that refers to it. +fn note_refs(txt: String) -> String { + rgx.replace_with(rgx.inline_notes_any_mark(), txt, fn(g) { + // the mark is shown as well as named in the target, and a star note's mark + // is an asterisk, which markdown would pair with the next one + keep("<sup>[" <> inert(g(0)) <> "](#note-" <> g(0) <> ")</sup>") + }) +} + +/// The font faces. Emphasis and strong are markdown's own; the rest have no +/// markdown and take the html element markdown itself would produce. +/// +/// The spaces inside a run are dropped rather than kept: the abstraction's +/// marker takes in the space that was beside the run, the text before it +/// already ends in one, and markdown's emphasis will not open on a space. +fn faces(txt: String) -> String { + [ + #(rgx.inline_emphasis(), "**", "**"), + #(rgx.inline_bold(), "**", "**"), + #(rgx.inline_italics(), "*", "*"), + #(rgx.inline_underscore(), "<u>", "</u>"), + #(rgx.inline_superscript(), "<sup>", "</sup>"), + #(rgx.inline_subscript(), "<sub>", "</sub>"), + #(rgx.inline_mono(), "`", "`"), + #(rgx.inline_strike(), "~~", "~~"), + #(rgx.inline_insert(), "<ins>", "</ins>"), + #(rgx.inline_cite(), "<cite>", "</cite>"), + ] + |> list.fold(txt, fn(acc, face) { + let #(pattern, open, close) = face + rgx.replace_with(pattern, acc, fn(g) { + keep(open) <> trim_spaces(g(0)) <> keep(close) + }) + }) +} + +fn trim_spaces(s: String) -> String { + case string.starts_with(s, " "), string.ends_with(s, " ") { + True, _ -> trim_spaces(string.drop_start(s, 1)) + _, True -> trim_spaces(string.drop_end(s, 1)) + _, _ -> s + } +} + +/// The line breaks the markup asked for, and the grouped indents. +fn breaks(txt: String, obj: Object) -> String { + case obj.is_a { + "code" -> txt + _ -> { + let txt = case obj.is_a == "group" || obj.is_a == "block" { + False -> txt + True -> + txt + |> rgx.replace_with(rgx.grouped_para_indent_hang(), _, fn(g) { + g(1) <> g(1) + }) + |> rgx.replace_with(rgx.grouped_para_bullet_at(), _, fn(g) { + g(0) <> g(0) <> "\u{25cf} " + }) + |> rgx.replace_with(rgx.grouped_para_bullet(), _, fn(_) { + "\u{25cf} " + }) + |> rgx.replace_with(rgx.grouped_para_indent(), _, fn(g) { + g(0) <> g(0) + }) + } + txt + |> regexp.replace(rgx.nbsp_char(), _, " ") + |> regexp.replace(rgx.br_line(), _, newline) + |> regexp.replace(rgx.br_line_inline(), _, newline) + |> regexp.replace(rgx.br_line_spaced(), _, newlines) + |> regexp.replace(rgx.line_break(), _, newline) + |> regexp.replace(rgx.mark_internal_site_lnk(), _, "") + } + } +} + +/// The object a table of contents entry points at, where that is an object +/// number. +fn internal_target(obj: Object) -> String { + regexp.scan(rgx.any_internal_target(), obj.text) + |> list.fold("", fn(found, m) { + case found { + "" -> { + let frag = rgx.group(m.submatches, 0) + case int.parse(frag) { + Ok(_) -> frag + Error(_) -> "" + } + } + _ -> found + } + }) +} + +fn at(items: List(String), index: Int, fallback: String) -> String { + case items, index { + [], _ -> fallback + [first, ..], 0 -> first + [_, ..rest], n -> at(rest, n - 1, fallback) + } +} diff --git a/gleam/src/write/rgx.gleam b/gleam/src/write/rgx.gleam new file mode 100644 index 0000000..71add21 --- /dev/null +++ b/gleam/src/write/rgx.gleam @@ -0,0 +1,282 @@ +//// The internal markup, as compiled patterns. +//// +//// An object's text is *spine's internal representation*, not the author's +//// markup and not any output: the markers are single Unicode characters +//// chosen not to occur in text. A writer's first job is to turn them into +//// something the target format understands, so every writer needs these +//// and they live in one place. +//// +//// The characters are load bearing. Do not "tidy" them into ASCII. + +import gleam/list +import gleam/option +import gleam/string +import gleam/regexp.{type Regexp} + +/// Compile, or crash: these are constants, and a pattern that does not +/// compile is a defect in this file rather than anything a caller can +/// handle. +fn re(pattern: String) -> Regexp { + let assert Ok(r) = + regexp.compile( + pattern, + regexp.Options(case_insensitive: False, multi_line: True), + ) + r +} + +pub fn nbsp_char() -> Regexp { + re("░") +} + +pub fn br_line() -> Regexp { + re("\\s*┘\\s*") +} + +pub fn br_line_inline() -> Regexp { + re("\\s*┙\\s*") +} + +pub fn br_line_spaced() -> Regexp { + re("\\s*┚\\s*") +} + +/// The font faces, each `⑆<mark>┨ text ┣<mark>`, paired with the markup +/// spine's text output writes them back as. +pub fn font_faces() -> List(#(Regexp, String, String)) { + [ + #(re("⑆[-]┨(.+?)┣[-]"), "-{", "}-"), + #(re("⑆[+]┨(.+?)┣[+]"), "+{", "}+"), + #(re("⑆[‖]┨(.+?)┣[‖]"), "\"{", "}\""), + #(re("⑆[*]┨(.+?)┣[*]"), "!{", "}!"), + #(re("⑆[!]┨(.+?)┣[!]"), "*{", "}*"), + #(re("⑆[/]┨(.+?)┣[/]"), "/{", "}/"), + #(re("⑆[_]┨(.+?)┣[_]"), "_{", "}_"), + #(re("⑆\\^┨(.+?)┣\\^"), "^{", "}^"), + #(re("⑆[,]┨(.+?)┣[,]"), ",{", "},"), + #(re("⑆[■]┨(.+?)┣[■]"), "#{", "}#"), + ] +} + +/// A link: capture 1 is the text, capture 3 the target without its leading +/// "#" where it has one. +pub fn inline_link() -> Regexp { + re("┥(.+?)┝┤(#?(\\S+?))├") +} + +pub fn inline_image() -> Regexp { + re( + "(┥)☼(([a-zA-Z0-9._-]+?\\.(?:jpg|gif|png)),w(\\d+)h(\\d+))\\s*(.*?┝┤.*?├)", + ) +} + +/// A note is carried inside the object that refers to it. +pub fn inline_notes_any() -> Regexp { + re("【.+?】") +} + +pub fn inline_notes_numbered() -> Regexp { + re("【(\\d+)\\s+(.+?)\\s*】") +} + +pub fn inline_superscript() -> Regexp { + re("⑆\\^┨(.+?)┣\\^") +} + +/// A table's text is rows and columns in one string. +pub fn table_delimiter_row() -> Regexp { + re("[ ]*\n") +} + +pub fn table_delimiter_col() -> Regexp { + re("[ ]*[┊][ ]*") +} + +pub fn trailing_backslash() -> Regexp { + re("\\s*\\\\") +} + +/// The font faces one at a time, for a writer that wraps each in its own +/// element rather than in a pair of markup characters. +pub fn inline_strike() -> Regexp { + re("⑆[-]┨(.+?)┣[-]") +} + +pub fn inline_insert() -> Regexp { + re("⑆[+]┨(.+?)┣[+]") +} + +pub fn inline_cite() -> Regexp { + re("⑆[‖]┨(.+?)┣[‖]") +} + +pub fn inline_emphasis() -> Regexp { + re("⑆[*]┨(.+?)┣[*]") +} + +pub fn inline_bold() -> Regexp { + re("⑆[!]┨(.+?)┣[!]") +} + +pub fn inline_italics() -> Regexp { + re("⑆[/]┨(.+?)┣[/]") +} + +pub fn inline_underscore() -> Regexp { + re("⑆[_]┨(.+?)┣[_]") +} + +pub fn inline_subscript() -> Regexp { + re("⑆[,]┨(.+?)┣[,]") +} + +pub fn inline_mono() -> Regexp { + re("⑆[■]┨(.+?)┣[■]") +} + +/// Replace every match, the replacement built from the match's submatches. +/// +/// `gleam/regexp` has no replace-with-function, so this is `scan` plus a walk +/// over the subject. It is the one piece of scaffolding the Gleam writers need +/// that the D ones get from Phobos. +pub fn replace_with( + pattern: Regexp, + subject: String, + build: fn(fn(Int) -> String) -> String, +) -> String { + apply(regexp.scan(pattern, subject), subject, build) +} + +fn apply( + matches: List(regexp.Match), + subject: String, + build: fn(fn(Int) -> String) -> String, +) -> String { + case matches { + [] -> subject + _ -> { + let #(out, rest) = + list.fold(matches, #("", subject), fn(state, m) { + let #(out, rest) = state + case string.split_once(rest, m.content) { + Error(_) -> #(out, rest) + Ok(#(before, after)) -> { + let g = fn(i) { group(m.submatches, i) } + #(out <> before <> build(g), after) + } + } + }) + out <> rest + } + } +} + +/// A note's mark may be a number or a run of stars or pluses, and both are +/// gathered into the endnotes section, so both need a reference. +pub fn inline_notes_any_mark() -> Regexp { + re("【(\\d+|(?:[*]|[+])+)\\s+(.+?)\\s*】") +} + +/// A link before its url has been substituted in: the number is an index into +/// the object's `.stow_link` list, not a target. +pub fn inline_link_number_only() -> Regexp { + re("(┥.+?┝)┤([0-9]+)├") +} + +pub fn inline_link_empty() -> Regexp { + re("┥(.+?)┝┤├") +} + +/// An anchor the markup placed in the text, ┃name┃. +pub fn inline_link_anchor() -> Regexp { + re("┃(\\S+?)┃") +} + +/// "¤" marks a link internal to the document collection, and ".fnSuffix" +/// stands where an output's file suffix goes. +pub fn mark_internal_site_lnk() -> Regexp { + re("¤") +} + +pub fn inline_seg_link() -> Regexp { + re("(¤)(?:.+?)\\.fnSuffix") +} + +/// The same target taken apart: the segment it names and whatever follows +/// it, which is what a segmented output needs. +pub fn internal_target() -> Regexp { + re("^¤(.+?)\\.fnSuffix(.*)$") +} + +/// An image's alt text, which the markup carries as a quoted string following +/// the image. +pub fn inline_image_alt() -> Regexp { + re("^\\s*[“\"]([^”\"]*)[”\"]") +} + +/// A line break the markup asked for, " \\\\". +pub fn line_break() -> Regexp { + re(" [\\\\]{2}") +} + +/// The leading markers of a line inside a group: indent, hang, bullet. +pub fn grouped_para_indent_hang() -> Regexp { + re("^_([0-9])_([0-9])[ ]") +} + +pub fn grouped_para_bullet_at() -> Regexp { + re("^_([1-9])[*] ") +} + +pub fn grouped_para_bullet() -> Regexp { + re("^_[*] ") +} + +pub fn grouped_para_indent() -> Regexp { + re("^_([1-9])[ ]") +} + +/// The characters html reserves. +pub fn ampersand() -> Regexp { + re("[&]") +} + +pub fn quotation() -> Regexp { + re("[\"]") +} + +pub fn less_than() -> Regexp { + re("[<]") +} + +pub fn greater_than() -> Regexp { + re("[>]") +} + +/// Whether a pattern matches at all, without caring where. +pub fn matches(pattern: Regexp, subject: String) -> Bool { + regexp.check(pattern, subject) +} + +/// The submatch at a position, "" when the group did not participate. +pub fn group(submatches: List(option.Option(String)), index: Int) -> String { + case at(submatches, index) { + option.Some(value) -> value + option.None -> "" + } +} + +fn at(items: List(option.Option(String)), index: Int) -> option.Option(String) { + case items, index { + [], _ -> option.None + [first, ..], 0 -> first + [_, ..rest], n -> at(rest, n - 1) + } +} + +/// An internal link target as the abstraction writes it, with or without the +/// segment prefix a segmented output would use. Read to decide whether a +/// table of contents entry points at anything a writer renders. +pub fn any_internal_target() -> Regexp { + re("┤(?:¤.+?\\.fnSuffix)?#([^├]+)├") +} diff --git a/gleam/src/write/text.gleam b/gleam/src/write/text.gleam new file mode 100644 index 0000000..2e44e3a --- /dev/null +++ b/gleam/src/write/text.gleam @@ -0,0 +1,399 @@ +//// The plain text writer: a function from the model to bytes. +//// +//// Given an `ocda/model.Document` and nothing else. It does not know where +//// the document came from, which is the point: the same call renders one +//// loaded from a `.ssp` and one loaded from a `.ocda.db`, and cannot tell +//// them apart. +//// +//// **This no longer matches spine's `--text` byte for byte, by decision.** +//// Byte identity was the first oracle and earned its place while there was +//// nothing else to check against; holding to it meant reproducing a dozen +//// things no one would choose. What replaced it is stricter where it matters: +//// the D and the Gleam writer produce the same bytes, the words of the output +//// equal the words of the html writer's objects as a multiset, every object +//// number appears exactly once, and the same bytes come from every input +//// form. +//// +//// Byte for byte the same as the D writer. What each changed from spine, and +//// why, is in that file's header and in the repository README. + +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/regexp +import gleam/string +import ocda/model.{type Document, type Object} +import write/rgx + +const newline = "\n" + +const newlines = "\n\n" + +/// Set to spine's own values so that a comparison is of the writer and not of +/// the signature. A writer that is not spine should say so here. +const generator_name = "spine" + +const generator_url = "https://sisudoc.org" + +pub fn write(doc: Document) -> String { + let depths = toc_depths(doc) + let body = + text_sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.filter(model.renders) + |> list.map(fn(obj) { object(obj, depths) }) + |> string.concat + body <> newline <> tail(doc) <> newline +} + +/// Which sections, in which order: head, toc, body, then each trailing +/// section that has more than one object. One object is the section's own +/// heading with nothing under it. +pub fn text_sections(doc: Document) -> List(String) { + let trailing = + ["glossary", "bibliography", "bookindex", "blurb"] + |> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 }) + list.append(["head", "toc", "body"], trailing) +} + +/// Rank the `indent_hang` values the table of contents uses. +/// +/// spine's `indent_hang` is not a depth: it reserves 1 to 3 for levels above +/// the body, so a document's toc uses 1 then jumps to 4, 5, 6. Indenting by +/// the number itself leaves the first body level six spaces in. Ranking the +/// values the document actually uses adapts to any document without a table +/// of special cases. +fn toc_depths(doc: Document) -> Dict(Int, Int) { + model.section(doc, "toc") + |> list.filter(model.renders) + |> list.map(fn(obj) { obj.indent_hang }) + |> list.unique + |> list.sort(int.compare) + |> list.index_map(fn(level, i) { #(level, i) }) + |> dict.from_list +} + +/// One object, by kind. +/// +/// The last case writes the object rather than dropping it: a kind this +/// dispatch has not heard of is more likely to be new than to be unwanted, +/// and an output that loses it silently cannot be checked. +fn object(obj: Object, depths: Dict(Int, Int)) -> String { + // an object with no text of its own writes nothing. A poem is such an + // object: it is a container that closes a run of verses and carries the + // *same* object number as the last of them, so writing it would put that + // number in the file twice - 15 times over in one sample. + // and a dummy heading writes nothing: it is generated furniture whose text + // repeats the heading above it, and every other writer here drops it. + // Spine's text keeps it, so a trailing section's name is in the file twice. + case string.trim(obj.text) == "" || obj.dummy_heading { + True -> "" + False -> + case obj.is_a { + "toc" -> toc_entry(obj, depths) + "table" -> table(obj) + "bookindex" -> index_entry(obj) + "code" -> + block(obj, regexp.replace(rgx.nbsp_char(), obj.text, " "), "") + "heading" -> newline <> general(obj) + _ -> general(obj) + } + } +} + +/// The usual case: the markers resolved, the notes gathered out of the +/// resolved text, and the rest written as text. +fn general(obj: Object) -> String { + let resolved = links(images(obj.text)) + block(obj, inline(resolved, obj), notes_of(resolved)) +} + +/// The common shape: the text, the notes it carried, and its number. +fn block(obj: Object, txt: String, notes: String) -> String { + txt <> notes <> newline <> ocn_mark(obj) <> newline +} + +/// The object's number, on a line of its own. +fn ocn_mark(obj: Object) -> String { + case obj.ocn { + 0 -> "" + n -> "「" <> int.to_string(n) <> "」" <> newline + } +} + +/// A note is carried inside the object that refers to it, 【<n> text】. The +/// reference becomes [n] where it stands and the note follows the object, +/// which is what makes the text readable in one pass. +fn notes_of(txt: String) -> String { + regexp.scan(rgx.inline_notes_any_mark(), txt) + |> list.map(fn(m) { + newlines + <> rgx.group(m.submatches, 0) + <> ". " + <> faces(breaks(rgx.group(m.submatches, 1))) + }) + |> string.concat +} + +/// A table of contents entry: two spaces per level, the text, and the object +/// it names. +fn toc_entry(obj: Object, depths: Dict(Int, Int)) -> String { + let indent = case dict.get(depths, obj.indent_hang) { + Ok(d) -> string.repeat(" ", d) + Error(_) -> "" + } + indent <> inline(links(images(obj.text)), obj) <> newline +} + +/// A book index entry, whose links are to objects rather than to segments. +fn index_entry(obj: Object) -> String { + let txt = regexp.replace(rgx.trailing_backslash(), obj.text, "") + inline(links(images(txt)), obj) <> newlines +} + +/// A table, in columns as wide as their widest cell needs. +/// +/// The object's text is the whole table: rows separated by newlines, columns +/// by ┊. A row of one cell in a table of more than one column is a note under +/// the table rather than a row of it. +fn table(obj: Object) -> String { + let aligns = string.split(obj.table_aligns, on: " ") + let #(rows, note) = + regexp.split(rgx.table_delimiter_row(), obj.text) + |> list.filter(fn(r) { string.trim(r) != "" }) + |> list.fold(#([], ""), fn(state, row) { + let #(rows, note) = state + let cells = + regexp.split(rgx.table_delimiter_col(), row) + |> list.map(fn(cell) { inline(links(images(cell)), obj) }) + case cells, obj.table_cols > 1 { + [only], True -> #(rows, note <> only <> newline) + _, _ -> #([cells, ..rows], note) + } + }) + let rows = list.reverse(rows) + let widths = + list.fold(rows, dict.new(), fn(acc, cells) { + list.index_fold(cells, acc, fn(a, cell, i) { + let w = display_width(cell) + case dict.get(a, i) { + Ok(had) if had >= w -> a + _ -> dict.insert(a, i, w) + } + }) + }) + let body = + rows + |> list.map(fn(cells) { + let n = list.length(cells) + cells + |> list.index_map(fn(cell, i) { + let w = case dict.get(widths, i) { + Ok(w) -> w + Error(_) -> display_width(cell) + } + let pad = string.repeat(" ", w - display_width(cell)) + let padded = case at(aligns, i, "l") { + "l" -> cell <> pad + _ -> pad <> cell + } + case i + 1 < n { + True -> padded <> " ┊ " + False -> padded + } + }) + |> string.concat + // stripped at the end of its own row, so a row that ends in an empty + // cell keeps the line it is on + |> strip_right + }) + |> list.map(fn(line) { line <> newline }) + |> string.concat + body <> note <> ocn_mark(obj) <> newline +} + +fn strip_right(s: String) -> String { + case string.ends_with(s, " ") || string.ends_with(s, "\t") { + True -> strip_right(string.drop_end(s, 1)) + False -> s + } +} + +/// How many columns a string occupies. +/// +/// One per character, and two for the ranges a terminal draws double width. +/// Neither a byte count nor a codepoint count lines up a table holding +/// Japanese, and this is the measurement the alignment means. +fn display_width(s: String) -> Int { + string.to_utf_codepoints(s) + |> list.fold(0, fn(w, c) { + case wide(string.utf_codepoint_to_int(c)) { + True -> w + 2 + False -> w + 1 + } + }) +} + +fn wide(n: Int) -> Bool { + { n >= 0x1100 && n <= 0x115f } + || { n >= 0x2e80 && n <= 0x303e } + || { n >= 0x3041 && n <= 0x33ff } + || { n >= 0x3400 && n <= 0x4dbf } + || { n >= 0x4e00 && n <= 0x9fff } + || { n >= 0xa000 && n <= 0xa4cf } + || { n >= 0xac00 && n <= 0xd7a3 } + || { n >= 0xf900 && n <= 0xfaff } + || { n >= 0xfe30 && n <= 0xfe6f } + || { n >= 0xff00 && n <= 0xff60 } + || { n >= 0xffe0 && n <= 0xffe6 } + || { n >= 0x20000 && n <= 0x3fffd } +} + +/// An object's text as plain text. +/// +/// The order is the one every writer here uses: images before links, because +/// an image sits inside a link; then the faces, then the breaks. A note +/// reference becomes [n] and the note itself is gathered by `notes`. +fn inline(txt: String, obj: Object) -> String { + txt + |> rgx.replace_with(rgx.inline_notes_any_mark(), _, fn(g) { + "[" <> g(0) <> "]" + }) + // an anchor the markup placed in the text has no rendering of its own; in a + // text file there is nothing for it to be + |> regexp.replace(rgx.inline_link_anchor(), _, "") + |> faces + |> group_indents(obj) + |> breaks +} + +/// The line-leading markers of a grouped object: an indent, a hanging indent's +/// inner level, a bullet. Spine leaves them in the text as "_1 " and "_* ", +/// which read as markup that was not translated. +fn group_indents(txt: String, obj: Object) -> String { + case obj.is_a == "group" || obj.is_a == "block" { + False -> txt + True -> + txt + |> rgx.replace_with(rgx.grouped_para_indent_hang(), _, fn(g) { + string.repeat(" ", 2) <> g(1) + }) + |> rgx.replace_with(rgx.grouped_para_bullet_at(), _, fn(g) { + g(0) <> g(0) <> "\u{25cf} " + }) + |> rgx.replace_with(rgx.grouped_para_bullet(), _, fn(_) { "\u{25cf} " }) + |> rgx.replace_with(rgx.grouped_para_indent(), _, fn(g) { g(0) <> g(0) }) + } +} + +/// An image becomes its file name in braces, which is spine's own markup for +/// one, and the words beside it stay where they are. Spine's text writes the +/// file name alone and loses the caption. +/// +/// The quotation marks around an alt string are the markup's delimiters and not +/// the author's punctuation, so they come off here as they do in the html +/// writer. +fn images(txt: String) -> String { + rgx.replace_with(rgx.inline_image(), txt, fn(g) { + let post = g(5) + let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) { + [m, ..] -> #(rgx.group(m.submatches, 0), case + string.split_once(post, m.content) + { + Ok(#(_, after)) -> after + Error(_) -> post + }) + [] -> #("", post) + } + g(0) <> "{" <> g(2) <> "}" <> alt <> rest + }) +} + +/// A link becomes its text, and the object it points at after it. +/// +/// Only when the target *is* an object: a name, a segment or a url means +/// nothing to a reader of a text file, where "≫12" is a number they can +/// search for. So the toc line for the endnotes reads "Endnotes" and not +/// "Endnotes ≫endnotes". +fn links(txt: String) -> String { + rgx.replace_with(rgx.inline_link(), txt, fn(g) { + case is_object_number(g(2)) { + True -> g(0) <> " ≫" <> g(2) + False -> g(0) + } + }) + |> regexp.replace(rgx.mark_internal_site_lnk(), _, "") +} + +/// A target that is an object number, and not a name or a url. +fn is_object_number(s: String) -> Bool { + case s, int.parse(s) { + "", _ -> False + "0", _ -> False + _, Ok(_) -> True + _, Error(_) -> False + } +} + +/// The font faces become the markup they were written as, which is what makes +/// the text re-editable rather than stripped. +fn faces(txt: String) -> String { + [ + #(rgx.inline_strike(), "-{", "}-"), + #(rgx.inline_insert(), "+{", "}+"), + #(rgx.inline_cite(), "\"{", "}\""), + #(rgx.inline_emphasis(), "!{", "}!"), + #(rgx.inline_bold(), "*{", "}*"), + #(rgx.inline_italics(), "/{", "}/"), + #(rgx.inline_underscore(), "_{", "}_"), + #(rgx.inline_superscript(), "^{", "}^"), + #(rgx.inline_subscript(), ",{", "},"), + #(rgx.inline_mono(), "#{", "}#"), + ] + |> list.fold(txt, fn(acc, face) { + let #(pattern, open, close) = face + rgx.replace_with(pattern, acc, fn(g) { open <> g(0) <> close }) + }) +} + +/// The line breaks the markup asked for, and the non-breaking space. +fn breaks(txt: String) -> String { + txt + |> regexp.replace(rgx.nbsp_char(), _, " ") + |> regexp.replace(rgx.br_line(), _, newline) + |> regexp.replace(rgx.br_line_inline(), _, newline) + |> regexp.replace(rgx.br_line_spaced(), _, newlines) + |> regexp.replace(rgx.line_break(), _, newline) +} + +fn at(items: List(String), index: Int, fallback: String) -> String { + case items, index { + [], _ -> fallback + [first, ..], 0 -> first + [_, ..rest], n -> at(rest, n - 1, fallback) + } +} + +/// The metadata block at the foot of the file. +/// +/// Two values are not in the abstraction: the generating program's name and +/// its home page. They describe the program, not the document, so the model +/// has no place for them and this writer names itself. +fn tail(doc: Document) -> String { + [ + #("Title", "title.full"), + #("Author", "creator.author"), + #("Published", "date.published"), + #("Copyright", "rights.copyright"), + #("License", "rights.license"), + ] + |> list.map(fn(pair) { + let #(label, key) = pair + case model.meta(doc, key) { + "" -> "" + value -> label <> ": " <> value <> newlines + } + }) + |> string.concat + |> string.append(generator_name <> newline <> generator_url) +} diff --git a/gleam/src/write/typst.gleam b/gleam/src/write/typst.gleam new file mode 100644 index 0000000..ada1bdb --- /dev/null +++ b/gleam/src/write/typst.gleam @@ -0,0 +1,909 @@ +//// The typst writer: a document as typst markup, for a pdf. +//// +//// The fourth format and the first that is not itself the output: what this +//// writes is a .typ file, and `typst compile` turns it into a pdf. A writer's +//// job is model -> bytes; making a pdf is a typesetter's. +//// +//// The judge is *typst itself*, and it is a strict one for the reason that +//// matters here: a link to a label that does not exist is a compile error. So +//// "it compiles" means "every internal link resolves", which in the html +//// writer took a checker of its own. +//// +//// Set against spine's xelatex path, measured rather than asserted: no +//// preamble, no .sty files, no second pass, a toolchain of 108 MB where +//// texlive-combined-full is 6.6 GB (nix closures), a tagged pdf where +//// spine's says "Tagged: no", and 56,990 CJK characters on the page where +//// spine's Japanese pdf has none at all. +//// +//// Byte for byte the same as the D writer. Where this looks like D rather +//// than like Gleam - the codepoint walk in `escape`, the set threaded through +//// every function in place of D's module variable - that is why. + +import gleam/dict +import gleam/int +import gleam/list +import gleam/regexp +import gleam/set.{type Set} +import gleam/string +import ocda/model.{type Document, type Object} +import write/rgx + +const generator_name = "spine" + +/// The labels this document defines, and the ones already placed. +/// +/// D keeps the second in a module variable; here it is threaded, because the +/// order in which labels are claimed decides the bytes and a thread makes +/// that order visible. +pub type Labels { + Labels(defined: Set(String), placed: Set(String)) +} + +pub fn write(doc: Document) -> String { + let defined = defined_targets(doc) + // an OCN label is placed by `ocn` on the number in the margin, always and + // for every numbered object, so it is claimed before anything else. An + // anchor or an identifier that happens to carry the same name is then + // dropped rather than defining the label twice, which typst rejects: + // free_culture has a heading whose identifier is "2" and an object whose + // OCN is 2. + let placed = + sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.filter(fn(obj) { model.renders(obj) && obj.ocn != 0 }) + |> list.map(fn(obj) { label_of(int.to_string(obj.ocn)) }) + |> set.from_list + let labels = Labels(defined, placed) + let #(body, _) = + sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.filter(model.renders) + |> list.fold(#("", labels), fn(state, obj) { + let #(out, lab) = state + let #(s, lab) = object(obj, doc, lab) + #(out <> guard_block_starts(s), lab) + }) + preamble(doc) <> body +} + +/// Which sections, in which order. +/// +/// head, toc, body, then glossary, bibliography, bookindex and blurb where +/// each has more than one object. *Not endnotes*: a note is a footnote here, +/// at the foot of the page that refers to it, so a gathered endnotes section +/// would put every note in the document twice. +pub fn sections(doc: Document) -> List(String) { + let trailing = + ["glossary", "bibliography", "bookindex", "blurb"] + |> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 }) + list.append(["head", "toc", "body"], trailing) +} + +/// Every label this document will define. +/// +/// A link to an undefined label does not compile, so the writer has to know +/// what it will define before it writes the first link. +pub fn defined_targets(doc: Document) -> Set(String) { + sections(doc) + |> list.flat_map(fn(name) { model.section(doc, name) }) + |> list.filter(model.renders) + |> list.fold(set.new(), fn(acc, obj) { + let acc = case obj.ocn { + 0 -> acc + n -> set.insert(acc, label_of(int.to_string(n))) + } + let acc = case obj.identifier { + "" -> acc + id -> set.insert(acc, label_of(id)) + } + let acc = case obj.heading_lev_anchor { + "" -> acc + a -> set.insert(acc, label_of(a)) + } + let acc = + list.fold(obj.anchor_tags, acc, fn(a, tag) { set.insert(a, label_of(tag)) }) + regexp.scan(rgx.inline_link_anchor(), obj.text) + |> list.fold(acc, fn(a, m) { + set.insert(a, label_of(rgx.group(m.submatches, 0))) + }) + }) +} + +/// A label name typst will accept, from a name the markup chose. Prefixed, +/// because a label that begins with a digit is asking for trouble and every +/// ocn does. +/// +/// Codepoints, not graphemes, so that the two writers cannot part company +/// over a combining mark. +pub fn label_of(name: String) -> String { + "t" + <> { + string.to_utf_codepoints(name) + |> list.map(fn(c) { + case is_label_code(string.utf_codepoint_to_int(c)) { + True -> string.from_utf_codepoints([c]) + False -> "_" + } + }) + |> string.concat + } +} + +fn is_label_code(n: Int) -> Bool { + n == 0x5f + || n == 0x2d + || n == 0x2e + || { n >= 97 && n <= 122 } + || { n >= 65 && n <= 90 } + || { n >= 48 && n <= 57 } +} + +fn preamble(doc: Document) -> String { + let language = case dict.get(doc.source, "language") { + Ok(l) -> l + Error(_) -> "en" + } + let parts = string.split(string.replace(language, "_", "-"), "-") + let #(lang, region) = case parts { + [l, r, ..] -> #(l, r) + [l] -> #(l, "") + [] -> #("en", "") + } + let title = case model.meta(doc, "title.full") { + "" -> model.meta(doc, "title.main") + t -> t + } + "// " + <> generator_name + <> ": the object-centric document abstraction, written as typst\n" + <> "// Generated from the abstraction. Compile with: typst compile <this file>\n\n" + // the pdf's own metadata, from the document's own metadata. date is the + // document's, never the clock's: a document typeset twice is one document + <> "#set document(\n title: " + <> str(title) + <> ",\n" + <> case model.meta(doc, "creator.author") { + "" -> "" + a -> " author: " <> str(a) <> ",\n" + } + <> " date: none,\n)\n" + // the left margin is wide because the object numbers live in it + <> "#set page(\n paper: \"a4\",\n" + <> " margin: (left: 3.4cm, right: 2cm, top: 2.2cm, bottom: 2.2cm),\n" + <> " numbering: \"1\",\n)\n" + <> "#set text(\n font: (\"DejaVu Serif\", \"Noto Sans CJK JP\"),\n" + <> " size: 10pt,\n lang: " + <> str(lang) + <> case region { + "" -> "" + r -> ",\n region: " <> str(r) + } + <> ",\n)\n" + <> "#set par(justify: true, leading: 0.65em)\n" + // the author's quotation marks and dashes, not typst's + <> "#set smartquote(enabled: false)\n" + <> "#set heading(numbering: none)\n" + <> "#show heading: it => block(above: 1.4em, below: 0.7em, it)\n" + <> "#show raw.where(block: true): it => block(\n" + <> " fill: luma(245), inset: 6pt, radius: 2pt, width: 100%, it)\n\n" + // the object number, in the margin, beside the object's first line, and + // carrying the label every citation of the object points at. One `place` + // with a negative offset, where spine's latex uses \marginpar and loses it + // inside a float or a footnote. + <> "#let ocn(n, body) = block(breakable: true, width: 100%, {\n" + <> " place(left, dx: -1.8cm, dy: 0.2em,\n" + <> " text(size: 7pt, fill: luma(130))[#n#label(\"t\" + str(n))])\n" + <> " body\n})\n" + <> "#let obj(body) = block(breakable: true, width: 100%, body)\n" + // monospace as a font rather than as `raw`: raw() takes a *string*, so it + // cannot carry the nested markup an object may have inside a monospace run + <> "#let mono(body) = text(font: (\"DejaVu Sans Mono\",), body)\n" + <> "#let anchor(name) = [#box(width: 0pt)#label(name)]\n\n" +} + +fn object(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + case obj.is_a { + "heading" -> heading(obj, doc, lab) + "toc" -> toc_entry(obj, doc, lab) + "para" | "blurb" | "glossary" | "bibliography" | "bookindex" -> + para(obj, doc, lab) + "quote" -> quote(obj, doc, lab) + "group" | "block" -> group(obj, doc, lab) + "verse" | "poem" -> verse(obj, doc, lab) + "code" -> #(code(obj), lab) + "table" -> table(obj, doc, lab) + _ -> #("", lab) + } +} + +/// Eight markup levels, eight heading levels. typst has no six-level +/// ceiling, so the document's own depth goes through unchanged - which the +/// html writer could not do and had to say so with aria-level. +fn heading_level(level: String) -> Int { + case level { + "A" -> 1 + "B" -> 2 + "C" -> 3 + "D" -> 4 + "1" -> 5 + "2" -> 6 + "3" -> 7 + "4" -> 8 + _ -> 1 + } +} + +fn heading(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + case obj.dummy_heading { + // a dummy heading is generated furniture whose text repeats the heading + // above it; its anchors are what the table of contents wants + True -> { + let #(a, lab) = anchors(obj, lab) + #(case a { + "" -> "" + _ -> a <> "\n\n" + }, lab) + } + False -> { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) { + "" -> #("", lab) + _ -> { + let #(a, lab) = anchors(obj, lab) + #( + wrap( + obj, + "#heading(level: " + <> int.to_string(heading_level(obj.heading_level)) + <> ")[" + <> a + <> body + <> "]", + ) + <> "\n\n", + lab, + ) + } + } + } + } +} + +/// A table of contents entry: its indent, the heading as a link, a dotted +/// leader, and **the object number as the locator**. +/// +/// Not a page number, and not typst's #outline(), which would make one. A page +/// number is a fact about this typesetting: change the paper, the orientation, +/// the type size, the leading or the margins and every number in the table +/// moves. The object number does not. So the reader is given the reference that +/// holds across every output of this document and across every setting of this +/// one, and it is the same number the margin carries and a citation uses. +/// +/// The text and the number are both links to the object. +fn toc_entry(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + case obj.dummy_heading { + True -> #("", lab) + False -> { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) != "" && points_at_something(obj, lab) { + False -> #("", lab) + True -> { + let indent = case obj.indent_hang > 1 { + True -> obj.indent_hang - 1 + False -> 0 + } + let locator = case toc_target(obj, lab) { + "" -> "" + n -> + " #box(width: 1fr, inset: (x: 3pt), repeat[.]) #link(label(" + <> str(label_of(n)) + <> "))[" + <> n + <> "]" + } + #( + "#block(inset: (left: " + <> int.to_string(indent * 8) + <> "pt))[" + <> body + <> locator + <> "]\n\n", + lab, + ) + } + } + } + } +} + +/// The object number a table of contents entry points at, where it points at +/// one. An entry may name a section by its anchor instead, and then there is +/// no number to show. +fn toc_target(obj: Object, lab: Labels) -> String { + regexp.scan(rgx.any_internal_target(), obj.text) + |> list.fold("", fn(found, m) { + case found { + "" -> { + let frag = rgx.group(m.submatches, 0) + case int.parse(frag), set.contains(lab.defined, label_of(frag)) { + Ok(_), True -> frag + _, _ -> "" + } + } + _ -> found + } + }) +} + +fn para(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + case obj.dummy_heading { + True -> #("", lab) + False -> { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) { + "" -> #("", lab) + _ -> { + let #(a, lab) = anchors(obj, lab) + let bullet = case obj.bullet { + True -> "\u{25cf}~" + False -> "" + } + let content = case obj.indent_base > 0 { + True -> + "#block(inset: (left: " + <> int.to_string(obj.indent_base * 12) + <> "pt))[" + <> a + <> bullet + <> body + <> "]" + False -> a <> bullet <> body + } + #(wrap(obj, content) <> "\n\n", lab) + } + } + } + } +} + +fn quote(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) { + "" -> #("", lab) + _ -> { + let #(a, lab) = anchors(obj, lab) + #( + wrap(obj, "#block(inset: (left: 18pt, right: 18pt))[" <> a <> body <> "]") + <> "\n\n", + lab, + ) + } + } +} + +/// group and block: one object whose text holds its own lines. +fn group(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) { + "" -> #("", lab) + _ -> { + let #(a, lab) = anchors(obj, lab) + #(wrap(obj, "#block[" <> a <> body <> "]") <> "\n\n", lab) + } + } +} + +/// verse: every line break kept, and no justification, because a line of +/// verse ends where the poet ended it. +fn verse(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + let #(body, lab) = inline(obj, doc, lab) + case string.trim(body) { + "" -> #("", lab) + _ -> { + let #(a, lab) = anchors(obj, lab) + #( + wrap( + obj, + "#block(inset: (left: 12pt))[#set par(justify: false);" <> a <> body <> "]", + ) + <> "\n\n", + lab, + ) + } + } +} + +/// code: nothing inside it interpreted, which is what a raw block is for. +/// +/// The one substitution is the non-breaking space marker spine uses inside +/// code to hold a leading indent. The html writer turns it into a space, so +/// this does too; left as it stands it is a lozenge in the middle of a +/// program. +fn code(obj: Object) -> String { + let lang = case obj.code_syntax { + "" -> "" + s -> "lang: " <> str(s) <> ", " + } + let txt = regexp.replace(rgx.nbsp_char(), obj.text, " ") + wrap(obj, "#raw(block: true, " <> lang <> str(txt) <> ")") <> "\n\n" +} + +/// A table. The declared column widths become fractions, which is what a +/// width relative to the others means, and a header row becomes one. +fn table(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + let rows = + regexp.split(rgx.table_delimiter_row(), obj.text) + |> list.filter(fn(r) { string.trim(r) != "" }) + let widths = string.split(obj.table_widths, on: " ") + let aligns = string.split(obj.table_aligns, on: " ") + let cols = int.max(obj.table_cols, 1) + let columns = case widths { + [first, ..] if first != "" -> + widths |> list.map(fn(w) { w <> "fr" }) |> string.join(", ") + _ -> list.repeat("1fr", cols) |> string.join(", ") + } + let align = + list.repeat("", cols) + |> list.index_map(fn(_, i) { align_name(at(aligns, i, "l")) }) + |> string.join(", ") + let #(body, lab) = + rows + |> list.index_map(fn(row, row_idx) { #(row, row_idx) }) + |> list.fold(#("", lab), fn(state, pair) { + let #(out, lab) = state + let #(row, row_idx) = pair + let header = row_idx == 0 && obj.table_header + let #(cells, lab) = + regexp.split(rgx.table_delimiter_col(), row) + |> list.fold(#("", lab), fn(cs, cell) { + let #(so_far, lab) = cs + let #(c, lab) = inline_text(cell, obj, doc, lab) + #( + so_far + <> case header { + True -> " table.cell(fill: luma(240))[#strong[" <> c <> "]],\n" + False -> " [" <> c <> "],\n" + }, + lab, + ) + }) + #(out <> cells, lab) + }) + #( + wrap( + obj, + "#table(\n columns: (" + <> columns + <> "),\n align: (" + <> align + <> "),\n stroke: 0.4pt + luma(180),\n inset: 4pt,\n" + <> body + <> ")", + ) + <> "\n\n", + lab, + ) +} + +fn align_name(a: String) -> String { + case a { + "c" -> "center" + "r" -> "right" + _ -> "left" + } +} + +fn at(items: List(String), index: Int, fallback: String) -> String { + case items, index { + [], _ -> fallback + [first, ..], 0 -> first + [_, ..rest], n -> at(rest, n - 1, fallback) + } +} + +/// An object in its block, with its number in the margin when it has one. +fn wrap(obj: Object, content: String) -> String { + case obj.ocn { + 0 -> "#obj[" <> content <> "]" + n -> "#ocn(" <> int.to_string(n) <> ")[" <> content <> "]" + } +} + +/// The named targets an object carries, as labels with no output. +fn anchors(obj: Object, lab: Labels) -> #(String, Labels) { + let own = case obj.ocn { + 0 -> "" + n -> label_of(int.to_string(n)) + } + list.append([obj.identifier, obj.heading_lev_anchor], obj.anchor_tags) + |> list.fold(#("", lab), fn(state, name) { + let #(out, lab) = state + case name { + "" -> state + _ -> { + let l = label_of(name) + // the ocn's own label is placed on the number in the margin, so it is + // not repeated here + case l == own || set.contains(lab.placed, l) { + True -> state + False -> #( + out <> "#anchor(" <> str(l) <> ")", + Labels(..lab, placed: set.insert(lab.placed, l)), + ) + } + } + } + }) +} + +fn inline(obj: Object, doc: Document, lab: Labels) -> #(String, Labels) { + inline_text(obj.text, obj, doc, lab) +} + +fn inline_text( + txt: String, + obj: Object, + doc: Document, + lab: Labels, +) -> #(String, Labels) { + let txt = notes(txt, obj, lab) + let txt = images(txt) + let txt = links(txt, obj, lab) + let txt = font_face(txt) + let #(txt, lab) = inline_anchors(txt, lab) + let txt = group_indents(txt, obj) + let txt = breaks(txt, obj) + let _ = doc + #(unprotect(escape(txt)), lab) +} + +// The markers that protect a replacement from the escaping that follows. A +// private-use pair, because they must not occur in a document. +const keep_open = 0xe000 + +const keep_close = 0xe001 + +fn keep(typst_source: String) -> String { + cp(keep_open) <> typst_source <> cp(keep_close) +} + +fn unprotect(txt: String) -> String { + txt |> string.replace(cp(keep_open), "") |> string.replace(cp(keep_close), "") +} + +fn cp(n: Int) -> String { + let assert Ok(c) = string.utf_codepoint(n) + string.from_utf_codepoints([c]) +} + +/// Everything outside a protected run is escaped. +/// +/// The rule is deliberately blunt: every character typst can read as syntax +/// is escaped wherever it occurs, not only where it would be read that way. A +/// narrower rule kept failing, because typst continues a function call across +/// whatever follows it: `#emph[a](b)` reads "(b)" as a second argument list +/// and `#emph[a].b` as a field access, so a "(" or a "." of the author's own, +/// landing after any call this writer emits, is a compile error. Escaping them +/// everywhere makes the set closed, and a closed set is checkable. +/// +/// The protection nests, so it is counted and not a flag. A link built by +/// `links` is protected, and a line break inside its text is then protected +/// again by `breaks`. +/// +/// Codepoints rather than graphemes, because the D writer walks codepoints +/// and the two must agree byte for byte. +fn escape(txt: String) -> String { + let #(out, _, _) = + string.to_utf_codepoints(txt) + |> list.fold(#([], 0, 0), fn(state, c) { + let #(acc, depth, previous) = state + let n = string.utf_codepoint_to_int(c) + case n == keep_open, n == keep_close, depth > 0 { + True, _, _ -> #(acc, depth + 1, previous) + _, True, _ -> #(acc, int.max(depth - 1, 0), 0) + _, _, True -> #([c, ..acc], depth, previous) + _, _, False -> + case is_syntax(n), n == 0x2d && previous == 0x2d { + True, _ -> #([c, backslash(), ..acc], depth, n) + // a second dash would be an en dash, so it is escaped + _, True -> #([c, backslash(), ..acc], depth, n) + _, _ -> #([c, ..acc], depth, n) + } + } + }) + string.from_utf_codepoints(list.reverse(out)) +} + +/// "\", "#", "[", "]", "*", "_", "$", "@", "<", ">", "`", "~", "(", ")", +/// ".", "/" and the quotation mark. A "." also begins an ellipsis and a "/" a +/// comment, both of which this covers. The quotation mark is not typst syntax +/// in markup and is escaped anyway, so that a bare one in the written source +/// always delimits a string literal, which is what `guard_block_starts` +/// reads. +fn is_syntax(n: Int) -> Bool { + case n { + 0x5c | 0x23 | 0x5b | 0x5d | 0x2a | 0x5f | 0x24 | 0x40 -> True + 0x3c | 0x3e | 0x60 | 0x7e | 0x28 | 0x29 | 0x2e | 0x2f -> True + 0x22 -> True + _ -> False + } +} + +fn backslash() -> UtfCodepoint { + let assert Ok(c) = string.utf_codepoint(0x5c) + c +} + +/// A typst string literal. +fn str(s: String) -> String { + "\"" + <> { + s + |> string.replace("\\", "\\\\") + |> string.replace("\"", "\\\"") + |> string.replace("\n", "\\n") + } + <> "\"" +} + +/// A note becomes a footnote *at the point that refers to it*, carrying the +/// document's own number rather than typst's count, and a way back to the +/// object it came from. +fn notes(txt: String, obj: Object, lab: Labels) -> String { + let back = case obj.ocn { + 0 -> "" + n -> { + let l = label_of(int.to_string(n)) + case set.contains(lab.defined, l) { + True -> " #link(label(" <> str(l) <> "))[#sym.arrow.l.hook]" + False -> "" + } + } + } + rgx.replace_with(rgx.inline_notes_any_mark(), txt, fn(g) { + // the note's own text is content in its own right, so it goes through the + // same pipeline - links, faces, escaping - and nothing else + let body = + escape(breaks(font_face(links(images(g(1)), obj, lab)), obj)) + keep( + "#footnote(numbering: _ => " + <> str(g(0)) + <> ")[" + <> unprotect(body) + <> back + <> "]", + ) + }) +} + +/// A link. An internal target becomes a link to a label, and only when the +/// label will exist: typst refuses to compile a link to one that will not, +/// which is the check and so also the constraint. +fn links(txt: String, obj: Object, lab: Labels) -> String { + let txt = + rgx.replace_with(rgx.inline_link_number_only(), txt, fn(g) { + let url = case int.parse(g(1)) { + Ok(n) -> at(obj.stow_link, n, "") + Error(_) -> "" + } + g(0) <> "┤" <> url <> "├" + }) + let txt = rgx.replace_with(rgx.inline_link_empty(), txt, fn(g) { g(0) }) + let txt = + rgx.replace_with(rgx.inline_link(), txt, fn(g) { + let text = escape(font_face(g(0))) + // an internal link to another segment: this is one document, so the + // segment is this document and only the fragment matters + let target = case string.starts_with(g(1), "¤") { + False -> g(1) + True -> + case regexp.scan(rgx.internal_target(), g(1)) { + [m, ..] -> rgx.group(m.submatches, 1) + [] -> g(1) + } + } + case string.starts_with(target, "#"), target { + True, _ -> { + let l = label_of(string.drop_start(target, 1)) + keep(case set.contains(lab.defined, l) { + True -> "#link(label(" <> str(l) <> "))[" <> unprotect(text) <> "]" + False -> unprotect(text) + }) + } + False, "" -> keep(unprotect(text)) + False, _ -> + keep("#link(" <> str(target) <> ")[" <> unprotect(text) <> "]") + } + }) + regexp.replace(rgx.mark_internal_site_lnk(), txt, "") +} + +/// An image, with its own width where the document declared one. +/// +/// Capped at the text width: the declared width is in pixels and a 600 pixel +/// figure read as 600 points is wider than the page. +fn images(txt: String) -> String { + rgx.replace_with(rgx.inline_image(), txt, fn(g) { + let width = case int.parse(g(3)) { + Ok(w) if w > 0 -> "width: " <> int.to_string(int.min(w, 440)) <> "pt" + _ -> "width: 100%" + } + let post = g(5) + let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) { + [m, ..] -> #(rgx.group(m.submatches, 0), case + string.split_once(post, m.content) + { + Ok(#(_, after)) -> after + Error(_) -> post + }) + [] -> #("", post) + } + let img = + "#image(" + <> str("image/" <> g(2)) + <> ", " + <> width + <> case alt { + "" -> "" + a -> ", alt: " <> str(a) + } + <> ")" + // the opening link marker is kept, exactly as the html writer keeps it: + // an image in the markup sits inside a link whose target may be empty or + // may be a url, and the link handling that follows needs the pair. + g(0) <> keep(img) <> rest + }) +} + +/// The font faces, as typst functions rather than typst markup: a function +/// call cannot be broken by the character beside it, and the markup form can. +fn font_face(txt: String) -> String { + [ + #(rgx.inline_emphasis(), "emph"), + #(rgx.inline_bold(), "strong"), + #(rgx.inline_italics(), "emph"), + #(rgx.inline_underscore(), "underline"), + #(rgx.inline_superscript(), "super"), + #(rgx.inline_subscript(), "sub"), + #(rgx.inline_mono(), "mono"), + #(rgx.inline_strike(), "strike"), + #(rgx.inline_insert(), "underline"), + #(rgx.inline_cite(), "emph"), + ] + |> list.fold(txt, fn(acc, face) { + let #(pattern, fname) = face + rgx.replace_with(pattern, acc, fn(g) { + keep("#" <> fname <> "[") <> g(0) <> keep("]") + }) + }) +} + +/// An anchor the markup placed in the text. +fn inline_anchors(txt: String, lab: Labels) -> #(String, Labels) { + case regexp.scan(rgx.inline_link_anchor(), txt) { + [] -> #(txt, lab) + matches -> { + let #(out, rest, lab) = + list.fold(matches, #("", txt, lab), fn(state, m) { + let #(out, rest, lab) = state + case string.split_once(rest, m.content) { + Error(_) -> state + Ok(#(before, after)) -> { + let l = label_of(rgx.group(m.submatches, 0)) + case + set.contains(lab.defined, l) && !set.contains(lab.placed, l) + { + False -> #(out <> before, after, lab) + True -> #( + out <> before <> keep("#anchor(" <> str(l) <> ")"), + after, + Labels(..lab, placed: set.insert(lab.placed, l)), + ) + } + } + } + }) + #(out <> rest, lab) + } + } +} + +/// The line breaks the markup asked for. Not inside code, whose text is +/// preserved as it stands. +fn breaks(txt: String, obj: Object) -> String { + case obj.is_a { + "code" -> txt + _ -> { + let br = keep("#linebreak()") + txt + |> regexp.replace(rgx.br_line(), _, br) + |> regexp.replace(rgx.br_line_inline(), _, br) + |> regexp.replace(rgx.br_line_spaced(), _, br <> br) + |> regexp.replace(rgx.line_break(), _, br) + |> regexp.replace(rgx.nbsp_char(), _, " ") + } + } +} + +/// The line-leading markers of a grouped object: an indent as a measurement, +/// a hanging indent's inner level, a bullet. +/// +/// Spine writes runs of non-breaking spaces here, up to eighteen of them for +/// one line. A measurement says the same intent in a way a typesetter can act +/// on, and the author's text keeps its own spaces. +fn group_indents(txt: String, obj: Object) -> String { + case obj.is_a == "group" || obj.is_a == "block" { + False -> txt + True -> + txt + |> rgx.replace_with(rgx.grouped_para_indent_hang(), _, fn(g) { + pad(g(1)) + }) + |> rgx.replace_with(rgx.grouped_para_bullet_at(), _, fn(g) { + pad(g(0)) <> keep("\u{25cf}") <> " " + }) + |> rgx.replace_with(rgx.grouped_para_bullet(), _, fn(_) { + keep("\u{25cf}") <> " " + }) + |> rgx.replace_with(rgx.grouped_para_indent(), _, fn(g) { pad(g(0)) }) + } +} + +fn pad(n: String) -> String { + keep("#h(" <> n <> " * 12pt)") +} + +/// A "- ", "+ " or "= " that begins a line or a content block. +/// +/// typst reads those three as a list item, an enumeration item and a heading, +/// but only in that position, so they are guarded there rather than escaped in +/// every hyphen of the document. An enumeration also starts at "1. ", which +/// needs nothing: the escaper takes every "." already. +fn guard_block_starts(s: String) -> String { + walk_guard(string.to_graphemes(s), [], False) +} + +// and not inside a string literal. A code object is written as +// #raw(block: true, "..."), and a guard inserted in there is a backslash in +// the middle of somebody's program rather than an escape: one sample has +// "[+ " in a shell snippet, and the html writer shows no backslash. +fn walk_guard( + rest: List(String), + acc: List(String), + in_string: Bool, +) -> String { + case rest, in_string { + [], _ -> string.concat(list.reverse(acc)) + ["\\", next, ..more], True -> walk_guard(more, [next, "\\", ..acc], True) + ["\"", ..more], True -> walk_guard(more, ["\"", ..acc], False) + [c, ..more], True -> walk_guard(more, [c, ..acc], True) + ["\"", ..more], False -> walk_guard(more, ["\"", ..acc], True) + [c, m, " ", ..more], False -> + case { c == "[" || c == "\n" } && { m == "-" || m == "+" || m == "=" } { + True -> walk_guard(more, [" ", m, "\\", c, ..acc], False) + False -> walk_guard([m, " ", ..more], [c, ..acc], False) + } + [c, ..more], False -> walk_guard(more, [c, ..acc], False) + } +} + +/// Whether a table of contents entry points at anything this writer renders. +/// +/// A table of contents names what is in the document, so an entry whose only +/// target is gone is not an entry: the endnotes are footnotes here, so the +/// section they had is gone and so is the line that pointed at it. +/// +/// Decided from the *target*, not from the rendered text. An earlier version +/// dropped any entry that came out without a link in it, and so dropped +/// gpl3's copyright line, whose markup nests one link inside another - which +/// the flat markers cannot express, so no writer resolves it and the entry has +/// no link to show. +fn points_at_something(obj: Object, lab: Labels) -> Bool { + case regexp.scan(rgx.any_internal_target(), obj.text) { + [] -> True + matches -> + list.any(matches, fn(m) { + set.contains(lab.defined, label_of(rgx.group(m.submatches, 0))) + }) + } +} diff --git a/gleam/src/write/zip.gleam b/gleam/src/write/zip.gleam new file mode 100644 index 0000000..05cb56a --- /dev/null +++ b/gleam/src/write/zip.gleam @@ -0,0 +1,132 @@ +//// Writing a zip, stored only. +//// +//// An epub is a zip with rules: the first entry must be "mimetype", stored, +//// with no extra field, so that a reading system can identify the file from +//// its first bytes without inflating anything. Everything else may be +//// deflated and does not have to be. +//// +//// So: stored only. Deflate would buy compression on a sample writer's +//// output, which is not what is being demonstrated, and it would put a second +//// thing in the way of the two writers agreeing byte for byte. +//// +//// No timestamps. Every entry is dated 1980-01-01, the epoch of the format, +//// which is what makes the same document produce the same bytes twice. + +import gleam/bit_array +import gleam/list + +pub type Entry { + Entry(name: String, data: BitArray) +} + +const method_stored = 0 + +/// 2.0, which stored requires +const version_needed = 20 + +const dos_time = 0 + +/// 1980-01-01 +const dos_date = 0x0021 + +/// The archive, entries in the order given. +pub fn write(entries: List(Entry)) -> BitArray { + let #(local, offsets, crcs, _) = + list.fold(entries, #(<<>>, [], [], 0), fn(state, e) { + let #(acc, offsets, crcs, at) = state + let crc = crc32(e.data) + let header = local_header(e, crc) + let size = bit_array.byte_size(header) + bit_array.byte_size(e.data) + #( + bit_array.concat([acc, header, e.data]), + [at, ..offsets], + [crc, ..crcs], + at + size, + ) + }) + let offsets = list.reverse(offsets) + let crcs = list.reverse(crcs) + let cd_offset = bit_array.byte_size(local) + let central = + list.zip(entries, list.zip(crcs, offsets)) + |> list.map(fn(t) { + let #(e, #(crc, offset)) = t + central_entry(e, crc, offset) + }) + |> bit_array.concat + let cd_size = bit_array.byte_size(central) + bit_array.concat([ + local, + central, + end_record(list.length(entries), cd_size, cd_offset), + ]) +} + +fn local_header(e: Entry, crc: Int) -> BitArray { + let name = <<e.name:utf8>> + let size = bit_array.byte_size(e.data) + bit_array.concat([ + <<0x04034b50:little-size(32)>>, + <<version_needed:little-size(16)>>, + // flags: none; the name is ascii + <<0:little-size(16)>>, + <<method_stored:little-size(16)>>, + <<dos_time:little-size(16)>>, + <<dos_date:little-size(16)>>, + <<crc:little-size(32)>>, + <<size:little-size(32)>>, + <<size:little-size(32)>>, + <<bit_array.byte_size(name):little-size(16)>>, + // no extra field, which "mimetype" requires + <<0:little-size(16)>>, + name, + ]) +} + +fn central_entry(e: Entry, crc: Int, offset: Int) -> BitArray { + let name = <<e.name:utf8>> + let size = bit_array.byte_size(e.data) + bit_array.concat([ + <<0x02014b50:little-size(32)>>, + // made by + <<version_needed:little-size(16)>>, + <<version_needed:little-size(16)>>, + <<0:little-size(16)>>, + <<method_stored:little-size(16)>>, + <<dos_time:little-size(16)>>, + <<dos_date:little-size(16)>>, + <<crc:little-size(32)>>, + <<size:little-size(32)>>, + <<size:little-size(32)>>, + <<bit_array.byte_size(name):little-size(16)>>, + // extra, comment, disk, internal attributes + <<0:little-size(16)>>, + <<0:little-size(16)>>, + <<0:little-size(16)>>, + <<0:little-size(16)>>, + // external attributes + <<0:little-size(32)>>, + <<offset:little-size(32)>>, + name, + ]) +} + +fn end_record(count: Int, cd_size: Int, cd_offset: Int) -> BitArray { + bit_array.concat([ + <<0x06054b50:little-size(32)>>, + // this disk, and the disk the central directory starts on + <<0:little-size(16)>>, + <<0:little-size(16)>>, + <<count:little-size(16)>>, + <<count:little-size(16)>>, + <<cd_size:little-size(32)>>, + <<cd_offset:little-size(32)>>, + // no archive comment + <<0:little-size(16)>>, + ]) +} + +/// Erlang has crc32 in the zlib module, which is built in: the NIF is for +/// compression, and the checksum is not. +@external(erlang, "erlang", "crc32") +fn crc32(data: BitArray) -> Int diff --git a/gleam/test/ocda_write_test.gleam b/gleam/test/ocda_write_test.gleam new file mode 100644 index 0000000..433dfdc --- /dev/null +++ b/gleam/test/ocda_write_test.gleam @@ -0,0 +1,233 @@ +import gleam/list +import gleam/string +import gleeunit +import gleeunit/should +import ocda/load_ssp +import ocda/model +import gleam/bit_array +import write/epub +import write/html +import write/text + +pub fn main() { + gleeunit.main() +} + +/// A document small enough to reason about, carrying the things the writer +/// has to get right: a heading, an object with a note inside it, an object +/// with no ocn, and a trailing section with only its own heading. +const minimal = "% SiSU Document Abstraction v2.0 +% Source: example.sst + +@meta { + title.full: An Example + creator.author: A Writer + date.published: 2026-09-25 +} + +@head { +[0] heading :A the_title +.part: frontmatter +| An Example + +} + +@toc { +[0] toc +.indent: 0 4 +| ┥Chapter One┝┤2├ + +} + +@body { +[1] heading :1 c1 +.part: body +| Chapter One + +[2] para +.part: body +.parent: 1 +| A paragraph with a note【1 The note itself.】in it. + +} + +@glossary { +[0] heading :A glossary +.part: backmatter +| Glossary + +} +" + +fn render() -> String { + text.write(load_ssp.parse(minimal)) +} + +/// head, toc, body. Not glossary: it holds its own heading and nothing else, +/// and spine omits a trailing section of one object. +pub fn sections_test() { + let doc = load_ssp.parse(minimal) + text.text_sections(doc) |> should.equal(["head", "toc", "body"]) + render() |> string.contains(contain: "Glossary") |> should.be_false +} + +/// A heading is preceded by a blank line; an object with ocn 0 gets no mark. +pub fn heading_test() { + render() |> string.contains(contain: "\nAn Example\n\n") |> should.be_true + render() |> string.contains(contain: "「0」") |> should.be_false +} + +/// The ocn follows the object, in its own brackets. +pub fn ocn_test() { + render() |> string.contains(contain: "「1」") |> should.be_true + render() |> string.contains(contain: "「2」") |> should.be_true +} + +/// A note is carried inside the object that refers to it: the reference +/// becomes [n] in place and the note is appended after the object. +pub fn note_test() { + let out = render() + out |> string.contains(contain: "a note[1]in it.") |> should.be_true + out |> string.contains(contain: "1. The note itself.") |> should.be_true +} + +/// A toc entry's link becomes its text and, its target being an object, +/// the ocn it points at; one level in use, so no indent. +pub fn toc_test() { + render() + |> string.contains(contain: "\nChapter One ≫2\n") + |> should.be_true +} + +/// The footer names the document and then the program. +pub fn tail_test() { + let out = render() + out |> string.contains(contain: "Title: An Example") |> should.be_true + out |> string.contains(contain: "Author: A Writer") |> should.be_true + out |> string.contains(contain: "Published: 2026-09-25") |> should.be_true + // no copyright or licence in this document, so no line for either + out |> string.contains(contain: "Copyright:") |> should.be_false + out |> string.ends_with("spine\nhttps://sisudoc.org\n") |> should.be_true +} + +/// The writer is given the model and nothing else, so an object it has no +/// case for writes nothing rather than failing. +pub fn unknown_kind_writes_nothing_test() { + let doc = load_ssp.parse(minimal) + let body = model.section(doc, "body") + body |> list.length |> should.equal(2) +} + +// ---- the html writer ----------------------------------------------------- + +fn render_html() -> String { + html.write(load_ssp.parse(minimal)) +} + +/// A level A heading is h1. Spine writes <h0>, which is not an element. +pub fn heading_element_test() { + let out = render_html() + out |> string.contains(contain: "<h1 id=\"the_title\">") |> should.be_true + out |> string.contains(contain: "<h0") |> should.be_false +} + +/// One charset declaration, and no placeholder metadata: a document with no +/// publisher gets no dc.publisher rather than the word "Publisher". +pub fn head_test() { + let out = render_html() + out |> string.contains(contain: "<meta charset=\"utf-8\" />") |> should.be_true + out |> string.contains(contain: "text/plain") |> should.be_false + out |> string.contains(contain: "dc.publisher") |> should.be_false + out |> string.contains(contain: "content=\"year\"") |> should.be_false +} + +/// The object number is a span, not a label, and links to itself. +pub fn ocn_mark_test() { + render_html() + |> string.contains(contain: "<span class=\"ocn\"><a href=\"#2\" class=\"lnkocn\">2</a></span>") + |> should.be_true +} + +/// A note reference points at the gathered note and carries the dpub role. +pub fn noteref_test() { + render_html() + |> string.contains( + contain: "<a id=\"noteref_1\" href=\"#note_1\" role=\"doc-noteref\">1</a>", + ) + |> should.be_true +} + +/// A section carries its dpub-aria role and no invented id. +pub fn section_test() { + let out = render_html() + out |> string.contains(contain: "<section class=\"toc\" role=\"doc-toc\">") |> should.be_true + out |> string.contains(contain: "id=\"section_toc\"") |> should.be_false +} + +/// It is self-contained: the stylesheet is in the file, not linked out of it. +pub fn self_contained_test() { + let out = render_html() + out |> string.contains(contain: "<style>") |> should.be_true + out |> string.contains(contain: "html_scroll.css") |> should.be_false +} + +/// An ampersand in the text becomes &, and a quotation mark in an +/// attribute becomes ". Erlang's re:replace reads "&" in a replacement +/// *string* as the whole match, which turned "&" into the match itself; +/// every replacement here goes through the function form instead. +pub fn escaping_test() { + let doc = + load_ssp.parse( + "% SiSU Document Abstraction v2.0 + +@body { +[1] para +.part: body +| Smith & Jones said \"hello\" <here> + +} +", + ) + let out = html.write(doc) + out |> string.contains(contain: "Smith & Jones") |> should.be_true + out |> string.contains(contain: "<here>") |> should.be_true + out |> string.contains(contain: "\"quot;") |> should.be_false +} + +// ---- the epub writer ----------------------------------------------------- + +/// The container is first and stored: a reading system identifies an epub +/// from the first bytes of the file, so "mimetype" has to be there and +/// uncompressed. +pub fn epub_mimetype_first_test() { + let bytes = epub.write(load_ssp.parse(minimal), "test/nothing.ssp") + // local file header, then the name at offset 30 + let assert Ok(head) = bit_array.slice(bytes, 0, 4) + head |> should.equal(<<0x50, 0x4b, 0x03, 0x04>>) + let assert Ok(name) = bit_array.slice(bytes, 30, 8) + name |> should.equal(<<"mimetype":utf8>>) + let assert Ok(kind) = bit_array.slice(bytes, 38, 20) + kind |> should.equal(<<"application/epub+zip":utf8>>) +} + +/// Every entry has the same date, so that the same document gives the same +/// epub twice. +pub fn epub_is_reproducible_test() { + let doc = load_ssp.parse(minimal) + epub.write(doc, "test/nothing.ssp") + |> should.equal(epub.write(doc, "test/nothing.ssp")) +} + +/// The segments are grouped by name, not by run: a section's objects are not +/// always contiguous, and two zip entries of one name is an epubcheck error. +pub fn epub_segments_are_unique_test() { + let segs = epub.segments(load_ssp.parse(minimal)) + let names = list.map(segs, fn(s) { s.file }) + list.length(names) |> should.equal(list.length(list.unique(names))) +} + +/// A `.ssp` outside a pod cannot give images, and the writer asks before it +/// emits one: an `<img>` whose file is not in the manifest is an error. +pub fn epub_images_test() { + epub.images_named(load_ssp.parse(minimal)) |> should.equal([]) +} |