summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--README.md38
-rw-r--r--dlang/dub.json2
-rw-r--r--dlang/flake.nix6
-rw-r--r--dlang/src/ocda/load.d17
-rw-r--r--dlang/src/ocda/load_pod.d115
-rw-r--r--dlang/src/ocda/zip.d115
-rw-r--r--dlang/src/ocda/zstd.d79
-rw-r--r--gleam/src/ocda/load.gleam38
-rw-r--r--gleam/src/ocda/load_pod.gleam138
-rw-r--r--gleam/src/ocda/zip.gleam170
-rw-r--r--gleam/test/ocda_lib_test.gleam21
11 files changed, 711 insertions, 28 deletions
diff --git a/README.md b/README.md
index efa9e70..59e3d96 100644
--- a/README.md
+++ b/README.md
@@ -102,13 +102,37 @@ serialisations, and naming it is the useful part.
## What it reads, and what it does not
-| | |
-|---|---|
-| `.ssp` | yes |
-| `.ocda.db` | yes, any language of it |
-| `.sisupod` | not yet: unwrap the archive and read the `.ssp` inside it |
-| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work |
-| a url | no |
+| | | |
+|---|---|---|
+| `.ssp` | yes | |
+| `.ocda.db` | yes, any language of it | |
+| `.sisupod` | yes, any language it carries | D only, see below |
+| `.zip` pod | yes, any language it carries | |
+| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work | |
+| a url | no | |
+
+A pod carries the abstraction only when spine was asked for it: `--pod2`
+puts it in, `--source` does not. A pod without one is refused with a message
+saying which flag makes a pod this library can read, because that is a fact
+about the pod rather than about the library.
+
+**Over the reference collection, a `.sisupod` gives the same model as the
+`.ssp` beside it: 36 of 36 document-languages, byte identical dumps.** A
+writer over the library therefore gained a third input form without being
+touched, which is the point of the dispatch being in one place.
+
+### the one thing the Gleam library cannot do
+
+A `.sisupod` is **one zstd frame wrapping** the zip. The D library reads both
+through a declared `libzstd`. The Gleam library reads the **zip and not the
+frame**, because the only zstd on the BEAM is a NIF that clones zstd from
+github in its own build and so cannot be built in a sandbox without patching
+it.
+
+So a `.zip` pod is read in both languages, a `.sisupod` in D only, and the
+Gleam refusal says why and says `zstd -d` rather than claiming the file is
+malformed. It is a packaging limit, not anything about the format, and it is
+the only place the two libraries differ in what they accept.
`ocda/load` is the one place in each language that decides what a path is,
so a new input form is added once and every reader and writer gains it.
diff --git a/dlang/dub.json b/dlang/dub.json
index f49ef85..373e5bd 100644
--- a/dlang/dub.json
+++ b/dlang/dub.json
@@ -9,5 +9,5 @@
"targetType": "library",
"sourcePaths": [ "./src" ],
"importPaths": [ "./src" ],
- "libs": [ "sqlite3" ]
+ "libs": [ "sqlite3", "zstd" ]
}
diff --git a/dlang/flake.nix b/dlang/flake.nix
index 26dbe2d..2ad3b1c 100644
--- a/dlang/flake.nix
+++ b/dlang/flake.nix
@@ -21,11 +21,11 @@
version = "0.1.0";
src = self;
nativeBuildInputs = [ compiler ];
- buildInputs = [ pkgs.sqlite ];
+ buildInputs = [ pkgs.sqlite pkgs.zstd ];
buildPhase = ''
runHook preBuild
export HOME=$TMPDIR
- ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3
+ ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 -L-lzstd
runHook postBuild
'';
installPhase = ''
@@ -57,7 +57,7 @@
};
devShells.${system}.default = pkgs.mkShell {
- packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite ];
+ packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ];
};
};
}
diff --git a/dlang/src/ocda/load.d b/dlang/src/ocda/load.d
index 0f630bc..89c8dcc 100644
--- a/dlang/src/ocda/load.d
+++ b/dlang/src/ocda/load.d
@@ -17,16 +17,24 @@ import std.algorithm : endsWith;
import ocda.model;
import ocda.load_ssp;
import ocda.load_db;
+import ocda.load_pod;
enum Source {
ssp, // .ssp, the abstraction as text
ocda_db, // .ocda.db, the abstraction as sqlite
+ pod, // .sisupod or .zip, an archive carrying a .ssp
unknown,
}
Source sourceOf(string path) {
if (path.endsWith(".ocda.db")) { return Source.ocda_db; }
if (path.endsWith(".ssp")) { return Source.ssp; }
+ /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote
+ before, and every pod already published carries that name, so both
+ are read. What is inside decides how it is opened, not the name.
+ +/
+ if (path.endsWith(".sisupod")) { return Source.pod; }
+ if (path.endsWith(".zip")) { return Source.pod; }
if (path.endsWith(".db")) { return Source.ocda_db; }
return Source.unknown;
}
@@ -35,6 +43,7 @@ string sourceName(Source s) {
final switch (s) {
case Source.ssp: return ".ssp (the abstraction as text)";
case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)";
+ case Source.pod: return "a pod archive carrying a .ssp";
case Source.unknown: return "not an abstraction this library reads";
}
}
@@ -44,15 +53,17 @@ class OcdaLoadError : Exception {
}
/+ ↓ load whatever the path is into the model.
- lang applies only to a .ocda.db, which holds every language of its
- document; a .ssp is one language and the argument is ignored.
+ lang names which language to take from a .ocda.db, which holds
+ every language of its document, and from a pod, which may carry more
+ than one .ssp. A .ssp is one language by construction and ignores it.
+/
OcdaDocument load(string path, string lang = "", bool any_version = false) {
final switch (sourceOf(path)) {
case Source.ssp: return parseSspFile(path);
case Source.ocda_db: return loadDbFile(path, lang, any_version);
+ case Source.pod: return loadPodFile(path, lang);
case Source.unknown:
throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown)
- ~ " (expected .ssp or .ocda.db)");
+ ~ " (expected .ssp, .ocda.db, .sisupod or .zip)");
}
}
diff --git a/dlang/src/ocda/load_pod.d b/dlang/src/ocda/load_pod.d
new file mode 100644
index 0000000..75993be
--- /dev/null
+++ b/dlang/src/ocda/load_pod.d
@@ -0,0 +1,115 @@
+module ocda.load_pod;
+
+/+ ↓ the abstraction out of a pod archive.
+ .
+ A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same
+ archive unwrapped, which is what spine published before it wrapped them.
+ Either way, a pod built with --pod2 carries the abstraction inside it,
+ at media/abstraction/<uid>.<lang>.ssp, and that is what is read.
+ .
+ So this is a third input form and *not* a third model: the .ssp inside
+ is parsed by ocda.load_ssp and comes back as the same value. A writer
+ over the library gains .sisupod without being touched, which is the
+ whole reason the dispatch is in one place.
+ .
+ A pod built without --pod2 carries markup and no abstraction. That is
+ refused, and the refusal says which flag makes a pod this reader can
+ read, because "no abstraction in it" is a fact about the pod rather
+ than about the reader.
++/
+
+import std.algorithm : endsWith, sort;
+import std.array : array;
+import std.file : read;
+import std.string : indexOf;
+import ocda.model;
+import ocda.load_ssp;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ a limit on what a frame may expand to.
+ A reader of an artefact it did not make declines to be told how much
+ memory to allocate. 512 MiB is far above the largest pod in the
+ reference collection (4 MiB) and far below anything that matters.
++/
+enum size_t max_pod_size = 512 * 1024 * 1024;
+
+class OcdaPodError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ which languages of the abstraction the pod carries, in sorted order +/
+string[] podLanguages(string path) {
+ auto bytes = archiveBytes(path);
+ string[] out_;
+ foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) {
+ string lang = langOfSspName(e.name);
+ if (lang.length > 0) { out_ ~= lang; }
+ }
+ return out_.sort.array;
+}
+
+/+ ↓ load one language of the abstraction a pod carries.
+ lang == "" takes the only one there is, and where there is a choice it
+ says so rather than choosing, as the .ocda.db loader does.
++/
+OcdaDocument loadPodFile(string path, string lang = "") {
+ auto bytes = archiveBytes(path);
+ auto entries = zipEntries(bytes);
+ auto ssps = zipFindSuffix(entries, ".ssp");
+ if (ssps.length == 0) {
+ throw new OcdaPodError(path ~ ": carries no .ssp."
+ ~ " A pod carries the abstraction only when spine was asked for it:"
+ ~ " spine --pod2 puts it in, spine --source does not.");
+ }
+ ZipEntry chosen = ssps[0];
+ if (lang.length > 0) {
+ bool found = false;
+ foreach (e; ssps) {
+ if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; }
+ }
+ if (!found) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": no language '" ~ lang
+ ~ "'; it holds: " ~ have);
+ }
+ } else if (ssps.length > 1) {
+ string have;
+ foreach (i, e; ssps) {
+ have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name);
+ }
+ throw new OcdaPodError(path ~ ": holds more than one language ("
+ ~ have ~ "); name the one wanted");
+ }
+ auto ssp = zipRead(bytes, chosen);
+ return parseSsp(cast(string) ssp.idup);
+}
+
+/+ ↓ the archive, unwrapped if it was wrapped.
+ The bytes decide, not the name: a .sisupod is a zstd frame and a .zip
+ is not, and a file misnamed either way still reads.
++/
+private ubyte[] archiveBytes(string path) {
+ auto bytes = cast(ubyte[]) read(path);
+ return zstdIsFrame(bytes)
+ ? zstdDecompress(bytes, max_pod_size)
+ : bytes;
+}
+
+/+ ↓ <uid>.<lang>.ssp -> <lang> +/
+private string langOfSspName(string name) {
+ if (!name.endsWith(".ssp")) { return ""; }
+ string stem = name[0 .. $ - 4];
+ auto dot = stem.lastIndexOf('.');
+ return (dot > 0) ? stem[dot + 1 .. $] : "";
+}
+
+private ptrdiff_t lastIndexOf(string s, char c) {
+ for (ptrdiff_t i = s.length - 1; i >= 0; i--) {
+ if (s[i] == c) { return i; }
+ }
+ return -1;
+}
diff --git a/dlang/src/ocda/zip.d b/dlang/src/ocda/zip.d
new file mode 100644
index 0000000..9dff6f4
--- /dev/null
+++ b/dlang/src/ocda/zip.d
@@ -0,0 +1,115 @@
+module ocda.zip;
+
+/+ ↓ enough of the zip format to find one file in a pod and read it.
+ .
+ Not a zip library. A pod archive holds a handful of entries, spine
+ writes them **stored** (uncompressed) so that the outer zstd frame does
+ the compressing once over the whole thing, and what is wanted from it
+ here is one named entry. So: read the central directory, find the
+ entry, return its bytes.
+ .
+ Deflate is *refused rather than mishandled*. Phobos can inflate, but a
+ library that says it read an entry it in fact skipped is worse than one
+ that says it cannot, and spine's own archives do not need it.
++/
+
+import std.algorithm : endsWith;
+import std.conv : to;
+
+class OcdaZipError : Exception {
+ this(string msg) { super(msg); }
+}
+
+struct ZipEntry {
+ string name;
+ uint method; // 0 stored, 8 deflate
+ uint compressed;
+ uint uncompressed;
+ uint local_offset;
+}
+
+/+ ↓ the entries, from the central directory at the end of the file +/
+ZipEntry[] zipEntries(const(ubyte)[] bytes) {
+ enum uint eocd_sig = 0x06054b50;
+ enum uint cd_sig = 0x02014b50;
+ if (bytes.length < 22) {
+ throw new OcdaZipError("too short to be a zip archive");
+ }
+ /+ ↓ the end of central directory record is last, and may be followed by a
+ comment of up to 65535 bytes, so it is searched for backwards +/
+ ptrdiff_t eocd = -1;
+ ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0;
+ for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) {
+ if (u32(bytes, i) == eocd_sig) { eocd = i; break; }
+ }
+ if (eocd < 0) {
+ throw new OcdaZipError("no end of central directory record;"
+ ~ " it does not look like a zip archive");
+ }
+ uint count = u16(bytes, eocd + 10);
+ uint offset = u32(bytes, eocd + 16);
+ ZipEntry[] out_;
+ size_t p = offset;
+ foreach (_; 0 .. count) {
+ if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) {
+ throw new OcdaZipError("central directory entry "
+ ~ out_.length.to!string ~ " is malformed");
+ }
+ ZipEntry e;
+ e.method = u16(bytes, p + 10);
+ e.compressed = u32(bytes, p + 20);
+ e.uncompressed = u32(bytes, p + 24);
+ uint name_len = u16(bytes, p + 28);
+ uint extra_len = u16(bytes, p + 30);
+ uint cmnt_len = u16(bytes, p + 32);
+ e.local_offset = u32(bytes, p + 42);
+ e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup;
+ out_ ~= e;
+ p += 46 + name_len + extra_len + cmnt_len;
+ }
+ return out_;
+}
+
+/+ ↓ one entry's bytes, from its local header +/
+ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) {
+ enum uint local_sig = 0x04034b50;
+ size_t p = e.local_offset;
+ if (p + 30 > bytes.length || u32(bytes, p) != local_sig) {
+ throw new OcdaZipError(e.name ~ ": local header is malformed");
+ }
+ uint name_len = u16(bytes, p + 26);
+ uint extra_len = u16(bytes, p + 28);
+ size_t start = p + 30 + name_len + extra_len;
+ size_t end = start + e.compressed;
+ if (end > bytes.length) {
+ throw new OcdaZipError(e.name ~ ": runs past the end of the archive");
+ }
+ if (e.method != 0) {
+ throw new OcdaZipError(e.name ~ ": compression method "
+ ~ e.method.to!string ~ "; this reader takes only stored entries,"
+ ~ " which is what spine writes");
+ }
+ return bytes[start .. end].dup;
+}
+
+/+ ↓ the first entry whose name ends this way, or none.
+ Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid.
++/
+ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) {
+ ZipEntry[] out_;
+ foreach (e; entries) {
+ if (e.name.endsWith(suffix)) { out_ ~= e; }
+ }
+ return out_;
+}
+
+private uint u16(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i] | (cast(uint) b[i + 1] << 8);
+}
+
+private uint u32(const(ubyte)[] b, size_t i) {
+ return cast(uint) b[i]
+ | (cast(uint) b[i + 1] << 8)
+ | (cast(uint) b[i + 2] << 16)
+ | (cast(uint) b[i + 3] << 24);
+}
diff --git a/dlang/src/ocda/zstd.d b/dlang/src/ocda/zstd.d
new file mode 100644
index 0000000..401f1a3
--- /dev/null
+++ b/dlang/src/ocda/zstd.d
@@ -0,0 +1,79 @@
+module ocda.zstd;
+
+/+ ↓ libzstd, declared rather than bound.
+ .
+ A .sisupod is one zstd frame wrapping a zip archive, so reading one
+ needs exactly two things from the library: is this a frame, and what is
+ in it. Both are declared here and the C library is linked, which is the
+ same arrangement ocda.sqlite uses: no vendored tree, no generated
+ header, and the prototypes say what is being called.
++/
+
+extern (C) @nogc nothrow private {
+ size_t ZSTD_decompress(void* dst, size_t dstCapacity,
+ const(void)* src, size_t compressedSize);
+ ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
+ uint ZSTD_isError(size_t code);
+ const(char)* ZSTD_getErrorName(size_t code);
+}
+
+/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/
+private enum ulong contentSizeUnknown = ulong.max;
+private enum ulong contentSizeError = ulong.max - 1;
+
+/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/
+private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD];
+
+class OcdaZstdError : Exception {
+ this(string msg) { super(msg); }
+}
+
+/+ ↓ does this begin with a zstd frame?
+ Asked of the bytes rather than of the filename, because what is inside
+ decides how a thing is opened. A .sisupod is a frame; a .zip published
+ before spine wrapped them is not, and both are read.
++/
+bool zstdIsFrame(const(ubyte)[] bytes) {
+ if (bytes.length < zstd_magic.length) { return false; }
+ foreach (i, b; zstd_magic) {
+ if (bytes[i] != b) { return false; }
+ }
+ return true;
+}
+
+/+ ↓ one frame in, its contents out.
+ max_size is a refusal rather than a buffer size: a frame declares how
+ much it expands to, and a frame that claims more than the caller is
+ willing to hold is not decompressed at all. A reader of an artefact it
+ did not make wants that limit.
++/
+ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) {
+ if (!zstdIsFrame(bytes)) {
+ throw new OcdaZstdError("not a zstd frame");
+ }
+ ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length);
+ if (declared == contentSizeError) {
+ throw new OcdaZstdError("zstd frame: could not read its content size");
+ }
+ if (declared == contentSizeUnknown) {
+ throw new OcdaZstdError("zstd frame declares no content size;"
+ ~ " this reader takes only frames that do");
+ }
+ if (declared > max_size) {
+ import std.conv : to;
+ throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string
+ ~ " bytes, over the " ~ max_size.to!string ~ " byte limit");
+ }
+ auto out_ = new ubyte[](cast(size_t) declared);
+ size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length);
+ if (ZSTD_isError(got)) {
+ import std.string : fromStringz;
+ throw new OcdaZstdError("zstd: "
+ ~ ZSTD_getErrorName(got).fromStringz.idup);
+ }
+ if (got != declared) {
+ throw new OcdaZstdError("zstd frame expanded to a different size"
+ ~ " than it declared");
+ }
+ return out_;
+}
diff --git a/gleam/src/ocda/load.gleam b/gleam/src/ocda/load.gleam
index 1bbf869..2a82ac9 100644
--- a/gleam/src/ocda/load.gleam
+++ b/gleam/src/ocda/load.gleam
@@ -10,8 +10,10 @@
//// than a reader or a sample writer needs. This library reads
//// abstractions.
+import gleam/list
import gleam/string
import ocda/load_db
+import ocda/load_pod
import ocda/load_ssp
import ocda/model.{type Document}
@@ -20,21 +22,26 @@ pub type Source {
Ssp
/// `.ocda.db`, the abstraction as sqlite
OcdaDb
+ /// `.sisupod` or `.zip`, an archive carrying a `.ssp`
+ Pod
Unknown
}
+/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote
+/// before, and every pod already published carries that name, so both are
+/// recognised. What is inside decides how it is opened, not the name.
+const suffixes = [
+ #(".ocda.db", OcdaDb),
+ #(".ssp", Ssp),
+ #(".sisupod", Pod),
+ #(".zip", Pod),
+ #(".db", OcdaDb),
+]
+
pub fn source_of(path: String) -> Source {
- case string.ends_with(path, ".ocda.db") {
- True -> OcdaDb
- False ->
- case string.ends_with(path, ".ssp") {
- True -> Ssp
- False ->
- case string.ends_with(path, ".db") {
- True -> OcdaDb
- False -> Unknown
- }
- }
+ case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) {
+ Ok(#(_, source)) -> source
+ Error(_) -> Unknown
}
}
@@ -42,14 +49,16 @@ pub fn source_name(source: Source) -> String {
case source {
Ssp -> ".ssp (the abstraction as text)"
OcdaDb -> ".ocda.db (the abstraction as sqlite)"
+ Pod -> "a pod archive carrying a .ssp"
Unknown -> "not an abstraction this library reads"
}
}
/// Load whatever the path is into the model.
///
-/// `lang` applies only to a `.ocda.db`, which holds every language of its
-/// document; a `.ssp` is one language and the argument is ignored.
+/// `lang` names which language to take from a `.ocda.db`, which holds
+/// every language of its document, and from a pod, which may carry more than
+/// one `.ssp`. A `.ssp` is one language by construction and ignores it.
pub fn load(
path: String,
lang: String,
@@ -58,12 +67,13 @@ pub fn load(
case source_of(path) {
Ssp -> load_ssp.load_file(path)
OcdaDb -> load_db.load_file(path, lang, any_version)
+ Pod -> load_pod.load_file(path, lang)
Unknown ->
Error(
path
<> ": "
<> source_name(Unknown)
- <> " (expected .ssp or .ocda.db)",
+ <> " (expected .ssp, .ocda.db, .sisupod or .zip)",
)
}
}
diff --git a/gleam/src/ocda/load_pod.gleam b/gleam/src/ocda/load_pod.gleam
new file mode 100644
index 0000000..35483c6
--- /dev/null
+++ b/gleam/src/ocda/load_pod.gleam
@@ -0,0 +1,138 @@
+//// The abstraction out of a pod archive.
+////
+//// A pod built with `--pod2` carries the abstraction inside it, at
+//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is
+//// a third input form and *not* a third model: the `.ssp` inside is parsed by
+//// `ocda/load_ssp` and comes back as the same value. A writer over the
+//// library gains pods without being touched, which is the whole reason the
+//// dispatch is in one place.
+////
+//// ## The one thing this cannot do
+////
+//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip
+//// and **not the frame**, because there is no zstd on the BEAM without a NIF,
+//// and the one that exists, `ezstd`, clones zstd from github in its build, so
+//// it cannot be built in a sandbox without patching it. The D library has
+//// both, through a declared `libzstd`.
+////
+//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says
+//// why rather than pretending the file is malformed. It is the only place the
+//// two libraries differ in what they accept, and it is a packaging limit
+//// rather than anything about the format.
+
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load_ssp
+import ocda/model.{type Document}
+import ocda/zip
+import simplifile
+
+/// The magic number of a zstd frame, little endian 0xFD2FB528.
+const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>>
+
+/// Which languages of the abstraction the pod carries, sorted.
+pub fn languages(path: String) -> Result(List(String), String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ zip.find_suffix(entries, ".ssp")
+ |> list.filter_map(fn(e) { lang_of(e.name) })
+ |> list.sort(string.compare)
+ |> Ok
+}
+
+/// Load one language of the abstraction a pod carries.
+///
+/// `lang` == "" takes the only one there is, and where there is a choice it
+/// says so rather than choosing, as the `.ocda.db` loader does.
+pub fn load_file(path: String, lang: String) -> Result(Document, String) {
+ use bytes <- result.try(archive_bytes(path))
+ use entries <- result.try(zip.entries(bytes))
+ let ssps = zip.find_suffix(entries, ".ssp")
+ use chosen <- result.try(case ssps, lang {
+ [], _ ->
+ Error(
+ path
+ <> ": carries no .ssp. A pod carries the abstraction only when spine"
+ <> " was asked for it: spine --pod2 puts it in, spine --source does not.",
+ )
+ [only], "" -> Ok(only)
+ _, "" ->
+ Error(
+ path
+ <> ": holds more than one language ("
+ <> string.join(langs(ssps), " ")
+ <> "); name the one wanted",
+ )
+ _, wanted ->
+ case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) {
+ Ok(found) -> Ok(found)
+ Error(_) ->
+ Error(
+ path
+ <> ": no language '"
+ <> wanted
+ <> "'; it holds: "
+ <> string.join(langs(ssps), " "),
+ )
+ }
+ })
+ use content <- result.try(zip.read(bytes, chosen))
+ case bit_array_to_string(content) {
+ Ok(text) -> Ok(load_ssp.parse(text))
+ Error(_) -> Error(chosen.name <> ": not valid UTF-8")
+ }
+}
+
+fn langs(entries: List(zip.Entry)) -> List(String) {
+ list.filter_map(entries, fn(e) { lang_of(e.name) })
+}
+
+/// `<uid>.<lang>.ssp` -> `<lang>`
+fn lang_of(name: String) -> Result(String, Nil) {
+ case string.ends_with(name, ".ssp") {
+ False -> Error(Nil)
+ True -> {
+ let stem = string.drop_end(name, 4)
+ case string.split(stem, on: ".") |> list.last {
+ Ok(lang) if lang != stem -> Ok(lang)
+ _ -> Error(Nil)
+ }
+ }
+ }
+}
+
+/// The bytes decide, not the name: a file misnamed either way is treated by
+/// what it holds.
+fn archive_bytes(path: String) -> Result(BitArray, String) {
+ case simplifile.read_bits(path) {
+ Error(e) -> Error(path <> ": " <> simplifile.describe_error(e))
+ Ok(bytes) ->
+ case is_zstd_frame(bytes) {
+ False -> Ok(bytes)
+ True ->
+ Error(
+ path
+ <> " is a zstd frame wrapping the archive, and this library has no"
+ <> " zstd: the only one on the BEAM is a NIF that fetches zstd in"
+ <> " its own build, so it cannot be built sandboxed. Unwrap it"
+ <> " first (zstd -d) and read the archive, or use the D library,"
+ <> " which reads both.",
+ )
+ }
+ }
+}
+
+fn is_zstd_frame(bytes: BitArray) -> Bool {
+ case bytes {
+ <<head:bytes-size(4), _:bits>> -> head == zstd_magic
+ _ -> False
+ }
+}
+
+@external(erlang, "unicode", "characters_to_binary")
+fn bit_array_to_string_ffi(b: BitArray) -> String
+
+fn bit_array_to_string(b: BitArray) -> Result(String, Nil) {
+ Ok(bit_array_to_string_ffi(b))
+}
diff --git a/gleam/src/ocda/zip.gleam b/gleam/src/ocda/zip.gleam
new file mode 100644
index 0000000..5ce6e09
--- /dev/null
+++ b/gleam/src/ocda/zip.gleam
@@ -0,0 +1,170 @@
+//// Enough of the zip format to find one file in a pod and read it.
+////
+//// Not a zip library. A pod archive holds a handful of entries, spine writes
+//// them **stored** (uncompressed) so that the outer zstd frame does the
+//// compressing once over the whole thing, and what is wanted from it here is
+//// one named entry.
+////
+//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could
+//// inflate it, but a library that says it read an entry it in fact skipped is
+//// worse than one that says it cannot, and spine's own archives do not need
+//// it.
+
+import gleam/bit_array
+import gleam/int
+import gleam/list
+import gleam/result
+import gleam/string
+
+pub type Entry {
+ Entry(
+ name: String,
+ /// 0 stored, 8 deflate
+ method: Int,
+ compressed: Int,
+ uncompressed: Int,
+ local_offset: Int,
+ )
+}
+
+const eocd_sig = 0x06054b50
+
+const cd_sig = 0x02014b50
+
+const local_sig = 0x04034b50
+
+/// The entries, from the central directory at the end of the file.
+pub fn entries(bytes: BitArray) -> Result(List(Entry), String) {
+ let size = bit_array.byte_size(bytes)
+ use _ <- result.try(case size >= 22 {
+ True -> Ok(Nil)
+ False -> Error("too short to be a zip archive")
+ })
+ use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size)))
+ use count <- result.try(u16(bytes, eocd + 10))
+ use offset <- result.try(u32(bytes, eocd + 16))
+ read_central(bytes, offset, count, [])
+}
+
+/// The end of central directory record is last, and may be followed by a
+/// comment of up to 65535 bytes, so it is searched for backwards.
+fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) {
+ case at < lowest {
+ True ->
+ Error(
+ "no end of central directory record; it does not look like a zip archive",
+ )
+ False ->
+ case u32(bytes, at) {
+ Ok(sig) if sig == eocd_sig -> Ok(at)
+ _ -> find_eocd(bytes, at - 1, lowest)
+ }
+ }
+}
+
+fn lowest(size: Int) -> Int {
+ case size > 22 + 65_535 {
+ True -> size - 22 - 65_535
+ False -> 0
+ }
+}
+
+fn read_central(
+ bytes: BitArray,
+ at: Int,
+ remaining: Int,
+ acc: List(Entry),
+) -> Result(List(Entry), String) {
+ case remaining {
+ 0 -> Ok(list.reverse(acc))
+ _ -> {
+ use sig <- result.try(u32(bytes, at))
+ use _ <- result.try(case sig == cd_sig {
+ True -> Ok(Nil)
+ False ->
+ Error(
+ "central directory entry "
+ <> int.to_string(list.length(acc))
+ <> " is malformed",
+ )
+ })
+ use method <- result.try(u16(bytes, at + 10))
+ use compressed <- result.try(u32(bytes, at + 20))
+ use uncompressed <- result.try(u32(bytes, at + 24))
+ use name_len <- result.try(u16(bytes, at + 28))
+ use extra_len <- result.try(u16(bytes, at + 30))
+ use cmnt_len <- result.try(u16(bytes, at + 32))
+ use local_offset <- result.try(u32(bytes, at + 42))
+ use name <- result.try(text(bytes, at + 46, name_len))
+ let entry =
+ Entry(
+ name: name,
+ method: method,
+ compressed: compressed,
+ uncompressed: uncompressed,
+ local_offset: local_offset,
+ )
+ read_central(
+ bytes,
+ at + 46 + name_len + extra_len + cmnt_len,
+ remaining - 1,
+ [entry, ..acc],
+ )
+ }
+ }
+}
+
+/// One entry's bytes, from its local header.
+pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) {
+ use sig <- result.try(u32(bytes, entry.local_offset))
+ use _ <- result.try(case sig == local_sig {
+ True -> Ok(Nil)
+ False -> Error(entry.name <> ": local header is malformed")
+ })
+ use name_len <- result.try(u16(bytes, entry.local_offset + 26))
+ use extra_len <- result.try(u16(bytes, entry.local_offset + 28))
+ use _ <- result.try(case entry.method {
+ 0 -> Ok(Nil)
+ m ->
+ Error(
+ entry.name
+ <> ": compression method "
+ <> int.to_string(m)
+ <> "; this reader takes only stored entries, which is what spine writes",
+ )
+ })
+ let start = entry.local_offset + 30 + name_len + extra_len
+ case bit_array.slice(bytes, start, entry.compressed) {
+ Ok(slice) -> Ok(slice)
+ Error(_) -> Error(entry.name <> ": runs past the end of the archive")
+ }
+}
+
+/// The entries whose names end this way, in archive order. Used to find
+/// media/abstraction/<uid>.<lang>.ssp without knowing the uid.
+pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) {
+ list.filter(entries, fn(e) { string.ends_with(e.name, suffix) })
+}
+
+fn u16(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 2) {
+ Ok(<<v:little-size(16)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn u32(bytes: BitArray, at: Int) -> Result(Int, String) {
+ case bit_array.slice(bytes, at, 4) {
+ Ok(<<v:little-size(32)>>) -> Ok(v)
+ _ -> Error("read past the end of the archive at " <> int.to_string(at))
+ }
+}
+
+fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) {
+ case bit_array.slice(bytes, at, len) {
+ Ok(slice) ->
+ bit_array.to_string(slice)
+ |> result.replace_error("an entry name is not valid UTF-8")
+ Error(_) -> Error("an entry name runs past the end of the archive")
+ }
+}
diff --git a/gleam/test/ocda_lib_test.gleam b/gleam/test/ocda_lib_test.gleam
index 692150a..02fe463 100644
--- a/gleam/test/ocda_lib_test.gleam
+++ b/gleam/test/ocda_lib_test.gleam
@@ -1,5 +1,6 @@
import gleam/dict
import gleam/list
+import simplifile
import gleam/string
import gleeunit
import gleeunit/should
@@ -125,3 +126,23 @@ pub fn property_test() {
@external(erlang, "maps", "size")
fn dict_size(d: dict.Dict(a, b)) -> Int
+
+/// What a path is, including the two archive forms.
+pub fn pod_source_test() {
+ load.source_of("a/b.sisupod") |> should.equal(load.Pod)
+ load.source_of("a/b.zip") |> should.equal(load.Pod)
+}
+
+/// A zstd-wrapped archive is refused, and the refusal says why rather than
+/// claiming the file is malformed. This is the one place the D and Gleam
+/// libraries differ in what they accept, and it is a packaging limit: the
+/// only zstd on the BEAM is a NIF that fetches zstd in its own build.
+pub fn zstd_refusal_test() {
+ let path = "test/fixture-zstd-frame.sisupod"
+ let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>)
+ let assert Error(message) = load.load(path, "", False)
+ message |> string.contains(contain: "zstd") |> should.be_true
+ message |> string.contains(contain: "zstd -d") |> should.be_true
+ let _ = simplifile.delete(path)
+ Nil
+}