diff options
| -rw-r--r-- | README.md | 38 | ||||
| -rw-r--r-- | dlang/dub.json | 2 | ||||
| -rw-r--r-- | dlang/flake.nix | 6 | ||||
| -rw-r--r-- | dlang/src/ocda/load.d | 17 | ||||
| -rw-r--r-- | dlang/src/ocda/load_pod.d | 115 | ||||
| -rw-r--r-- | dlang/src/ocda/zip.d | 115 | ||||
| -rw-r--r-- | dlang/src/ocda/zstd.d | 79 | ||||
| -rw-r--r-- | gleam/src/ocda/load.gleam | 38 | ||||
| -rw-r--r-- | gleam/src/ocda/load_pod.gleam | 138 | ||||
| -rw-r--r-- | gleam/src/ocda/zip.gleam | 170 | ||||
| -rw-r--r-- | gleam/test/ocda_lib_test.gleam | 21 |
11 files changed, 711 insertions, 28 deletions
@@ -102,13 +102,37 @@ serialisations, and naming it is the useful part. ## What it reads, and what it does not -| | | -|---|---| -| `.ssp` | yes | -| `.ocda.db` | yes, any language of it | -| `.sisupod` | not yet: unwrap the archive and read the `.ssp` inside it | -| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work | -| a url | no | +| | | | +|---|---|---| +| `.ssp` | yes | | +| `.ocda.db` | yes, any language of it | | +| `.sisupod` | yes, any language it carries | D only, see below | +| `.zip` pod | yes, any language it carries | | +| `.sst`, `.ssm` markup | **no, and not planned**: that is a parser, which is spine's work | | +| a url | no | | + +A pod carries the abstraction only when spine was asked for it: `--pod2` +puts it in, `--source` does not. A pod without one is refused with a message +saying which flag makes a pod this library can read, because that is a fact +about the pod rather than about the library. + +**Over the reference collection, a `.sisupod` gives the same model as the +`.ssp` beside it: 36 of 36 document-languages, byte identical dumps.** A +writer over the library therefore gained a third input form without being +touched, which is the point of the dispatch being in one place. + +### the one thing the Gleam library cannot do + +A `.sisupod` is **one zstd frame wrapping** the zip. The D library reads both +through a declared `libzstd`. The Gleam library reads the **zip and not the +frame**, because the only zstd on the BEAM is a NIF that clones zstd from +github in its own build and so cannot be built in a sandbox without patching +it. + +So a `.zip` pod is read in both languages, a `.sisupod` in D only, and the +Gleam refusal says why and says `zstd -d` rather than claiming the file is +malformed. It is a packaging limit, not anything about the format, and it is +the only place the two libraries differ in what they accept. `ocda/load` is the one place in each language that decides what a path is, so a new input form is added once and every reader and writer gains it. diff --git a/dlang/dub.json b/dlang/dub.json index f49ef85..373e5bd 100644 --- a/dlang/dub.json +++ b/dlang/dub.json @@ -9,5 +9,5 @@ "targetType": "library", "sourcePaths": [ "./src" ], "importPaths": [ "./src" ], - "libs": [ "sqlite3" ] + "libs": [ "sqlite3", "zstd" ] } diff --git a/dlang/flake.nix b/dlang/flake.nix index 26dbe2d..2ad3b1c 100644 --- a/dlang/flake.nix +++ b/dlang/flake.nix @@ -21,11 +21,11 @@ version = "0.1.0"; src = self; nativeBuildInputs = [ compiler ]; - buildInputs = [ pkgs.sqlite ]; + buildInputs = [ pkgs.sqlite pkgs.zstd ]; buildPhase = '' runHook preBuild export HOME=$TMPDIR - ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 + ${compilerBin} -I=src -of=${tool} src/ocda/*.d test/${tool}.d -L-lsqlite3 -L-lzstd runHook postBuild ''; installPhase = '' @@ -57,7 +57,7 @@ }; devShells.${system}.default = pkgs.mkShell { - packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite ]; + packages = [ pkgs.ldc pkgs.dub pkgs.dtools pkgs.sqlite pkgs.zstd ]; }; }; } diff --git a/dlang/src/ocda/load.d b/dlang/src/ocda/load.d index 0f630bc..89c8dcc 100644 --- a/dlang/src/ocda/load.d +++ b/dlang/src/ocda/load.d @@ -17,16 +17,24 @@ import std.algorithm : endsWith; import ocda.model; import ocda.load_ssp; import ocda.load_db; +import ocda.load_pod; enum Source { ssp, // .ssp, the abstraction as text ocda_db, // .ocda.db, the abstraction as sqlite + pod, // .sisupod or .zip, an archive carrying a .ssp unknown, } Source sourceOf(string path) { if (path.endsWith(".ocda.db")) { return Source.ocda_db; } if (path.endsWith(".ssp")) { return Source.ssp; } + /+ ↓ .sisupod is the archive spine writes now; .zip is what it wrote + before, and every pod already published carries that name, so both + are read. What is inside decides how it is opened, not the name. + +/ + if (path.endsWith(".sisupod")) { return Source.pod; } + if (path.endsWith(".zip")) { return Source.pod; } if (path.endsWith(".db")) { return Source.ocda_db; } return Source.unknown; } @@ -35,6 +43,7 @@ string sourceName(Source s) { final switch (s) { case Source.ssp: return ".ssp (the abstraction as text)"; case Source.ocda_db: return ".ocda.db (the abstraction as sqlite)"; + case Source.pod: return "a pod archive carrying a .ssp"; case Source.unknown: return "not an abstraction this library reads"; } } @@ -44,15 +53,17 @@ class OcdaLoadError : Exception { } /+ ↓ load whatever the path is into the model. - lang applies only to a .ocda.db, which holds every language of its - document; a .ssp is one language and the argument is ignored. + lang names which language to take from a .ocda.db, which holds + every language of its document, and from a pod, which may carry more + than one .ssp. A .ssp is one language by construction and ignores it. +/ OcdaDocument load(string path, string lang = "", bool any_version = false) { final switch (sourceOf(path)) { case Source.ssp: return parseSspFile(path); case Source.ocda_db: return loadDbFile(path, lang, any_version); + case Source.pod: return loadPodFile(path, lang); case Source.unknown: throw new OcdaLoadError(path ~ ": " ~ sourceName(Source.unknown) - ~ " (expected .ssp or .ocda.db)"); + ~ " (expected .ssp, .ocda.db, .sisupod or .zip)"); } } diff --git a/dlang/src/ocda/load_pod.d b/dlang/src/ocda/load_pod.d new file mode 100644 index 0000000..75993be --- /dev/null +++ b/dlang/src/ocda/load_pod.d @@ -0,0 +1,115 @@ +module ocda.load_pod; + +/+ ↓ the abstraction out of a pod archive. + . + A .sisupod is one zstd frame wrapping a stored zip; a .zip is the same + archive unwrapped, which is what spine published before it wrapped them. + Either way, a pod built with --pod2 carries the abstraction inside it, + at media/abstraction/<uid>.<lang>.ssp, and that is what is read. + . + So this is a third input form and *not* a third model: the .ssp inside + is parsed by ocda.load_ssp and comes back as the same value. A writer + over the library gains .sisupod without being touched, which is the + whole reason the dispatch is in one place. + . + A pod built without --pod2 carries markup and no abstraction. That is + refused, and the refusal says which flag makes a pod this reader can + read, because "no abstraction in it" is a fact about the pod rather + than about the reader. ++/ + +import std.algorithm : endsWith, sort; +import std.array : array; +import std.file : read; +import std.string : indexOf; +import ocda.model; +import ocda.load_ssp; +import ocda.zip; +import ocda.zstd; + +/+ ↓ a limit on what a frame may expand to. + A reader of an artefact it did not make declines to be told how much + memory to allocate. 512 MiB is far above the largest pod in the + reference collection (4 MiB) and far below anything that matters. ++/ +enum size_t max_pod_size = 512 * 1024 * 1024; + +class OcdaPodError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ which languages of the abstraction the pod carries, in sorted order +/ +string[] podLanguages(string path) { + auto bytes = archiveBytes(path); + string[] out_; + foreach (e; zipFindSuffix(zipEntries(bytes), ".ssp")) { + string lang = langOfSspName(e.name); + if (lang.length > 0) { out_ ~= lang; } + } + return out_.sort.array; +} + +/+ ↓ load one language of the abstraction a pod carries. + lang == "" takes the only one there is, and where there is a choice it + says so rather than choosing, as the .ocda.db loader does. ++/ +OcdaDocument loadPodFile(string path, string lang = "") { + auto bytes = archiveBytes(path); + auto entries = zipEntries(bytes); + auto ssps = zipFindSuffix(entries, ".ssp"); + if (ssps.length == 0) { + throw new OcdaPodError(path ~ ": carries no .ssp." + ~ " A pod carries the abstraction only when spine was asked for it:" + ~ " spine --pod2 puts it in, spine --source does not."); + } + ZipEntry chosen = ssps[0]; + if (lang.length > 0) { + bool found = false; + foreach (e; ssps) { + if (langOfSspName(e.name) == lang) { chosen = e; found = true; break; } + } + if (!found) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": no language '" ~ lang + ~ "'; it holds: " ~ have); + } + } else if (ssps.length > 1) { + string have; + foreach (i, e; ssps) { + have ~= (i > 0 ? " " : "") ~ langOfSspName(e.name); + } + throw new OcdaPodError(path ~ ": holds more than one language (" + ~ have ~ "); name the one wanted"); + } + auto ssp = zipRead(bytes, chosen); + return parseSsp(cast(string) ssp.idup); +} + +/+ ↓ the archive, unwrapped if it was wrapped. + The bytes decide, not the name: a .sisupod is a zstd frame and a .zip + is not, and a file misnamed either way still reads. ++/ +private ubyte[] archiveBytes(string path) { + auto bytes = cast(ubyte[]) read(path); + return zstdIsFrame(bytes) + ? zstdDecompress(bytes, max_pod_size) + : bytes; +} + +/+ ↓ <uid>.<lang>.ssp -> <lang> +/ +private string langOfSspName(string name) { + if (!name.endsWith(".ssp")) { return ""; } + string stem = name[0 .. $ - 4]; + auto dot = stem.lastIndexOf('.'); + return (dot > 0) ? stem[dot + 1 .. $] : ""; +} + +private ptrdiff_t lastIndexOf(string s, char c) { + for (ptrdiff_t i = s.length - 1; i >= 0; i--) { + if (s[i] == c) { return i; } + } + return -1; +} diff --git a/dlang/src/ocda/zip.d b/dlang/src/ocda/zip.d new file mode 100644 index 0000000..9dff6f4 --- /dev/null +++ b/dlang/src/ocda/zip.d @@ -0,0 +1,115 @@ +module ocda.zip; + +/+ ↓ enough of the zip format to find one file in a pod and read it. + . + Not a zip library. A pod archive holds a handful of entries, spine + writes them **stored** (uncompressed) so that the outer zstd frame does + the compressing once over the whole thing, and what is wanted from it + here is one named entry. So: read the central directory, find the + entry, return its bytes. + . + Deflate is *refused rather than mishandled*. Phobos can inflate, but a + library that says it read an entry it in fact skipped is worse than one + that says it cannot, and spine's own archives do not need it. ++/ + +import std.algorithm : endsWith; +import std.conv : to; + +class OcdaZipError : Exception { + this(string msg) { super(msg); } +} + +struct ZipEntry { + string name; + uint method; // 0 stored, 8 deflate + uint compressed; + uint uncompressed; + uint local_offset; +} + +/+ ↓ the entries, from the central directory at the end of the file +/ +ZipEntry[] zipEntries(const(ubyte)[] bytes) { + enum uint eocd_sig = 0x06054b50; + enum uint cd_sig = 0x02014b50; + if (bytes.length < 22) { + throw new OcdaZipError("too short to be a zip archive"); + } + /+ ↓ the end of central directory record is last, and may be followed by a + comment of up to 65535 bytes, so it is searched for backwards +/ + ptrdiff_t eocd = -1; + ptrdiff_t lowest = bytes.length > (22 + 65535) ? bytes.length - (22 + 65535) : 0; + for (ptrdiff_t i = bytes.length - 22; i >= lowest; i--) { + if (u32(bytes, i) == eocd_sig) { eocd = i; break; } + } + if (eocd < 0) { + throw new OcdaZipError("no end of central directory record;" + ~ " it does not look like a zip archive"); + } + uint count = u16(bytes, eocd + 10); + uint offset = u32(bytes, eocd + 16); + ZipEntry[] out_; + size_t p = offset; + foreach (_; 0 .. count) { + if (p + 46 > bytes.length || u32(bytes, p) != cd_sig) { + throw new OcdaZipError("central directory entry " + ~ out_.length.to!string ~ " is malformed"); + } + ZipEntry e; + e.method = u16(bytes, p + 10); + e.compressed = u32(bytes, p + 20); + e.uncompressed = u32(bytes, p + 24); + uint name_len = u16(bytes, p + 28); + uint extra_len = u16(bytes, p + 30); + uint cmnt_len = u16(bytes, p + 32); + e.local_offset = u32(bytes, p + 42); + e.name = cast(string) bytes[p + 46 .. p + 46 + name_len].idup; + out_ ~= e; + p += 46 + name_len + extra_len + cmnt_len; + } + return out_; +} + +/+ ↓ one entry's bytes, from its local header +/ +ubyte[] zipRead(const(ubyte)[] bytes, ZipEntry e) { + enum uint local_sig = 0x04034b50; + size_t p = e.local_offset; + if (p + 30 > bytes.length || u32(bytes, p) != local_sig) { + throw new OcdaZipError(e.name ~ ": local header is malformed"); + } + uint name_len = u16(bytes, p + 26); + uint extra_len = u16(bytes, p + 28); + size_t start = p + 30 + name_len + extra_len; + size_t end = start + e.compressed; + if (end > bytes.length) { + throw new OcdaZipError(e.name ~ ": runs past the end of the archive"); + } + if (e.method != 0) { + throw new OcdaZipError(e.name ~ ": compression method " + ~ e.method.to!string ~ "; this reader takes only stored entries," + ~ " which is what spine writes"); + } + return bytes[start .. end].dup; +} + +/+ ↓ the first entry whose name ends this way, or none. + Used to find media/abstraction/<uid>.<lang>.ssp without knowing the uid. ++/ +ZipEntry[] zipFindSuffix(ZipEntry[] entries, string suffix) { + ZipEntry[] out_; + foreach (e; entries) { + if (e.name.endsWith(suffix)) { out_ ~= e; } + } + return out_; +} + +private uint u16(const(ubyte)[] b, size_t i) { + return cast(uint) b[i] | (cast(uint) b[i + 1] << 8); +} + +private uint u32(const(ubyte)[] b, size_t i) { + return cast(uint) b[i] + | (cast(uint) b[i + 1] << 8) + | (cast(uint) b[i + 2] << 16) + | (cast(uint) b[i + 3] << 24); +} diff --git a/dlang/src/ocda/zstd.d b/dlang/src/ocda/zstd.d new file mode 100644 index 0000000..401f1a3 --- /dev/null +++ b/dlang/src/ocda/zstd.d @@ -0,0 +1,79 @@ +module ocda.zstd; + +/+ ↓ libzstd, declared rather than bound. + . + A .sisupod is one zstd frame wrapping a zip archive, so reading one + needs exactly two things from the library: is this a frame, and what is + in it. Both are declared here and the C library is linked, which is the + same arrangement ocda.sqlite uses: no vendored tree, no generated + header, and the prototypes say what is being called. ++/ + +extern (C) @nogc nothrow private { + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); +} + +/+ ↓ the two values ZSTD_getFrameContentSize returns in place of a size +/ +private enum ulong contentSizeUnknown = ulong.max; +private enum ulong contentSizeError = ulong.max - 1; + +/+ ↓ the magic number of a zstd frame, little endian 0xFD2FB528 +/ +private static immutable ubyte[4] zstd_magic = [0x28, 0xB5, 0x2F, 0xFD]; + +class OcdaZstdError : Exception { + this(string msg) { super(msg); } +} + +/+ ↓ does this begin with a zstd frame? + Asked of the bytes rather than of the filename, because what is inside + decides how a thing is opened. A .sisupod is a frame; a .zip published + before spine wrapped them is not, and both are read. ++/ +bool zstdIsFrame(const(ubyte)[] bytes) { + if (bytes.length < zstd_magic.length) { return false; } + foreach (i, b; zstd_magic) { + if (bytes[i] != b) { return false; } + } + return true; +} + +/+ ↓ one frame in, its contents out. + max_size is a refusal rather than a buffer size: a frame declares how + much it expands to, and a frame that claims more than the caller is + willing to hold is not decompressed at all. A reader of an artefact it + did not make wants that limit. ++/ +ubyte[] zstdDecompress(const(ubyte)[] bytes, size_t max_size) { + if (!zstdIsFrame(bytes)) { + throw new OcdaZstdError("not a zstd frame"); + } + ulong declared = ZSTD_getFrameContentSize(bytes.ptr, bytes.length); + if (declared == contentSizeError) { + throw new OcdaZstdError("zstd frame: could not read its content size"); + } + if (declared == contentSizeUnknown) { + throw new OcdaZstdError("zstd frame declares no content size;" + ~ " this reader takes only frames that do"); + } + if (declared > max_size) { + import std.conv : to; + throw new OcdaZstdError("zstd frame expands to " ~ declared.to!string + ~ " bytes, over the " ~ max_size.to!string ~ " byte limit"); + } + auto out_ = new ubyte[](cast(size_t) declared); + size_t got = ZSTD_decompress(out_.ptr, out_.length, bytes.ptr, bytes.length); + if (ZSTD_isError(got)) { + import std.string : fromStringz; + throw new OcdaZstdError("zstd: " + ~ ZSTD_getErrorName(got).fromStringz.idup); + } + if (got != declared) { + throw new OcdaZstdError("zstd frame expanded to a different size" + ~ " than it declared"); + } + return out_; +} diff --git a/gleam/src/ocda/load.gleam b/gleam/src/ocda/load.gleam index 1bbf869..2a82ac9 100644 --- a/gleam/src/ocda/load.gleam +++ b/gleam/src/ocda/load.gleam @@ -10,8 +10,10 @@ //// than a reader or a sample writer needs. This library reads //// abstractions. +import gleam/list import gleam/string import ocda/load_db +import ocda/load_pod import ocda/load_ssp import ocda/model.{type Document} @@ -20,21 +22,26 @@ pub type Source { Ssp /// `.ocda.db`, the abstraction as sqlite OcdaDb + /// `.sisupod` or `.zip`, an archive carrying a `.ssp` + Pod Unknown } +/// `.sisupod` is the archive spine writes now; `.zip` is what it wrote +/// before, and every pod already published carries that name, so both are +/// recognised. What is inside decides how it is opened, not the name. +const suffixes = [ + #(".ocda.db", OcdaDb), + #(".ssp", Ssp), + #(".sisupod", Pod), + #(".zip", Pod), + #(".db", OcdaDb), +] + pub fn source_of(path: String) -> Source { - case string.ends_with(path, ".ocda.db") { - True -> OcdaDb - False -> - case string.ends_with(path, ".ssp") { - True -> Ssp - False -> - case string.ends_with(path, ".db") { - True -> OcdaDb - False -> Unknown - } - } + case list.find(suffixes, fn(s) { string.ends_with(path, s.0) }) { + Ok(#(_, source)) -> source + Error(_) -> Unknown } } @@ -42,14 +49,16 @@ pub fn source_name(source: Source) -> String { case source { Ssp -> ".ssp (the abstraction as text)" OcdaDb -> ".ocda.db (the abstraction as sqlite)" + Pod -> "a pod archive carrying a .ssp" Unknown -> "not an abstraction this library reads" } } /// Load whatever the path is into the model. /// -/// `lang` applies only to a `.ocda.db`, which holds every language of its -/// document; a `.ssp` is one language and the argument is ignored. +/// `lang` names which language to take from a `.ocda.db`, which holds +/// every language of its document, and from a pod, which may carry more than +/// one `.ssp`. A `.ssp` is one language by construction and ignores it. pub fn load( path: String, lang: String, @@ -58,12 +67,13 @@ pub fn load( case source_of(path) { Ssp -> load_ssp.load_file(path) OcdaDb -> load_db.load_file(path, lang, any_version) + Pod -> load_pod.load_file(path, lang) Unknown -> Error( path <> ": " <> source_name(Unknown) - <> " (expected .ssp or .ocda.db)", + <> " (expected .ssp, .ocda.db, .sisupod or .zip)", ) } } diff --git a/gleam/src/ocda/load_pod.gleam b/gleam/src/ocda/load_pod.gleam new file mode 100644 index 0000000..35483c6 --- /dev/null +++ b/gleam/src/ocda/load_pod.gleam @@ -0,0 +1,138 @@ +//// The abstraction out of a pod archive. +//// +//// A pod built with `--pod2` carries the abstraction inside it, at +//// `media/abstraction/<uid>.<lang>.ssp`, and that is what is read. So this is +//// a third input form and *not* a third model: the `.ssp` inside is parsed by +//// `ocda/load_ssp` and comes back as the same value. A writer over the +//// library gains pods without being touched, which is the whole reason the +//// dispatch is in one place. +//// +//// ## The one thing this cannot do +//// +//// A `.sisupod` is **one zstd frame wrapping** the zip. This reads the zip +//// and **not the frame**, because there is no zstd on the BEAM without a NIF, +//// and the one that exists, `ezstd`, clones zstd from github in its build, so +//// it cannot be built in a sandbox without patching it. The D library has +//// both, through a declared `libzstd`. +//// +//// So: a `.zip` pod is read, a `.sisupod` is refused, and the refusal says +//// why rather than pretending the file is malformed. It is the only place the +//// two libraries differ in what they accept, and it is a packaging limit +//// rather than anything about the format. + +import gleam/list +import gleam/result +import gleam/string +import ocda/load_ssp +import ocda/model.{type Document} +import ocda/zip +import simplifile + +/// The magic number of a zstd frame, little endian 0xFD2FB528. +const zstd_magic = <<0x28, 0xB5, 0x2F, 0xFD>> + +/// Which languages of the abstraction the pod carries, sorted. +pub fn languages(path: String) -> Result(List(String), String) { + use bytes <- result.try(archive_bytes(path)) + use entries <- result.try(zip.entries(bytes)) + zip.find_suffix(entries, ".ssp") + |> list.filter_map(fn(e) { lang_of(e.name) }) + |> list.sort(string.compare) + |> Ok +} + +/// Load one language of the abstraction a pod carries. +/// +/// `lang` == "" takes the only one there is, and where there is a choice it +/// says so rather than choosing, as the `.ocda.db` loader does. +pub fn load_file(path: String, lang: String) -> Result(Document, String) { + use bytes <- result.try(archive_bytes(path)) + use entries <- result.try(zip.entries(bytes)) + let ssps = zip.find_suffix(entries, ".ssp") + use chosen <- result.try(case ssps, lang { + [], _ -> + Error( + path + <> ": carries no .ssp. A pod carries the abstraction only when spine" + <> " was asked for it: spine --pod2 puts it in, spine --source does not.", + ) + [only], "" -> Ok(only) + _, "" -> + Error( + path + <> ": holds more than one language (" + <> string.join(langs(ssps), " ") + <> "); name the one wanted", + ) + _, wanted -> + case list.find(ssps, fn(e) { lang_of(e.name) == Ok(wanted) }) { + Ok(found) -> Ok(found) + Error(_) -> + Error( + path + <> ": no language '" + <> wanted + <> "'; it holds: " + <> string.join(langs(ssps), " "), + ) + } + }) + use content <- result.try(zip.read(bytes, chosen)) + case bit_array_to_string(content) { + Ok(text) -> Ok(load_ssp.parse(text)) + Error(_) -> Error(chosen.name <> ": not valid UTF-8") + } +} + +fn langs(entries: List(zip.Entry)) -> List(String) { + list.filter_map(entries, fn(e) { lang_of(e.name) }) +} + +/// `<uid>.<lang>.ssp` -> `<lang>` +fn lang_of(name: String) -> Result(String, Nil) { + case string.ends_with(name, ".ssp") { + False -> Error(Nil) + True -> { + let stem = string.drop_end(name, 4) + case string.split(stem, on: ".") |> list.last { + Ok(lang) if lang != stem -> Ok(lang) + _ -> Error(Nil) + } + } + } +} + +/// The bytes decide, not the name: a file misnamed either way is treated by +/// what it holds. +fn archive_bytes(path: String) -> Result(BitArray, String) { + case simplifile.read_bits(path) { + Error(e) -> Error(path <> ": " <> simplifile.describe_error(e)) + Ok(bytes) -> + case is_zstd_frame(bytes) { + False -> Ok(bytes) + True -> + Error( + path + <> " is a zstd frame wrapping the archive, and this library has no" + <> " zstd: the only one on the BEAM is a NIF that fetches zstd in" + <> " its own build, so it cannot be built sandboxed. Unwrap it" + <> " first (zstd -d) and read the archive, or use the D library," + <> " which reads both.", + ) + } + } +} + +fn is_zstd_frame(bytes: BitArray) -> Bool { + case bytes { + <<head:bytes-size(4), _:bits>> -> head == zstd_magic + _ -> False + } +} + +@external(erlang, "unicode", "characters_to_binary") +fn bit_array_to_string_ffi(b: BitArray) -> String + +fn bit_array_to_string(b: BitArray) -> Result(String, Nil) { + Ok(bit_array_to_string_ffi(b)) +} diff --git a/gleam/src/ocda/zip.gleam b/gleam/src/ocda/zip.gleam new file mode 100644 index 0000000..5ce6e09 --- /dev/null +++ b/gleam/src/ocda/zip.gleam @@ -0,0 +1,170 @@ +//// Enough of the zip format to find one file in a pod and read it. +//// +//// Not a zip library. A pod archive holds a handful of entries, spine writes +//// them **stored** (uncompressed) so that the outer zstd frame does the +//// compressing once over the whole thing, and what is wanted from it here is +//// one named entry. +//// +//// Deflate is *refused rather than mishandled*. Erlang's `zlib` could +//// inflate it, but a library that says it read an entry it in fact skipped is +//// worse than one that says it cannot, and spine's own archives do not need +//// it. + +import gleam/bit_array +import gleam/int +import gleam/list +import gleam/result +import gleam/string + +pub type Entry { + Entry( + name: String, + /// 0 stored, 8 deflate + method: Int, + compressed: Int, + uncompressed: Int, + local_offset: Int, + ) +} + +const eocd_sig = 0x06054b50 + +const cd_sig = 0x02014b50 + +const local_sig = 0x04034b50 + +/// The entries, from the central directory at the end of the file. +pub fn entries(bytes: BitArray) -> Result(List(Entry), String) { + let size = bit_array.byte_size(bytes) + use _ <- result.try(case size >= 22 { + True -> Ok(Nil) + False -> Error("too short to be a zip archive") + }) + use eocd <- result.try(find_eocd(bytes, size - 22, lowest(size))) + use count <- result.try(u16(bytes, eocd + 10)) + use offset <- result.try(u32(bytes, eocd + 16)) + read_central(bytes, offset, count, []) +} + +/// The end of central directory record is last, and may be followed by a +/// comment of up to 65535 bytes, so it is searched for backwards. +fn find_eocd(bytes: BitArray, at: Int, lowest: Int) -> Result(Int, String) { + case at < lowest { + True -> + Error( + "no end of central directory record; it does not look like a zip archive", + ) + False -> + case u32(bytes, at) { + Ok(sig) if sig == eocd_sig -> Ok(at) + _ -> find_eocd(bytes, at - 1, lowest) + } + } +} + +fn lowest(size: Int) -> Int { + case size > 22 + 65_535 { + True -> size - 22 - 65_535 + False -> 0 + } +} + +fn read_central( + bytes: BitArray, + at: Int, + remaining: Int, + acc: List(Entry), +) -> Result(List(Entry), String) { + case remaining { + 0 -> Ok(list.reverse(acc)) + _ -> { + use sig <- result.try(u32(bytes, at)) + use _ <- result.try(case sig == cd_sig { + True -> Ok(Nil) + False -> + Error( + "central directory entry " + <> int.to_string(list.length(acc)) + <> " is malformed", + ) + }) + use method <- result.try(u16(bytes, at + 10)) + use compressed <- result.try(u32(bytes, at + 20)) + use uncompressed <- result.try(u32(bytes, at + 24)) + use name_len <- result.try(u16(bytes, at + 28)) + use extra_len <- result.try(u16(bytes, at + 30)) + use cmnt_len <- result.try(u16(bytes, at + 32)) + use local_offset <- result.try(u32(bytes, at + 42)) + use name <- result.try(text(bytes, at + 46, name_len)) + let entry = + Entry( + name: name, + method: method, + compressed: compressed, + uncompressed: uncompressed, + local_offset: local_offset, + ) + read_central( + bytes, + at + 46 + name_len + extra_len + cmnt_len, + remaining - 1, + [entry, ..acc], + ) + } + } +} + +/// One entry's bytes, from its local header. +pub fn read(bytes: BitArray, entry: Entry) -> Result(BitArray, String) { + use sig <- result.try(u32(bytes, entry.local_offset)) + use _ <- result.try(case sig == local_sig { + True -> Ok(Nil) + False -> Error(entry.name <> ": local header is malformed") + }) + use name_len <- result.try(u16(bytes, entry.local_offset + 26)) + use extra_len <- result.try(u16(bytes, entry.local_offset + 28)) + use _ <- result.try(case entry.method { + 0 -> Ok(Nil) + m -> + Error( + entry.name + <> ": compression method " + <> int.to_string(m) + <> "; this reader takes only stored entries, which is what spine writes", + ) + }) + let start = entry.local_offset + 30 + name_len + extra_len + case bit_array.slice(bytes, start, entry.compressed) { + Ok(slice) -> Ok(slice) + Error(_) -> Error(entry.name <> ": runs past the end of the archive") + } +} + +/// The entries whose names end this way, in archive order. Used to find +/// media/abstraction/<uid>.<lang>.ssp without knowing the uid. +pub fn find_suffix(entries: List(Entry), suffix: String) -> List(Entry) { + list.filter(entries, fn(e) { string.ends_with(e.name, suffix) }) +} + +fn u16(bytes: BitArray, at: Int) -> Result(Int, String) { + case bit_array.slice(bytes, at, 2) { + Ok(<<v:little-size(16)>>) -> Ok(v) + _ -> Error("read past the end of the archive at " <> int.to_string(at)) + } +} + +fn u32(bytes: BitArray, at: Int) -> Result(Int, String) { + case bit_array.slice(bytes, at, 4) { + Ok(<<v:little-size(32)>>) -> Ok(v) + _ -> Error("read past the end of the archive at " <> int.to_string(at)) + } +} + +fn text(bytes: BitArray, at: Int, len: Int) -> Result(String, String) { + case bit_array.slice(bytes, at, len) { + Ok(slice) -> + bit_array.to_string(slice) + |> result.replace_error("an entry name is not valid UTF-8") + Error(_) -> Error("an entry name runs past the end of the archive") + } +} diff --git a/gleam/test/ocda_lib_test.gleam b/gleam/test/ocda_lib_test.gleam index 692150a..02fe463 100644 --- a/gleam/test/ocda_lib_test.gleam +++ b/gleam/test/ocda_lib_test.gleam @@ -1,5 +1,6 @@ import gleam/dict import gleam/list +import simplifile import gleam/string import gleeunit import gleeunit/should @@ -125,3 +126,23 @@ pub fn property_test() { @external(erlang, "maps", "size") fn dict_size(d: dict.Dict(a, b)) -> Int + +/// What a path is, including the two archive forms. +pub fn pod_source_test() { + load.source_of("a/b.sisupod") |> should.equal(load.Pod) + load.source_of("a/b.zip") |> should.equal(load.Pod) +} + +/// A zstd-wrapped archive is refused, and the refusal says why rather than +/// claiming the file is malformed. This is the one place the D and Gleam +/// libraries differ in what they accept, and it is a packaging limit: the +/// only zstd on the BEAM is a NIF that fetches zstd in its own build. +pub fn zstd_refusal_test() { + let path = "test/fixture-zstd-frame.sisupod" + let assert Ok(_) = simplifile.write_bits(path, <<0x28, 0xB5, 0x2F, 0xFD, 0>>) + let assert Error(message) = load.load(path, "", False) + message |> string.contains(contain: "zstd") |> should.be_true + message |> string.contains(contain: "zstd -d") |> should.be_true + let _ = simplifile.delete(path) + Nil +} |
