diff options
| -rw-r--r-- | dlang/src/ocda/dump.d | 2 | ||||
| -rw-r--r-- | dlang/src/ocda/images.d | 100 | ||||
| -rw-r--r-- | dlang/src/ocda/load_db.d | 5 | ||||
| -rw-r--r-- | dlang/src/ocda/load_ssp.d | 5 | ||||
| -rw-r--r-- | dlang/src/ocda/model.d | 2 | ||||
| -rw-r--r-- | dlang/src/ocda/sqlite.d | 21 | ||||
| -rw-r--r-- | gleam/src/ocda/dump.gleam | 2 | ||||
| -rw-r--r-- | gleam/src/ocda/images.gleam | 135 | ||||
| -rw-r--r-- | gleam/src/ocda/load_db.gleam | 7 | ||||
| -rw-r--r-- | gleam/src/ocda/load_ssp.gleam | 2 | ||||
| -rw-r--r-- | gleam/src/ocda/model.gleam | 4 |
11 files changed, 283 insertions, 2 deletions
diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d index b16a209..7a4f11a 100644 --- a/dlang/src/ocda/dump.d +++ b/dlang/src/ocda/dump.d @@ -79,6 +79,8 @@ string dumpDocument(ref OcdaDocument doc) { kv("part", obj.part); kv("is_of_type", obj.is_of_type); kv("segment_html_is", obj.segment_html_is); + kv("segment_epub", obj.segment_epub); + kv("segment_epub_is", obj.segment_epub_is); kn("parent", obj.parent_ocn); kn("last_descendant", obj.last_descendant_ocn); kv("children", ints(obj.children)); diff --git a/dlang/src/ocda/images.d b/dlang/src/ocda/images.d new file mode 100644 index 0000000..b3a9652 --- /dev/null +++ b/dlang/src/ocda/images.d @@ -0,0 +1,100 @@ +module ocda.images; + +/+ ↓ an image's bytes, from whichever source the document came from. + . + The model *names* images, with digest, byte size and pixel size; it does + not hold them. That is right - an abstraction of a document is not a + bundle of its assets - but a writer of any format that shows images + needs the bytes, so something has to know where they are. This is that + something, and it is in the library rather than in a writer because the + answer depends on the *input form* and not on the output. + . + Three answers, and one refusal: + . + .ocda.db the files table, role 'image', by name + .sisupod the archive entry whose path ends in /image/<name> + .zip the same + .ssp beside it: a pod puts the abstraction in + media/abstraction/ and its images in media/image/, so the + path is documented rather than guessed + . + A .ssp that is not in a pod has nowhere to look, and that is reported as + absent rather than as an error: a reader that cannot find an image + should say so and carry on, because the document is still readable. ++/ + +import std.algorithm : endsWith; +import std.array : array; +import std.file : exists, isFile, read; +import std.path : buildPath, dirName; +import ocda.load; +import ocda.sqlite; +import ocda.zip; +import ocda.zstd; + +/+ ↓ the bytes, or an empty slice when the source has no such image +/ +ubyte[] imageBytes(string source, string name) { + final switch (sourceOf(source)) { + case Source.ocda_db: return fromDb(source, name); + case Source.pod: return fromArchive(source, name); + case Source.ssp: return fromBeside(source, name); + case Source.unknown: return []; + } +} + +/+ ↓ can this source give images at all? + A writer asks before it decides whether to emit them, because an <img> + pointing at a file it did not include is worse than no image. ++/ +bool canGiveImages(string source) { + final switch (sourceOf(source)) { + case Source.ocda_db: return true; + case Source.pod: return true; + case Source.ssp: return imageDirBeside(source).length > 0; + case Source.unknown: return false; + } +} + +private ubyte[] fromDb(string path, string name) { + auto db = Db.open(path); + scope(exit) { db.close(); } + if (!db.has("files")) { return []; } + /+ ↓ the name is from the abstraction, not from a caller, and the quote + doubling is what keeps it a value rather than syntax +/ + return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '" + ~ sqlQuote(name) ~ "'"); +} + +private ubyte[] fromArchive(string path, string name) { + auto bytes = cast(ubyte[]) read(path); + if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); } + auto entries = zipEntries(bytes); + foreach (e; entries) { + if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); } + } + return []; +} + +private ubyte[] fromBeside(string path, string name) { + string dir = imageDirBeside(path); + if (dir.length == 0) { return []; } + string file = buildPath(dir, name); + return file.exists && file.isFile ? cast(ubyte[]) read(file) : []; +} + +/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/ +private string imageDirBeside(string path) { + string dir = path.dirName; + if (!dir.endsWith("abstraction")) { return ""; } + string image_dir = buildPath(dir.dirName, "image"); + return image_dir.exists ? image_dir : ""; +} + +/+ ↓ a single quote doubled is how sql says a single quote +/ +private string sqlQuote(string s) { + string out_; + foreach (c; s) { + out_ ~= (c == '\'') ? "''" : [c]; + } + return out_; +} diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d index f81112a..dec0626 100644 --- a/dlang/src/ocda/load_db.d +++ b/dlang/src/ocda/load_db.d @@ -151,7 +151,8 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) coalesce(table_aligns,''), coalesce(table_header,0), coalesce(code_linenumbers,0), coalesce(heading_ancestors_text,''), coalesce(sha256,''), - coalesce(text,''), id, coalesce(heading_lev_anchor,'') + coalesce(text,''), id, coalesce(heading_lev_anchor,''), + coalesce(segment_epub,''), coalesce(segment_epub_is,'') FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string ~ " ORDER BY CASE section WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3 @@ -189,6 +190,8 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false) obj.anchor = r[26]; obj.segment_html_is = r[27]; obj.heading_lev_anchor = r[37]; + obj.segment_epub = r[38]; + obj.segment_epub_is = r[39]; obj.table_cols = r[28].to!int; obj.table_widths = jsonWords(r[29]); obj.table_aligns = jsonWords(r[30]); diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d index 962e8f6..34d156c 100644 --- a/dlang/src/ocda/load_ssp.d +++ b/dlang/src/ocda/load_ssp.d @@ -198,6 +198,11 @@ private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unkn case "section": obj.own_section = value; break; case "identifier": obj.identifier = value; break; case "segment_html_is": obj.segment_html_is = value; break; + /+ ↓ the epub segments at heading levels 1 to 4 where the html + segments at level 4 alone, so the two are different files and + the abstraction records both +/ + case "segment_epub": obj.segment_epub = value; break; + case "segment_epub_is": obj.segment_epub_is = value; break; case "table_aligns": obj.table_aligns = value; break; case "stow_link": obj.stow_link ~= value; break; case "anchor_tag": obj.anchor_tags ~= value; break; diff --git a/dlang/src/ocda/model.d b/dlang/src/ocda/model.d index 8ee0043..403502a 100644 --- a/dlang/src/ocda/model.d +++ b/dlang/src/ocda/model.d @@ -20,6 +20,8 @@ struct OcdaObject { string is_of_type; // para, block, comment: how the text behaves string own_section; // .section, when it differs from the block it sits in string segment_html_is;// the html segment this object sits in + string segment_epub; // the epub segment this heading opens + string segment_epub_is;// the epub segment this object sits in string table_aligns; // l c r per column string[] stow_link; // links extracted from the object string[] anchor_tags; // named targets the markup placed on this object diff --git a/dlang/src/ocda/sqlite.d b/dlang/src/ocda/sqlite.d index 1552fc4..6f3753c 100644 --- a/dlang/src/ocda/sqlite.d +++ b/dlang/src/ocda/sqlite.d @@ -28,6 +28,8 @@ extern (C) { int sqlite3_column_int(sqlite3_stmt* stmt, int col); int sqlite3_column_type(sqlite3_stmt* stmt, int col); const(char)* sqlite3_errmsg(sqlite3* db); + const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col); + int sqlite3_column_bytes(sqlite3_stmt* stmt, int col); } enum SQLITE_OK = 0; @@ -94,6 +96,25 @@ struct Db { return (r.length > 0 && r[0].length > 0) ? r[0][0] : ""; } + /+ ↓ the first column of the first row as bytes, for a blob. + Separate from one(), because a blob is not text: an image read as text + stops at the first zero byte, and a png's ninth byte is zero. + +/ + ubyte[] blob(string sql) { + sqlite3_stmt* stmt; + if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) { + throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup + ~ " [" ~ sql ~ "]"); + } + scope(exit) { sqlite3_finalize(stmt); } + if (sqlite3_step(stmt) != SQLITE_ROW) { return []; } + int n = sqlite3_column_bytes(stmt, 0); + if (n <= 0) { return []; } + auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0); + if (p is null) { return []; } + return p[0 .. n].dup; + } + /+ ↓ does a table or view of this name exist +/ bool has(string name) { return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0"; diff --git a/gleam/src/ocda/dump.gleam b/gleam/src/ocda/dump.gleam index 2b05751..a75da67 100644 --- a/gleam/src/ocda/dump.gleam +++ b/gleam/src/ocda/dump.gleam @@ -147,6 +147,8 @@ fn object( |> kv("part", obj.part) |> kv("is_of_type", obj.is_of_type) |> kv("segment_html_is", obj.segment_html_is) + |> kv("segment_epub", obj.segment_epub) + |> kv("segment_epub_is", obj.segment_epub_is) |> kn("parent", obj.parent_ocn) |> kn("last_descendant", obj.last_descendant_ocn) |> kv("children", ints(obj.children)) diff --git a/gleam/src/ocda/images.gleam b/gleam/src/ocda/images.gleam new file mode 100644 index 0000000..2f3a56e --- /dev/null +++ b/gleam/src/ocda/images.gleam @@ -0,0 +1,135 @@ +//// An image's bytes, from whichever source the document came from. +//// +//// The model *names* images, with digest, byte size and pixel size; it does +//// not hold them. That is right - an abstraction of a document is not a +//// bundle of its assets - but a writer of any format that shows images needs +//// the bytes, so something has to know where they are. This is that +//// something, and it is in the library rather than in a writer because the +//// answer depends on the *input form* and not on the output. +//// +//// | `.ocda.db` | the `files` table, role 'image', by name | +//// | `.zip` | the archive entry whose path ends in `/image/<name>` | +//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot | +//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` | + +import gleam/bit_array +import gleam/dynamic/decode +import gleam/list +import gleam/result +import gleam/string +import ocda/load +import ocda/zip +import simplifile +import sqlight + +/// The bytes, or an error saying why not. +pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) { + case load.source_of(source) { + load.OcdaDb -> from_db(source, name) + load.Pod -> from_archive(source, name) + load.Ssp -> from_beside(source, name) + load.Unknown -> Error(source <> ": not a source this library reads") + } +} + +/// Can this source give images at all? +/// +/// A writer asks before it decides whether to emit them, because an `<img>` +/// pointing at a file it did not include is worse than no image. +pub fn can_give_images(source: String) -> Bool { + case load.source_of(source) { + load.OcdaDb -> True + load.Pod -> zip_readable(source) + load.Ssp -> image_dir_beside(source) != "" + load.Unknown -> False + } +} + +fn zip_readable(path: String) -> Bool { + case simplifile.read_bits(path) { + Error(_) -> False + Ok(bytes) -> + case zip.entries(bytes) { + Ok(_) -> True + Error(_) -> False + } + } +} + +fn from_db(path: String, name: String) -> Result(BitArray, String) { + case sqlight.open("file:" <> path <> "?mode=ro") { + Error(e) -> Error(path <> ": " <> e.message) + Ok(conn) -> { + let out = + sqlight.query( + "select data from files where role = 'image' and name = ?1", + on: conn, + with: [sqlight.text(name)], + expecting: blob(), + ) + |> result.replace_error(path <> ": could not read " <> name) + let _ = sqlight.close(conn) + case out { + Ok([data, ..]) -> Ok(data) + Ok([]) -> Error(name <> ": not in this database") + Error(e) -> Error(e) + } + } + } +} + +/// A blob column, read as bytes rather than as text: an image read as text +/// stops at the first zero byte, and a png's ninth byte is zero. +fn blob() -> decode.Decoder(BitArray) { + use v <- decode.field(0, decode.bit_array) + decode.success(v) +} + +fn from_archive(path: String, name: String) -> Result(BitArray, String) { + use bytes <- result.try( + simplifile.read_bits(path) + |> result.replace_error(path <> ": cannot read"), + ) + use entries <- result.try(zip.entries(bytes)) + case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) { + Error(_) -> Error(name <> ": not in this archive") + Ok(entry) -> zip.read(bytes, entry) + } +} + +fn from_beside(path: String, name: String) -> Result(BitArray, String) { + case image_dir_beside(path) { + "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it") + dir -> + simplifile.read_bits(dir <> "/" <> name) + |> result.replace_error(dir <> "/" <> name <> ": cannot read") + } +} + +/// `media/abstraction/<x>.ssp` -> `media/image/` +fn image_dir_beside(path: String) -> String { + let parts = string.split(path, on: "/") + let dir = case list.reverse(parts) { + [_file, ..rest] -> string.join(list.reverse(rest), "/") + [] -> "" + } + case string.ends_with(dir, "abstraction") { + False -> "" + True -> { + let parent = case list.reverse(string.split(dir, on: "/")) { + [_, ..rest] -> string.join(list.reverse(rest), "/") + [] -> "" + } + let image_dir = parent <> "/image" + case simplifile.is_directory(image_dir) { + Ok(True) -> image_dir + _ -> "" + } + } + } +} + +/// Unused here, kept so that a caller can size a read before making it. +pub fn byte_size(b: BitArray) -> Int { + bit_array.byte_size(b) +} diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam index e919615..a5f044b 100644 --- a/gleam/src/ocda/load_db.gleam +++ b/gleam/src/ocda/load_db.gleam @@ -243,6 +243,8 @@ fn objects( use text <- decode.field(35, decode.string) use id <- decode.field(36, decode.int) use hla <- decode.field(37, decode.string) + use seg_epub <- decode.field(38, decode.string) + use seg_epub_is <- decode.field(39, decode.string) decode.success(#( id, Object( @@ -280,6 +282,8 @@ fn objects( segment_next: segment_next, anchor: anchor, heading_lev_anchor: hla, + segment_epub: seg_epub, + segment_epub_is: seg_epub_is, table_cols: table_cols, table_widths: json_words(table_widths), table_header: table_header != 0, @@ -345,7 +349,8 @@ fn object_sql(doc_id: Int) -> String { coalesce(table_aligns,''), coalesce(table_header,0), coalesce(code_linenumbers,0), coalesce(heading_ancestors_text,''), coalesce(sha256,''), - coalesce(text,''), id, coalesce(heading_lev_anchor,'') + coalesce(text,''), id, coalesce(heading_lev_anchor,''), + coalesce(segment_epub,''), coalesce(segment_epub_is,'') from objects where doc_id = " <> int.to_string(doc_id) <> " order by case section when 'head' then 1 when 'toc' then 2 when 'body' then 3 diff --git a/gleam/src/ocda/load_ssp.gleam b/gleam/src/ocda/load_ssp.gleam index 83af6e2..b48e1c5 100644 --- a/gleam/src/ocda/load_ssp.gleam +++ b/gleam/src/ocda/load_ssp.gleam @@ -262,6 +262,8 @@ fn apply_property( "section" -> #(Object(..object, own_section: value), True) "identifier" -> #(Object(..object, identifier: value), True) "segment_html_is" -> #(Object(..object, segment_html_is: value), True) + "segment_epub" -> #(Object(..object, segment_epub: value), True) + "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True) "table_aligns" -> #(Object(..object, table_aligns: value), True) "stow_link" -> #( Object(..object, stow_link: list.append(object.stow_link, [value])), diff --git a/gleam/src/ocda/model.gleam b/gleam/src/ocda/model.gleam index a45cf6e..050a97b 100644 --- a/gleam/src/ocda/model.gleam +++ b/gleam/src/ocda/model.gleam @@ -85,6 +85,8 @@ pub type Object { /// the object's own section, when it differs from the block it sits in own_section: String, segment_html_is: String, + segment_epub: String, + segment_epub_is: String, table_aligns: String, stow_link: List(String), anchor_tags: List(String), @@ -130,6 +132,8 @@ pub fn new_object() -> Object { is_of_type: "", own_section: "", segment_html_is: "", + segment_epub: "", + segment_epub_is: "", table_aligns: "", stow_link: [], anchor_tags: [], |
