summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--dlang/src/ocda/dump.d2
-rw-r--r--dlang/src/ocda/images.d100
-rw-r--r--dlang/src/ocda/load_db.d5
-rw-r--r--dlang/src/ocda/load_ssp.d5
-rw-r--r--dlang/src/ocda/model.d2
-rw-r--r--dlang/src/ocda/sqlite.d21
-rw-r--r--gleam/src/ocda/dump.gleam2
-rw-r--r--gleam/src/ocda/images.gleam135
-rw-r--r--gleam/src/ocda/load_db.gleam7
-rw-r--r--gleam/src/ocda/load_ssp.gleam2
-rw-r--r--gleam/src/ocda/model.gleam4
11 files changed, 283 insertions, 2 deletions
diff --git a/dlang/src/ocda/dump.d b/dlang/src/ocda/dump.d
index b16a209..7a4f11a 100644
--- a/dlang/src/ocda/dump.d
+++ b/dlang/src/ocda/dump.d
@@ -79,6 +79,8 @@ string dumpDocument(ref OcdaDocument doc) {
kv("part", obj.part);
kv("is_of_type", obj.is_of_type);
kv("segment_html_is", obj.segment_html_is);
+ kv("segment_epub", obj.segment_epub);
+ kv("segment_epub_is", obj.segment_epub_is);
kn("parent", obj.parent_ocn);
kn("last_descendant", obj.last_descendant_ocn);
kv("children", ints(obj.children));
diff --git a/dlang/src/ocda/images.d b/dlang/src/ocda/images.d
new file mode 100644
index 0000000..b3a9652
--- /dev/null
+++ b/dlang/src/ocda/images.d
@@ -0,0 +1,100 @@
+module ocda.images;
+
+/+ ↓ an image's bytes, from whichever source the document came from.
+ .
+ The model *names* images, with digest, byte size and pixel size; it does
+ not hold them. That is right - an abstraction of a document is not a
+ bundle of its assets - but a writer of any format that shows images
+ needs the bytes, so something has to know where they are. This is that
+ something, and it is in the library rather than in a writer because the
+ answer depends on the *input form* and not on the output.
+ .
+ Three answers, and one refusal:
+ .
+ .ocda.db the files table, role 'image', by name
+ .sisupod the archive entry whose path ends in /image/<name>
+ .zip the same
+ .ssp beside it: a pod puts the abstraction in
+ media/abstraction/ and its images in media/image/, so the
+ path is documented rather than guessed
+ .
+ A .ssp that is not in a pod has nowhere to look, and that is reported as
+ absent rather than as an error: a reader that cannot find an image
+ should say so and carry on, because the document is still readable.
++/
+
+import std.algorithm : endsWith;
+import std.array : array;
+import std.file : exists, isFile, read;
+import std.path : buildPath, dirName;
+import ocda.load;
+import ocda.sqlite;
+import ocda.zip;
+import ocda.zstd;
+
+/+ ↓ the bytes, or an empty slice when the source has no such image +/
+ubyte[] imageBytes(string source, string name) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return fromDb(source, name);
+ case Source.pod: return fromArchive(source, name);
+ case Source.ssp: return fromBeside(source, name);
+ case Source.unknown: return [];
+ }
+}
+
+/+ ↓ can this source give images at all?
+ A writer asks before it decides whether to emit them, because an <img>
+ pointing at a file it did not include is worse than no image.
++/
+bool canGiveImages(string source) {
+ final switch (sourceOf(source)) {
+ case Source.ocda_db: return true;
+ case Source.pod: return true;
+ case Source.ssp: return imageDirBeside(source).length > 0;
+ case Source.unknown: return false;
+ }
+}
+
+private ubyte[] fromDb(string path, string name) {
+ auto db = Db.open(path);
+ scope(exit) { db.close(); }
+ if (!db.has("files")) { return []; }
+ /+ ↓ the name is from the abstraction, not from a caller, and the quote
+ doubling is what keeps it a value rather than syntax +/
+ return db.blob("SELECT data FROM files WHERE role = 'image' AND name = '"
+ ~ sqlQuote(name) ~ "'");
+}
+
+private ubyte[] fromArchive(string path, string name) {
+ auto bytes = cast(ubyte[]) read(path);
+ if (zstdIsFrame(bytes)) { bytes = zstdDecompress(bytes, 512 * 1024 * 1024); }
+ auto entries = zipEntries(bytes);
+ foreach (e; entries) {
+ if (e.name.endsWith("/image/" ~ name)) { return zipRead(bytes, e); }
+ }
+ return [];
+}
+
+private ubyte[] fromBeside(string path, string name) {
+ string dir = imageDirBeside(path);
+ if (dir.length == 0) { return []; }
+ string file = buildPath(dir, name);
+ return file.exists && file.isFile ? cast(ubyte[]) read(file) : [];
+}
+
+/+ ↓ media/abstraction/<x>.ssp -> media/image/ +/
+private string imageDirBeside(string path) {
+ string dir = path.dirName;
+ if (!dir.endsWith("abstraction")) { return ""; }
+ string image_dir = buildPath(dir.dirName, "image");
+ return image_dir.exists ? image_dir : "";
+}
+
+/+ ↓ a single quote doubled is how sql says a single quote +/
+private string sqlQuote(string s) {
+ string out_;
+ foreach (c; s) {
+ out_ ~= (c == '\'') ? "''" : [c];
+ }
+ return out_;
+}
diff --git a/dlang/src/ocda/load_db.d b/dlang/src/ocda/load_db.d
index f81112a..dec0626 100644
--- a/dlang/src/ocda/load_db.d
+++ b/dlang/src/ocda/load_db.d
@@ -151,7 +151,8 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false)
coalesce(table_aligns,''), coalesce(table_header,0),
coalesce(code_linenumbers,0),
coalesce(heading_ancestors_text,''), coalesce(sha256,''),
- coalesce(text,''), id, coalesce(heading_lev_anchor,'')
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,'')
FROM objects WHERE doc_id = " ~ chosen.doc_id.to!string
~ " ORDER BY CASE section
WHEN 'head' THEN 1 WHEN 'toc' THEN 2 WHEN 'body' THEN 3
@@ -189,6 +190,8 @@ OcdaDocument loadDbFile(string path, string lang = "", bool any_version = false)
obj.anchor = r[26];
obj.segment_html_is = r[27];
obj.heading_lev_anchor = r[37];
+ obj.segment_epub = r[38];
+ obj.segment_epub_is = r[39];
obj.table_cols = r[28].to!int;
obj.table_widths = jsonWords(r[29]);
obj.table_aligns = jsonWords(r[30]);
diff --git a/dlang/src/ocda/load_ssp.d b/dlang/src/ocda/load_ssp.d
index 962e8f6..34d156c 100644
--- a/dlang/src/ocda/load_ssp.d
+++ b/dlang/src/ocda/load_ssp.d
@@ -198,6 +198,11 @@ private void parseProperty(string line, ref OcdaObject obj, ref OcdaUnknown unkn
case "section": obj.own_section = value; break;
case "identifier": obj.identifier = value; break;
case "segment_html_is": obj.segment_html_is = value; break;
+ /+ ↓ the epub segments at heading levels 1 to 4 where the html
+ segments at level 4 alone, so the two are different files and
+ the abstraction records both +/
+ case "segment_epub": obj.segment_epub = value; break;
+ case "segment_epub_is": obj.segment_epub_is = value; break;
case "table_aligns": obj.table_aligns = value; break;
case "stow_link": obj.stow_link ~= value; break;
case "anchor_tag": obj.anchor_tags ~= value; break;
diff --git a/dlang/src/ocda/model.d b/dlang/src/ocda/model.d
index 8ee0043..403502a 100644
--- a/dlang/src/ocda/model.d
+++ b/dlang/src/ocda/model.d
@@ -20,6 +20,8 @@ struct OcdaObject {
string is_of_type; // para, block, comment: how the text behaves
string own_section; // .section, when it differs from the block it sits in
string segment_html_is;// the html segment this object sits in
+ string segment_epub; // the epub segment this heading opens
+ string segment_epub_is;// the epub segment this object sits in
string table_aligns; // l c r per column
string[] stow_link; // links extracted from the object
string[] anchor_tags; // named targets the markup placed on this object
diff --git a/dlang/src/ocda/sqlite.d b/dlang/src/ocda/sqlite.d
index 1552fc4..6f3753c 100644
--- a/dlang/src/ocda/sqlite.d
+++ b/dlang/src/ocda/sqlite.d
@@ -28,6 +28,8 @@ extern (C) {
int sqlite3_column_int(sqlite3_stmt* stmt, int col);
int sqlite3_column_type(sqlite3_stmt* stmt, int col);
const(char)* sqlite3_errmsg(sqlite3* db);
+ const(void)* sqlite3_column_blob(sqlite3_stmt* stmt, int col);
+ int sqlite3_column_bytes(sqlite3_stmt* stmt, int col);
}
enum SQLITE_OK = 0;
@@ -94,6 +96,25 @@ struct Db {
return (r.length > 0 && r[0].length > 0) ? r[0][0] : "";
}
+ /+ ↓ the first column of the first row as bytes, for a blob.
+ Separate from one(), because a blob is not text: an image read as text
+ stops at the first zero byte, and a png's ninth byte is zero.
+ +/
+ ubyte[] blob(string sql) {
+ sqlite3_stmt* stmt;
+ if (sqlite3_prepare_v2(_handle, sql.toStringz, -1, &stmt, null) != SQLITE_OK) {
+ throw new SqliteError("prepare: " ~ sqlite3_errmsg(_handle).fromStringz.idup
+ ~ " [" ~ sql ~ "]");
+ }
+ scope(exit) { sqlite3_finalize(stmt); }
+ if (sqlite3_step(stmt) != SQLITE_ROW) { return []; }
+ int n = sqlite3_column_bytes(stmt, 0);
+ if (n <= 0) { return []; }
+ auto p = cast(const(ubyte)*) sqlite3_column_blob(stmt, 0);
+ if (p is null) { return []; }
+ return p[0 .. n].dup;
+ }
+
/+ ↓ does a table or view of this name exist +/
bool has(string name) {
return one("SELECT count(*) FROM sqlite_master WHERE name = '" ~ name ~ "'") != "0";
diff --git a/gleam/src/ocda/dump.gleam b/gleam/src/ocda/dump.gleam
index 2b05751..a75da67 100644
--- a/gleam/src/ocda/dump.gleam
+++ b/gleam/src/ocda/dump.gleam
@@ -147,6 +147,8 @@ fn object(
|> kv("part", obj.part)
|> kv("is_of_type", obj.is_of_type)
|> kv("segment_html_is", obj.segment_html_is)
+ |> kv("segment_epub", obj.segment_epub)
+ |> kv("segment_epub_is", obj.segment_epub_is)
|> kn("parent", obj.parent_ocn)
|> kn("last_descendant", obj.last_descendant_ocn)
|> kv("children", ints(obj.children))
diff --git a/gleam/src/ocda/images.gleam b/gleam/src/ocda/images.gleam
new file mode 100644
index 0000000..2f3a56e
--- /dev/null
+++ b/gleam/src/ocda/images.gleam
@@ -0,0 +1,135 @@
+//// An image's bytes, from whichever source the document came from.
+////
+//// The model *names* images, with digest, byte size and pixel size; it does
+//// not hold them. That is right - an abstraction of a document is not a
+//// bundle of its assets - but a writer of any format that shows images needs
+//// the bytes, so something has to know where they are. This is that
+//// something, and it is in the library rather than in a writer because the
+//// answer depends on the *input form* and not on the output.
+////
+//// | `.ocda.db` | the `files` table, role 'image', by name |
+//// | `.zip` | the archive entry whose path ends in `/image/<name>` |
+//// | `.sisupod` | the same, if the frame can be unwrapped, which on the BEAM it cannot |
+//// | `.ssp` | beside it: a pod puts the abstraction in `media/abstraction/` and its images in `media/image/` |
+
+import gleam/bit_array
+import gleam/dynamic/decode
+import gleam/list
+import gleam/result
+import gleam/string
+import ocda/load
+import ocda/zip
+import simplifile
+import sqlight
+
+/// The bytes, or an error saying why not.
+pub fn image_bytes(source: String, name: String) -> Result(BitArray, String) {
+ case load.source_of(source) {
+ load.OcdaDb -> from_db(source, name)
+ load.Pod -> from_archive(source, name)
+ load.Ssp -> from_beside(source, name)
+ load.Unknown -> Error(source <> ": not a source this library reads")
+ }
+}
+
+/// Can this source give images at all?
+///
+/// A writer asks before it decides whether to emit them, because an `<img>`
+/// pointing at a file it did not include is worse than no image.
+pub fn can_give_images(source: String) -> Bool {
+ case load.source_of(source) {
+ load.OcdaDb -> True
+ load.Pod -> zip_readable(source)
+ load.Ssp -> image_dir_beside(source) != ""
+ load.Unknown -> False
+ }
+}
+
+fn zip_readable(path: String) -> Bool {
+ case simplifile.read_bits(path) {
+ Error(_) -> False
+ Ok(bytes) ->
+ case zip.entries(bytes) {
+ Ok(_) -> True
+ Error(_) -> False
+ }
+ }
+}
+
+fn from_db(path: String, name: String) -> Result(BitArray, String) {
+ case sqlight.open("file:" <> path <> "?mode=ro") {
+ Error(e) -> Error(path <> ": " <> e.message)
+ Ok(conn) -> {
+ let out =
+ sqlight.query(
+ "select data from files where role = 'image' and name = ?1",
+ on: conn,
+ with: [sqlight.text(name)],
+ expecting: blob(),
+ )
+ |> result.replace_error(path <> ": could not read " <> name)
+ let _ = sqlight.close(conn)
+ case out {
+ Ok([data, ..]) -> Ok(data)
+ Ok([]) -> Error(name <> ": not in this database")
+ Error(e) -> Error(e)
+ }
+ }
+ }
+}
+
+/// A blob column, read as bytes rather than as text: an image read as text
+/// stops at the first zero byte, and a png's ninth byte is zero.
+fn blob() -> decode.Decoder(BitArray) {
+ use v <- decode.field(0, decode.bit_array)
+ decode.success(v)
+}
+
+fn from_archive(path: String, name: String) -> Result(BitArray, String) {
+ use bytes <- result.try(
+ simplifile.read_bits(path)
+ |> result.replace_error(path <> ": cannot read"),
+ )
+ use entries <- result.try(zip.entries(bytes))
+ case list.find(entries, fn(e) { string.ends_with(e.name, "/image/" <> name) }) {
+ Error(_) -> Error(name <> ": not in this archive")
+ Ok(entry) -> zip.read(bytes, entry)
+ }
+}
+
+fn from_beside(path: String, name: String) -> Result(BitArray, String) {
+ case image_dir_beside(path) {
+ "" -> Error(name <> ": a .ssp outside a pod has no image directory beside it")
+ dir ->
+ simplifile.read_bits(dir <> "/" <> name)
+ |> result.replace_error(dir <> "/" <> name <> ": cannot read")
+ }
+}
+
+/// `media/abstraction/<x>.ssp` -> `media/image/`
+fn image_dir_beside(path: String) -> String {
+ let parts = string.split(path, on: "/")
+ let dir = case list.reverse(parts) {
+ [_file, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ case string.ends_with(dir, "abstraction") {
+ False -> ""
+ True -> {
+ let parent = case list.reverse(string.split(dir, on: "/")) {
+ [_, ..rest] -> string.join(list.reverse(rest), "/")
+ [] -> ""
+ }
+ let image_dir = parent <> "/image"
+ case simplifile.is_directory(image_dir) {
+ Ok(True) -> image_dir
+ _ -> ""
+ }
+ }
+ }
+}
+
+/// Unused here, kept so that a caller can size a read before making it.
+pub fn byte_size(b: BitArray) -> Int {
+ bit_array.byte_size(b)
+}
diff --git a/gleam/src/ocda/load_db.gleam b/gleam/src/ocda/load_db.gleam
index e919615..a5f044b 100644
--- a/gleam/src/ocda/load_db.gleam
+++ b/gleam/src/ocda/load_db.gleam
@@ -243,6 +243,8 @@ fn objects(
use text <- decode.field(35, decode.string)
use id <- decode.field(36, decode.int)
use hla <- decode.field(37, decode.string)
+ use seg_epub <- decode.field(38, decode.string)
+ use seg_epub_is <- decode.field(39, decode.string)
decode.success(#(
id,
Object(
@@ -280,6 +282,8 @@ fn objects(
segment_next: segment_next,
anchor: anchor,
heading_lev_anchor: hla,
+ segment_epub: seg_epub,
+ segment_epub_is: seg_epub_is,
table_cols: table_cols,
table_widths: json_words(table_widths),
table_header: table_header != 0,
@@ -345,7 +349,8 @@ fn object_sql(doc_id: Int) -> String {
coalesce(table_aligns,''), coalesce(table_header,0),
coalesce(code_linenumbers,0),
coalesce(heading_ancestors_text,''), coalesce(sha256,''),
- coalesce(text,''), id, coalesce(heading_lev_anchor,'')
+ coalesce(text,''), id, coalesce(heading_lev_anchor,''),
+ coalesce(segment_epub,''), coalesce(segment_epub_is,'')
from objects where doc_id = " <> int.to_string(doc_id) <> "
order by case section
when 'head' then 1 when 'toc' then 2 when 'body' then 3
diff --git a/gleam/src/ocda/load_ssp.gleam b/gleam/src/ocda/load_ssp.gleam
index 83af6e2..b48e1c5 100644
--- a/gleam/src/ocda/load_ssp.gleam
+++ b/gleam/src/ocda/load_ssp.gleam
@@ -262,6 +262,8 @@ fn apply_property(
"section" -> #(Object(..object, own_section: value), True)
"identifier" -> #(Object(..object, identifier: value), True)
"segment_html_is" -> #(Object(..object, segment_html_is: value), True)
+ "segment_epub" -> #(Object(..object, segment_epub: value), True)
+ "segment_epub_is" -> #(Object(..object, segment_epub_is: value), True)
"table_aligns" -> #(Object(..object, table_aligns: value), True)
"stow_link" -> #(
Object(..object, stow_link: list.append(object.stow_link, [value])),
diff --git a/gleam/src/ocda/model.gleam b/gleam/src/ocda/model.gleam
index a45cf6e..050a97b 100644
--- a/gleam/src/ocda/model.gleam
+++ b/gleam/src/ocda/model.gleam
@@ -85,6 +85,8 @@ pub type Object {
/// the object's own section, when it differs from the block it sits in
own_section: String,
segment_html_is: String,
+ segment_epub: String,
+ segment_epub_is: String,
table_aligns: String,
stow_link: List(String),
anchor_tags: List(String),
@@ -130,6 +132,8 @@ pub fn new_object() -> Object {
is_of_type: "",
own_section: "",
segment_html_is: "",
+ segment_epub: "",
+ segment_epub_is: "",
table_aligns: "",
stow_link: [],
anchor_tags: [],