ocda db: now carry markup source, conf & manifest

The database already carried the images, because without them no output
can be produced. It now carries the markup as well, and with it the
document's configuration and the manifest as the author wrote it.

That is the difference between a serialised abstraction and a document
source. An abstraction can be rendered but not re-parsed, and a reader
who wants to correct a sentence needs the sentence as written. With
these a pod can be written back out of the sqlite-file.

Three new values in the existing role column, no schema change: the
format stays 2.0.

Source rows are named by their path within the pod,
media/text/<lang>/<file>, and not by bare filename as images are. Every
language of a document has a file of the same name, so bare names would
collide under UNIQUE(role, name) and nine of ten would be dropped
without a word. The path is also what a pod materialised from this
database has to be told.

Bytes stored as read with the digest over them, so that each row is
checkable against the line source.digest was built from.

Also build.spine_version, a file level row saying which spine wrote the
file: not a property of the document, and the one thing a file cannot be
asked for afterwards. The reader skips the build. prefix as it skips
schema. and translation., or it would come back inside the document
header and the round trip would differ.

(assisted by Claude-Code)

2 files changed, 86 insertions, 1 deletions

diff --git a/src/sisudoc/ocda/abstraction/db_in.d b/src/sisudoc/ocda/abstraction/db_in.d
index 7f1b2c3..53ac95d 100644
--- a/src/sisudoc/ocda/abstraction/db_in.d
+++ b/src/sisudoc/ocda/abstraction/db_in.d
@@ -204,7 +204,10 @@ template spineAbstractionDbRead() {
         string _kk = _k["doc_has.".length .. $];
         doc.doc_has[_kk] = _v;
         doc.doc_has_order ~= _kk;
-      } else if (_k.startsWith("schema.")) {
+      } else if (_k.startsWith("schema.") || _k.startsWith("build.")) {
+        /+ ↓ what the file is and what wrote it. Neither is part of the
+             document, and only the format version is wanted here.
+        +/
         if (_k == "schema.version") {
           doc.format = "% SiSU Document Abstraction v" ~ _v;
         }
diff --git a/src/sisudoc/outputs/io_out/sqlite_ocda_db.d b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d
index aa91e89..d30295a 100644
--- a/src/sisudoc/outputs/io_out/sqlite_ocda_db.d
+++ b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d
@@ -240,6 +240,22 @@ template spineAbstractionDb() {
       -- output writers must have in order to produce a document. bytes are
       -- stored exactly as read, so sha256 matches the digest the abstraction
       -- was built with
+      --
+      --   image     what the output writers need to produce a document.
+      --             named by its bare filename, as the abstraction refers
+      --             to it, and shared by every language.
+      --   source    the markup: the document file and the files it inserts,
+      --             one set per language. named by its path within the pod,
+      --             media/text/<lang>/<file>, because the bare names repeat
+      --             across languages and because that is where a pod
+      --             materialised from this database has to put them.
+      --   conf      conf/document_make, the document's own configuration.
+      --   manifest  pod.manifest, as the author wrote it.
+      --
+      -- source, conf and manifest are what make this a document source and
+      -- not only a serialised abstraction: with them a pod can be written
+      -- back out of the file and rebuilt from the markup, rather than only
+      -- rendered from the objects.
       CREATE TABLE IF NOT EXISTS files (
         id      INTEGER PRIMARY KEY,
         role    TEXT NOT NULL,
@@ -379,6 +395,12 @@ template spineAbstractionDb() {
     +/
     insertFileMeta("schema.name", "sisu-abstraction-db");
     insertFileMeta("schema.version", ssp_format_version);
+    /+ ↓ which spine wrote the file. Not a property of the document and not
+         in the .ssp: it says what to blame when a database made months ago
+         reads oddly, and it is the one thing a file cannot be asked for
+         after the fact.
+    +/
+    insertFileMeta("build.spine_version", doc_matters.generator_program.ver);
     insertMeta("source.filename", ssp_doc.source);
     insertBlock("source.", ssp_doc.source_info_order, ssp_doc.source_info);
     /+ ↓ the digest of the .ssp this database was built from. a file cannot
@@ -727,6 +749,66 @@ template spineAbstractionDb() {
         file_stmt.execute();
         file_stmt.reset();
       }
+      /+ ↓ the markup, the configuration and the manifest.
+           Carried for the same reason the images are, and for one more: a
+           database that holds the markup is a document source, and a pod can
+           be written back out of it. The abstraction alone can be rendered
+           but not re-parsed, and a reader who wants to correct a sentence
+           needs the sentence as the author wrote it.
+           .
+           Stored as read, with the digest over those bytes, which is the
+           digest the abstraction was built with: source.digest in the header
+           is the sha256 over one line per markup file, and each line is
+           checkable against the row written here.
+      +/
+      void _carry(string _role, string _name, string _path) {
+        import std.typecons : Nullable;
+        if (_path.length == 0 || _name.length == 0) { return; }
+        ubyte[] _bytes;
+        try {
+          if (_path.exists && _path.isFile) { _bytes = cast(ubyte[]) _path.read; }
+        } catch (Exception ex) {
+          _bytes = [];
+        }
+        if (_bytes.length == 0) {
+          if (doc_matters.opt.action.vox_gt_1) {
+            writeln("  ", _role, " not carried into abstraction db: ", _path);
+          }
+          return;
+        }
+        file_stmt.bind(":role",   _role);
+        file_stmt.bind(":name",   _name);
+        file_stmt.bind(":bytes",  cast(long) _bytes.length);
+        file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string);
+        file_stmt.bind(":width",  Nullable!int());
+        file_stmt.bind(":height", Nullable!int());
+        file_stmt.bind(":data",   _bytes);
+        file_stmt.execute();
+        file_stmt.reset();
+      }
+      {
+        import std.path : baseName;
+        /+ ↓ the markup of this language. The writer runs once per language,
+             so the set of source rows fills up as the languages go by, and
+             UNIQUE(role, name) over the pod relative path is what keeps a
+             second run from writing them twice.
+        +/
+        string _text_dir = "media/text/" ~ doc_matters.src.lng ~ "/";
+        _carry("source", _text_dir ~ doc_matters.src.filename,
+          doc_matters.src.file_with_absolute_path.to!string);
+        foreach (_insert; doc_matters.srcs.file_insert_list) {
+          _carry("source", _text_dir ~ _insert.baseName, _insert);
+        }
+        /+ ↓ the configuration and the manifest, one of each for the whole
+             document however many languages it has
+        +/
+        _carry("conf", "conf/document_make",
+          doc_matters.src.conf_dir_path.to!string ~ "/document_make");
+        if (doc_matters.src.is_pod) {
+          _carry("manifest", "pod.manifest",
+            doc_matters.pod.manifest_file_with_path);
+        }
+      }
       file_stmt.finalize();
     }