diff options
Diffstat (limited to 'org/document_source_conversions.org')
| -rw-r--r-- | org/document_source_conversions.org | 238 |
1 files changed, 238 insertions, 0 deletions
diff --git a/org/document_source_conversions.org b/org/document_source_conversions.org new file mode 100644 index 0000000..cde085e --- /dev/null +++ b/org/document_source_conversions.org @@ -0,0 +1,238 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) object-centric document abstraction +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+SUMMARY: process markup document, create document abstraction +#+FILETAGS: :spine:abstraction: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] + +* document conversions +** write a sisupod from a database + +#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/pod_from_db.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/++ + a pod, written back out of a database<br><br> + . + the inverse of the carrying: markup, configuration, manifest and images + out of a .ocda.db and onto the filesystem as a pod tree<br><br> + . + [sisudoc.ocda.abstraction.pod_from_db] ++/ +module sisudoc.ocda.abstraction.pod_from_db; +@safe: +/+ ↓ a database that carries its source can give the pod back. + . + This writes the tree and stops. It does not parse, and it does not produce + output: what it leaves behind is a pod directory like any other, and what + happens next is whatever would have happened to the original. That is the + intent. A materialiser that also rendered would be a second path to every + output format, and would provide opportunity for the two to drift; a + materialiser that only writes files means the document is built by the + same code over the same bytes, and "identical output" is guaranteed as a + consequence (rather than an aspirational). + . + What is written: + . + <dest>/<pod>/pod.manifest role='manifest' + <dest>/<pod>/conf/document_make role='conf' + <dest>/<pod>/media/text/<lang>/<file> role='source' + <dest>/<pod>/media/image/<file> role='image' + . + The pod's name comes from the database's own filename: <doc>.ocda.db is + named by doc_uid_out_no_lang, which is the pod name and the document's + filename joined by ":" when they differ and the one name when they do not. + Splitting on ":" inverts that rule exactly, so a pod written here + recomputes the uid the database was named by, and every output file lands + on the name it had before. Rename the database and the pod is named + accordingly, which is the behaviour to expect of a file named for its + document. + . + Every name is checked before anything is created, and every file is checked + against the digest stored with it. A database can be downloaded, so its + names are attacker controlled: a name that climbs out of the pod is + refused, and one bad name refuses the whole artefact rather than skipping + one file, which is what the zip reader does with a zip. A digest that does + not match is refused outright here, unlike the images, which are written + with a warning: an image that is wrong makes a document that looks wrong, + while markup that is wrong makes a document that *is* wrong, silently and + in its text. ++/ +template spinePodFromDb() { + /+ ↓ narrow imports: this is mixed in, and both std.file and std.stdio + define write, which is an ambiguity the mixing scope inherits + +/ + import std.array : array; + import std.conv : to; + import std.file : exists, mkdirRecurse, fileWrite = write; + import std.path : baseName, chainPath, dirName; + import std.stdio : writeln; + import sisudoc.ocda.io_in.carried_names; + import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; + mixin spineCarriedNames; + mixin spineAbstractionDbRead _dbr; + struct ST_PodMaterialised { + string pod_dir; // where the pod was written, "" when it was not + string note; // why not, when it was not + size_t files; // how many were written + bool ok; + } + /+ ↓ the pod's name, from the database's own filename +/ + string podNameFromDbPath(string _db_file) { + import std.algorithm : endsWith, findSplit; + import sisudoc.ocda.meta.defaults : InternalMarkup; + mixin InternalMarkup _mkup_; + auto _mkup = _mkup_.InlineMarkup(); + string _stem = _db_file.baseName; + foreach (_sfx; [".ocda.db", ".db"]) { + if (_stem.endsWith(_sfx)) { _stem = _stem[0 .. $ - _sfx.length]; break; } + } + if (auto _s = _stem.findSplit(_mkup.uid_sep)) { return _s[0]; } + return _stem; + } + /+ ↓ where each role is written within the pod. + image is the one role whose rows are named by bare filename, because that + is how the abstraction refers to an image; the rest carry their path + within the pod and are written at it. + +/ + private string _relPathFor(string _role, string _name) { + switch (_role) { + case "image": return "media/image/" ~ _name; + case "source": + case "conf": + case "manifest": return _name; + default: return ""; + } + } + @trusted ST_PodMaterialised podFromDb(O)( + string _db_file, + string _dest_root, + O _opt_action, + ) { + import std.digest : toHexString; + import std.digest.sha : sha256Of; + ST_PodMaterialised _out; + if (!_db_file.exists) { + _out.note = "no such file"; + return _out; + } + _dbr.ST_ArtefactFile[] _files; + foreach (_role; ["manifest", "conf", "source", "image"]) { + _files ~= _dbr.dbReadFiles(_db_file, _role); + } + if (_files.length == 0) { + _out.note = "carries no source: written by a spine older than the" + ~ " format that carries markup, or written from an artefact"; + return _out; + } + bool _has_source = false; + foreach (_f; _files) { if (_f.role == "source") { _has_source = true; } } + if (!_has_source) { + _out.note = "carries images but no markup, so no pod can be written" + ~ " from it"; + return _out; + } + string _pod_dir = (_dest_root.chainPath(podNameFromDbPath(_db_file)) + .array).to!string; + /+ ↓ every name checked before anything is created or written +/ + foreach (_f; _files) { + string _rel = _relPathFor(_f.role, _f.name); + if (_rel.length == 0) { + _out.note = "carries a file of a role spine does not write: " + ~ _f.role; + return _out; + } + string _bad = (_f.role == "image") + ? validateCarriedFileName(_f.name) + : validateCarriedPath(_f.name); + if (_bad.length > 0) { + _out.note = "carries a file spine will not write: " ~ _bad; + return _out; + } + } + /+ ↓ and every digest, before anything is created or written. markup that + does not match what was recorded with it is refused, not warned about: + the difference would be in the text of the document and nothing + downstream would notice. + +/ + foreach (_f; _files) { + if (_f.sha256.length == 0) { continue; } + string _got = _f.data.sha256Of.toHexString.to!string; + if (_got != _f.sha256) { + _out.note = _f.role ~ " " ~ _f.name + ~ " does not match the digest recorded with it (" + ~ _f.sha256 ~ " expected, " ~ _got ~ " found)"; + return _out; + } + } + foreach (_f; _files) { + string _path = (_pod_dir.chainPath(_relPathFor(_f.role, _f.name)) + .array).to!string; + /+ ↓ the check on the check: the name rules above already forbid a + path that climbs, so this can only fire if they were loosened + +/ + if (!(carriedPathIsWithin(_pod_dir, _path))) { + _out.note = _f.name ~ " resolves outside the pod"; + return _out; + } + try { + _path.dirName.mkdirRecurse; + fileWrite(_path, _f.data); + } catch (Exception ex) { + _out.note = "could not write " ~ _f.name ~ ": " ~ ex.msg; + return _out; + } + _out.files += 1; + } + _out.pod_dir = _pod_dir; + _out.ok = true; + if (_opt_action.vox_gt_1) { + writeln(" pod from ", _db_file.baseName, ": ", _out.files, + " file(s) in ", _out.pod_dir); + } + return _out; + } +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ + |