-*- mode: org -*- #+TITLE: spine (doc_reform) output pod #+DESCRIPTION: documents - structuring, publishing in multiple formats & search #+FILETAGS: :spine:output:source:pod: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 - 2023 Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args :exports code #+PROPERTY: header-args+ :noweb yes #+PROPERTY: header-args+ :results no #+PROPERTY: header-args+ :cache no #+PROPERTY: header-args+ :padline no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - [[./doc-reform.org][doc-reform.org]] [[./][org/]] - [[./output_hub.org][output_hub]] * pod ** _module template_ :module: #+HEADER: :tangle "../src/doc_reform/io_out/source_pod.d" #+HEADER: :noweb yes #+BEGIN_SRC d <<doc_header_including_copyright_and_license>> module doc_reform.io_out.source_pod; template spinePod() { <<output_imports>> void spinePod(T)(T doc_matters) { <<source_pod_init>> <<source_pod_archive_zip>> try { <<source_pod_mkdirs>> <<source_pod_archive_prepare_for_zip>> <<source_pod_copy_info>> <<source_pod_zip>> } catch (ErrnoException ex) { // Handle error } } } #+END_SRC ** init *** imports #+NAME: output_imports #+BEGIN_SRC d import doc_reform.meta.rgx_files, doc_reform.io_out; import std.digest.sha, std.file, std.outbuffer, std.zip, std.conv : to; import doc_reform.io_out.create_zip_file, doc_reform.io_out.xmls; #+END_SRC *** init #+NAME: source_pod_init #+BEGIN_SRC d debug(asserts) { // static assert(is(typeof(doc_matters) == tuple)); } mixin spineRgxFiles; string pwd = doc_matters.env.pwd; auto src_path_info = doc_matters.src_path_info; auto pth_dr_doc_src = doc_matters.src_path_info; auto pths_pod = spinePathsPods!()(doc_matters); mixin spineLanguageCodes; auto lang = Lang(); static auto rgx_files = RgxFiles(); assert (doc_matters.src.filename.match(rgx_files.src_fn)); #+END_SRC *** pod zip archive #+NAME: source_pod_archive_zip #+BEGIN_SRC d @system auto pod_archive(Z)( string _source_type, string _data_in, string _pth_out, Z zip ) { auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = _pth_out; auto zip_data = new OutBuffer(); switch (_source_type) { case "file_path_bin": zip_data.write(cast(char[]) ((_data_in).read)); goto default; case "file_path_text": zip_data.write((_data_in).readText); goto default; case "string": zip_data.write(_data_in); goto default; default: zip_arc_member_file.expandedData = zip_data.toBytes(); zip.addMember(zip_arc_member_file); } return zip; } #+END_SRC ** mkdir :mkdir: #+NAME: source_pod_mkdirs #+BEGIN_SRC d /+ create directory structure +/ if (!exists(pths_pod.pod_dir_())) { // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) pths_pod.pod_dir_().mkdirRecurse; } if (doc_matters.opt.action.source_or_pod) { if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); } if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; } } #+END_SRC ** copy :copy: #+NAME: source_pod_archive_prepare_for_zip #+BEGIN_SRC d if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln(__LINE__, ": ", doc_matters.src.filename, " -> ", pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod ); } auto zip = new ZipArchive(); auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; { /+ bundle images +/ foreach (image; doc_matters.srcs.image_list) { debug(podimages) { writeln( pth_dr_doc_src.image_root.to!string, "/", image, " -> ", pths_pod.image_root(doc_matters.src.filename).zpod, "/", image ); } auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; auto fn_src_out_pod_zip_base = pths_pod.image_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ image; auto fn_src_out_filesystem = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ image; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (image): ", fn_src_in); } } } } { /+ bundle dr_document_make +/ auto fn_src_in = ((doc_matters.src.is_pod) ? doc_matters.src.conf_dir_path : pth_dr_doc_src.conf_root).to!string ~ "/" ~ "dr_document_make"; auto fn_src_out_pod_zip_base = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make"; auto fn_src_out_filesystem = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ "dr_document_make"; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); } } } { /+ pod manifest +/ auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; auto fn_src_out_pod_zip_base = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; auto fn_src_out_filesystem = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path auto fn_src_out_inside_pod = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src in found: ", fn_src_in); } filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; filelist_src_zpod_arr ~= fn_src_out_inside_pod; { import dyaml; auto pod_filelist_yaml_string = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); Node _pmy; string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { try { _pmy = Loader.fromString(_pm).load(); } catch (ErrnoException ex) { } catch (Throwable) { writeln("ERROR failed to read config file content, not parsed as yaml"); } writeln("pod filename: ", _pmy["doc"]["filename"].get!string); writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); writeln("pod languages: ", doc_matters.src.language); // foreach(string _l; _pmy["doc"]["language"]) { // writeln("language: ", _l); // } } if (doc_matters.opt.action.source_or_pod) { pod_filelist_yaml_string.writeln(_pm); } if (doc_matters.opt.action.pod) { zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); } } } } { /+ bundle primary file (.ssm/.sst) +/ auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; auto fn_src_out_pod_zip_base = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; auto fn_src_out_filesystem = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: auto fn_src_out_inside_pod = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src in found: ", fn_src_in); } filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; filelist_src_zpod_arr ~= fn_src_out_inside_pod; string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); if (auto _x = fn_src_in.match(_rgx)){ if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { string _path_to_pod = _x.captures["path_to_pod"]; string _podname = _x.captures["podname"]; string _root_to_lang = _x.captures["from_root"]; string _language = _x.captures["language"]; string _filename = _x.captures["filename"]; foreach (_lang; doc_matters.pod.manifest_list_of_languages) { string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } } } else { zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); } } } { /+ bundle insert files (.ssi) +/ if (doc_matters.srcs.file_insert_list.length > 0) { auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); foreach (insert_file; doc_matters.srcs.file_insert_list) { debug(pod) { writeln( insert_file, " -> ", pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).zpod ); } if (auto _x = insert_file.match(_rgx)){ if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { string _path_to_pod = _x.captures["path_to_pod"]; string _podname = _x.captures["podname"]; string _root_to_lang = _x.captures["from_root"]; string _language = _x.captures["language"]; string _filename = _x.captures["filename"]; foreach (_lang; doc_matters.pod.manifest_list_of_languages) { string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; if (exists(_pth_mkup_src_in)) { if (doc_matters.opt.action.source_or_pod) { auto fn_src_out_filesystem // you need to change language sources = pths_pod.fn_doc_insert( doc_matters.src.filename, // doc_matters.src.filename _pth_mkup_src_in, // insert_file _lang, ).filesystem_open_zpod.to!string; _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere } if (doc_matters.opt.action.pod) { zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); } } } } } else { auto fn_src_in = insert_file; auto fn_src_out_pod_zip_base = pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).zpod.to!string; auto fn_src_out_filesystem = pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).filesystem_open_zpod.to!string; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); } } } } } } { auto fn_src_in = doc_matters.src.filename; if (doc_matters.opt.action.pod) { if (exists(doc_matters.src.file_with_absolute_path)) { createZipFile!()(fn_pod, zip.build()); } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } if (!(exists(fn_pod))) { writeln("WARNING failed to create pod zip archive: ", fn_pod); } } } #+END_SRC ** sha256 of pod.zip, zip debug, read zip archive #+NAME: source_pod_copy_info #+BEGIN_SRC d if (exists(fn_pod)) { try { if (doc_matters.opt.action.vox_gt0 && doc_matters.opt.action.pod) { auto data = (cast(byte[]) (fn_pod).read); if (doc_matters.opt.action.vox_gt1) { writeln(" ", doc_matters.src.filename, " > "); } if (doc_matters.opt.action.pod) { writefln("%s\n. %-(%02x%)::%s . %s.zip", fn_pod, data.sha256Of, data.length, doc_matters.src.filename_base); } } if (doc_matters.opt.action.debug_do_pod) { try { auto zipped = new ZipArchive((fn_pod).read); foreach (filename, member; zipped.directory) { auto data = zipped.expand(member); writeln(". ", ((data).sha256Of).toHexString, "::", data.length, " . ", filename); } } catch (ZipException ex) { // Handle errors } } } catch (ErrnoException ex) { // Handle errors } } #+END_SRC ** source pod zip UNUSED #+NAME: source_pod_zip #+BEGIN_SRC d // source pod zip #+END_SRC * document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+BEGIN_SRC txt /+ - Name: Spine, Doc Reform [a part of] - Description: documents, structuring, processing, publishing, search - static content generator - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 - 2023 Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: Spine (SiSU), a framework for document structuring, publishing and search Copyright (C) Ralph Amissah This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFERO General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at these locations: [https://www.fsf.org/licensing/licenses/agpl.html] [https://www.gnu.org/licenses/agpl.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.doc_reform.org] [https://www.sisudoc.org] - Git [https://git.sisudoc.org/projects/?p=software/spine.git;a=summary] +/ #+END_SRC * __END__ ** pod gitignore *** pod/{pods/} #+NAME: pod_gitignore #+BEGIN_SRC sh # git ls-files --others --exclude-from=.git/info/exclude ,* !.gitignore !COPYRIGHT !README !shell.nix !packages.nix !version.txt !pod/ !pod/.dr/config_local_site # pod.manifest !pod/*/ !pod/*/pod.manifest # conf/sisu_document_make !pod/*/conf/ !pod/*/conf/sisu_document_make # media/text !pod/*/media/ !pod/*/media/text/ !pod/*/media/text/*/ !pod/*/media/text/*/*.sst !pod/*/media/text/*/*.ssm !pod/*/media/text/*/*.ssi !pod/*/media/text/*/index.html.in # media/image !pod/*/media/image/ !pod/*/media/image/*.png !pod/*/media/image/*.jpg !pod/*/media/image/*.PNG !pod/*/media/image/*.JPG # tools/po4a !pod/*/tools/ !pod/*/tools/Makefile !pod/*/tools/bin/ !pod/*/tools/bin/*.sh !pod/*/tools/po4a/ !pod/*/tools/po4a/pot/ !pod/*/tools/po4a/pot/*.sst.pot !pod/*/tools/po4a/pot/*.ssm.pot !pod/*/tools/po4a/pot/*.ssi.pot !pod/*/tools/po4a/pot/index.html.in.pot !pod/*/tools/po4a/po/ !pod/*/tools/po4a/po/*/ !pod/*/tools/po4a/po/*/*.sst.po !pod/*/tools/po4a/po/*/*.ssm.po !pod/*/tools/po4a/po/*/*.ssi.po !pod/*/tools/po4a/po/*/index.html.in.po # depreciate, separate from media/text use tools !pod/*/media/text/Makefile !pod/*/media/text/bin/ !pod/*/media/text/bin/*.sh !pod/*/media/text/pot/ !pod/*/media/text/pot/*.sst.pot !pod/*/media/text/pot/*.ssm.pot !pod/*/media/text/pot/*.ssi.pot !pod/*/media/text/pot/index.html.in.pot !pod/*/media/text/po/ !pod/*/media/text/po/*/ !pod/*/media/text/po/*/*.sst.po !pod/*/media/text/po/*/*.ssm.po !pod/*/media/text/po/*/*.ssi.po !pod/*/media/text/po/*/index.html.in.po # skip .dub/** tmp/** *_.org *_.d *_.txt *_ *~ \#* *.\#* #+END_SRC *** document pod structure #+NAME: pod_gitignore #+BEGIN_SRC sh # git ls-files --others --exclude-from=.git/info/exclude ,* !.gitignore !pod.manifest !conf/ !conf/sisu_document_make !media/ !media/text/ !media/text/*/ !media/text/*/*.sst !media/text/*/*.ssm !media/text/*/*.ssi !media/image/ !media/image/*.png !media/image/*.jpg !media/image/*.PNG !media/image/*.JPG #!media/** #!*.sst #!*.ssm #!*.ssi #!**/*.sst #!**/*.ssm #!**/*.ssi #+END_SRC *** pod po4a structure #+NAME: pod_gitignore #+BEGIN_SRC sh # git ls-files --others --exclude-from=.git/info/exclude ,* !.gitignore !pot/ !pot/*.sst.pot !pot/*.ssm.pot !pot/*.ssi.pot !po/ !po/*/ !po/*/*.sst.po !po/*/*.ssm.po !po/*/*.ssi.po #+END_SRC