diff options
| -rw-r--r-- | org/in_source_files.org | 2 | ||||
| -rw-r--r-- | org/ocda.org | 15 | ||||
| -rw-r--r-- | org/ocda_functions.org | 33 | ||||
| -rw-r--r-- | org/out_metadata.org | 47 | ||||
| -rw-r--r-- | org/out_src_pod.org | 697 | ||||
| -rw-r--r-- | org/output_show.org | 9 | ||||
| -rw-r--r-- | org/spine.org | 5 | ||||
| -rw-r--r-- | src/sisudoc/io_in/read_source_files.d | 2 | ||||
| -rw-r--r-- | src/sisudoc/io_out/metadata.d | 47 | ||||
| -rw-r--r-- | src/sisudoc/io_out/source_pod.d | 635 | ||||
| -rw-r--r-- | src/sisudoc/meta/metadoc.d | 5 | ||||
| -rw-r--r-- | src/sisudoc/meta/metadoc_from_src.d | 15 | ||||
| -rw-r--r-- | src/sisudoc/meta/metadoc_from_src_functions.d | 5 | ||||
| -rw-r--r-- | src/sisudoc/meta/metadoc_show_summary.d | 9 |
14 files changed, 817 insertions, 709 deletions
diff --git a/org/in_source_files.org b/org/in_source_files.org index 8df3ded..674709a 100644 --- a/org/in_source_files.org +++ b/org/in_source_files.org @@ -390,7 +390,7 @@ template spineRawMarkupContent() { ST_doc_parts ret; { ret.header_raw = st.header; - ret.sourcefile_body_content = sourcefile_body_content; + ret.sourcefile_body_content = sourcefile_body_content; ret.insert_file_list = insert_file_list_get; ret.images_list = images_list_get; ret.doc_digest = dig; diff --git a/org/ocda.org b/org/ocda.org index 946dcb9..dca0098 100644 --- a/org/ocda.org +++ b/org/ocda.org @@ -1068,20 +1068,7 @@ if (the_document_body_section.length > 1) { // writeln("body"); obj = _links(obj); } } -auto image_list = (_images.sort()).uniq; // also get digest on each image here? // workon -if (_images.length > 0) { - foreach (img; image_list) { - try { // also get sha digest on image file - // read_image - auto data = (cast(byte[]) (manifested.src.image_dir_path ~ "/" ~ img).read); - // calculate, digest, hash - writeln(img, "\n", data.sha256Of.toHexString, "::", data.length, " ", img); - writeln(data.sha256Of.toHexString, " ", img, " ", data.length); - } catch (Exception ex) { - writeln("WARNING, image not found: ", img, "\n ", manifested.src.image_dir_path ~ "/" ~ img); - } - } -} +auto image_list = (_images.sort()).uniq; // endnotes optional only one 1~ level if (the_document_endnotes_section.length > 1) { // writeln("endnotes"); dom_structure_markedup_tags_status_buffer = dom_structure_markedup_tags_status.dup; diff --git a/org/ocda_functions.org b/org/ocda_functions.org index ecc8b2f..070b842 100644 --- a/org/ocda_functions.org +++ b/org/ocda_functions.org @@ -3197,16 +3197,39 @@ pure ObjGenericComposite obj_dom_set_collapsed_tags()( #+NAME: ocdaFunc_obj_digest #+HEADER: :noweb yes #+BEGIN_SRC d +<<ocdaFunc_obj_digest_return>> +#+END_SRC + +****** return object digest + +#+NAME: ocdaFunc_obj_digest_return +#+HEADER: :noweb yes +#+BEGIN_SRC d // ↓ - object digest pure ubyte[32] obj_digest()( ObjGenericComposite obj, ) { obj.metainfo.sha256 = obj.text.sha256Of; - // if (obj.metainfo.is_a == "heading") { - // writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a, " ", obj.metainfo.heading_lev_markup); - // } else { - // writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a); - // } + return obj.metainfo.sha256; +} +// ↑ - object digest +#+END_SRC + +****** screen show/debug object digest + +#+NAME: ocdaFunc_obj_digest_debug +#+HEADER: :noweb yes +#+BEGIN_SRC d +// ↓ - object digest +ubyte[32] obj_digest()( + ObjGenericComposite obj, +) { + obj.metainfo.sha256 = obj.text.sha256Of; + if (obj.metainfo.is_a == "heading") { + writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a, " ", obj.metainfo.heading_lev_markup); + } else { + writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a); + } return obj.metainfo.sha256; } // ↑ - object digest diff --git a/org/out_metadata.org b/org/out_metadata.org index 289b702..d43adf2 100644 --- a/org/out_metadata.org +++ b/org/out_metadata.org @@ -26,9 +26,9 @@ #+BEGIN_SRC d <<doc_header_including_copyright_and_license>> module sisudoc.io_out.metadata; -@safe: +// @safe: template outputMetadata() { - void outputMetadata(T)( T doc_matters) { + void outputMetadata(T)(T doc_matters) { <<curated_html_search>> <<output_imports>> <<curated_html_themes_0>> @@ -139,13 +139,6 @@ if (!(doc_matters.conf_make_meta.meta.notes_summary.empty)) { } else if (doc_matters.opt.action.debug_do) { writeln("WARNING no summary of text provided in document header ", doc_matters.src.filename_base); } -metadata_ ~= "<hr /><p class=\"lev1\">source: " ~ doc_matters.src.filename_base ~ "</p>"; -if (doc_matters.opt.action.html_link_markup_source) { - metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".zip\" class=\"lnkicon\">" - ~ " 🫛 zipped </a>| " - ~ "<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ "/\" class=\"lnkicon\">" - ~ " 🫛 tree </a>] "; -} metadata_ ~= "<p class=\"lev1\">● outputs: [ html: <a href=\"" ~ doc_matters.src.filename_base ~ ".html\" class=\"lnkicon\">" ~ " ▤ scroll </a> " ~ "|<a href=\"" ~ doc_matters.src.filename_base ~ "/toc.html\" class=\"lnkicon\">" @@ -173,18 +166,32 @@ if ((doc_matters.opt.action.html_link_pdf) || (doc_matters.opt.action.html_link_ ~ " □ pdf (U.S. letter) </a>] "; } metadata_ ~= "</p>"; -metadata_ ~= "<hr /><p class=\"lev0\">Digests:</p>"; -metadata_ ~= "<p class=\"lev1\"><tt>" +if (doc_matters.opt.action.html_link_markup_source) { + metadata_ ~= "<hr /><p class=\"lev1\">source: " ~ doc_matters.src.filename_base ~ "</p>"; + metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".zip\" class=\"lnkicon\">" + ~ " 🫛 zipped </a>| " + ~ "<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ "/\" class=\"lnkicon\">" + ~ " 🫛 tree </a>] "; + metadata_ ~= "<p class=\"lev1\">● source digests:" + ~ " [ <a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt\" class=\"lnkizipcon\">" + ~ " # digests </a>]</p>"; + auto pths_pod = spinePathsPods!()(doc_matters); + auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; + if (doc_matters.opt.action.pod) { + try { // get sha digest for pod + metadata_ ~= "<p class=\"lev2\">"; + auto data = (cast(byte[]) (fn_pod).read); // prevents code from being safe + metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ "</tt> - " ~ doc_matters.src.filename_base ~ ".zip"; + metadata_ ~= "</p>"; + } catch (Exception ex) { + writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); + } + } +} +metadata_ ~= "<p class=\"lev2\"><tt>" ~ doc_matters.doc_digest.markup_doc.toHexString - ~ "</tt> - src doc digest" - ~ "</p>" - ~ "<p class=\"lev1\"><tt>" - ~ doc_matters.doc_digest.header.toHexString - ~ "</tt> - doc header" - ~ "</p>" - ~ "<p class=\"lev1\"><tt>" - ~ doc_matters.doc_digest.text.toHexString - ~ "</tt> - doc text" + ~ "</tt> - " + ~ doc_matters.src.filename ~ "</p>"; if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>"; diff --git a/org/out_src_pod.org b/org/out_src_pod.org index c6143cb..da2b050 100644 --- a/org/out_src_pod.org +++ b/org/out_src_pod.org @@ -31,16 +31,29 @@ template spinePod() { <<output_imports>> void spinePod(T)(T doc_matters) { <<source_pod_init>> - <<source_pod_archive_zip>> try { - <<source_pod_mkdirs>> - <<source_pod_archive_prepare_for_zip>> - <<source_pod_copy_info>> - <<source_pod_zip>> + { + pod_archive_directory_tree(doc_matters, pths_pod); + } + auto t = pod_zip_make_ready(doc_matters, pths_pod); + static assert(t.length==3); + auto zip = t[0]; + auto fn_pod = t[1]; + auto _digests = t[2]; + { + zipArchive(doc_matters, fn_pod, zip); + } { + zipArchiveDigest(doc_matters, fn_pod, _digests); + } } catch (ErrnoException ex) { // Handle error } } + <<source_pod_mkdirs>> + <<source_pod_archive_prepare_for_zip_and_get_digests>> + <<source_pod_archive_zip>> + <<zip_archive>> + <<pod_source_files_digests>> } #+END_SRC @@ -72,8 +85,6 @@ debug(asserts) { } mixin spineRgxFiles; string pwd = doc_matters.env.pwd; -auto src_path_info = doc_matters.src_path_info; -auto pth_dr_doc_src = doc_matters.src_path_info; auto pths_pod = spinePathsPods!()(doc_matters); mixin spineLanguageCodes; auto lang = Lang(); @@ -81,377 +92,423 @@ static auto rgx_files = RgxFiles(); assert (doc_matters.src.filename.match(rgx_files.src_fn)); #+END_SRC -*** pod zip archive +** source pod archive, (prepare for zip) :copy: -#+NAME: source_pod_archive_zip +#+NAME: source_pod_archive_prepare_for_zip_and_get_digests #+BEGIN_SRC d -@system auto pod_archive(Z)( - string _source_type, - string _data_in, - string _pth_out, - Z zip -) { - auto zip_arc_member_file = new ArchiveMember(); - zip_arc_member_file.name = _pth_out; - auto zip_data = new OutBuffer(); - switch (_source_type) { - case "file_path_bin": - zip_data.write(cast(char[]) ((_data_in).read)); - goto default; - case "file_path_text": - zip_data.write((_data_in).readText); - goto default; - case "string": - zip_data.write(_data_in); - goto default; - default: - zip_arc_member_file.expandedData = zip_data.toBytes(); - zip.addMember(zip_arc_member_file); - } - return zip; -} -#+END_SRC - -** mkdir :mkdir: - -#+NAME: source_pod_mkdirs -#+BEGIN_SRC d -/+ create directory structure +/ -if (!exists(pths_pod.pod_dir_())) { - // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) - pths_pod.pod_dir_().mkdirRecurse; -} -if (doc_matters.opt.action.source_or_pod) { - if (doc_matters.opt.action.vox_gt0) { - writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); - } - if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; +auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) { + auto pth_dr_doc_src = doc_matters.src_path_info; + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln(__LINE__, ": ", + doc_matters.src.filename, " -> ", + pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod + ); } - if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { - pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; - } -} -if (!exists(pths_pod.pod_dir_() ~ "/index.html")) { - import sisudoc.io_out.html_snippet; - mixin htmlSnippet; - auto f = File(pths_pod.pod_dir_() ~"/index.html", "w"); - f.writeln(format_html_blank_page_guide_home( - "../../css/html_scroll.css", - (doc_matters.opt.action.webserver_url_doc_root.length > 0) - ? doc_matters.opt.action.webserver_url_doc_root - : doc_matters.conf_make_meta.conf.w_srv_data_root_url, - "../../index.html", - )); -} -#+END_SRC - -** copy :copy: - -#+NAME: source_pod_archive_prepare_for_zip -#+BEGIN_SRC d -if (doc_matters.opt.action.debug_do_pod -&& doc_matters.opt.action.vox_gt1) { - writeln(__LINE__, ": ", - doc_matters.src.filename, " -> ", - pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod - ); -} -auto zip = new ZipArchive(); -auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; -{ /+ bundle images +/ - foreach (image; doc_matters.srcs.image_list) { - debug(podimages) { - writeln( - pth_dr_doc_src.image_root.to!string, "/", image, " -> ", - pths_pod.image_root(doc_matters.src.filename).zpod, "/", image - ); + auto zip = new ZipArchive(); // needed + auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; + string[string][string] _digests; + { // bundle images - get digest + foreach (image; doc_matters.srcs.image_list) { + debug(podimages) { + writeln( + pth_dr_doc_src.image_root.to!string, "/", image, " -> ", + pths_pod.image_root(doc_matters.src.filename).zpod, "/", image + ); + } + auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; + auto fn_src_out_pod_zip_base + = pths_pod.image_root(doc_matters.src.filename).zpod.to!string + ~ "/" ~ image; + auto fn_src_out_filesystem + = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ image; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (fn_src_in).read); + _digests["shared"]["images"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ image ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", image); + } + if (doc_matters.opt.action.source_or_pod) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); + } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (image): ", fn_src_in); + } + } } - auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; + } { // bundle dr_document_make + auto fn_src_in = ((doc_matters.src.is_pod) + ? doc_matters.src.conf_dir_path + : pth_dr_doc_src.conf_root).to!string + ~ "/" ~ "dr_document_make"; auto fn_src_out_pod_zip_base - = pths_pod.image_root(doc_matters.src.filename).zpod.to!string - ~ "/" ~ image; + = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make"; auto fn_src_out_filesystem - = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string - ~ "/" ~ image; + = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ "dr_document_make"; if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); - } + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (image): ", fn_src_in); + writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); } } - } -} { /+ bundle dr_document_make +/ - auto fn_src_in = ((doc_matters.src.is_pod) - ? doc_matters.src.conf_dir_path - : pth_dr_doc_src.conf_root).to!string - ~ "/" ~ "dr_document_make"; - auto fn_src_out_pod_zip_base - = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make"; - auto fn_src_out_filesystem - = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string - ~ "/" ~ "dr_document_make"; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); - } - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); - } - } -} { /+ pod manifest +/ - auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; - auto fn_src_out_pod_zip_base - = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path - auto fn_src_out_inside_pod - = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path - string[] filelist_src_out_pod_arr; - string[] filelist_src_zpod_arr; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src in found: ", fn_src_in); - } - filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; - filelist_src_zpod_arr ~= fn_src_out_inside_pod; - { - import dyaml; - auto pod_filelist_yaml_string - = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); - Node _pmy; - string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - try { - _pmy = Loader.fromString(_pm).load(); - } catch (ErrnoException ex) { - } catch (Throwable) { - writeln("ERROR failed to read config file content, not parsed as yaml"); + } { // pod manifest + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path + auto fn_src_out_inside_pod + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src in found: ", fn_src_in); } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + { + import dyaml; + auto pod_filelist_yaml_string + = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); + Node _pmy; + string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + try { + _pmy = Loader.fromString(_pm).load(); + } catch (ErrnoException ex) { + } catch (Throwable) { + writeln("ERROR failed to read config file content, not parsed as yaml"); + } + writeln("pod filename: ", _pmy["doc"]["filename"].get!string); + writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); + writeln("pod languages: ", doc_matters.src.language); + // foreach(string _l; _pmy["doc"]["language"]) { + // writeln("language: ", _l); + // } + } + if (doc_matters.opt.action.source_or_pod) { + pod_filelist_yaml_string.writeln(_pm); } - writeln("pod filename: ", _pmy["doc"]["filename"].get!string); - writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); - writeln("pod languages: ", doc_matters.src.language); - // foreach(string _l; _pmy["doc"]["language"]) { - // writeln("language: ", _l); - // } + if (doc_matters.opt.action.pod) { + zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); + } + } + } + } { // bundle primary file (.ssm/.sst) - get digest + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: + auto fn_src_out_inside_pod + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { // what of language? + debug(io) { writeln("(io debug) src in found: ", fn_src_in); } + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (fn_src_in).read); + _digests["en"]["sst"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename; // FIX language issue + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename); } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; if (doc_matters.opt.action.source_or_pod) { - pod_filelist_yaml_string.writeln(_pm); + fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); - } - } - } -} { /+ bundle primary file (.ssm/.sst) +/ - auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; - auto fn_src_out_pod_zip_base - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: - auto fn_src_out_inside_pod - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: - string[] filelist_src_out_pod_arr; - string[] filelist_src_zpod_arr; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src in found: ", fn_src_in); - } - filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; - filelist_src_zpod_arr ~= fn_src_out_inside_pod; - string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); - if (auto _x = fn_src_in.match(_rgx)){ - if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { - string _path_to_pod = _x.captures["path_to_pod"]; - string _podname = _x.captures["podname"]; - string _root_to_lang = _x.captures["from_root"]; - string _language = _x.captures["language"]; - string _filename = _x.captures["filename"]; - foreach (_lang; doc_matters.pod.manifest_list_of_languages) { - string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); + if (auto _x = fn_src_in.match(_rgx)){ + if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { + string _path_to_pod = _x.captures["path_to_pod"]; + string _podname = _x.captures["podname"]; + string _root_to_lang = _x.captures["from_root"]; + string _language = _x.captures["language"]; + string _filename = _x.captures["filename"]; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + } } + } else { + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } - } else { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); - } - } -} { /+ bundle insert files (.ssi) +/ - if (doc_matters.srcs.file_insert_list.length > 0) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); - foreach (insert_file; doc_matters.srcs.file_insert_list) { - debug(pod) { - writeln( - insert_file, " -> ", - pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).zpod - ); + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); } - if (auto _x = insert_file.match(_rgx)){ - if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { - string _path_to_pod = _x.captures["path_to_pod"]; - string _podname = _x.captures["podname"]; - string _root_to_lang = _x.captures["from_root"]; - string _language = _x.captures["language"]; - string _filename = _x.captures["filename"]; - foreach (_lang; doc_matters.pod.manifest_list_of_languages) { - string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - if (exists(_pth_mkup_src_in)) { - if (doc_matters.opt.action.source_or_pod) { - auto fn_src_out_filesystem // you need to change language sources - = pths_pod.fn_doc_insert( - doc_matters.src.filename, // doc_matters.src.filename - _pth_mkup_src_in, // insert_file - _lang, - ).filesystem_open_zpod.to!string; - _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + } + } { // bundle insert files (.ssi) - get digest + if (doc_matters.srcs.file_insert_list.length > 0) { + auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); + foreach (insert_file; doc_matters.srcs.file_insert_list) { + debug(pod) { + writeln( + insert_file, " -> ", + pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod + ); + } + if (auto _x = insert_file.match(_rgx)){ + if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { + string _path_to_pod = _x.captures["path_to_pod"]; + string _podname = _x.captures["podname"]; + string _root_to_lang = _x.captures["from_root"]; + string _language = _x.captures["language"]; + string _filename = _x.captures["filename"]; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (_pth_mkup_src_in).read); + _digests[_language]["ssi"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ _filename ~ " - [" ~ _lang ~ "]" ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", _filename, " - [", _lang, "]"); } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); + string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + if (exists(_pth_mkup_src_in)) { + if (doc_matters.opt.action.source_or_pod) { + auto fn_src_out_filesystem // you need to change language sources + = pths_pod.fn_doc_insert( + doc_matters.src.filename, // doc_matters.src.filename + _pth_mkup_src_in, // insert_file + _lang, + ).filesystem_open_zpod.to!string; + _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere + } + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); + } } } } - } - } else { - auto fn_src_in = insert_file; - auto fn_src_out_pod_zip_base - = pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).filesystem_open_zpod.to!string; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); - } - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); - } } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); + auto fn_src_in = insert_file; + { // take DIGEST write to pod file digests.txt // FIX likely bug insert_file includes path, not what you wish to write + auto data = (cast(byte[]) (fn_src_in).read); + _digests["en"]["ssi"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ insert_file ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", insert_file); + } + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).filesystem_open_zpod.to!string; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } + if (doc_matters.opt.action.source_or_pod) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); + } } } } } } -} { + auto t = tuple(zip, fn_pod, _digests); + return t; +} +#+END_SRC + +** mkdir :mkdir: + +#+NAME: source_pod_mkdirs +#+BEGIN_SRC d +void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure + if (!exists(pths_pod.pod_dir_())) { + // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) + pths_pod.pod_dir_().mkdirRecurse; + } + if (doc_matters.opt.action.source_or_pod) { + if (doc_matters.opt.action.vox_gt0) { + writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); + } + if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { + pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; + } + } + if (!exists(pths_pod.pod_dir_() ~ "/index.html")) { + import sisudoc.io_out.html_snippet; + mixin htmlSnippet; + auto f = File(pths_pod.pod_dir_() ~"/index.html", "w"); + f.writeln(format_html_blank_page_guide_home( + "../../css/html_scroll.css", + (doc_matters.opt.action.webserver_url_doc_root.length > 0) + ? doc_matters.opt.action.webserver_url_doc_root + : doc_matters.conf_make_meta.conf.w_srv_data_root_url, + "../../index.html", + )); + } +} +#+END_SRC + +** pod zip archive + +#+NAME: source_pod_archive_zip +#+BEGIN_SRC d +@system auto pod_archive(Z)( + string _source_type, + string _data_in, + string _pth_out, + Z zip +) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = _pth_out; + auto zip_data = new OutBuffer(); + switch (_source_type) { + case "file_path_bin": + zip_data.write(cast(char[]) ((_data_in).read)); + goto default; + case "file_path_text": + zip_data.write((_data_in).readText); + goto default; + case "string": + zip_data.write(_data_in); + goto default; + default: + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + return zip; +} +#+END_SRC + +** zip archive + +#+NAME: zip_archive +#+BEGIN_SRC d +void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) { auto fn_src_in = doc_matters.src.filename; if (doc_matters.opt.action.pod) { if (exists(doc_matters.src.file_with_absolute_path)) { - createZipFile!()(fn_pod, zip.build()); - } else { - writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); - } - if (!(exists(fn_pod))) { - writeln("WARNING failed to create pod zip archive: ", fn_pod); - } + try { + createZipFile!()(fn_pod, zip.build()); + } catch (ErrnoException ex) { + // Handle errors + } + } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } + if (!(exists(fn_pod))) { writeln("WARNING failed to create pod zip archive: ", fn_pod); } } } #+END_SRC ** sha256 of pod.zip, zip debug, read zip archive -#+NAME: source_pod_copy_info +#+NAME: pod_source_files_digests #+BEGIN_SRC d -if (exists(fn_pod)) { +void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { + import sisudoc.io_out.paths_output; + auto pths_pod = spinePathsPods!()(doc_matters); try { - if (doc_matters.opt.action.vox_gt0 - && doc_matters.opt.action.pod) { - auto data = (cast(byte[]) (fn_pod).read); - if (doc_matters.opt.action.vox_gt1) { - writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); - } - if (doc_matters.opt.action.pod) { - writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename_base, ".zip"); - } + if (!exists(pths_pod.pod_dir_())) { + // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) + pths_pod.pod_dir_().mkdirRecurse; } - if (doc_matters.opt.action.debug_do_pod) { + } catch (ErrnoException ex) { + // Handle error + } + try { + writeln(pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); + string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt"; + writeln(_digest_fn); + auto f = File(_digest_fn, "w"); + if (exists(fn_pod)) { try { - auto zipped = new ZipArchive((fn_pod).read); - foreach (filename, member; zipped.directory) { - auto data = zipped.expand(member); - writeln(data.sha256Of.toHexString, "::", data.length, " . ", filename); + if (doc_matters.opt.action.vox_gt0 + && doc_matters.opt.action.pod) { + auto data = (cast(byte[]) (fn_pod).read); + if (doc_matters.opt.action.vox_gt1) { + writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); + } + if (doc_matters.opt.action.pod) { + auto _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); + writeln(_zip_digest); + f.writeln(_zip_digest); + } } - } catch (ZipException ex) { + } catch (ErrnoException ex) { // Handle errors } } + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + if (_lang in _digests) { + if (("sst" in _digests[_lang]) && (_digests[_lang]["sst"].length > 0)) { + writeln(_digests[_lang]["sst"]); + f.writeln(_digests[_lang]["sst"]); + } + if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) { + writeln(_digests[_lang]["ssi"]); + f.writeln(_digests[_lang]["ssi"]); + } + } + } + if ("shared" in _digests) { + if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) { + writeln(_digests["shared"]["images"]); + f.writeln(_digests["shared"]["images"]); + } + } } catch (ErrnoException ex) { - // Handle errors + // Handle error } } #+END_SRC -** source pod zip UNUSED - -#+NAME: source_pod_zip -#+BEGIN_SRC d -// source pod zip -#+END_SRC - * document header including copyright & license #+NAME: doc_header_including_copyright_and_license diff --git a/org/output_show.org b/org/output_show.org index 9c3f358..53e2d77 100644 --- a/org/output_show.org +++ b/org/output_show.org @@ -80,7 +80,7 @@ foreach (k; doc_matters.has.keys_seq.seg) { #+NAME: meta_metadoc_show_summary_document #+BEGIN_SRC d writefln( - "%s\n\"%s\", %s\n%s [%s]\n%s \n%s%s\n%s%s\n%s%s\n%s\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%s", + "%s\n\"%s\", %s\n%s [%s]\n%s \n%s - %s [%s]\n%s\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%s", markup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.conf_make_meta.meta.title_full, doc_matters.conf_make_meta.meta.creator_author, @@ -88,11 +88,8 @@ writefln( doc_matters.src.language, markup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.doc_digest.markup_doc.toHexString, - " - src doc digest", - doc_matters.doc_digest.header.toHexString, - " - doc header", - doc_matters.doc_digest.text.toHexString, - " - doc text", + doc_matters.src.filename, + doc_matters.src.language, markup.repeat_character_by_number_provided("-", char_repeat_number), "- toc arr length:", to!int(doc_abstraction["toc"].length), diff --git a/org/spine.org b/org/spine.org index e66ad47..8ccaa0f 100644 --- a/org/spine.org +++ b/org/spine.org @@ -1533,10 +1533,7 @@ if ((_opt_action.debug_do) } auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); -// writeln("src doc digest: ", _header_body_insertfilelist_imagelist.doc_digest.markup_doc.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.markup_doc.toHexString.length); -// writeln("header: ", _header_body_insertfilelist_imagelist.doc_digest.header.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.header.toHexString.length); -// writeln("text: ", _header_body_insertfilelist_imagelist.doc_digest.text.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.text.toHexString.length); -auto doc_digests = _header_body_insertfilelist_imagelist.doc_digest; +auto doc_digests = _header_body_insertfilelist_imagelist.doc_digest; // CHECK, REVIEW, discard likely, other route taken if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { diff --git a/src/sisudoc/io_in/read_source_files.d b/src/sisudoc/io_in/read_source_files.d index 80b3d71..b465cc3 100644 --- a/src/sisudoc/io_in/read_source_files.d +++ b/src/sisudoc/io_in/read_source_files.d @@ -148,7 +148,7 @@ template spineRawMarkupContent() { ST_doc_parts ret; { ret.header_raw = st.header; - ret.sourcefile_body_content = sourcefile_body_content; + ret.sourcefile_body_content = sourcefile_body_content; ret.insert_file_list = insert_file_list_get; ret.images_list = images_list_get; ret.doc_digest = dig; diff --git a/src/sisudoc/io_out/metadata.d b/src/sisudoc/io_out/metadata.d index 59329e7..95fc465 100644 --- a/src/sisudoc/io_out/metadata.d +++ b/src/sisudoc/io_out/metadata.d @@ -48,9 +48,9 @@ +/ module sisudoc.io_out.metadata; -@safe: +// @safe: template outputMetadata() { - void outputMetadata(T)( T doc_matters) { + void outputMetadata(T)(T doc_matters) { string inline_search_form(M)( M doc_matters, ) { @@ -475,13 +475,6 @@ string theme_light_1 = format(q"┃ } else if (doc_matters.opt.action.debug_do) { writeln("WARNING no summary of text provided in document header ", doc_matters.src.filename_base); } - metadata_ ~= "<hr /><p class=\"lev1\">source: " ~ doc_matters.src.filename_base ~ "</p>"; - if (doc_matters.opt.action.html_link_markup_source) { - metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".zip\" class=\"lnkicon\">" - ~ " 🫛 zipped </a>| " - ~ "<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ "/\" class=\"lnkicon\">" - ~ " 🫛 tree </a>] "; - } metadata_ ~= "<p class=\"lev1\">● outputs: [ html: <a href=\"" ~ doc_matters.src.filename_base ~ ".html\" class=\"lnkicon\">" ~ " ▤ scroll </a> " ~ "|<a href=\"" ~ doc_matters.src.filename_base ~ "/toc.html\" class=\"lnkicon\">" @@ -509,18 +502,32 @@ string theme_light_1 = format(q"┃ ~ " □ pdf (U.S. letter) </a>] "; } metadata_ ~= "</p>"; - metadata_ ~= "<hr /><p class=\"lev0\">Digests:</p>"; - metadata_ ~= "<p class=\"lev1\"><tt>" + if (doc_matters.opt.action.html_link_markup_source) { + metadata_ ~= "<hr /><p class=\"lev1\">source: " ~ doc_matters.src.filename_base ~ "</p>"; + metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".zip\" class=\"lnkicon\">" + ~ " 🫛 zipped </a>| " + ~ "<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ "/\" class=\"lnkicon\">" + ~ " 🫛 tree </a>] "; + metadata_ ~= "<p class=\"lev1\">● source digests:" + ~ " [ <a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt\" class=\"lnkizipcon\">" + ~ " # digests </a>]</p>"; + auto pths_pod = spinePathsPods!()(doc_matters); + auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; + if (doc_matters.opt.action.pod) { + try { // get sha digest for pod + metadata_ ~= "<p class=\"lev2\">"; + auto data = (cast(byte[]) (fn_pod).read); // prevents code from being safe + metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ "</tt> - " ~ doc_matters.src.filename_base ~ ".zip"; + metadata_ ~= "</p>"; + } catch (Exception ex) { + writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); + } + } + } + metadata_ ~= "<p class=\"lev2\"><tt>" ~ doc_matters.doc_digest.markup_doc.toHexString - ~ "</tt> - src doc digest" - ~ "</p>" - ~ "<p class=\"lev1\"><tt>" - ~ doc_matters.doc_digest.header.toHexString - ~ "</tt> - doc header" - ~ "</p>" - ~ "<p class=\"lev1\"><tt>" - ~ doc_matters.doc_digest.text.toHexString - ~ "</tt> - doc text" + ~ "</tt> - " + ~ doc_matters.src.filename ~ "</p>"; if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>"; diff --git a/src/sisudoc/io_out/source_pod.d b/src/sisudoc/io_out/source_pod.d index 3fd7a9a..6b880d9 100644 --- a/src/sisudoc/io_out/source_pod.d +++ b/src/sisudoc/io_out/source_pod.d @@ -68,327 +68,376 @@ template spinePod() { } mixin spineRgxFiles; string pwd = doc_matters.env.pwd; - auto src_path_info = doc_matters.src_path_info; - auto pth_dr_doc_src = doc_matters.src_path_info; auto pths_pod = spinePathsPods!()(doc_matters); mixin spineLanguageCodes; auto lang = Lang(); static auto rgx_files = RgxFiles(); assert (doc_matters.src.filename.match(rgx_files.src_fn)); - @system auto pod_archive(Z)( - string _source_type, - string _data_in, - string _pth_out, - Z zip - ) { - auto zip_arc_member_file = new ArchiveMember(); - zip_arc_member_file.name = _pth_out; - auto zip_data = new OutBuffer(); - switch (_source_type) { - case "file_path_bin": - zip_data.write(cast(char[]) ((_data_in).read)); - goto default; - case "file_path_text": - zip_data.write((_data_in).readText); - goto default; - case "string": - zip_data.write(_data_in); - goto default; - default: - zip_arc_member_file.expandedData = zip_data.toBytes(); - zip.addMember(zip_arc_member_file); + try { + { + pod_archive_directory_tree(doc_matters, pths_pod); + } + auto t = pod_zip_make_ready(doc_matters, pths_pod); + static assert(t.length==3); + auto zip = t[0]; + auto fn_pod = t[1]; + auto _digests = t[2]; + { + zipArchive(doc_matters, fn_pod, zip); + } { + zipArchiveDigest(doc_matters, fn_pod, _digests); } - return zip; + } catch (ErrnoException ex) { + // Handle error } - try { - /+ create directory structure +/ - if (!exists(pths_pod.pod_dir_())) { - // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) - pths_pod.pod_dir_().mkdirRecurse; + } + void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure + if (!exists(pths_pod.pod_dir_())) { + // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) + pths_pod.pod_dir_().mkdirRecurse; + } + if (doc_matters.opt.action.source_or_pod) { + if (doc_matters.opt.action.vox_gt0) { + writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); } - if (doc_matters.opt.action.source_or_pod) { - if (doc_matters.opt.action.vox_gt0) { - writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); - } - if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { - pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; - } - if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { - pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; - } + if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } - if (!exists(pths_pod.pod_dir_() ~ "/index.html")) { - import sisudoc.io_out.html_snippet; - mixin htmlSnippet; - auto f = File(pths_pod.pod_dir_() ~"/index.html", "w"); - f.writeln(format_html_blank_page_guide_home( - "../../css/html_scroll.css", - (doc_matters.opt.action.webserver_url_doc_root.length > 0) - ? doc_matters.opt.action.webserver_url_doc_root - : doc_matters.conf_make_meta.conf.w_srv_data_root_url, - "../../index.html", - )); + if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln(__LINE__, ": ", - doc_matters.src.filename, " -> ", - pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod - ); + if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } - auto zip = new ZipArchive(); - auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; - { /+ bundle images +/ - foreach (image; doc_matters.srcs.image_list) { - debug(podimages) { - writeln( - pth_dr_doc_src.image_root.to!string, "/", image, " -> ", - pths_pod.image_root(doc_matters.src.filename).zpod, "/", image - ); - } - auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; - auto fn_src_out_pod_zip_base - = pths_pod.image_root(doc_matters.src.filename).zpod.to!string - ~ "/" ~ image; - auto fn_src_out_filesystem - = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string - ~ "/" ~ image; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); - } - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (image): ", fn_src_in); - } - } + if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { + pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; + } + if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { + pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; + } + } + if (!exists(pths_pod.pod_dir_() ~ "/index.html")) { + import sisudoc.io_out.html_snippet; + mixin htmlSnippet; + auto f = File(pths_pod.pod_dir_() ~"/index.html", "w"); + f.writeln(format_html_blank_page_guide_home( + "../../css/html_scroll.css", + (doc_matters.opt.action.webserver_url_doc_root.length > 0) + ? doc_matters.opt.action.webserver_url_doc_root + : doc_matters.conf_make_meta.conf.w_srv_data_root_url, + "../../index.html", + )); + } + } + auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) { + auto pth_dr_doc_src = doc_matters.src_path_info; + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln(__LINE__, ": ", + doc_matters.src.filename, " -> ", + pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod + ); + } + auto zip = new ZipArchive(); // needed + auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; + string[string][string] _digests; + { // bundle images - get digest + foreach (image; doc_matters.srcs.image_list) { + debug(podimages) { + writeln( + pth_dr_doc_src.image_root.to!string, "/", image, " -> ", + pths_pod.image_root(doc_matters.src.filename).zpod, "/", image + ); } - } { /+ bundle dr_document_make +/ - auto fn_src_in = ((doc_matters.src.is_pod) - ? doc_matters.src.conf_dir_path - : pth_dr_doc_src.conf_root).to!string - ~ "/" ~ "dr_document_make"; + auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; auto fn_src_out_pod_zip_base - = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make"; + = pths_pod.image_root(doc_matters.src.filename).zpod.to!string + ~ "/" ~ image; auto fn_src_out_filesystem - = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string - ~ "/" ~ "dr_document_make"; + = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ image; if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (fn_src_in).read); + _digests["shared"]["images"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ image ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", image); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); + writeln("WARNING (io) src out NOT found (image): ", fn_src_in); } } - } { /+ pod manifest +/ - auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; - auto fn_src_out_pod_zip_base - = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path - auto fn_src_out_inside_pod - = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path - string[] filelist_src_out_pod_arr; - string[] filelist_src_zpod_arr; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src in found: ", fn_src_in); - } - filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; - filelist_src_zpod_arr ~= fn_src_out_inside_pod; - { - import dyaml; - auto pod_filelist_yaml_string - = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); - Node _pmy; - string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - try { - _pmy = Loader.fromString(_pm).load(); - } catch (ErrnoException ex) { - } catch (Throwable) { - writeln("ERROR failed to read config file content, not parsed as yaml"); - } - writeln("pod filename: ", _pmy["doc"]["filename"].get!string); - writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); - writeln("pod languages: ", doc_matters.src.language); - // foreach(string _l; _pmy["doc"]["language"]) { - // writeln("language: ", _l); - // } - } - if (doc_matters.opt.action.source_or_pod) { - pod_filelist_yaml_string.writeln(_pm); - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); - } - } + } + } { // bundle dr_document_make + auto fn_src_in = ((doc_matters.src.is_pod) + ? doc_matters.src.conf_dir_path + : pth_dr_doc_src.conf_root).to!string + ~ "/" ~ "dr_document_make"; + auto fn_src_out_pod_zip_base + = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make"; + auto fn_src_out_filesystem + = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string + ~ "/" ~ "dr_document_make"; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } + if (doc_matters.opt.action.source_or_pod) { + fn_src_in.copy(fn_src_out_filesystem); } - } { /+ bundle primary file (.ssm/.sst) +/ - auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; - auto fn_src_out_pod_zip_base - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: - auto fn_src_out_inside_pod - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: - string[] filelist_src_out_pod_arr; - string[] filelist_src_zpod_arr; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src in found: ", fn_src_in); + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); + } + } + } { // pod manifest + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path + auto fn_src_out_inside_pod + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src in found: ", fn_src_in); } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + { + import dyaml; + auto pod_filelist_yaml_string + = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); + Node _pmy; + string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + try { + _pmy = Loader.fromString(_pm).load(); + } catch (ErrnoException ex) { + } catch (Throwable) { + writeln("ERROR failed to read config file content, not parsed as yaml"); + } + writeln("pod filename: ", _pmy["doc"]["filename"].get!string); + writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); + writeln("pod languages: ", doc_matters.src.language); + // foreach(string _l; _pmy["doc"]["language"]) { + // writeln("language: ", _l); + // } } - filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; - filelist_src_zpod_arr ~= fn_src_out_inside_pod; - string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); + pod_filelist_yaml_string.writeln(_pm); } if (doc_matters.opt.action.pod) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); - if (auto _x = fn_src_in.match(_rgx)){ - if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { - string _path_to_pod = _x.captures["path_to_pod"]; - string _podname = _x.captures["podname"]; - string _root_to_lang = _x.captures["from_root"]; - string _language = _x.captures["language"]; - string _filename = _x.captures["filename"]; - foreach (_lang; doc_matters.pod.manifest_list_of_languages) { - string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); - } + zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); + } + } + } + } { // bundle primary file (.ssm/.sst) - get digest + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: + auto fn_src_out_inside_pod + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { // what of language? + debug(io) { writeln("(io debug) src in found: ", fn_src_in); } + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (fn_src_in).read); + _digests["en"]["sst"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename; // FIX language issue + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename); + } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; + if (doc_matters.opt.action.source_or_pod) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); + if (auto _x = fn_src_in.match(_rgx)){ + if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { + string _path_to_pod = _x.captures["path_to_pod"]; + string _podname = _x.captures["podname"]; + string _root_to_lang = _x.captures["from_root"]; + string _language = _x.captures["language"]; + string _filename = _x.captures["filename"]; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } - } else { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); + } else { + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } - } { /+ bundle insert files (.ssi) +/ - if (doc_matters.srcs.file_insert_list.length > 0) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); - foreach (insert_file; doc_matters.srcs.file_insert_list) { - debug(pod) { - writeln( - insert_file, " -> ", - pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).zpod - ); - } - if (auto _x = insert_file.match(_rgx)){ - if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { - string _path_to_pod = _x.captures["path_to_pod"]; - string _podname = _x.captures["podname"]; - string _root_to_lang = _x.captures["from_root"]; - string _language = _x.captures["language"]; - string _filename = _x.captures["filename"]; - foreach (_lang; doc_matters.pod.manifest_list_of_languages) { - string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - if (exists(_pth_mkup_src_in)) { - if (doc_matters.opt.action.source_or_pod) { - auto fn_src_out_filesystem // you need to change language sources - = pths_pod.fn_doc_insert( - doc_matters.src.filename, // doc_matters.src.filename - _pth_mkup_src_in, // insert_file - _lang, - ).filesystem_open_zpod.to!string; - _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); - } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); + } + } + } { // bundle insert files (.ssi) - get digest + if (doc_matters.srcs.file_insert_list.length > 0) { + auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); + foreach (insert_file; doc_matters.srcs.file_insert_list) { + debug(pod) { + writeln( + insert_file, " -> ", + pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod + ); + } + if (auto _x = insert_file.match(_rgx)){ + if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { + string _path_to_pod = _x.captures["path_to_pod"]; + string _podname = _x.captures["podname"]; + string _root_to_lang = _x.captures["from_root"]; + string _language = _x.captures["language"]; + string _filename = _x.captures["filename"]; + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (_pth_mkup_src_in).read); + _digests[_language]["ssi"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ _filename ~ " - [" ~ _lang ~ "]" ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", _filename, " - [", _lang, "]"); + } + string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; + if (exists(_pth_mkup_src_in)) { + if (doc_matters.opt.action.source_or_pod) { + auto fn_src_out_filesystem // you need to change language sources + = pths_pod.fn_doc_insert( + doc_matters.src.filename, // doc_matters.src.filename + _pth_mkup_src_in, // insert_file + _lang, + ).filesystem_open_zpod.to!string; + _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere + } + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + } + } else { + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); } } } + } + } else { + auto fn_src_in = insert_file; + { // take DIGEST write to pod file digests.txt // FIX likely bug insert_file includes path, not what you wish to write + auto data = (cast(byte[]) (fn_src_in).read); + _digests["en"]["ssi"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ insert_file ~ "\n"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", insert_file); + } + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc_insert( + doc_matters.src.filename, + insert_file, + doc_matters.src.language, + ).filesystem_open_zpod.to!string; + if (exists(fn_src_in)) { + debug(io) { writeln("(io debug) src out found: ", fn_src_in); } + if (doc_matters.opt.action.source_or_pod) { + fn_src_in.copy(fn_src_out_filesystem); + } + if (doc_matters.opt.action.pod) { + zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + } } else { - auto fn_src_in = insert_file; - auto fn_src_out_pod_zip_base - = pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).zpod.to!string; - auto fn_src_out_filesystem - = pths_pod.fn_doc_insert( - doc_matters.src.filename, - insert_file, - doc_matters.src.language, - ).filesystem_open_zpod.to!string; - if (exists(fn_src_in)) { - debug(io) { - writeln("(io debug) src out found: ", fn_src_in); - } - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); - } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); - } + if (doc_matters.opt.action.debug_do_pod + && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); } } } } - } { - auto fn_src_in = doc_matters.src.filename; - if (doc_matters.opt.action.pod) { - if (exists(doc_matters.src.file_with_absolute_path)) { - createZipFile!()(fn_pod, zip.build()); - } else { - writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); - } - if (!(exists(fn_pod))) { - writeln("WARNING failed to create pod zip archive: ", fn_pod); - } + } + } + auto t = tuple(zip, fn_pod, _digests); + return t; + } + @system auto pod_archive(Z)( + string _source_type, + string _data_in, + string _pth_out, + Z zip + ) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = _pth_out; + auto zip_data = new OutBuffer(); + switch (_source_type) { + case "file_path_bin": + zip_data.write(cast(char[]) ((_data_in).read)); + goto default; + case "file_path_text": + zip_data.write((_data_in).readText); + goto default; + case "string": + zip_data.write(_data_in); + goto default; + default: + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + return zip; + } + void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) { + auto fn_src_in = doc_matters.src.filename; + if (doc_matters.opt.action.pod) { + if (exists(doc_matters.src.file_with_absolute_path)) { + try { + createZipFile!()(fn_pod, zip.build()); + } catch (ErrnoException ex) { + // Handle errors } + } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } + if (!(exists(fn_pod))) { writeln("WARNING failed to create pod zip archive: ", fn_pod); } + } + } + void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { + import sisudoc.io_out.paths_output; + auto pths_pod = spinePathsPods!()(doc_matters); + try { + if (!exists(pths_pod.pod_dir_())) { + // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) + pths_pod.pod_dir_().mkdirRecurse; } + } catch (ErrnoException ex) { + // Handle error + } + try { + writeln(pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); + string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt"; + writeln(_digest_fn); + auto f = File(_digest_fn, "w"); if (exists(fn_pod)) { try { if (doc_matters.opt.action.vox_gt0 @@ -398,25 +447,33 @@ template spinePod() { writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); } if (doc_matters.opt.action.pod) { - writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename_base, ".zip"); - } - } - if (doc_matters.opt.action.debug_do_pod) { - try { - auto zipped = new ZipArchive((fn_pod).read); - foreach (filename, member; zipped.directory) { - auto data = zipped.expand(member); - writeln(data.sha256Of.toHexString, "::", data.length, " . ", filename); - } - } catch (ZipException ex) { - // Handle errors + auto _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); + writeln(_zip_digest); + f.writeln(_zip_digest); } } } catch (ErrnoException ex) { // Handle errors } } - // source pod zip + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { + if (_lang in _digests) { + if (("sst" in _digests[_lang]) && (_digests[_lang]["sst"].length > 0)) { + writeln(_digests[_lang]["sst"]); + f.writeln(_digests[_lang]["sst"]); + } + if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) { + writeln(_digests[_lang]["ssi"]); + f.writeln(_digests[_lang]["ssi"]); + } + } + } + if ("shared" in _digests) { + if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) { + writeln(_digests["shared"]["images"]); + f.writeln(_digests["shared"]["images"]); + } + } } catch (ErrnoException ex) { // Handle error } diff --git a/src/sisudoc/meta/metadoc.d b/src/sisudoc/meta/metadoc.d index 34212a5..25f28b6 100644 --- a/src/sisudoc/meta/metadoc.d +++ b/src/sisudoc/meta/metadoc.d @@ -90,10 +90,7 @@ template spineAbstraction() { } auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); - // writeln("src doc digest: ", _header_body_insertfilelist_imagelist.doc_digest.markup_doc.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.markup_doc.toHexString.length); - // writeln("header: ", _header_body_insertfilelist_imagelist.doc_digest.header.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.header.toHexString.length); - // writeln("text: ", _header_body_insertfilelist_imagelist.doc_digest.text.toHexString, " ", _header_body_insertfilelist_imagelist.doc_digest.text.toHexString.length); - auto doc_digests = _header_body_insertfilelist_imagelist.doc_digest; + auto doc_digests = _header_body_insertfilelist_imagelist.doc_digest; // CHECK, REVIEW, discard likely, other route taken if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { diff --git a/src/sisudoc/meta/metadoc_from_src.d b/src/sisudoc/meta/metadoc_from_src.d index c9112ef..70fec62 100644 --- a/src/sisudoc/meta/metadoc_from_src.d +++ b/src/sisudoc/meta/metadoc_from_src.d @@ -1037,20 +1037,7 @@ template docAbstraction() { obj = _links(obj); } } - auto image_list = (_images.sort()).uniq; // also get digest on each image here? // workon - if (_images.length > 0) { - foreach (img; image_list) { - try { // also get sha digest on image file - // read_image - auto data = (cast(byte[]) (manifested.src.image_dir_path ~ "/" ~ img).read); - // calculate, digest, hash - writeln(img, "\n", data.sha256Of.toHexString, "::", data.length, " ", img); - writeln(data.sha256Of.toHexString, " ", img, " ", data.length); - } catch (Exception ex) { - writeln("WARNING, image not found: ", img, "\n ", manifested.src.image_dir_path ~ "/" ~ img); - } - } - } + auto image_list = (_images.sort()).uniq; // endnotes optional only one 1~ level if (the_document_endnotes_section.length > 1) { // writeln("endnotes"); dom_structure_markedup_tags_status_buffer = dom_structure_markedup_tags_status.dup; diff --git a/src/sisudoc/meta/metadoc_from_src_functions.d b/src/sisudoc/meta/metadoc_from_src_functions.d index b5956c0..953e75a 100644 --- a/src/sisudoc/meta/metadoc_from_src_functions.d +++ b/src/sisudoc/meta/metadoc_from_src_functions.d @@ -2991,11 +2991,6 @@ template docAbstractionFunctions() { ObjGenericComposite obj, ) { obj.metainfo.sha256 = obj.text.sha256Of; - // if (obj.metainfo.is_a == "heading") { - // writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a, " ", obj.metainfo.heading_lev_markup); - // } else { - // writeln(obj.metainfo.sha256.toHexString, " ", obj.metainfo.ocn, " ", obj.metainfo.is_a); - // } return obj.metainfo.sha256; } // ↑ - object digest diff --git a/src/sisudoc/meta/metadoc_show_summary.d b/src/sisudoc/meta/metadoc_show_summary.d index 36ed3c1..92d476c 100644 --- a/src/sisudoc/meta/metadoc_show_summary.d +++ b/src/sisudoc/meta/metadoc_show_summary.d @@ -104,7 +104,7 @@ template spineMetaDocSummary() { } } writefln( - "%s\n\"%s\", %s\n%s [%s]\n%s \n%s%s\n%s%s\n%s%s\n%s\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%s", + "%s\n\"%s\", %s\n%s [%s]\n%s \n%s - %s [%s]\n%s\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%40-s%10-d\n%s", markup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.conf_make_meta.meta.title_full, doc_matters.conf_make_meta.meta.creator_author, @@ -112,11 +112,8 @@ template spineMetaDocSummary() { doc_matters.src.language, markup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.doc_digest.markup_doc.toHexString, - " - src doc digest", - doc_matters.doc_digest.header.toHexString, - " - doc header", - doc_matters.doc_digest.text.toHexString, - " - doc text", + doc_matters.src.filename, + doc_matters.src.language, markup.repeat_character_by_number_provided("-", char_repeat_number), "- toc arr length:", to!int(doc_abstraction["toc"].length), |