#!/usr/bin/env rdmd /+ - Name: Doc Reform - Description: documents, structuring, processing, publishing, search - static content generator - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 - 2019 Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: Doc Reform (SiSU), a framework for document structuring, publishing and search Copyright (C) Ralph Amissah This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFERO General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see [http://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at these locations: [http://www.fsf.org/licensing/licenses/agpl.html] [http://www.gnu.org/licenses/agpl.html] - Doc Reform (related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Hompages: [http://www.doc_reform.org] [http://www.sisudoc.org] - Git [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary] [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/html.rb;hb=HEAD] +/ module doc_reform.sisu_document_parser; import doc_reform.conf.compile_time_info, doc_reform.meta.metadoc; import std.datetime, std.getopt, std.file, std.path, std.process; import doc_reform.meta, doc_reform.meta.metadoc_summary, doc_reform.meta.metadoc_harvest, doc_reform.meta.metadoc_from_src, doc_reform.meta.conf_make_meta_structs, doc_reform.meta.conf_make_meta_toml, doc_reform.meta.conf_make_meta_json, doc_reform.meta.defaults, doc_reform.meta.doc_debugs, doc_reform.meta.rgx, doc_reform.source.paths_source, doc_reform.source.read_config_files, doc_reform.source.read_source_files, doc_reform.output.hub; import std.algorithm; import std.parallelism; mixin(import("version.txt")); mixin CompileTimeInfo; string program_name = "doc-reform"; /++ name "doc_reform" description "A SiSU inspired document parser writen in D." homepage "http://sisudoc.org" +/ void main(string[] args) { mixin DocReformRgxInit; mixin contentJSONtoDocReformStruct; mixin DocReformBiblio; mixin DocReformRgxInitFlags; mixin outputHub; struct Harvest { string title = ""; string[] author_arr = []; string author = ""; string author_surname = ""; string author_surname_fn = ""; string language = ""; string language_original = ""; string uid = ""; string date_published = ""; string[] topic_register_arr = []; string path_html_seg = ""; string path_html_scroll = ""; string path_epub = ""; string url_html_seg = ""; string url_html_scroll = ""; string url_epub = ""; } Harvest harvested; Harvest[] harvests; string flag_action; string arg_unrecognized; enum dAM { abstraction, matters } static auto rgx = Rgx(); scope(success) { writefln( "~ run complete, ok ~ (%s-%s.%s.%s, %s D:%s, %s %s)", program_name, _ver.major, _ver.minor, _ver.patch, __VENDOR__, __VERSION__, bits, os, ); } scope(failure) { debug(checkdoc) { stderr.writefln( "run failure", ); } } bool[string] opts = [ "abstraction" : false, "assertions" : false, "concordance" : false, "dark" : false, "debug" : false, "digest" : false, "epub" : false, "harvest" : false, "harvest-authors" : false, "harvest-topics" : false, "html" : false, "html-seg" : false, "html-scroll" : false, "latex" : false, "light" : false, "manifest" : false, "ocn" : true, "odf" : false, "odt" : false, "parallel" : false, "parallel-subprocesses" : false, "pdf" : false, "quiet" : false, "pod" : false, "serial" : false, "source" : false, "sqlite-discrete" : false, "sqlite-db-create" : false, "sqlite-db-drop" : false, "sqlite-db-recreate" : false, "sqlite-delete" : false, "sqlite-insert" : false, "sqlite-update" : false, "text" : false, "verbose" : false, "very-verbose" : false, "xhtml" : false, "section_toc" : true, "section_body" : true, "section_endnotes" : true, "section_glossary" : true, "section_biblio" : true, "section_bookindex" : true, "section_blurb" : true, "backmatter" : true, "skip-output" : false, "theme-dark" : false, "theme-light" : false, "workon" : false, ]; string[string] settings = [ "output-dir" : "", "site-config-dir" : "", "lang" : "all", "sqlite-filename" : "documents", ]; auto helpInfo = getopt(args, std.getopt.config.passThrough, "abstraction", "--abstraction document abstraction ", &opts["abstraction"], "assert", "--assert set optional assertions on", &opts["assertions"], "concordance", "--concordance file for document", &opts["concordance"], "dark", "--dark alternative dark theme", &opts["dark"], "debug", "--debug", &opts["debug"], "digest", "--digest hash digest for each object", &opts["digest"], "epub", "--epub process epub output", &opts["epub"], "harvest", "--harvest extract info on authors & topics from document header metadata", &opts["harvest"], "harvest-authors", "--harvest-authors extract info on authors from document header metadata", &opts["harvest-authors"], "harvest-topics", "--harvest-topics extract info on topics from document header metadata", &opts["harvest-topics"], "html", "--html process html output", &opts["html"], "html-seg", "--html-seg process html output", &opts["html-seg"], "html-scroll", "--html-seg process html output", &opts["html-scroll"], "latex", "--latex output for pdfs", &opts["latex"], "light", "--light default light theme", &opts["light"], "manifest", "--manifest process manifest output", &opts["manifest"], "ocn", "--ocn object cite numbers (default)", &opts["ocn"], "odf", "--odf open document format text (--odt)", &opts["odf"], "odt", "--odt open document format text", &opts["odt"], "parallel", "--parallel parallelisation", &opts["parallel"], "parallel-subprocesses", "--parallel-subprocesses nested parallelisation", &opts["parallel-subprocesses"], "quiet|q", "--quiet output to terminal", &opts["quiet"], "pdf", "--pdf latex output for pdfs", &opts["pdf"], "pod", "--pod doc reform pod source content bundled", &opts["pod"], "serial", "--serial serial processing", &opts["serial"], "source", "--source markup source text content", &opts["source"], "sqlite-discrete", "--sqlite process discrete sqlite output", &opts["sqlite-discrete"], "sqlite-db-create", "--sqlite-db-create create db, create tables", &opts["sqlite-db-create"], "sqlite-db-drop", "--sqlite-db-drop drop tables & db", &opts["sqlite-db-drop"], "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables", &opts["sqlite-db-recreate"], "sqlite-delete", "--sqlite-delete process sqlite output", &opts["sqlite-delete"], "sqlite-insert", "--sqlite-insert process sqlite output", &opts["sqlite-insert"], "sqlite-update", "--sqlite-update process sqlite output", &opts["sqlite-update"], "text", "--text process text output", &opts["text"], "txt", "--txt process text output", &opts["text"], "verbose|v", "--verbose output to terminal", &opts["verbose"], "very-verbose", "--very-verbose output to terminal", &opts["very-verbose"], "xhtml", "--xhtml process xhtml output", &opts["xhtml"], "section-toc", "--section-toc process table of contents (default)", &opts["section_toc"], "section-body", "--section-body process document body (default)", &opts["section_body"], "section-endnotes", "--section-endnotes process document endnotes (default)", &opts["section_endnotes"], "section-glossary", "--section-glossary process document glossary (default)", &opts["section_glossary"], "section-biblio", "--section-biblio process document biblio (default)", &opts["section_biblio"], "section-bookindex", "--section-bookindex process document bookindex (default)", &opts["section_bookindex"], "section-blurb", "--section-blurb process document blurb (default)", &opts["section_blurb"], "backmatter", "--section-backmatter process document backmatter (default)", &opts["backmatter"], "skip-output", "--skip-output", &opts["skip-output"], "theme-dark", "--theme-dark alternative dark theme", &opts["theme-dark"], "theme-light", "--theme-light default light theme", &opts["theme-light"], "workon", "--workon (reserved for some matters under development & testing)", &opts["workon"], "output-dir", "--output-dir=[dir path]", &settings["output-dir"], "site-config-dir", "--site-config-dir=[dir path]", &settings["site-config-dir"], "sqlite-filename", "--sqlite-filename=[filename].sqlite", &settings["sqlite-filename"], "lang", "--lang=[lang code e.g. =en or =en,es]", &settings["lang"], ); if (helpInfo.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInfo.options); } enum outTask { pod, source, sqlite, sqlite_multi, latex, odt, epub, html_scroll, html_seg, html_stuff } struct OptActions { bool assertions() { return opts["assertions"]; } bool concordance() { return opts["concordance"]; } bool css_theme_default() { bool _is_light; if (opts["light"] || opts["theme-light"]) { _is_light = true; } else if (opts["dark"] || opts["theme-dark"]) { _is_light = false; } else { _is_light = true; } return _is_light; } bool debug_do() { return opts["debug"]; } bool digest() { return opts["digest"]; } bool epub() { return opts["epub"]; } bool harvest() { bool _is = ( opts["harvest"] || opts["harvest-authors"] || opts["harvest-topics"] ) ? true : false; return _is; } bool harvest_authors() { return opts["harvest-authors"]; } bool harvest_topics() { return opts["harvest-topics"]; } bool html() { bool _is; if ( opts["html"] || opts["html-seg"] || opts["html-scroll"]) { _is = true; } else { _is = false; } return _is; } bool html_seg() { bool _is; if ( opts["html"] || opts["html-seg"]) { _is = true; } else { _is = false; } return _is; } bool html_scroll() { bool _is; if ( opts["html"] || opts["html-scroll"]) { _is = true; } else { _is = false; } return _is; } bool html_stuff() { bool _is; if (opts["html"] || opts["html-scroll"] || opts["html-seg"]) { _is = true; } else { _is = false; } return _is; } bool latex() { bool _is; if ( opts["latex"] || opts["pdf"]) { _is = true; } else { _is = false; } return _is; } bool odt() { bool _is; if ( opts["odf"] || opts["odt"]) { _is = true; } else { _is = false; } return _is; } bool manifest() { return opts["manifest"]; } bool ocn() { return opts["ocn"]; } bool quiet() { return opts["quiet"]; } bool pod() { return opts["pod"]; } bool source() { return opts["source"]; } bool sqlite_discrete() { return opts["sqlite-discrete"]; } bool sqlite_db_drop() { bool _is; if ( opts["sqlite-db-recreate"] || opts["sqlite-db-drop"]) { _is = true; } else { _is = false; } return _is; } bool sqlite_db_create() { bool _is; if ( opts["sqlite-db-recreate"] || opts["sqlite-db-create"]) { _is = true; } else { _is = false; } return _is; } bool sqlite_delete() { return opts["sqlite-delete"]; } bool sqlite_update() { bool _is; if (opts["sqlite-update"] || opts["sqlite-insert"]) { _is = true; } else { _is = false; } return _is; } bool sqlite_shared_db_action() { bool _is; if (opts["sqlite-db-recreate"] || opts["sqlite-db-create"] || opts["sqlite-delete"] || opts["sqlite-insert"] || opts["sqlite-update"] ) { _is = true; } else { _is = false; } return _is; } bool text() { return opts["text"]; } bool verbose() { bool _is; if (opts["verbose"] || opts["very-verbose"]) { _is = true; } else { _is = false; } return _is; } bool very_verbose() { return opts["very-verbose"]; } bool xhtml() { return opts["xhtml"]; } bool section_toc() { return opts["section_toc"]; } bool section_body() { return opts["section_body"]; } bool section_endnotes() { return opts["section_endnotes"]; } bool section_glossary() { return opts["section_glossary"]; } bool section_biblio() { return opts["section_biblio"]; } bool section_bookindex() { return opts["section_bookindex"]; } bool section_blurb() { return opts["section_blurb"]; } bool backmatter() { return opts["backmatter"]; } bool skip_output() { return opts["skip-output"]; } bool workon() { return opts["workon"]; } auto languages_set() { return settings["lang"].split(","); } auto output_dir_set() { return settings["output-dir"]; } auto sqlite_filename() { return settings["sqlite-filename"]; } bool parallelise() { bool _is; if (opts["parallel"] == true) { _is = true; if (sqlite_shared_db_action) { _is = false; } } else if (opts["parallel"] == false && opts["serial"] == true) { _is = false; } else if (opts["abstraction"] || concordance || html || epub || odt || latex || manifest || pod || source || sqlite_discrete ) { _is = true; } else { _is = false; } return _is; } bool parallelise_subprocesses() { return opts["parallel-subprocesses"]; } auto output_task_scheduler() { int[] schedule; if (pod) { schedule ~= outTask.pod; } if (source) { schedule ~= outTask.source; } if (sqlite_discrete) { schedule ~= outTask.sqlite; } if (epub) { schedule ~= outTask.epub; } if (html_scroll) { schedule ~= outTask.html_scroll; } if (html_seg) { schedule ~= outTask.html_seg; } if (html_stuff) { schedule ~= outTask.html_stuff; } if (odt) { schedule ~= outTask.odt; } if (latex) { schedule ~= outTask.latex; } return schedule.sort().uniq; } bool abstraction() { bool _is; if (opts["abstraction"] || concordance || source || pod || html || epub || odt || latex || manifest || sqlite_discrete || sqlite_delete || sqlite_update ) { _is = true; } else { _is = false; } return _is; } bool meta_processing_general() { bool _is; if (opts["abstraction"] || html || epub || odt || latex || sqlite_discrete || sqlite_update ) { _is = true; } else { _is = false; } return _is; } bool meta_processing_xml_dom() { bool _is; if (opts["abstraction"] || html || epub || odt || sqlite_discrete || sqlite_update ) { _is = true; } else { _is = false; } return _is; } } auto _opt_action = OptActions(); auto program_info() { struct ProgramInfo { string name() { return program_name; } string ver() { string ver_ = format( "%s.%s.%s", _ver.major, _ver.minor, _ver.patch, ); return ver_; } } return ProgramInfo(); } auto _env = [ "pwd" : environment["PWD"], "home" : environment["HOME"], ]; auto _manifest_start = PodManifest!()(""); auto _manifest_matter = PathMatters!()(_opt_action, _env, ""); auto _manifests = [ _manifest_matter ]; foreach(arg; args[1..$]) { _manifest_start = PodManifest!()(arg); if (arg.match(rgx.flag_action)) { flag_action ~= " " ~ arg; // flags not taken by getopt } else if ( !(arg.match(rgx.src_pth_sst_or_ssm)) && _manifest_start.pod_manifest_file_with_path && _opt_action.abstraction ) { string contents_location_raw_; string contents_location_; string sisudoc_txt_ = _manifest_start.pod_manifest_file_with_path; enforce( exists(sisudoc_txt_)!=0, "file not found: «" ~ sisudoc_txt_ ~ "»" ); if (exists(sisudoc_txt_)) { try { if (exists(sisudoc_txt_)) { contents_location_raw_ = sisudoc_txt_.readText; } } catch (ErrnoException ex) { } catch (FileException ex) { // Handle errors } if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+ foreach (m; contents_location_raw_.matchAll(rgx.pod_content_location)) { foreach (n; m.captures[2].matchAll(rgx.language_codes)) { contents_location_ ~= "media/text/" ~ n.captures[1].to!string ~ "/" ~ m.captures[1].to!string ~ "\n"; } } } else { contents_location_ = contents_location_raw_; } } else { writeln("manifest not found: ", sisudoc_txt_); } auto contents_locations_arr = (cast(char[]) contents_location_).split; auto tmp_dir_ = (sisudoc_txt_).dirName.array; foreach (contents_location; contents_locations_arr) { assert(contents_location.match(rgx.src_pth_sst_or_ssm), "not a recognised file: «" ~ contents_location ~ "»" ); auto contents_location_pth_ = (contents_location).to!string; Regex!(char) lang_rgx_ = regex(r"/(" ~ _opt_action.languages_set.join("|") ~ ")/"); if (_opt_action.languages_set[0] == "all" || (contents_location_pth_).match(lang_rgx_) ) { auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string; _manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr); _manifests ~= _manifest_matter; } } } else if (arg.match(rgx.src_pth_sst_or_ssm)) { if (exists(arg)==0) { writeln("ERROR >> Processing Skipped! File not found: ", arg); } else { _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg); _manifests ~= _manifest_matter; } } else if (arg.match(rgx.src_pth_zip)) { // fns_src ~= arg; // gather input markup source file names for processing } else { // anything remaining, unused arg_unrecognized ~= " " ~ arg; } } if (!(_opt_action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step0 commence → (without processing files)"); } outputHubOp!()(_env, _opt_action); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step0 complete"); } } if (_manifests.length > 1 // _manifests[0] initialized dummy element && _opt_action.abstraction) { if (_opt_action.parallelise) { // note cannot parallelise sqlite shared db foreach(manifest; parallel(_manifests[1..$])) { if (!empty(manifest.src.filename)) { scope(success) { if (!(_opt_action.quiet)) { writefln( "%s", "-- ~ document complete, ok ~ ------------------------------------", ); } } scope(failure) { debug(checkdoc) { stderr.writefln( "~ document run failure ~ (%s v%s)\n\t%s\n%s", __VENDOR__, __VERSION__, manifest.src.filename, "------------------------------------------------------------------", ); } } enforce( manifest.src.filename.match(rgx.src_pth_types), "not a sisu markup filename: «" ~ manifest.src.filename ~ "»" ); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("--->\nstepX commence → (document abstraction)"); } auto t = DocReformAbstraction!()(_env, program_info, _opt_action, manifest); static assert(!isTypeTuple!(t)); static assert(t.length==2); auto doc_abstraction = t[dAM.abstraction]; auto doc_matters = t[dAM.matters]; if ((doc_matters.opt.action.debug_do) || (doc_matters.opt.action.very_verbose) ) { writeln("- stepX complete"); } /+ ↓ debugs +/ if (doc_matters.opt.action.verbose) { DocReformMetaDocSummary!()(doc_abstraction, doc_matters); } if (doc_matters.opt.action.harvest) { if (doc_matters.opt.action.harvest_authors) { } if (doc_matters.opt.action.harvest_topics) { } Harvest[] DocReformMetaDocHarvests()( Harvest harvested, Harvest[] harvests, ) { harvests ~= harvested; return harvests; } harvested = DocReformMetaDocHarvest!()(doc_matters, harvested); harvests = DocReformMetaDocHarvests!()(harvested, harvests); } /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { DocReformDebugs!()(doc_abstraction, doc_matters); } /+ ↓ output hub +/ if (!(doc_matters.opt.action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step5 commence → (process outputs)"); } doc_abstraction.outputHub!()(doc_matters); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step5 complete"); } } scope(exit) { if (!(_opt_action.quiet)) { writefln( "processed file: %s (%s)", manifest.src.filename, manifest.src.language ); } destroy(manifest); } } else { /+ no recognized filename provided +/ writeln("no recognized filename"); break; // terminate, stop } } } else { foreach(manifest; _manifests[1..$]) { writeln("parallelisation off: actions include sqlite shared db"); if (!empty(manifest.src.filename)) { scope(success) { if (!(_opt_action.quiet)) { writefln( "%s", "-- ~ document complete, ok ~ ------------------------------------", ); } } scope(failure) { debug(checkdoc) { stderr.writefln( "~ document run failure ~ (%s v%s)\n\t%s\n%s", __VENDOR__, __VERSION__, manifest.src.filename, "------------------------------------------------------------------", ); } } enforce( manifest.src.filename.match(rgx.src_pth_types), "not a sisu markup filename: «" ~ manifest.src.filename ~ "»" ); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("--->\nstepX commence → (document abstraction)"); } auto t = DocReformAbstraction!()(_env, program_info, _opt_action, manifest); static assert(!isTypeTuple!(t)); static assert(t.length==2); auto doc_abstraction = t[dAM.abstraction]; auto doc_matters = t[dAM.matters]; if ((doc_matters.opt.action.debug_do) || (doc_matters.opt.action.very_verbose) ) { writeln("- stepX complete"); } /+ ↓ debugs +/ if (doc_matters.opt.action.verbose) { DocReformMetaDocSummary!()(doc_abstraction, doc_matters); } if (doc_matters.opt.action.harvest) { if (doc_matters.opt.action.harvest_authors) { } if (doc_matters.opt.action.harvest_topics) { } Harvest[] DocReformMetaDocHarvests()( Harvest harvested, Harvest[] harvests, ) { harvests ~= harvested; return harvests; } harvested = DocReformMetaDocHarvest!()(doc_matters, harvested); harvests = DocReformMetaDocHarvests!()(harvested, harvests); } /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { DocReformDebugs!()(doc_abstraction, doc_matters); } /+ ↓ output hub +/ if (!(doc_matters.opt.action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("step5 commence → (process outputs)"); } doc_abstraction.outputHub!()(doc_matters); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { writeln("- step5 complete"); } } scope(exit) { if (!(_opt_action.quiet)) { writefln( "processed file: %s (%s)", manifest.src.filename, manifest.src.language ); } destroy(manifest); } } else { /+ no recognized filename provided +/ writeln("no recognized filename"); break; // terminate, stop } } } } if (_opt_action.very_verbose && harvests.length > 0 ) { auto min_repeat_number = 42; string[] _document_topic_register; string[] _topic_register; string[] _sub_topic_register; Harvest[][string][string][string][string] subject_trees; string[] topics = []; string _auth = ""; foreach(k, doc_harvest; harvests) { _topic_register = []; foreach(topic; doc_harvest.topic_register_arr.sort) { _sub_topic_register = []; string _spaces; string[] subject_tree = topic.split(mkup.sep); switch (subject_tree.length) { case 1: subject_trees[subject_tree[0]]["_a"]["_a"]["_a"] ~= doc_harvest; break; case 2: subject_trees[subject_tree[0]][subject_tree[1]]["_a"]["_a"] ~= doc_harvest; break; case 3: subject_trees[subject_tree[0]][subject_tree[1]][subject_tree[2]]["_a"] ~= doc_harvest; break; case 4: subject_trees[subject_tree[0]][subject_tree[1]][subject_tree[2]][subject_tree[3]] ~= doc_harvest; break; default: break; } _topic_register ~= _sub_topic_register.join("\n"); } auto char_repeat_number = (doc_harvest.title.length + doc_harvest.author.length + 16); char_repeat_number = (char_repeat_number > min_repeat_number) ? char_repeat_number : min_repeat_number; _document_topic_register ~= format( "\"%s\", %s%s\n%s", doc_harvest.title, doc_harvest.author, (doc_harvest.date_published.length > 0) ? " (" ~ doc_harvest.date_published ~ ")" : "", _topic_register.sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable).release.join("\n"), ); } topics ~= format(q"┃ Metadata Harvest - Topics

Metadata Harvest - Topics (output organised by language & filetype)

[ HOME ] also see Metadata Harvest - Authors

A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z, 

English   

┃") ~ "\n"; char _prev_k = "_".to!char; int _kn; foreach(k0; subject_trees.keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k0.toUpper.to!(char[])[0] != _prev_k) { topics ~= format(q"┃

%s

┃", k0.toUpper.to!(char[])[0], k0.toUpper.to!(char[])[0], ); _prev_k = k0.toUpper.to!(char[])[0]; } if (k0 != "_a") { topics ~= format(q"┃

%s

┃", k0, k0,) ~ "\n"; writeln("", k0); if ("_a" in subject_trees[k0]) { foreach (t_a_; subject_trees[k0]["_a"]["_a"]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname, t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname, a, ); } } topics ~= format(q"┃

"%s" -%s┃", "url", t_a_.title, _auth, ) ~ "\n"; writeln("- ", t_a_.title, " - ", t_a_.author); } } foreach(k1; subject_trees[k0].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k1 != "_a") { topics ~= format(q"┃

%s

┃", k1, k1,) ~ "\n"; writeln(" ", k1); if ("_a" in subject_trees[k0][k1]) { foreach (t_a_; subject_trees[k0][k1]["_a"]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname, t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname, a, ); } } topics ~= format(q"┃

%s -%s┃", "url", t_a_.title, _auth, ) ~ "\n"; writeln(" - ", t_a_.title, " - ", t_a_.author); } } } foreach(k2; subject_trees[k0][k1].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k2 != "_a") { topics ~= format(q"┃

%s

┃", k2, k2,) ~ "\n"; writeln(" ", k2); if ("_a" in subject_trees[k0][k1][k2]) { foreach (t_a_; subject_trees[k0][k1][k2]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname, t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname, a, ); } } topics ~= format(q"┃

%s -%s┃", "url", t_a_.title, _auth, ) ~ "\n"; writeln(" - ", t_a_.title, " - ", t_a_.author); } } } foreach(k3; subject_trees[k0][k1][k2].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k3 != "_a") { topics ~= format(q"┃

%s

┃", k3, k3,) ~ "\n"; writeln(" ", k3); { foreach (t_a_; subject_trees[k0][k1][k2][k3] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃%s┃", t_a_.author_surname, t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname, a, ); } } topics ~= format(q"┃

%s -%s┃", "url", t_a_.title, _auth, ) ~ "\n"; writeln(" - ", t_a_.title, " - ", t_a_.author); } } } } } } } } topics ~= format(q"┃

┃") ~ "\n"; try { auto f = File("topics.html", "w"); foreach (o; topics) { f.writeln(o); } } catch (ErrnoException ex) { // Handle error } } if ((_opt_action.verbose || _opt_action.very_verbose) && harvests.length > 0 ) { string[] authors = []; authors ~= format(q"┃ Metadata Harvest - Authors

Metadata Harvest - Authors (output organised by language & filetype)

[ HOME ] also see Metadata Harvest - Topics

English   

A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z,  ┃") ~ "\n"; string[string] _au; string[] _auth_date_title; string[] _author_date_title; string _prev_auth = ""; char _prev_k = "_".to!char; foreach(doc_harvest; harvests .multiSort!( "toUpper(a.author_surname_fn) < toUpper(b.author_surname_fn)", "a.date_published < b.date_published", "a.title < b.title", SwapStrategy.unstable ) ) { if (doc_harvest.author_surname_fn != _prev_auth) { _au[doc_harvest.author_surname_fn] = format(q"┃

%s

%s "%s" [%s]

┃", doc_harvest.author_surname, doc_harvest.author_surname_fn, (doc_harvest.date_published.length > 0) ? doc_harvest.date_published : "", "url", doc_harvest.title, doc_harvest.language, ); _prev_auth = doc_harvest.author_surname_fn; } else { _au[doc_harvest.author_surname_fn] ~= format(q"┃

%s "%s" [%s]

┃", (doc_harvest.date_published.length > 0) ? doc_harvest.date_published : "", "url", doc_harvest.title, doc_harvest.language, ); } _author_date_title ~= format(q"┃%s %s "%s" [%s]┃", doc_harvest.author_surname_fn, (doc_harvest.date_published.length > 0) ? "(" ~ doc_harvest.date_published ~ ")" : "", doc_harvest.title, doc_harvest.language, ); // writeln(doc_harvest.author_date_title); } foreach (k; _au.keys.sort) { if (k.toUpper.to!(char[])[0] != _prev_k) { authors ~= format(q"┃

%s

┃", k.toUpper.to!(char[])[0], k.toUpper.to!(char[])[0], ); _prev_k = k.toUpper.to!(char[])[0]; } authors ~= _au[k]; } authors ~= format(q"┃
┃") ~ "\n"; try { auto f = File("authors.html", "w"); foreach (o; authors) { f.writeln(o); } } catch (ErrnoException ex) { // Handle error } foreach(_adt; _author_date_title.sort) { writeln(_adt); } } }