-*- mode: org -*- #+TITLE: spine (doc_manifest) metadata (multidocument) harvests #+DESCRIPTION: documents - structuring, various output representations & search #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 - 2020 Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args :exports code #+PROPERTY: header-args+ :noweb yes #+PROPERTY: header-args+ :eval no #+PROPERTY: header-args+ :results no #+PROPERTY: header-args+ :cache no #+PROPERTY: header-args+ :padline no * document harvest _summary_ :module:spine:metadoc_show_summary: - html ~dr/bin/spine-ldc -v --html --html-link-harvest --harvest \ --output=/var/www/html \ ~grotto/repo/git.repo/code/project-spine/doc-reform-markup/markup_samples/markup/pod/* ** _module template_ metadoc harvest #+BEGIN_SRC d :tangle "../src/doc_reform/meta/metadoc_harvest.d" module doc_reform.meta.metadoc_harvest; template spineMetaDocHarvest() { @safe auto spineMetaDocHarvest(T,H)( T doc_matters, H hvst, ) { <> mixin InternalMarkup; <> <> } } #+END_SRC ** init *** imports #+NAME: metadoc_harvest_imports #+BEGIN_SRC d import doc_reform.meta.defaults, doc_reform.meta.rgx; import std.array, std.exception, std.regex, std.stdio, std.string, std.typecons, std.uni, std.utf, std.conv : to; #+END_SRC *** initialize :report: #+NAME: metadoc_harvest_initialize #+BEGIN_SRC d static auto mkup = InlineMarkup(); #+END_SRC ** harvest summary #+NAME: meta_metadoc_harvest_summary #+BEGIN_SRC d auto min_repeat_number = 66; auto char_repeat_number = (doc_matters.conf_make_meta.meta.title_full.length + doc_matters.conf_make_meta.meta.creator_author.length + 4); char_repeat_number = (char_repeat_number > min_repeat_number) ? char_repeat_number : min_repeat_number; writefln( "%s\n\"%s\", %s\n%s\n%s\n%s", mkup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.conf_make_meta.meta.title_full, doc_matters.conf_make_meta.meta.creator_author, doc_matters.src.filename, doc_matters.conf_make_meta.meta.classify_topic_register_arr, mkup.repeat_character_by_number_provided("-", char_repeat_number), ); #+END_SRC ** return harvest #+NAME: meta_metadoc_harvest #+BEGIN_SRC d import doc_reform.io_out.paths_output; auto pth_html_abs = spinePathsHTML!()(doc_matters.output_path, doc_matters.src.language); auto pth_html_rel = spineDocRootTreeHTML!()(doc_matters.src.language); hvst.harvest.title = doc_matters.conf_make_meta.meta.title_full; hvst.harvest.author = doc_matters.conf_make_meta.meta.creator_author; hvst.harvest.author_surname = doc_matters.conf_make_meta.meta.creator_author_surname; hvst.harvest.author_surname_fn = doc_matters.conf_make_meta.meta.creator_author_surname_fn; hvst.harvest.author_arr = doc_matters.conf_make_meta.meta.creator_author_arr; hvst.harvest.language_original = doc_matters.conf_make_meta.meta.original_language; hvst.harvest.language = doc_matters.src.language; hvst.harvest.uid = doc_matters.src.doc_uid; hvst.harvest.date_published = doc_matters.conf_make_meta.meta.date_published; hvst.harvest.topic_register_arr = doc_matters.conf_make_meta.meta.classify_topic_register_arr; hvst.harvest.path_html_metadata = pth_html_rel.fn_metadata(doc_matters.src.filename); hvst.harvest.path_html_scroll = pth_html_rel.fn_scroll(doc_matters.src.filename); hvst.harvest.path_html_segtoc = pth_html_rel.fn_seg(doc_matters.src.filename, "toc"); hvst.harvest.path_abs_html_scroll = pth_html_abs.fn_scroll(doc_matters.src.filename); hvst.harvest.path_abs_html_segtoc = pth_html_abs.fn_seg(doc_matters.src.filename, "toc"); return hvst.harvest; #+END_SRC ** _module template_ metadoc harvest topics *** _module template_ metadoc harvest topics template #+BEGIN_SRC d :tangle "../src/doc_reform/meta/metadoc_harvests_topics.d" module doc_reform.meta.metadoc_harvests_topics; import std.algorithm, std.array, std.exception, std.regex, std.stdio, std.string, std.conv : to; import doc_reform.meta.defaults, doc_reform.meta.rgx; mixin spineHarvest; mixin InternalMarkup; template spineMetaDocHarvestsTopics() { static auto mkup = InlineMarkup(); void spineMetaDocHarvestsTopics(H,M,O)( H hvst, M _make_and_meta_struct, O _opt_action, ) { <> <> <> <> <> <> <> topics <> <> } } #+END_SRC *** order topic register #+NAME: harvested_topics #+BEGIN_SRC d auto min_repeat_number = 42; string[] _document_topic_register; string[] _topic_register; string[] _sub_topic_register; string[] topics = []; string _auth = ""; foreach(k, doc_harvest; hvst.harvests) { _topic_register = []; foreach(topic; doc_harvest.topic_register_arr.sort) { _sub_topic_register = []; string _spaces; string[] subject_tree = topic.split(mkup.sep); switch (subject_tree.length) { case 1: hvst.subject_trees[subject_tree[0]]["_a"]["_a"]["_a"] ~= doc_harvest; break; case 2: hvst.subject_trees[subject_tree[0]][subject_tree[1]]["_a"]["_a"] ~= doc_harvest; break; case 3: hvst.subject_trees[subject_tree[0]][subject_tree[1]][subject_tree[2]]["_a"] ~= doc_harvest; break; case 4: hvst.subject_trees[subject_tree[0]][subject_tree[1]][subject_tree[2]][subject_tree[3]] ~= doc_harvest; break; default: break; } _topic_register ~= _sub_topic_register.join("\n"); } auto char_repeat_number = (doc_harvest.title.length + doc_harvest.author.length + 16); char_repeat_number = (char_repeat_number > min_repeat_number) ? char_repeat_number : min_repeat_number; _document_topic_register ~= format( "\"%s\", %s%s\n%s", doc_harvest.title, doc_harvest.author, (doc_harvest.date_published.length > 0) ? " (" ~ doc_harvest.date_published ~ ")" : "", _topic_register.sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable).release.join("\n"), ); } #+END_SRC *** harvested topics html head #+NAME: harvested_topics_html_head_1 #+BEGIN_SRC d topics ~= format(q"┃ Metadata Harvest - Topics #+END_SRC *** harvested topics html head #+NAME: harvested_topics_html_head_2 #+BEGIN_SRC d

Metadata Harvest - Topics (output organised by language & filetype)

[ HOME ] [ Metadata Harvest - Authors ]

A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z, 

#+END_SRC *** harvested topics html head theme insert #+NAME: harvested_topics_html_head_theme #+BEGIN_SRC d ┃", _opt_action.css_theme_default ? theme_light_0 : theme_dark_0, _opt_action.css_theme_default ? theme_light_1 : theme_dark_1, ) ~ "\n"; #+END_SRC *** harvested topics html #+NAME: harvested_topics_html #+BEGIN_SRC d char _prev_k = "_".to!char; int _kn; foreach(k0; hvst.subject_trees.keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k0.toUpper.to!(char[])[0] != _prev_k) { topics ~= format(q"┃

%s

┃", k0.toUpper.to!(char[])[0], k0.toUpper.to!(char[])[0], ); _prev_k = k0.toUpper.to!(char[])[0]; } if (k0 != "_a") { topics ~= format(q"┃

%s

┃", k0.translate([' ' : "_"]), k0,) ~ "\n"; if (_opt_action.very_verbose) { writeln("", k0); } if ("_a" in hvst.subject_trees[k0]) { foreach (t_a_; hvst.subject_trees[k0]["_a"]["_a"]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname.translate([' ' : "_"]), t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname.translate([' ' : "_"]), a, ); } } topics ~= format(q"┃

"%s" - %s [ %s ]┃", t_a_.path_html_segtoc, t_a_.title, _auth, t_a_.path_html_metadata, t_a_.language, ) ~ "\n"; if (_opt_action.very_verbose) { writeln("- ", t_a_.title, " - ", t_a_.author); } } } foreach(k1; hvst.subject_trees[k0].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k1 != "_a") { topics ~= format(q"┃

%s

┃", k0.translate([' ' : "_"]), k1.translate([' ' : "_"]), k1,) ~ "\n"; if (_opt_action.very_verbose) { writeln(" ", k1); } if ("_a" in hvst.subject_trees[k0][k1]) { foreach (t_a_; hvst.subject_trees[k0][k1]["_a"]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname.translate([' ' : "_"]), t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname.translate([' ' : "_"]), a, ); } } topics ~= format(q"┃

%s - %s [ %s ]┃", t_a_.path_html_segtoc, t_a_.title, _auth, t_a_.path_html_metadata, t_a_.language, ) ~ "\n"; if (_opt_action.very_verbose) { writeln(" - ", t_a_.title, " - ", t_a_.author); } } } } foreach(k2; hvst.subject_trees[k0][k1].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k2 != "_a") { topics ~= format(q"┃

%s

┃", k0.translate([' ' : "_"]), k1.translate([' ' : "_"]), k2.translate([' ' : "_"]), k2,) ~ "\n"; if (_opt_action.very_verbose) { writeln(" ", k2); } if ("_a" in hvst.subject_trees[k0][k1][k2]) { foreach (t_a_; hvst.subject_trees[k0][k1][k2]["_a"] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃ %s┃", t_a_.author_surname.translate([' ' : "_"]), t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname.translate([' ' : "_"]), a, ); } } topics ~= format(q"┃

%s - %s [ %s ]┃", t_a_.path_html_segtoc, t_a_.title, _auth, t_a_.path_html_metadata, t_a_.language, ) ~ "\n"; if (_opt_action.very_verbose) { writeln(" - ", t_a_.title, " - ", t_a_.author); } } } } foreach(k3; hvst.subject_trees[k0][k1][k2].keys .sort!("toUpper(a) < toUpper(b)", SwapStrategy.unstable) ) { if (k3 != "_a") { topics ~= format(q"┃

%s

┃", k0.translate([' ' : "_"]), k1.translate([' ' : "_"]), k2.translate([' ' : "_"]), k3.translate([' ' : "_"]), k3,) ~ "\n"; if (_opt_action.very_verbose) { writeln(" ", k3); } { foreach (t_a_; hvst.subject_trees[k0][k1][k2][k3] .multiSort!("toUpper(a.title) < toUpper(b.title)", "a.author < b.author", SwapStrategy.unstable) ) { _auth = []; if (t_a_.author_arr.length < 2) { _auth = format(q"┃%s┃", t_a_.author_surname.translate([' ' : "_"]), t_a_.author, ); } else { foreach (a; t_a_.author_arr) { _auth ~= format(q"┃ %s,┃", t_a_.author_surname.translate([' ' : "_"]), a, ); } } topics ~= format(q"┃

%s - %s [ %s ]┃", t_a_.path_html_segtoc, t_a_.title, _auth, t_a_.path_html_metadata, t_a_.language, ) ~ "\n"; if (_opt_action.very_verbose) { writeln(" - ", t_a_.title, " - ", t_a_.author); } } } } } } } } } #+END_SRC *** harvested topics write #+NAME: harvested_topics_html_write #+BEGIN_SRC d import doc_reform.io_out.paths_output; auto out_pth = spinePathsHTML!()(_make_and_meta_struct.conf.output_path, ""); try { auto f = File(out_pth.harvest("topics.html"), "w"); foreach (o; topics) { f.writeln(o); } } catch (ErrnoException ex) { // Handle error } #+END_SRC ** _module template_ metadoc harvests authors *** _module template_ metadoc harvest authors #+BEGIN_SRC d :tangle "../src/doc_reform/meta/metadoc_harvests_authors.d" module doc_reform.meta.metadoc_harvests_authors; import std.algorithm, std.array, std.exception, std.regex, std.stdio, std.string, std.conv : to; import doc_reform.meta.defaults, doc_reform.meta.rgx; mixin spineHarvest; mixin InternalMarkup; template spineMetaDocHarvestsAuthors() { static auto mkup = InlineMarkup(); void spineMetaDocHarvestsAuthors(H,M,O)( H harvests, M _make_and_meta_struct, O _opt_action, ) { <> <> <> <> <> authors <> <> } } #+END_SRC *** harvested authors html head #+NAME: harvested_authors_html_head_1 #+BEGIN_SRC d string[] authors = []; authors ~= format(q"┃ Metadata Harvest - Authors #+END_SRC *** harvested authors html head #+NAME: harvested_authors_html_head_2 #+BEGIN_SRC d

Metadata Harvest - Authors (output organised by language & filetype)

[ HOME ] [ Metadata Harvest - Topics ]

A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z,  #+END_SRC *** harvested authors html theme inserts #+NAME: harvested_authors_html_head_theme #+BEGIN_SRC d ┃", _opt_action.css_theme_default ? theme_light_0 : theme_dark_0, _opt_action.css_theme_default ? theme_light_1 : theme_dark_1, ) ~ "\n"; string[string] _au; string[] _auth_date_title; string[] _author_date_title; string _prev_auth = ""; char _prev_k = "_".to!char; foreach(doc_harvest; harvests .multiSort!( "toUpper(a.author_surname_fn) < toUpper(b.author_surname_fn)", "a.date_published < b.date_published", "a.title < b.title", SwapStrategy.unstable ) ) { if (doc_harvest.author_surname_fn != _prev_auth) { _au[doc_harvest.author_surname_fn] = format(q"┃

%s

%s "%s" [ %s ]

┃", doc_harvest.author_surname.translate([' ' : "_"]), doc_harvest.author_surname_fn, (doc_harvest.date_published.length > 0) ? doc_harvest.date_published : "", doc_harvest.path_html_segtoc, doc_harvest.title, doc_harvest.path_html_metadata, doc_harvest.language, ); _prev_auth = doc_harvest.author_surname_fn; } else { _au[doc_harvest.author_surname_fn] ~= format(q"┃

%s "%s" [ %s ]

┃", (doc_harvest.date_published.length > 0) ? doc_harvest.date_published : "", doc_harvest.path_html_segtoc, doc_harvest.title, doc_harvest.path_html_metadata, doc_harvest.language, ); } _author_date_title ~= format(q"┃%s %s "%s" [ %s ]%s┃", doc_harvest.author_surname_fn, (doc_harvest.date_published.length > 0) ? "(" ~ doc_harvest.date_published ~ ")" : "", doc_harvest.title, doc_harvest.path_html_metadata, doc_harvest.language, (_opt_action.very_verbose) ? "\n " ~ doc_harvest.path_abs_html_scroll : "", ); } foreach (k; _au.keys.sort) { if (k.toUpper.to!(char[])[0] != _prev_k) { authors ~= format(q"┃

%s

┃", k.toUpper.to!(char[])[0], k.toUpper.to!(char[])[0], ); _prev_k = k.toUpper.to!(char[])[0]; } authors ~= _au[k]; } #+END_SRC *** harvested authors write #+NAME: harvested_authors_html_write #+BEGIN_SRC d import doc_reform.io_out.paths_output; auto out_pth = spinePathsHTML!()(_make_and_meta_struct.conf.output_path, ""); try { auto f = File(out_pth.harvest("authors.html"), "w"); foreach (o; authors) { f.writeln(o); } } catch (ErrnoException ex) { // Handle error } if (_opt_action.verbose || _opt_action.very_verbose ) { foreach(_adt; _author_date_title.sort) { writeln(_adt); } } #+END_SRC ** harvested authors & topics shared html *** themes **** head #+NAME: harvested_html_themes #+BEGIN_SRC d string theme_dark_0 = format(q"┃ body { color : #CCCCCC; background : #000000; background-color : #000000; } a:link { color : #FFFFFF; text-decoration : none; } a:visited { color : #999999; text-decoration : none; } a:hover { color : #000000; background-color : #555555; } a:hover img { background-color : #000000; } a:active { color : #888888; text-decoration : underline; } a.lev0:hover { color : #FFFFFF; background-color : #000000; } a.lev1:hover { color : #FFFFFF; background : #333333; } a.lev2:hover { color : #FFFFFF; background : #555555; } a.lev3:hover { color : #FFFFFF; background : #777777; } ┃"); string theme_light_0 = format(q"┃ body { color : #000000; background : #FFFFFF; background-color : #FFFFFF; } a:link { color : #003399; text-decoration : none; } a:visited { color : #003399; text-decoration : none; } a:hover { color : #000000; background-color : #f9f9aa; } a:hover img { background-color : #FFFFFF; } a:active { color : #003399; text-decoration : underline; } a.lev0:hover { color : #000000; background-color : #FFFFFF; } a.lev1:hover { color : #FFFFFF; background : #444444; } a.lev2:hover { background : #888888; } a.lev3:hover { background : #BBBBBB; } ┃"); #+END_SRC **** levels #+NAME: harvested_html_themes #+BEGIN_SRC d string theme_dark_1 = format(q"┃ h1 { color : #FFFFFF; background : #000000; } p.letter { color : #FFFFFF; background : #333333; } p.lev0 { color : #FFFFFF; background : #000000; } p.lev1 { color : #FFFFFF; background : #333333; } p.lev2 { background : #555555; } p.lev3 { background : #777777; } p.lev4 { background : #AAAAAA; } p.lev5 { } ┃"); string theme_light_1 = format(q"┃ h1 { color : #FFFFFF; background : #000088; } p.letter { color : #FFFFFF; background : #880000; } p.lev0 { color : #FFFFFF; background : #000000; } p.lev1 { color : #FFFFFF; background : #444444; } p.lev2 { background : #888888; } p.lev3 { background : #BBBBBB; } p.lev4 { background : #EEEEEE; } p.lev5 { } ┃"); #+END_SRC *** harvested html head #+NAME: harvested_html_head #+BEGIN_SRC d #+END_SRC *** harvested html bottom #+NAME: harvested_html_bottom #+BEGIN_SRC d ~= format(q"┃
┃") ~ "\n"; #+END_SRC