0.7.1 re-arranging for unittests; tuple tests; read in files; obt

4 files changed, 112 insertions, 99 deletions

diff --git a/org/ao_abstract_doc_source.org b/org/ao_abstract_doc_source.org
index 282223d..05825e6 100644
--- a/org/ao_abstract_doc_source.org
+++ b/org/ao_abstract_doc_source.org
@@ -2528,7 +2528,6 @@ struct ObjInlineMarkupMunge {
 #+name: ao_emitters
 #+BEGIN_SRC d
 struct ObjInlineMarkup {
-// struct ObjInlineMarkup : AssertObjInlineMarkup {
   auto munge = ObjInlineMarkupMunge();
   string[string] obj_txt;
 #+END_SRC
@@ -2548,7 +2547,7 @@ struct ObjInlineMarkup {
     switch (obj_["is"]) {
     case "heading":
       static __gshared string anchor_tag = "";
-      // WORK ON, you still need to ensure that level 1 anchor_tags are unique
+      // TODO WORK ON, you still need to ensure that level 1 anchor_tags are unique
       obj_txt["munge"]=_configured_auto_heading_numbering_and_segment_anchor_tags(obj_txt["munge"], obj_, dochead_make_aa);
       obj_txt["munge"]=_make_segment_anchor_tags_if_none_provided(obj_txt["munge"], obj_["lev"]);
       if (auto m = match(obj_txt["munge"], rgx.heading_anchor_tag)) {
@@ -3349,41 +3348,6 @@ struct BookIndexReportSection {
 ***** book index struct close
 #+name: ao_emitters
 #+BEGIN_SRC d
-  auto bookindex_build_section_(
-    string[][string][string] bookindex_unordered_hashes
-  ) {
-    auto mainkeys =
-      bookindex_unordered_hashes.byKey.array.sort().release;
-    string bi_tmp;
-    string[] bookindex_section;
-    // int bi_num;
-    // writeln(mainkeys.length);
-    foreach (mainkey; mainkeys) {
-      bi_tmp = "_0_1 !{" ~ mainkey ~ "}! ";
-      foreach (ref_; bookindex_unordered_hashes[mainkey]["_a"]) {
-        auto go = replaceAll(ref_, rgx.book_index_go, "$1");
-        bi_tmp ~= " {" ~ ref_ ~ "}#" ~ go ~ ", ";
-      }
-      bi_tmp ~= " \\\\\n    ";
-      bookindex_unordered_hashes[mainkey].remove("_a");
-      auto subkeys =
-        bookindex_unordered_hashes[mainkey].byKey.array.sort().release;
-      foreach (subkey; subkeys) {
-        bi_tmp ~= subkey ~ ", ";
-        // bi_tmp ~= "  " ~ subkey ~ ", ";
-        foreach (ref_; bookindex_unordered_hashes[mainkey][subkey]) {
-          auto go = replaceAll(ref_, rgx.book_index_go, "$1");
-          bi_tmp ~= " {" ~ ref_ ~ "}#" ~ go ~ ", ";
-        }
-        bi_tmp ~= " \\\\\n    ";
-        ++skn;
-      }
-      bi_tmp = replaceFirst(bi_tmp, rgx.trailing_linebreak, "");
-      bookindex_section ~= bi_tmp;
-      ++mkn;
-    }
-    return bookindex_section;
-  }
 }
 #+END_SRC
 
diff --git a/org/ao_conf_make_meta.org b/org/ao_conf_make_meta.org
index ae028f2..b6fcb92 100644
--- a/org/ao_conf_make_meta.org
+++ b/org/ao_conf_make_meta.org
@@ -404,7 +404,7 @@ body {
     }
   }
   auto t = tuple(dochead_meta, dochead_make);
-  static assert(!isTypeTuple!(t));
+  static assert(t.length==2);
   return t;
 }
 #+END_SRC
@@ -605,7 +605,7 @@ private auto headerSDLangToAAmake(Tag header_sdlang, string[string][string] doch
   dochead_make = sdlangToAAmake(dochead_make, header_sdlang);
   auto dochead_meta = sdlangToAAmake(meta_aa, header_sdlang);
   auto t = tuple(dochead_make, dochead_meta);
-  static assert(!isTypeTuple!(t));
+  static assert(t.length==2);
   return t;
 }
 #+END_SRC
@@ -651,6 +651,7 @@ template SiSUheaderExtractHub() {
       ? (head_native.headerNativeToAA(header_src))
       : (head_sdlang.headerSDLangToAA(header_src, conf_doc_make_aa));
       static assert(!isTypeTuple!(header_make_and_meta_tuple));
+      static assert(header_make_and_meta_tuple.length==2);
       return header_make_and_meta_tuple;
     }
   }
diff --git a/org/ao_read_source_files.org b/org/ao_read_source_files.org
index 11647fb..be7a3a5 100644
--- a/org/ao_read_source_files.org
+++ b/org/ao_read_source_files.org
@@ -121,7 +121,7 @@ final private auto configSDLang(string conf_sdl) {
 * get markup source, read file                                :source:markup:
 ** [#A] read file, source string                                     :string:
 
-#+name: ao_markup_source_raw
+#+name: ao_markup_source_raw_read_file_source_string
 #+BEGIN_SRC d
 final private string readInMarkupSource(in string fn_src) {
   enforce(
@@ -189,7 +189,7 @@ catch (ErrnoException ex) {
 here you split document header and body, an array.length == 2
 split is on first match of level A~ (which is required)
 
-#+name: ao_markup_source_raw
+#+name: ao_markup_source_raw_doc_header_and_content_split
 #+BEGIN_SRC d
 final private char[][] header0Content1(in string src_text) {
   /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
@@ -207,7 +207,7 @@ final private char[][] header0Content1(in string src_text) {
 
 ** source line array                                                  :array:
 
-#+name: ao_markup_source_raw
+#+name: ao_markup_source_raw_source_line_array
 #+BEGIN_SRC d
 final private char[][] markupSourceLineArray(in char[] src_text) {
   char[][] source_line_arr =
@@ -220,14 +220,26 @@ final private char[][] markupSourceLineArray(in char[] src_text) {
 - used for regular .sst files; master .ssm files and; .ssi inserts
 - regex is passed for relevant enforce match
 
-#+name: ao_markup_source_raw
+*** read in file
+
+#+name: ao_markup_source_raw_read_in_file
 #+BEGIN_SRC d
-auto markupSourceHeaderContentRawLineTupleArray(in string fn_src, Regex!(char) rgx_file ) {
+auto markupSourceReadIn(in string fn_src) {
+  auto rgx = Rgx();
   enforce(
-    match(fn_src, rgx_file),
+    match(fn_src, rgx.src_pth),
     "not a sisu markup filename"
   );
   auto source_txt_str = readInMarkupSource(fn_src);
+  return source_txt_str;
+}
+#+END_SRC
+
+*** tuple header and body content
+
+#+name: ao_markup_source_raw_tuple_of_header_and_body
+#+BEGIN_SRC d
+auto markupSourceHeaderContentRawLineTupleArray(in string source_txt_str) {
   auto hc = header0Content1(source_txt_str);
   auto header = hc[0];
   char[] la;
@@ -240,7 +252,13 @@ auto markupSourceHeaderContentRawLineTupleArray(in string fn_src, Regex!(char) r
   );
   return t;
 }
-final char[][] markupSourceContentRawLineArray(in string fn_src, Regex!(char) rgx_file ) {
+#+END_SRC
+
+*** get insert source line array
+
+#+name: ao_markup_source_raw_get_insert_source_line_array
+#+BEGIN_SRC d
+final char[][] getInsertMarkupSourceContentRawLineArray(in string fn_src, Regex!(char) rgx_file) {
   enforce(
     match(fn_src, rgx_file),
     "not a sisu markup filename"
@@ -316,7 +334,7 @@ if (type1["curly_code"] == 1) {
     to!string(markup_src_file_path ~ insert_sub_pth ~ insert_fn);
   auto raw = MarkupRawUnit();
   auto markup_sourcesubfile_insert_content =
-    raw.markupSourceContentRawLineArray(fn_src_insert, rgx.src_fn_find_inserts);
+    raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx.src_fn_find_inserts);
   debug(insert) {                              // insert file
     tell_l("red", line);
     tell_l("red", fn_src_insert);
@@ -395,7 +413,7 @@ if (type["curly_code"] == 1) {
   //   raw.markupSourceHeaderContentRawLineTupleArray(fn_src, rgx.src_pth);
   }
   auto markup_sourcefile_insert_content =
-    raw.markupSourceContentRawLineArray(fn_src_insert, rgx.src_fn_find_inserts);
+    raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx.src_fn_find_inserts);
   debug(insert) {                              // insert file
     tell_l("red", line);
     tell_l("red", fn_src_insert);
@@ -456,22 +474,28 @@ template SiSUmarkupRaw() {
   mixin RgxInit;
   auto rgx = Rgx();
   struct MarkupRaw {
-    auto sourceContent(in string fn_src) {
+    final sourceContent(in string fn_src) {
+      auto raw = MarkupRawUnit();
+      auto source_txt_str =
+        raw.markupSourceReadIn(fn_src);
+      return source_txt_str;
+    }
+    final auto sourceContentSplitIntoHeaderAndBody(in string source_txt_str, in string fn_src="") {
       auto raw = MarkupRawUnit();
       auto t =
-        raw.markupSourceHeaderContentRawLineTupleArray(fn_src, rgx.src_pth);
+        raw.markupSourceHeaderContentRawLineTupleArray(source_txt_str);
       auto header_raw = t[0];
-      auto sourcefile_content = t[1];
-      if (match(fn_src, rgx.src_fn_master)) {
+      auto sourcefile_body_content = t[1];
+      if (match(fn_src, rgx.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
         auto ins = Inserts();
-        sourcefile_content =
-          ins.scan_master_src_for_insert_files_and_import_content(sourcefile_content, fn_src);
-        // auto ins = SiSUdocInserts.Inserts();
+        sourcefile_body_content =
+          ins.scan_master_src_for_insert_files_and_import_content(sourcefile_body_content, fn_src);
       }
       t = tuple(
         header_raw,
-        sourcefile_content
+        sourcefile_body_content
       );
+      static assert(t.length==2);
       return t;
     }
   }
@@ -479,7 +503,12 @@ template SiSUmarkupRaw() {
   struct MarkupRawUnit {
     private import std.file;
     // enum State { off, on }
-    <<ao_markup_source_raw>>
+    <<ao_markup_source_raw_read_file_source_string>>
+    <<ao_markup_source_raw_doc_header_and_content_split>>
+    <<ao_markup_source_raw_source_line_array>>
+    <<ao_markup_source_raw_read_in_file>>
+    <<ao_markup_source_raw_tuple_of_header_and_body>>
+    <<ao_markup_source_raw_get_insert_source_line_array>>
   }
   struct Inserts {
     private import ao_defaults;    // ao_defaults.d
@@ -496,12 +525,12 @@ template SiSUmarkupRaw() {
       <<ao_inserts_scan_post>>
     }
     auto scan_master_src_for_insert_files_and_import_content(
-      char[][] sourcefile_content,
+      char[][] sourcefile_body_content,
       string fn_src
     ) {
       mixin SiSUrgxInitFlags;
       <<ao_master_doc_scan_for_insert_filenames>>
-      foreach (line; sourcefile_content) {
+      foreach (line; sourcefile_body_content) {
         <<ao_master_doc_scan_for_insert_filenames_loop>>
       } // end src doc loop
       <<ao_master_doc_scan_for_insert_filenames_post>>
diff --git a/org/sdp.org b/org/sdp.org
index 47cecf8..ebf427e 100644
--- a/org/sdp.org
+++ b/org/sdp.org
@@ -24,7 +24,7 @@ struct Version {
   int minor;
   int patch;
 }
-enum ver = Version(0, 7, 0);
+enum ver = Version(0, 7, 1);
 #+END_SRC
 
 * sdp.d   sisu document parser                                       :sdp.d:
@@ -131,13 +131,21 @@ std.path,
 std.stdio,
 std.variant,
 
-**** version.txt                                                    :version:
+**** mixins
+***** version.txt                                                   :version:
 
 #+NAME: sdp_version_mixin
 #+BEGIN_SRC d
 mixin(import("version.txt"));
 #+END_SRC
 
+***** std mixins
+#+NAME: std_mixins
+#+BEGIN_SRC d
+mixin CompileTimeInfo;
+mixin RgxInit;
+#+END_SRC
+
 *** mixin                                                             :mixin:
 
 #+NAME: sdp_mixin
@@ -289,7 +297,7 @@ auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make);
 *** filename provided [+1]                                     :file:process:
 **** scope (loop)                                                     :scope:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_scope
 #+BEGIN_SRC d
 scope(success) {
   debug(checkdoc) {
@@ -314,39 +322,47 @@ enforce(
 );
 #+END_SRC
 
-**** [#A] read file                                               :file:read:
+**** [#A] read in raw file (using filename with path)             :file:read:
 [[./ao_markup_source_raw.org][ao_markup_source_raw]]
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_read_sisu_markup_file
 #+BEGIN_SRC d
 /+ ↓ read file +/
-auto header_and_content_tuple = raw.sourceContent(fn_src);
-static assert(!isTypeTuple!(header_and_content_tuple));
-auto header = header_and_content_tuple[0];
-auto sourcefile_content = header_and_content_tuple[1];
-debug(header_and_content) {
+auto read_in_file_string = raw.sourceContent(fn_src);
+#+END_SRC
+
+**** [#A] split raw content file into: doc header & doc content   :file:read:
+
+#+NAME: sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body
+#+BEGIN_SRC d
+/+ ↓ file tuple of header and content +/
+auto header_and_body_tuple = raw.sourceContentSplitIntoHeaderAndBody(read_in_file_string, fn_src);
+auto header = header_and_body_tuple[0];
+auto content_body = header_and_body_tuple[1];
+debug(header_and_body) {
   writeln(header);
-  writeln(header_and_content_tuple.length);
-  writeln(sourcefile_content[0]);
+  writeln(header_and_body_tuple.length);
+  writeln(content_body[0]);
 }
 #+END_SRC
 
-**** [#A] read doc header: metadata & make         :doc:header:metadata:make:
+**** [#A] split doc header into: metadata & make   :doc:header:metadata:make:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta
 #+BEGIN_SRC d
+/+ ↓ split header into make and meta +/
 auto header_make_and_meta_tuple = head.headerContentAA(header, conf_doc_make_aa);
 static assert(!isTypeTuple!(header_make_and_meta_tuple));
-string[string][string] dochead_make_aa = header_make_and_meta_tuple[0];
-string[string][string] dochead_meta_aa = header_make_and_meta_tuple[1];
+string[string][string] dochead_make = header_make_and_meta_tuple[0];
+string[string][string] dochead_meta = header_make_and_meta_tuple[1];
 #+END_SRC
 
 **** [#A] processing: document abstraction, tuple                :processing:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_document_abstraction
 #+BEGIN_SRC d
-/+ ↓ process document, return abstraction as tuple +/
-auto t = abs.abstract_doc_source(sourcefile_content, dochead_make_aa, dochead_meta_aa);
+/+ ↓ document abstraction: process document, return abstraction as tuple +/
+auto t = abs.abstract_doc_source(content_body, dochead_make, dochead_meta);
 static assert(!isTypeTuple!(t));
 auto doc_ao_contents = t[0]; // contents ~ endnotes ~ bookindex;
 // static assert(!isIterable!(doc_ao_contents));
@@ -355,19 +371,18 @@ auto doc_ao_biblio = t[2];
 // destroy(t);
 #+END_SRC
 
-**** TODO debug                                                       :debug:
-***** [#A] debug document parts (checkdoc)                         :checkdoc:
+**** TODO debug (document parts, checkdoc)                   :debug:checkdoc:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_debugs_checkdoc
 #+BEGIN_SRC d
-/+ ↓ document parts +/
+/+ ↓ debugs +/
 debug(checkdoc) { // checkbook & dumpdoc
   dbg.abstract_doc_source_debugs(
     doc_ao_contents,
     doc_ao_bookindex_unordered_hashes,
     doc_ao_biblio,
-    dochead_make_aa,
-    dochead_meta_aa,
+    dochead_make,
+    dochead_meta,
     fn_src,
     opt_action_bool
   );
@@ -376,15 +391,15 @@ debug(checkdoc) { // checkbook & dumpdoc
 
 **** TODO process outputs                                           :outputs:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_selected_output
 #+BEGIN_SRC d
 /+ ↓ output hub +/
 output.hub(
   doc_ao_contents,
   doc_ao_bookindex_unordered_hashes,
   doc_ao_biblio,
-  dochead_make_aa,
-  dochead_meta_aa,
+  dochead_make,
+  dochead_meta,
   fn_src,
   opt_action_bool
 );
@@ -392,7 +407,7 @@ output.hub(
 
 **** scope (on loop exit)                                        :scope:exit:
 
-#+NAME: sdp_each_file_do
+#+NAME: sdp_each_file_do_scope_exit
 #+BEGIN_SRC d
 scope(exit) {
   debug(checkdoc) {
@@ -401,7 +416,7 @@ scope(exit) {
       fn_src
     );
   }
-  destroy(sourcefile_content);
+  destroy(content_body);
   destroy(t);
   destroy(doc_ao_contents);
   // destroy(doc_ao_make_aa);
@@ -422,21 +437,18 @@ break;
 #+END_SRC
 
 * tangles (code structure)                                           :tangle:
-** sdp                                                                :sdp.d:
-*** TODO src/sdp.d
+** TODO sdp src/sdp.d                                               :sdp.d:
 
 #+BEGIN_SRC d  :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd
 /+
   sdp
-  sdp.d
 +/
 <<imports_sdp>>
 <<imports_sdlang>>
 <<imports_std>>
 <<sdp_output_selection>>
 <<sdp_version_mixin>>
-mixin CompileTimeInfo;
-mixin RgxInit;
+<<std_mixins>>
 /++ A SiSU document parser writen in D. +/
 void main(string[] args) {
   <<sdp_mixin>>
@@ -444,7 +456,14 @@ void main(string[] args) {
   <<sdp_conf_files>>
   foreach(fn_src; fns_src) {
     if (!empty(fn_src)) {
-      <<sdp_each_file_do>>
+      <<sdp_each_file_do_scope>>
+      <<sdp_each_file_do_read_sisu_markup_file>>
+      <<sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body>>
+      <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>>
+      <<sdp_each_file_do_document_abstraction>>
+      <<sdp_each_file_do_debugs_checkdoc>>
+      <<sdp_each_file_do_selected_output>>
+      <<sdp_each_file_do_scope_exit>>
     } else {
       <<sdp_no_filename_provided>>
     }
@@ -640,12 +659,12 @@ democratizing_innovation.eric_von_hippel.sst
 #+END_SRC
 
 ** read markup files
-**** regular .sst
+*** regular .sst
 relatively straight forward
-**** master .ssm
+*** master .ssm
 master files have been able to read in inser files .ssi and regular files .sst
-***** reading in .ssi files is straightforward
-***** reading in .sst files is more problematic
+**** reading in .ssi files is straightforward
+**** reading in .sst files is more problematic
 .sst files have their own root (structure)
 either
 - the root needs to be disabled - not used