0.8.2 yaml pod.manifest, read site config first

- yaml (configuration & metadata)
  site config,
  pod:
    pod.manifest,
    doc header:
      make,
      meta
- pod.manifest in yaml breaks
  - older program reading new pod and
  - newer program reading old pod
- read config first,
  - get site default language in case it is absent
    in pod.manifest
  -  ensure config is not read too many times
     (it is at present, revisit)

5 files changed, 202 insertions, 45 deletions

diff --git a/src/doc_reform/doc_reform.d b/src/doc_reform/doc_reform.d
index 2e388d1..40f1a05 100755
--- a/src/doc_reform/doc_reform.d
+++ b/src/doc_reform/doc_reform.d
@@ -580,7 +580,16 @@ void main(string[] args) {
   ];
   auto _manifest_matter = PathMatters!()(_opt_action, _env, "");
   auto _manifests = [ _manifest_matter ];
+  ConfCompositePlus _make_and_meta_struct_tst;
   foreach(arg; args[1..$]) {
+    _manifest_matter = PathMatters!()(_opt_action, _env, arg); // gather input markup source file names for processing
+    auto _config_local_site_struct = readConfigSite!()(_manifest_matter, _env);
+    { /+ local site config +/
+      if (_config_local_site_struct.filetype == "yaml") {
+        import doc_reform.meta.conf_make_meta_yaml;
+        _make_and_meta_struct_tst = _config_local_site_struct.configParseYAMLreturnDocReformStruct!()(_make_and_meta_struct_tst, _manifest_matter);
+      }
+    }
     auto _manifest_start = PodManifest!()(arg);
     if (arg.match(rgx.flag_action)) {
       flag_action ~= " " ~ arg;   // flags not taken by getopt
@@ -600,21 +609,104 @@ void main(string[] args) {
       if (exists(sisudoc_txt_)) {
         try {
           if (exists(sisudoc_txt_)) {
-            pod_manifest_root_content_paths_to_markup_location_raw_ = sisudoc_txt_.readText;
+            import dyaml;
+            try {
+              Node pod_manifest_yaml = Loader.fromFile(sisudoc_txt_).load();
+              if ("doc" in pod_manifest_yaml) {
+                if (pod_manifest_yaml["doc"].type.mapping
+                  && pod_manifest_yaml["doc"].tag.match(rgx.yaml_tag_is_map)
+                ) {
+                  if ("path" in pod_manifest_yaml["doc"]) {
+                    if (pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_seq)) {
+                      foreach (string _path; pod_manifest_yaml["doc"]["path"]) {
+                        markup_contents_location_ ~= _path ~ "\n";
+                        pod_manifest_root_content_paths_to_markup_location_raw_ ~=
+                          _path ~ "\n";
+                      }
+                    } else if (
+                      pod_manifest_yaml["doc"]["path"].type.string
+                      && pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_str)
+                    ) {
+                      markup_contents_location_ = pod_manifest_yaml["doc"]["path"].get!string;
+                      pod_manifest_root_content_paths_to_markup_location_raw_ =
+                        pod_manifest_yaml["doc"]["path"].get!string;
+                    }
+                  }
+                  if ("filename" in pod_manifest_yaml["doc"]) {
+                    if (pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_seq)) {
+                      foreach (string _filename; pod_manifest_yaml["doc"]["filename"]) {
+                        if ("language" in pod_manifest_yaml["doc"]) {
+                          if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) {
+                            foreach (string _lang; pod_manifest_yaml["doc"]["language"]) {
+                              markup_contents_location_ ~=
+                                "media/text/"
+                                ~ _lang ~ "/"
+                                ~ _filename ~ "\n";
+                            }
+                          } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)
+                          ) {
+                            markup_contents_location_ =
+                              "media/text/"
+                              ~ pod_manifest_yaml["doc"]["language"].get!string
+                              ~ "/" ~ _filename ~ "\n";
+                          } else {
+                            string _lang_default = "en";
+                            markup_contents_location_ ~=
+                              "media/text/"
+                              ~ _lang_default ~ "/"
+                              ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                          }
+                        } else {
+                          string _lang_default = "en";
+                          markup_contents_location_ ~=
+                            "media/text/"
+                            ~ _lang_default ~ "/"
+                            ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                        }
+                      }
+                    } else if (
+                      pod_manifest_yaml["doc"]["filename"].type.string
+                      && pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_str)
+                    ) {
+                      if ("language" in pod_manifest_yaml["doc"]) {
+                        if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) {
+                          foreach (string _lang; pod_manifest_yaml["doc"]["language"]) {
+                            markup_contents_location_ ~=
+                              "media/text/"
+                              ~ _lang ~ "/"
+                              ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                          }
+                        } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)) {
+                          markup_contents_location_ =
+                            "media/text/"
+                            ~ pod_manifest_yaml["doc"]["language"].get!string
+                            ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                        } else {
+                          string _lang_default = "en";
+                          markup_contents_location_ ~=
+                            "media/text/"
+                            ~ _lang_default ~ "/"
+                            ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                        }
+                      } else {
+                        string _lang_default = "en";
+                        markup_contents_location_ ~=
+                          "media/text/"
+                          ~ _lang_default ~ "/"
+                          ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                      }
+                    }
+                  }
+                }
+              }
+  
+            } catch (ErrnoException ex) {
+            }
           }
         } catch (ErrnoException ex) {
         } catch (FileException ex) {
           // Handle errors
         }
-        if (pod_manifest_root_content_paths_to_markup_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+
-          foreach (m; pod_manifest_root_content_paths_to_markup_location_raw_.matchAll(rgx.pod_content_location)) {
-            foreach (n; m.captures[2].matchAll(rgx.language_codes)) {
-              markup_contents_location_ ~= "media/text/" ~ n.captures[1].to!string ~ "/" ~ m.captures[1].to!string ~ "\n";
-            }
-          }
-        } else { // (file name with path \n)+
-          markup_contents_location_ = pod_manifest_root_content_paths_to_markup_location_raw_;
-        }
       } else {
         writeln("manifest not found: ", sisudoc_txt_);
       }
diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d
index 8f16528..a4a402b 100644
--- a/src/doc_reform/output/hub.d
+++ b/src/doc_reform/output/hub.d
@@ -18,11 +18,23 @@ template outputHub() {
     enum outTask { pod, source, sqlite, sqlite_multi, latex, odt, epub, html_scroll, html_seg, html_stuff }
     void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) {
       auto msg = Msg!()(doc_matters);
-      if (sched == outTask.pod) {
-        msg.v("doc reform source processing... ");
+      if (sched == outTask.source
+        || sched == outTask.pod
+      ) {
+        if (sched == outTask.source) {
+          msg.v("doc reform source processing... ");
+        }
+        if (sched == outTask.pod) {
+          msg.v("doc reform source pod processing... ");
+        }
         import doc_reform.output.source_pod;
         DocReformPod!()(doc_matters);
-        msg.vv("doc reform source done");
+        if (sched == outTask.source) {
+          msg.vv("doc reform source done");
+        }
+        if (sched == outTask.pod) {
+          msg.vv("doc reform source pod done");
+        }
       }
       if (sched == outTask.epub) {
         msg.v("epub3 processing... ");
diff --git a/src/doc_reform/output/source_pod.d b/src/doc_reform/output/source_pod.d
index 73c8d47..bf07715 100644
--- a/src/doc_reform/output/source_pod.d
+++ b/src/doc_reform/output/source_pod.d
@@ -91,7 +91,8 @@ template DocReformPod() {
               zip.addMember(zip_arc_member_file);
             }
           } else {
-            if (doc_matters.opt.action.verbose) {
+            if (doc_matters.opt.action.debug_do
+            && doc_matters.opt.action.verbose) {
               writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
             }
           }
@@ -122,19 +123,19 @@ template DocReformPod() {
             zip.addMember(zip_arc_member_file);
           }
         } else {
-          if (doc_matters.opt.action.verbose
-          || doc_matters.opt.action.debug_do) {
+          if (doc_matters.opt.action.debug_do
+          && doc_matters.opt.action.verbose) {
             writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
           }
         }
-      } { /+ bundle primary file +/
+      } { /+ pod manifest +/
         auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
         auto fn_src_out_pod_zip_base
-          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
+          = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string;
         auto fn_src_out_filesystem
-          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // without root path:
+          = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path
         auto fn_src_out_inside_pod
-          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // without root path:
+          = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path
         string[] filelist_src_out_pod_arr;
         string[] filelist_src_zpod_arr;
         if (exists(fn_src_in)) {
@@ -143,14 +144,51 @@ template DocReformPod() {
           }
           filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
           filelist_src_zpod_arr ~= fn_src_out_inside_pod;
-          if (doc_matters.opt.action.source) {
-            auto filelist
+          {
+            import dyaml;
+            auto pod_filelist_yaml_string
               = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
-            foreach (source_pth_and_fn; filelist_src_zpod_arr) {
-              filelist.writeln(source_pth_and_fn);
+            Node _pmy;
+            string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
+            if (doc_matters.opt.action.debug_do
+            && doc_matters.opt.action.verbose) {
+              _pmy = Loader.fromString(_pm).load();
+              writeln("pod filename:  ", _pmy["doc"]["filename"].get!string);
+              writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string);
+              writeln("pod languages: ", doc_matters.src.language);
+              // foreach(string _l; _pmy["doc"]["language"]) {
+              //   writeln("language:      ", _l);
+              // }
             }
-            fn_src_in.copy(fn_src_out_filesystem);
+            if (doc_matters.opt.action.source) {
+              pod_filelist_yaml_string.writeln(_pm);
+            }
+            if (doc_matters.opt.action.pod) {
+              auto zip_arc_member_file = new ArchiveMember();
+              zip_arc_member_file.name = fn_src_out_pod_zip_base;
+              auto zip_data = new OutBuffer();
+              zip_data.write(_pm);
+              zip_arc_member_file.expandedData = zip_data.toBytes();
+              zip.addMember(zip_arc_member_file);
+            }
+          }
+        }
+      } { /+ bundle primary file (.ssm/.sst) +/
+        auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
+        auto fn_src_out_pod_zip_base
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
+        auto fn_src_out_filesystem
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path:
+        auto fn_src_out_inside_pod
+          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
+        string[] filelist_src_out_pod_arr;
+        string[] filelist_src_zpod_arr;
+        if (exists(fn_src_in)) {
+          debug(io) {
+            writeln("(io debug) src in found: ", fn_src_in);
           }
+          filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
+          filelist_src_zpod_arr ~= fn_src_out_inside_pod;
           if (doc_matters.opt.action.pod) {
             auto zip_arc_member_file = new ArchiveMember();
             zip_arc_member_file.name = fn_src_out_pod_zip_base;
@@ -160,12 +198,12 @@ template DocReformPod() {
             zip.addMember(zip_arc_member_file);
           }
         } else {
-          if (doc_matters.opt.action.verbose
-          || doc_matters.opt.action.debug_do) {
+          if (doc_matters.opt.action.debug_do
+          && doc_matters.opt.action.verbose) {
             writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
           }
         }
-      } { /+ bundle insert files +/
+      } { /+ bundle insert files (.ssi) +/
         if (doc_matters.srcs.file_insert_list.length > 0) {
           foreach (insert_file; doc_matters.srcs.file_insert_list) {
             debug(pod) {
@@ -208,8 +246,8 @@ template DocReformPod() {
                 createZipFile!()(fn_pod, zip.build());
               }
             } else {
-              if (doc_matters.opt.action.verbose
-              || doc_matters.opt.action.debug_do) {
+              if (doc_matters.opt.action.debug_do
+              && doc_matters.opt.action.verbose) {
                 writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
               }
             }
@@ -230,7 +268,8 @@ template DocReformPod() {
       }
       if (exists(fn_pod)) {
         try {
-          if (!(doc_matters.opt.action.quiet)) {
+          if (!(doc_matters.opt.action.quiet)
+          && doc_matters.opt.action.pod) {
             auto data = (cast(byte[]) (fn_pod).read);
             if (doc_matters.opt.action.verbose) {
               writeln(" ", doc_matters.src.filename, " > ");
@@ -244,8 +283,7 @@ template DocReformPod() {
                 auto data = zipped.expand(member);
                 writeln("> ", filename, " length ", data.length);
               }
-            }
-            catch (ZipException ex) {
+            } catch (ZipException ex) {
               // Handle errors
             }
             if (doc_matters.src.filename == "dr_doc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") {
@@ -269,14 +307,12 @@ template DocReformPod() {
               );
             }
           }
-        }
-        catch (ErrnoException ex) {
+        } catch (ErrnoException ex) {
           // Handle errors
         }
       }
       
-    }
-    catch (ErrnoException ex) {
+    } catch (ErrnoException ex) {
       // Handle error
     }
   }
diff --git a/src/doc_reform/source/paths_source.d b/src/doc_reform/source/paths_source.d
index b1622bd..86338af 100644
--- a/src/doc_reform/source/paths_source.d
+++ b/src/doc_reform/source/paths_source.d
@@ -17,7 +17,7 @@ template PodManifest() {
   mixin DocReformRgxInit;
   static auto rgx = Rgx();
   auto PodManifest(P)(
-    P   _pth
+    P   _pth=""
   ) {
     struct ManifestFile_ {
       string pod_manifest_filename() {
@@ -370,9 +370,8 @@ template PathMatters() {
               if (!exists(_output_path)) {
                 try {
                   _output_path.mkdirRecurse;
-                }
-                // catch (ErrnoException ex) {
-                catch (Exception ex) {
+                // } catch (ErrnoException ex) {
+                } catch (Exception ex) {
                   // Handle error
                 }
               }
@@ -581,6 +580,10 @@ template DocReformPathsPods() {
           pth ~ " == "  ~ _base_dir_pod ~ "/" ~ base_filename_(fn_src) ~ "?");
         return pth;
       }
+      string output_pod_manifest_file(string fn_src) {
+        string pth = base_filesystem_(fn_src).chainPath("pod.manifest").array;
+        return pth;
+      }
       string base_pod_(string fn_src) {
         string pth = _base_dir_pod.chainPath(base_filename_(fn_src)).array; // change this
         return pth;
@@ -625,8 +628,8 @@ template DocReformPathsPods() {
         return _pods();
       }
       auto pod_root(string fn_src) {
-        auto pth_1_ = "";
-        auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array;
+        auto pth_1_ = "pod";
+        auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array; // "dr_doc"
         struct _pods {
           auto zpod() {
             return pth_1_;
@@ -667,9 +670,24 @@ template DocReformPathsPods() {
         }
         return _pods();
       }
+      auto pod_manifest(string fn_src) {
+        auto pod_root_ = pod_root(fn_src);
+        auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("pod.manifest")).asNormalizedPath).array;
+        auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("pod.manifest")).asNormalizedPath).array;
+        struct _pods {
+          auto zpod() {
+            return pth_1_;
+          }
+          auto filesystem_open_zpod() {
+            assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
+            return pth_2_;
+          }
+        }
+        return _pods();
+      }
       auto media_root(string fn_src) {
         auto pod_root_ = pod_root(fn_src);
-        auto pth_1_ = "media";
+        auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("media")).asNormalizedPath).array;
         auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("media")).asNormalizedPath).array;
         struct _pods {
           auto zpod() {
diff --git a/src/doc_reform/source/read_config_files.d b/src/doc_reform/source/read_config_files.d
index 787f6e0..6c652fb 100644
--- a/src/doc_reform/source/read_config_files.d
+++ b/src/doc_reform/source/read_config_files.d
@@ -136,7 +136,6 @@ static template configReadDocYAML() {
     doc_reform.source.paths_source,
     std.file,
     std.path;
-  import yaml;
   final YAMLDocument configReadDocYAML(M,E)(M _manifest, E _env) {
     string _configuration = configReadInDocYAML!()(_manifest, _env);
     auto _conf_file_details = ConfigFilePaths!()(_manifest, _env);