-*- mode: org -*-
#+TITLE:       sisudoc spine (doc_reform) markup source raw
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+FILETAGS:    :spine:sourcefile:read:
#+AUTHOR:      Ralph Amissah
#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT:   Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE:    en
#+STARTUP:     content hideblocks hidestars noindent entitiespretty
#+PROPERTY:    header-args+ :eval never-export :exports code
#+PROPERTY:    header-args+ :noweb yes :padline no
#+PROPERTY:    header-args+ :results silent :cache no
#+PROPERTY:    header-args+ :mkdirp yes
#+OPTIONS:     H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
- magic single double-quote → " ← FIX changes hilighting behavior (occuring
  after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX

- [[./doc-reform.org][doc-reform.org]]  [[./][org/]]

* A. get _config file_, read in (.readText) [#A]
** _module template_ :module:config_files:

#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/++
  read configuration files<BR>
  - read config files<BR>
  meta_config_files.d
+/
module sisudoc.ocda.io_in.read_config_files;
@safe:
import std.file;
import std.path;
import sisudoc.ocda.meta;
import sisudoc.ocda.io_in.paths_source;
import sisudoc.ocda.meta.rgx_files;
import sisudoc.ocda.meta.rgx;
<<meta_config_file_hub_read_site_config>>
<<meta_config_file_hub_read_document_config>>
<<meta_config_file_hub_read_site_yaml>>
#+END_SRC

*** read config files (config local site & dr document make) (yaml)
**** site configuration

SEE NOTES on configuration hierarchy in spine.org

#+NAME: meta_config_file_hub_read_site_config
#+BEGIN_SRC d
template readConfigSite() {
  @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) {
    mixin spineRgxIn;
    static auto rgx = RgxI();
    string conf_filename = "NONE";
    string config_file_str;
    string default_config_file_str = format(q"┃
flag:
  act0:                  "--html"
  act1:                  "--html --epub"
output:
  path:                  "%s"
default:
  language:              "en"
  papersize:             "a4"
  text_wrap:             "80"
  digest:                "sha256"
webserv:
  http:                  "%s"
  host:                  "%s"
  data_http:             "%s"
  data_host:             "%s"
  data_root_url:         "%s"
  data_root_path:        "%s"
  data_root_part:        ""
  images_root_part:      "image"
  cgi_search_form_title: "%s"
  cgi_http:              "%s"
  cgi_host:              "%s"
  cgi_bin_url:           "%s"
  cgi_bin_subpath:       "%s"
  cgi_bin_path:          "%s"
  cgi_search_script:     "%s"
  cgi_port:              ""
  cgi_user:              ""
  cgi_action:            "%s"
  db_sqlite_path:        "%s"
  db_sqlite_filename:    "%s"
  db_pg_table:           ""
  db_pg_user:            ""
┃",
  _cfg.processing_path_doc_root, // doc root
  _cfg.http_request_type,        // http
  _cfg.http_host,                // host / domain
  _cfg.http_request_type,        // data "http" or "https"
  _cfg.http_host,                // data domain "localhost"
  _cfg.www_url_doc_root,         // data root url "http://locahost" "https://sisudoc.org"
  _cfg.processing_path_doc_root, // data root path
  _cfg.cgi_search_form_title,    // cgi title // e.g. "≅ SiSU Spine search"
  _cfg.http_request_type,        // cgi http
  _cfg.http_host,                // cgi host
  _cfg.cgi_url_root,             // cgi bin url
  _cfg.cgi_bin_subpath,          // cgi bin path
  _cfg.cgi_bin_root,             // cgi bin path
  _cfg.cgi_filename,             // cgi filename
  _cfg.cgi_url_action,           // cgi action
  _cfg.db_sqlite_path,           // sqlite db path
  _cfg.db_sqlite_filename,       // sqlite db filename
);
    foreach(conf_fn; [_conf_file_details.config_filename_site]) {
      foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) {
        char[] conf_file;
        conf_filename = conf_fn;
        if (exists(pth)) {
          auto f_attrib = pth.getLinkAttributes;
          if (
            _conf_file_details.possible_config_path_locations.config_local_site.length == 1
            && f_attrib.attrIsFile
          ) {
            conf_file = pth.to!(char[]);
            conf_filename = pth.baseName;
          } else if (f_attrib.attrIsDir) {
            conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
            conf_filename = conf_fn;
          }
          try {
            if (exists(conf_file)) {
              if (conf_file.getLinkAttributes.attrIsFile) {
                if (_opt_action.vox_gt_2 || _opt_action.debug_do) {
                  writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)");
                }
                config_file_str = conf_file.readText;
                break;
              }
            }
          } catch (ErrnoException ex) {
          } catch (FileException ex) {
          }
        }
      }
      if (config_file_str.length > 0) { break; }
    }
    if (config_file_str.length > 0) {
      import dyaml;
      Node yaml_root;
      try {
        yaml_root = Loader.fromString(config_file_str).load();
      } catch (Throwable) {
        import std.stdio;
        writeln("ERROR failed to read config file content, not parsed as yaml, program default used");
        conf_filename = "VIRTUAL";
        config_file_str = default_config_file_str;
      }
    }
    if (config_file_str.length == 0) { /+ use dummy default config file +/
      // writeln("WARNING config file NOT found, default provided");
      conf_filename = "VIRTUAL";
      config_file_str = default_config_file_str;
    }
    struct _ConfContent {
      string filename() {
        return conf_filename;
      }
      string filetype() {
        string _ft = "";
        if (content.match(rgx.yaml_config)) {
          _ft = "yaml";
        }
        return _ft;
      }
      string content() {
        return config_file_str;
      }
    }
    return _ConfContent();
  }
}
#+END_SRC

**** document make/config

#+NAME: meta_config_file_hub_read_document_config
#+HEADER: :noweb yes
#+BEGIN_SRC d
static template readConfigDoc() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  @system final auto readConfigDoc(M,E)(M _manifested, E _env) {
    mixin spineRgxIn;
    static auto rgx = RgxI();
    mixin spineRgxFiles;
    static auto rgx_files = RgxFiles();
    string config_file_str;
    string conf_filename = "NONE";
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make;
    foreach(conf_fn; [_conf_file_details.config_filename_document]) {
      foreach(pth; possible_config_path_locations) {
        char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
        conf_filename = conf_fn;
        if (config_file_str.length > 0) {
          break;
        }
        try {
          if (exists(conf_file)) {
            if (conf_file.getLinkAttributes.attrIsFile) {
              config_file_str = conf_file.readText;
              break;
            }
          }
        } catch (ErrnoException ex) {
        } catch (FileException ex) {
        }
      }
      if (config_file_str.length > 0) { break; }
    }
    struct _ConfContent {
      string filename() {
        return conf_filename;
      }
      string content() {
        return config_file_str;
      }
      string filetype() {
        string _ft = "";
        if (content.match(rgx.yaml_config)) {
          _ft = "yaml";
        }
        return _ft;
      }
    }
    return _ConfContent();
  }
}
#+END_SRC

*** YAML config (config local site & dr document make) :file:config:hub:

#+NAME: meta_config_file_hub_read_site_yaml
#+HEADER: :noweb yes
#+BEGIN_SRC d
static template configReadSiteYAML() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) {
    string _configuration = configReadInSiteYAML!()(_manifested, _env);
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string _conf_yaml_fn = _conf_file_details.config_filename_site;
    YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
    return _yaml_conf;
  }
}
static template configReadDocYAML() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) {
    string _configuration = configReadInDocYAML!()(_manifested, _env);
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string _conf_yaml_fn = _conf_file_details.config_filename_document;
    YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
    return _yaml_conf;
  }
}
#+END_SRC

* B. get _markup source_, read file (.readText) [#A]:module:source_files:
** _module template_

#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/++
  module source_read_source_files;<BR>
  - open markup files<BR>
  - if master file scan for addional files to import/insert
+/
module sisudoc.ocda.io_in.read_source_files;
@safe:
template spineRawMarkupContent() {
  import std.digest.sha;
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  mixin spineRgxIn;
  static auto rgx = RgxI();
  mixin spineRgxFiles;
  static auto rgx_files = RgxFiles();
  string[] _images=[];
  /+ ↓ the images a markup source shows.
       matched on the image markup itself ({ name.png }image, and the url and
       ~^ forms), ensure a filename-shaped word in prose is not mistaken for
       one, and over every match (so a line showing two images reports two).
  +/
  string[] _extract_images(S)(S content_block) {
    string[] images_;
    string _content_block = content_block.to!string;
    foreach (m; _content_block.matchAll(rgx.smid_image)) {
      images_ ~= m["image"].to!string;
    }
    return images_;
  }
  auto rawsrc = RawMarkupContent();
  struct ST_contents_inserts_images {
    char[][]   contents;
    string[]   insert_files;
    string[]   images;
  }
  struct ST_header_content_inserts_images {
    char[]     header;
    char[][]   src_txt;
    string[]   insert_files;
    string[]   images;
  }
  struct ST_doc_digest {
    ubyte[32]  markup_doc;
    ubyte[32]  header;
    ubyte[32]  text;
  }
  struct ST_doc_parts {
    char[]        header_raw;
    char[][]      sourcefile_body_content;
    string[]      insert_file_list;
    string[]      images_list;
    ST_doc_digest doc_digest;
  }
  ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) {
    ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct
      = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src);
    return _0_header_1_body_content_2_insert_filelist_struct;
  }
  struct RawMarkupContent {
    final sourceContent(in string fn_src) {
      auto raw = MarkupRawUnit();
      string source_txt_str
        = raw.markupSourceReadIn(fn_src);
      return source_txt_str;
    }
    final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)(
      O         _opt_action,
      in string source_txt_str,
      in string fn_src=""
    ) {
      auto raw = MarkupRawUnit();
      string[] insert_file_list_get;
      string[] images_list_get;
      ST_header_content_inserts_images st
        = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str);
      char[] header_raw = st.header;
      char[][] sourcefile_body_content = st.src_txt;
      if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
        auto ins = Inserts();
        ST_contents_inserts_images _cii
          = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
        sourcefile_body_content = _cii.contents;
        insert_file_list_get = _cii.insert_files.dup;
        images_list_get = _cii.images.dup;
      } else if (_opt_action.source_or_pod) {
        auto ins = Inserts();
        ST_contents_inserts_images _cii
          = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
        images_list_get = _cii.images.dup;
      } // image_list, if path could take sha256 digests already here?
      string header_type = "";
      ST_doc_digest dig;
      {
        dig.markup_doc = source_txt_str.sha256Of;
        dig.header = st.header.sha256Of;
        dig.text = sourcefile_body_content.sha256Of;
      }
      ST_doc_parts ret;
      {
        ret.header_raw = st.header;
        ret.sourcefile_body_content = sourcefile_body_content;
        ret.insert_file_list = insert_file_list_get;
        ret.images_list = images_list_get;
        ret.doc_digest = dig;
      }
      return ret;
    }
  }
  struct MarkupRawUnit {
    import std.digest.sha;
    import std.file;
    <<meta_markup_source_raw_read_file_source_string>>
    <<meta_markup_source_raw_doc_header_and_content_split>>
    <<meta_markup_source_raw_source_line_array>>
    <<meta_markup_source_raw_read_in_file>>
    <<meta_markup_source_raw_struct_of_header_and_body>>
    <<meta_markup_source_raw_get_insert_source_line_array>>
  }
  struct Inserts {
    struct ST_contents_and_images {
      char[][]   insert_contents;
      string[]   images;
    }
    ST_contents_and_images scan_subdoc_source(O)(
      O        _opt_action,
      char[][] markup_sourcefile_insert_content,
      string   fn_src
    ) {
      <<meta_inserts_scan>>
      foreach (line; markup_sourcefile_insert_content) {
        <<meta_inserts_scan_loop>>
      } // end src subdoc (inserts) loop
      <<meta_inserts_scan_post>>
    }
    ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)(
      O        _opt_action,
      char[][] sourcefile_body_content,
      string   fn_src
    ) {
      import std.algorithm;
      <<meta_master_doc_scan_for_insert_filenames>>
      foreach (line; sourcefile_body_content) {
        <<meta_master_doc_scan_for_insert_filenames_loop>>
      } // end src doc loop
      <<meta_master_doc_scan_for_insert_filenames_post>>
    }
  }
}
#+END_SRC

** get markup source, read file :source:markup:
*** read file, source string [#A] :string:

#+NAME: meta_markup_source_raw_read_file_source_string
#+BEGIN_SRC d
final private string readInMarkupSource(in char[] fn_src) {
  enforce(
    exists(fn_src) != 0,
    "file not found: «" ~
    fn_src ~ "»"
  );
  string source_txt_str;
  try {
    if (exists(fn_src)) {
      if (fn_src.getLinkAttributes.attrIsFile) {
        source_txt_str = fn_src.readText;
      } else {
      }
    }
  } catch (ErrnoException ex) {
  } catch (UTFException ex) {
    // Handle validation errors
  } catch (FileException ex) {
    // Handle errors
  }
  std.utf.validate(source_txt_str);
  return source_txt_str;
}
#+END_SRC

*** document header & content, array.length == 2 [#A] :array:

here you split document header and body, an array.length == 2
split is on first match of level A~ (which is required)

#+NAME: meta_markup_source_raw_doc_header_and_content_split
#+BEGIN_SRC d
@trusted final private char[][] header0Content1(in string src_text) { // cast(char[])
  /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
  char[][] header_and_content;
  /+ ↓ a utf-8 byte order mark at the start of the file is kept out of
       the split, and so out of the yaml header. Left in, it becomes
       part of the first key, so "title:" arrives as "title:" and
       every value under that key is silently lost: the document keeps
       its text and loses its title. It is not stripped at read time,
       because source_txt_str is what source.digest is taken over and
       that digest names the file as it sits on disk.
  +/
  string _src = src_text;
  enum string _bom = "";
  if (_src.length >= _bom.length && _src[0.._bom.length] == _bom) {
    _src = _src[_bom.length..$];
  }
  auto m = (cast(char[]) _src).matchFirst(rgx.heading_a);
  header_and_content ~= m.pre;
  header_and_content ~= m.hit ~ m.post;
  assert(header_and_content.length == 2,
    "document markup is broken, header body split == "
    ~ header_and_content.length.to!string
    ~ "; (header / body array split should == 2 (split is on level A~))"
  );
  return header_and_content;
}
#+END_SRC

*** source line array :array:

#+NAME: meta_markup_source_raw_source_line_array
#+BEGIN_SRC d
@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[])
  char[][] source_line_arr
    = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding);
  return source_line_arr;
}
#+END_SRC

*** source content raw line array :array:
- used for regular .sst files; master .ssm files and; .ssi inserts
- regex is passed for relevant enforce match
**** read in file

#+NAME: meta_markup_source_raw_read_in_file
#+BEGIN_SRC d
string markupSourceReadIn(in string fn_src) {
  static auto rgx_files = RgxFiles();
  enforce(
    fn_src.match(rgx_files.src_pth_sst_or_ssm),
    "not a dr markup filename: «" ~
    fn_src ~ "»"
  );
  string source_txt_str = readInMarkupSource(fn_src);
  return source_txt_str;
}
#+END_SRC

**** tuple (a) header, (b) body content, (c) file insert list & (d) image list?

- header
- body content
- file insert list
- [image list?]

#+NAME: meta_markup_source_raw_struct_of_header_and_body
#+BEGIN_SRC d
ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) {
  string[] file_insert_list = [];
  string[] images_list = [];
  char[][] hc = header0Content1(source_txt_str);
  char[] header = hc[0];
  char[] source_txt = hc[1];
  char[][] source_line_arr = markupSourceLineArray(source_txt);
  ST_header_content_inserts_images ret;
  {
    ret.header          = header;
    ret.src_txt         = source_line_arr;
    ret.insert_files    = file_insert_list;
    ret.images          = images_list;
  }
  return ret;
}
#+END_SRC

**** get insert source line array

#+NAME: meta_markup_source_raw_get_insert_source_line_array
#+BEGIN_SRC d
final char[][] getInsertMarkupSourceContentRawLineArray(
  in char[]    fn_src_insert,
  Regex!(char) rgx_file
) {
  enforce(
    fn_src_insert.match(rgx_file),
    "not a dr markup filename: «" ~
    fn_src_insert  ~ "»"
  );
  string source_txt_str = readInMarkupSource(fn_src_insert);
  char[][] source_line_arr = markupSourceLineArray(source_txt_str);
  return source_line_arr;
}
#+END_SRC

** get markup source, master file & inserts :masterfile:inserts:
*** scan inserts (sub-document) source :scan_insert_src:
**** scan subdoc source

#+NAME: meta_inserts_scan
#+BEGIN_SRC d
char[][] contents_insert;
int code_block_status     = 0;
enum codeBlock { off, curly, tic, quotemarks }
auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
auto markup_src_file_path = fn_pth_full.captures[1];
#+END_SRC

**** loop insert (sub-document)

#+NAME: meta_inserts_scan_loop
#+BEGIN_SRC d
if (code_block_status == codeBlock.curly) {
  if (line.matchFirst(rgx.block_curly_code_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_curly_code_open)) {
  code_block_status   = codeBlock.curly;
  contents_insert ~= line;
} else if (code_block_status == codeBlock.quotemarks) {
  if (line.matchFirst(rgx.block_quotemarks_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (code_block_status == codeBlock.tic) {
  if (line.matchFirst(rgx.block_tic_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
  code_block_status   = codeBlock.quotemarks;
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_tic_code_open)) {
  code_block_status   = codeBlock.tic;
  contents_insert ~= line;
} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
  auto insert_fn = m.captures[2];
  auto insert_sub_pth = m.captures[1];
  auto fn_src_insert
    = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
  auto raw = MarkupRawUnit();
  auto markup_sourcesubfile_insert_content
    = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
  debug(insert_file) {
    writeln(line);
    writeln(fn_src_insert);
    writeln(
      "  length contents insert array: ",
      markup_sourcesubfile_insert_content.length
    );
  }
  if (_opt_action.source_or_pod) {
    _images ~= _extract_images(markup_sourcesubfile_insert_content);
  }
  auto ins = Inserts();
  /+
    - 1. load file
    - 2. read lines
    - 3. scan lines
      - a. if filename insert, and insert filename
         - repeat 1
      - b. else
         - add line to new array;
         - build image list, search for any image files to add to image list
  +/
} else {
  contents_insert ~= line; // images to extract for image list?
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(line);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
}
#+END_SRC

**** post loop

#+NAME: meta_inserts_scan_post
#+BEGIN_SRC d
ST_contents_and_images ret;
{
  ret.insert_contents = contents_insert;
  ret.images          = _images;
}
return ret;
#+END_SRC

*** scan document source :scan_src:
**** scan doc source

#+NAME: meta_master_doc_scan_for_insert_filenames
#+BEGIN_SRC d
char[][] contents;
int code_block_status     = 0;
enum codeBlock { off, curly, tic, quotemarks }
auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
auto markup_src_file_path = fn_pth_full.captures[1];
char[][] contents_insert;
string[] _images          =[];
string[] insert_file_list =[];
#+END_SRC

**** include inserts: _loop master_ scan for inserts (insert documents)

#+NAME: meta_master_doc_scan_for_insert_filenames_loop
#+BEGIN_SRC d
if (code_block_status == codeBlock.curly) {
  if (line.matchFirst(rgx.block_curly_code_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (line.matchFirst(rgx.block_curly_code_open)) {
  code_block_status = codeBlock.curly;
  contents ~= line;
} else if (code_block_status == codeBlock.quotemarks) {
  if (line.matchFirst(rgx.block_quotemarks_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (code_block_status == codeBlock.tic) {
  if (line.matchFirst(rgx.block_tic_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
  code_block_status = codeBlock.quotemarks;
  contents ~= line;
} else if (line.matchFirst(rgx.block_tic_code_open)) {
  code_block_status = codeBlock.tic;
  contents ~= line;
} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
  auto insert_fn      = m.captures[2];
  auto insert_sub_pth = m.captures[1];
  auto fn_src_insert
    = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
    insert_file_list ~= fn_src_insert.to!string;
  auto raw = MarkupRawUnit();
  /+ TODO +/
  auto markup_sourcefile_insert_content
    = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
  debug(insert_file) {
    writeln(line);
    writeln(fn_src_insert);
    writeln(
      "  length contents insert array: ",
      markup_sourcefile_insert_content.length
    );
  }
  auto ins = Inserts();
  ST_contents_and_images contents_insert_st = ins.scan_subdoc_source(
    _opt_action,
    markup_sourcefile_insert_content,
    fn_src_insert.to!string
  );
  contents ~= contents_insert_st.insert_contents;
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(contents_insert_st.images);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
  /+
    - 1. load file
    - 2. read lines
    - 3. scan lines
      - a. if filename insert, and insert filename
         - repeat 1
      - b. else
         - add line to new array;
         - build image list, search for any image files to add to image list
  +/
} else {
  contents ~= line;
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(line);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
}
#+END_SRC

**** post loop

#+NAME: meta_master_doc_scan_for_insert_filenames_post
#+BEGIN_SRC d
string[] images = [];
foreach(i; uniq(_images.sort())) {
  images ~= i;
}
debug(insert_file) {
  writeln(__LINE__);
  writeln(contents.length);
}
ST_contents_inserts_images ret;
{
  ret.contents = contents;
  ret.insert_files = insert_file_list;
  ret.images = images;
}
return ret;
#+END_SRC

* carried names

#+HEADER: :tangle "../src/sisudoc/ocda/io_in/carried_names.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/++
  module carried_names;<BR>
  - validate the names of files carried inside an artefact, before any of
    them is written to disk<BR>
  - one set of rules for a bare filename, one for a relative path, and a
    containment check for either
+/
module sisudoc.ocda.io_in.carried_names;
@safe:
/+ ↓ names that arrive from outside, and decide where a file gets written
     .
     A pod zip and a .ocda.db both carry files by name, and both can arrive
     over https. A name read out of either is attacker-controlled: it can be
     absolute, it can climb with "..", and chainPath silently drops
     everything before an absolute segment, so "/etc/x" does not land under
     the directory it was joined to. read_zip_pod.d has guarded against this
     since the zip reader was written; the database reader did not, which is
     what this module is for.
     .
     Two rules, because the two roles differ:
     - an image is carried under a bare filename and belongs in one
       directory, so a directory part in the name is itself the error;
     - markup, conf and manifest are carried under their path inside the pod
       (media/text/de/about_manual.ssi), so the rule there is the zip
       reader's: relative, no climbing, bounded depth.
     .
     Both are followed by carriedPathIsWithin() at the point of writing.
     That is deliberate duplication: the name rules are the check, and
     containment is the check on the check.
+/
template spineCarriedNames() {
  import std.algorithm : canFind;
  /+ ↓ imports stay narrow on purpose. This template is mixed in beside
       others, and a template-level import lands in the mixing scope: an
       "import std.array : split" here takes over the UFCS lookup for
       split(ctRegex) at every mixin site and breaks it. Nothing below needs
       split, and nothing below should acquire it.
  +/
  import std.array     : array;
  import std.conv      : to;
  import std.path      : asNormalizedPath, isAbsolute;
  import std.regex;
  import std.string    : indexOf;
  /+ ↓ limits, the zip reader's where they overlap +/
  enum size_t MAX_CARRIED_NAME_LENGTH = 255;               /+ one path component +/
  enum size_t MAX_CARRIED_PATH_LENGTH = 1024;              /+ the whole path +/
  enum size_t MAX_CARRIED_PATH_DEPTH  = 10;                /+ as read_zip_pod.d +/
  /+ ↓ allowed characters: a filename has no separator, a path has "/" +/
  static auto rgx_safe_carried_name = ctRegex!(`^[a-zA-Z0-9._ -]+$`);
  static auto rgx_safe_carried_path = ctRegex!(`^[a-zA-Z0-9._/ -]+$`);
  /+ ↓ the checks both rules share, "" when the name is acceptable +/
  private string _carriedNameCommon(string _name) {
    if (_name.length == 0) {
      return "empty carried name";
    }
    if (_name.indexOf('\0') >= 0) {
      return "null byte in carried name";
    }
    if (_name.canFind("\\")) {
      return "backslash in carried name: " ~ _name;
    }
    if (_name.canFind("..")) {
      return "path traversal in carried name: " ~ _name;
    }
    if (_name.isAbsolute || _name[0] == '/') {
      return "absolute path in carried name: " ~ _name;
    }
    return "";
  }
  /+ ↓ a bare filename, as an image is carried: no directory part at all +/
  string validateCarriedFileName(string _name) {
    string _common = _carriedNameCommon(_name);
    if (_common.length > 0) {
      return _common;
    }
    if (_name.length > MAX_CARRIED_NAME_LENGTH) {
      return "carried filename too long: " ~ _name;
    }
    if (_name.canFind("/")) {
      return "directory part in carried filename: " ~ _name;
    }
    if (_name[0] == '.') {
      return "carried filename begins with a dot: " ~ _name;
    }
    if (!(_name.matchFirst(rgx_safe_carried_name))) {
      return "disallowed characters in carried filename: " ~ _name;
    }
    return "";
  }
  /+ ↓ a relative path inside a pod, as markup and conf are carried +/
  string validateCarriedPath(string _name) {
    string _common = _carriedNameCommon(_name);
    if (_common.length > 0) {
      return _common;
    }
    if (_name.length > MAX_CARRIED_PATH_LENGTH) {
      return "carried path too long: " ~ _name;
    }
    if (_name[$-1] == '/') {
      return "carried path names a directory: " ~ _name;
    }
    size_t _depth = 0;
    foreach (_c; _name) {
      if (_c == '/') {
        _depth++;
      }
    }
    if (_depth > MAX_CARRIED_PATH_DEPTH) {
      return "carried path too deep: " ~ _name;
    }
    /+ ↓ each component in turn, walked rather than split, so that this
         module needs no import that would leak into a mixing scope
    +/
    size_t _from = 0;
    foreach (_i; 0 .. _name.length + 1) {
      if (_i < _name.length && _name[_i] != '/') {
        continue;
      }
      string _component = _name[_from .. _i];
      if (_component.length == 0) {
        return "empty component in carried path: " ~ _name;
      }
      if (_component.length > MAX_CARRIED_NAME_LENGTH) {
        return "component too long in carried path: " ~ _name;
      }
      if (_component[0] == '.') {
        return "component begins with a dot in carried path: " ~ _name;
      }
      _from = _i + 1;
    }
    if (!(_name.matchFirst(rgx_safe_carried_path))) {
      return "disallowed characters in carried path: " ~ _name;
    }
    return "";
  }
  /+ ↓ does _path, normalised, still sit inside _root? the check on the check +/
  bool carriedPathIsWithin(string _root, string _path) {
    auto _canonical_root = (_root.asNormalizedPath).array.to!string;
    auto _canonical_path = (_path.asNormalizedPath).array.to!string;
    if (_canonical_root.length == 0
      || _canonical_path.length <= _canonical_root.length
    ) {
      return false;
    }
    if (_canonical_path[0.._canonical_root.length] != _canonical_root) {
      return false;
    }
    return (_canonical_path[_canonical_root.length] == '/');
  }
}
#+END_SRC

* org includes
** project version

#+NAME: spine_version
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
#+END_SRC

** year

#+NAME: year
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
#+END_SRC

** document header including copyright & license

#+NAME: doc_header_including_copyright_and_license
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
#+END_SRC

* __END__