path: root/org/sdp.org

#+TITLE:       sdp hub
#+AUTHOR:      Ralph Amissah
#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+KEYWORDS
#+LANGUAGE:    en
#+STARTUP:     indent content
#+OPTIONS:     H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS:     TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS:     author:nil email:nil creator:nil timestamp:nil
#+PROPERTY:    header-args :padline no :exports code :noweb yes
#+EXPORT_SELECT_TAGS:  export
#+EXPORT_EXCLUDE_TAGS: noexport
#+FILETAGS:            :sdp:rel:hub:
#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n)

[[../maker.org][maker.org makefile]]  [[./][org/]]
* 0. version.txt (set version)                                      :version:
** set program version

#+NAME: version_txt
#+BEGIN_SRC d  :tangle ../views/version.txt
/+ obt - org generated file +/
struct Version {
  int major;
  int minor;
  int patch;
}
enum ver = Version(0, 20, 0);
#+END_SRC

#+NAME: version_txt

** compilation restrictions (supported compilers)

http://dlang.org/spec/version.html#predefined-versions

#+BEGIN_SRC d  :tangle ../views/version.txt
version (Posix) {
  version (DigitalMars) {
  } else version (LDC) {
  } else version (GNU) {
  } else {
    static assert (0, "Unsupported D compiler");
  }
} else {
  static assert (0, "Unsupported D compiler");
}
#+END_SRC

* 1. sdp (sisu document parser)                                         :sdp:

- deal with imports
- get options
  - get command line instructions
  - read config instructions
- process files as instructed by options
  - read in file
  - process file
  - output

** 0. sdp src/sdp                                                 :template:

#+BEGIN_SRC d  :tangle ../src/sdp/sdp.d :shebang #!/usr/bin/env rdmd
/+
  sdp: sisu document parser
       a SiSU document parser writen in D
       see http://sisudoc.org.
+/
module sdp.sisu_document_parser;
import
  sdp.conf.compile_time_info,
  sdp.meta.metadoc;
<<imports_sdp>>
<<mixin_sdp_version>>
<<mixin_pre_main>>
/++ A SiSU document parser writen in D. +/
void main(string[] args) {
  <<sdp_mixin>>
  <<sdp_args>>
  <<sdp_env>>
  <<sdp_do_selected>>
  if (fns_src.length > 0) {
    foreach(fn_src; fns_src) {
      if (!empty(fn_src)) {
        <<sdp_each_file_do_scope>>
        <<sdp_abstraction>>
        <<sdp_each_file_do_debugs_checkdoc>>
        <<sdp_each_file_do_selected_output>>
        <<sdp_each_file_do_scope_exit>>
      } else {
        <<sdp_no_filename_provided>>
      }
    }
  }
}
unittest {
  /++
  name        "sdp"
  description "A SiSU document parser writen in D."
  homepage    "http://sisudoc.org"
  +/
}
#+END_SRC

** 1. pre-loop init                                                   :init:
*** init
**** imports                                                      :import:
***** sdp                                                           :sdp:

#+NAME: imports_sdp
#+BEGIN_SRC d
import
  std.getopt,
  std.process;
import
  sdp.meta,
  sdp.meta.metadoc_summary,
  sdp.meta.metadoc_from_src,
  sdp.meta.conf_make_meta,
  // sdp.meta.conf_make_meta_native,
  sdp.meta.conf_make_meta_sdlang,
  sdp.meta.conf_make_meta_composite,
  sdp.meta.defaults,
  sdp.meta.doc_debugs,
  sdp.meta.read_config_files,
  sdp.meta.read_source_files,
  sdp.meta.rgx,
  sdp.output.hub,
  sdp.output.paths_source;
#+END_SRC

****** notes
├── src
│   ├── sdp.d
│   └── sdp
│       ├── metadoc_from_src.d
│       ├── ...
│       └── compile_time_info.d
└── views
    └── version.txt

[[./meta_abstraction.org][meta_abstraction]]
[[./meta_conf_make_meta.org][meta_conf_make_meta]]
[[./meta_defaults.org][meta_defaults]]
[[./meta_output_debugs.org][meta_output_debugs]]
[[./meta_read_source_files.org][meta_read_source_files]]
[[./compile_time_info.org][compile time info]]
[[./output.org][output]]
[[./sdp.org][sdp]]

keep up to date, configuration in ../maker.org
check:
- http://github.com/Abscissa/SDLang-D
- https://github.com/abscissa/libInputVisitor

sdlang.parser,
sdlang.exceptions;

std.conv,
std.variant,

**** mixins                                                        :mixin:
***** version.txt                                               :version:

#+NAME: mixin_sdp_version
#+BEGIN_SRC d
mixin(import("version.txt"));
#+END_SRC

***** pre main mixins
#+NAME: mixin_pre_main
#+BEGIN_SRC d
mixin CompileTimeInfo;
#+END_SRC

***** sdp "main" mixins                                             :sdp:

#+NAME: sdp_mixin
#+BEGIN_SRC d
mixin SiSUrgxInit;
mixin SiSUregisters;
mixin SiSUextractSDLang;
mixin SiSUnode;
mixin SiSUbiblio;
mixin SiSUrgxInitFlags;
mixin outputHub;
#+END_SRC

**** init                                                           :init:

#+NAME: sdp_args
#+BEGIN_SRC d
string[] fns_src;
string flag_action;
string arg_unrecognized;
enum dAM { abstraction, matters }
static auto rgx = Rgx();
#+END_SRC

*** scope (run complete)                                            :scope:

#+NAME: sdp_args
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)",
      ver.major, ver.minor, ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "run failure",
    );
  }
}
#+END_SRC

*** config files and command line arguements
**** getopt args for loop                                    :args:getopt:

look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]

#+NAME: sdp_args
#+BEGIN_SRC d
bool[string] opts = [
  "assertions"         : false,
  "concordance"        : false,
  "debug"              : false,
  "digest"             : false,
  "docbook"            : false,
  "epub"               : false,
  "html"               : false,
  "html-seg"           : false,
  "html-scroll"        : false,
  "manifest"           : false,
  "ocn"                : true,
  "odt"                : false,
  "pdf"                : false,
  "postgresql"         : false,
  "qrcode"             : false,
  "sisupod"            : false,
  "source"             : false,
  "sqlite-discrete"    : false,
  "sqlite-update"      : false,
  "sqlite-create"      : false,
  "sqlite-drop"        : false,
  "text"               : false,
  "verbose"            : false,
  "xhtml"              : false,
  "xml-dom"            : false,
  "xml-sax"            : false,
  "section_toc"        : true,
  "section_body"       : true,
  "section_endnotes"   : true,
  "section_glossary"   : true,
  "section_biblio"     : true,
  "section_bookindex"  : true,
  "section_blurb"      : true,
  "backmatter"         : true,
  "skip-output"        : false,
];
auto helpInfo = getopt(args,
  std.getopt.config.passThrough,
  "assert",             "--assert set optional assertions on",                        &opts["assertions"],
  "concordance",        "--concordance file for document",                            &opts["concordance"],
  "debug",              "--debug only relevant when debug options compiled in",       &opts["debug"],
  "digest",             "--digest hash digest for each object",                       &opts["digest"],
  "docbook",            "--docbook process docbook output",                           &opts["docbook"],
  "epub",               "--epub process epub output",                                 &opts["epub"],
  "html",               "--html process html output",                                 &opts["html"],
  "html-seg",           "--html-seg process html output",                             &opts["html-seg"],
  "html-scroll",        "--html-seg process html output",                             &opts["html-scroll"],
  "manifest",           "--manifest process manifest output",                         &opts["manifest"],
  "ocn",                "--ocn object cite numbers (default)",                        &opts["ocn"],
  "odf",                "--odf process odf:odt output",                               &opts["odt"],
  "odt",                "--odt process odf:odt output",                               &opts["odt"],
  "pdf",                "--pdf process pdf output",                                   &opts["pdf"],
  "pg",                 "--pg process postgresql output",                             &opts["postgresql"],
  "postgresql",         "--postgresql process postgresql output",                     &opts["postgresql"],
  "qrcode",             "--qrcode with document metadata",                            &opts["qrcode"],
  "sisupod",            "--sisupod sisupod source content bundled",                   &opts["sisupod"],
  "source",             "--source markup source text content",                        &opts["source"],
  "sqlite-discrete",    "--sqlite process discrete sqlite output",                    &opts["sqlite-discrete"],
  "sqlite-create",      "--sqlite-create create db, create tables",                   &opts["sqlite-create"],
  "sqlite-drop",        "--sqlite-drop drop tables & db",                             &opts["sqlite-drop"],
  "sqlite-update",      "--sqlite process sqlite output",                             &opts["sqlite-update"],
  "text",               "--text process text output",                                 &opts["text"],
  "txt",                "--txt process text output",                                  &opts["text"],
  "verbose|v",          "--verbose output to terminal",                               &opts["verbose"],
  "xhtml",              "--xhtml process xhtml output",                               &opts["xhtml"],
  "xml-dom",            "--xml-dom process xml dom output",                           &opts["xml-dom"],
  "xml-sax",            "--xml-sax process xml sax output",                           &opts["xml-sax"],
  "section-toc",        "--section-toc process table of contents (default)",          &opts["section_toc"],
  "section-body",       "--section-body process document body (default)",             &opts["section_body"],
  "section-endnotes",   "--section-endnotes process document endnotes (default)",     &opts["section_endnotes"],
  "section-glossary",   "--section-glossary process document glossary (default)",     &opts["section_glossary"],
  "section-biblio",     "--section-biblio process document biblio (default)",         &opts["section_biblio"],
  "section-bookindex",  "--section-bookindex process document bookindex (default)",   &opts["section_bookindex"],
  "section-blurb",      "--section-blurb process document blurb (default)",           &opts["section_blurb"],
  "backmatter",         "--section-backmatter process document backmatter (default)", &opts["backmatter"],
  "skip-output",        "--skip-output",                                              &opts["skip-output"],
);
if (helpInfo.helpWanted) {
  defaultGetoptPrinter("Some information about the program.", helpInfo.options);
}
foreach(arg; args[1..$]) {
  if (arg.match(rgx.flag_action)) {
    flag_action ~= " " ~ arg;   // flags not taken by getopt
  } else if (arg.match(rgx.src_pth)) {
    fns_src ~= arg;             // gather input markup source file names for processing
  } else if (arg.match(rgx.src_pth_contents)) {
    import std.file,
      std.path;
    string contents_location_;
    try {
      if (exists(arg)) {
        contents_location_ = arg.readText;
      }
    }
    catch (ErrnoException ex) {
    }
    catch (FileException ex) {
      // Handle errors
    }
    auto contents_locations_arr =
      (cast(char[]) contents_location_).split;
    auto tmp_dir_ = (arg).dirName.array;
    foreach (contents_location; contents_locations_arr) {
      assert(contents_location.match(rgx.src_pth),
        "not a recognised file: " ~
         contents_location
      );
      auto contents_location_pth_ = (contents_location).to!string;
      fns_src ~= (((tmp_dir_).chainPath(contents_location_pth_)).array).to!(char[]);
    }
  } else if (arg.match(rgx.src_pth_zip)) {
    // fns_src ~= arg;             // gather input markup source file names for processing
  } else {                      // anything remaining, unused
    arg_unrecognized ~= " " ~ arg;
  }
}
#+END_SRC

**** environment                                             :environment:

#+NAME: sdp_env
#+BEGIN_SRC d
auto env = [
  "pwd" : environment["PWD"],
  "home" : environment["HOME"],
];
#+END_SRC

**** TODO config files (load & read) (so far only SDLang)   :config:files:

#+NAME: sdp_conf_files
#+BEGIN_SRC d
auto sdl_root_config_share = configRead!()("config_share", env);
auto sdl_root_config_local = configRead!()("config_local", env);
auto conf_files_composite_make = confFilesSDLtoStruct!()(sdl_root_config_share, sdl_root_config_local);
#+END_SRC

** 2a. actions independed of processing files
#+NAME: sdp_do_selected
#+BEGIN_SRC d
if (!(opts["skip-output"])) {
  outputHubOp!()(opts);
}
#+END_SRC

** _2b. processing: loop each file_ [+2]                          :loop:files:
*** scope (loop)                                                    :scope:

#+NAME: sdp_each_file_do_scope
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "%s\n%s",
      "~ document complete, ok ~",
      "------------------------------------------------------------------",
    );
  }
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "~ document run failure ~ (%s  v%s)\n\t%s",
      __VENDOR__, __VERSION__,
      fn_src
    );
  }
}
enforce(
  fn_src.match(rgx.src_pth_types),
  "not a sisu markup filename: <<" ~
  fn_src ~ ">>"
);
#+END_SRC

*** 1. _document abstraction_ [#A]

- return tuple of:
  - doc_abstraction (the document)
  - doc_matters

#+NAME: sdp_abstraction
#+BEGIN_SRC d
auto t = SiSUabstraction!()(fn_src, opts, env);
static assert(!isTypeTuple!(t));
static assert(t.length==2);
auto doc_abstraction = t[dAM.abstraction];
auto doc_matters = t[dAM.matters];
#+END_SRC

*** 2. _output processing_ (post abstraction processing)
**** 0. abstraction _print summary_                    :abstraction:summary:

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
if (doc_matters.opt_action["verbose"]) {
  SiSUabstractionSummary!()(doc_abstraction, doc_matters);
}
#+END_SRC

**** 1. _debug_ (document parts, checkdoc)                  :debug:checkdoc:
- [[./meta_output_debugs.org][meta_output_debugs]]

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
if ((doc_matters.opt_action["debug"])
|| (doc_matters.opt_action["verbose"])
) {
  SiSUdebugs!()(doc_abstraction, doc_matters);
}
#+END_SRC

**** 2. _process outputs_                                          :outputs:
- [[./output_hub.org][output_hub]]

#+NAME: sdp_each_file_do_selected_output
#+BEGIN_SRC d
/+ ↓ output hub +/
if (!(opts["skip-output"])) {
  outputHub!()(doc_abstraction, doc_matters);
}
#+END_SRC

*** scope (on loop exit)                                       :scope:exit:

#+NAME: sdp_each_file_do_scope_exit
#+BEGIN_SRC d
scope(exit) {
  debug(checkdoc) {
    writefln(
      "processed file: %s",
      fn_src
    );
  }
  destroy(fn_src);
}
#+END_SRC

** +2c. no valid filename provided+
#+NAME: sdp_no_filename_provided
#+BEGIN_SRC d
/+ no recognized filename provided +/
writeln("no recognized filename");
break; // terminate, stop
#+END_SRC

* 2. _document abstraction functions_                  :module:sdp:abstraction:
** 0. module template

#+BEGIN_SRC d  :tangle ../src/sdp/meta/metadoc.d
module sdp.meta.metadoc;
template SiSUabstraction() {
  <<imports_sdp>>
  <<sdp_mixin>>
  enum headBody { header, body_content, insert_filelist }
  enum makeMeta { make, meta }
  enum docAbst  { doc_abstraction, section_keys, segnames, segnames_0_4, images }
  static auto rgx = Rgx();
  auto SiSUabstraction(Fn,O,E)(
    Fn fn_src,
    O opts,
    E env,
  ){
    <<sdp_conf_files>>
    <<sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body>>
    <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>>
    <<sdp_each_file_do_document_abstraction>>
    <<sdp_each_file_do_document_matters>>
    auto t = tuple(doc_abstraction, doc_matters);
    static assert(t.length==2);
    return t;
  }
}
#+END_SRC

** 1. (a) _read in raw file_ (b) split content into: _doc header & doc content_
- [[./meta_read_source_files.org][meta_read_source_files]]

- read in the source marked up document and
  - split the document into:
    - document header
    - document body
  - if a master document make a list of insert files
- return a tuple of:
  - header
  - body
  - insert file list

#+NAME: sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body
#+BEGIN_SRC d
/+ ↓ read file (filename with path) +/
/+ ↓ file tuple of header and content +/
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step1 commence → (get document header & body & insert files)");
}
auto _header_body_inserts =
  SiSUrawMarkupContent!()(fn_src);
static assert(!isTypeTuple!(_header_body_inserts));
static assert(_header_body_inserts.length==3);
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step1 complete");
}
debug(header_and_body) {
  writeln(header);
  writeln(_header_body_inserts.length);
  writeln(_header_body_inserts.length[headBody.body_content][0]);
}
#+END_SRC

** 2. _document metadata_ & _make instructions_       :doc:header:metadata:make:
- [[./meta_conf_make_meta.org][meta_conf_make_meta]]

- read document header, split into:
  - metadata
  - make instructions
- read config files
  - consolidate make instructions
- return tuple of:
  - document metadata
  - make instructions (from configuration files & document header make
    instructions)

#+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta
#+BEGIN_SRC d
/+ ↓ split header into make and meta +/
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step2 commence → (doc header: make & meta as struct)");
}
auto _make_and_meta_struct =
  docHeaderMakeAndMetaTupExtractAndConvertToStruct!()(conf_files_composite_make, _header_body_inserts[headBody.header]); // breakage ...
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step2 complete");
}
#+END_SRC

** 3. _document abstraction, tuple_ (pre-output-processing)       :processing:
- [[./meta_abstraction.org][meta_abstraction]]

- prepare the document abstraction used in downstream processing

- return tuple of:
  - document abstraction (_the_document_ or doc_abstraction)
  - document abstraction keys
    - (head, toc, body, endnotes, glossary, bibliography, bookindex, blurb,
      tail)
    - (transfer to _doc_matters_)
  - segnames for html epub (transfer to _doc_matters_)
  - image list (transfer to _doc_matters_)

#+NAME: sdp_each_file_do_document_abstraction
#+BEGIN_SRC d
/+ ↓ document abstraction: process document, return abstraction as tuple +/
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step3 commence → (document abstraction (da); da keys; segnames; doc_matters)");
}
auto da = SiSUdocAbstraction!()(
  _header_body_inserts[headBody.body_content],
  _make_and_meta_struct,
  opts
);
static assert(!isTypeTuple!(da));
static assert(da.length==5);
auto doc_abstraction = da[docAbst.doc_abstraction]; /+ head ~ toc ~ body ~ endnotes_seg ~ glossary ~ bibliography ~ bookindex ~ blurb; +/
auto _document_section_keys_sequenced = da[docAbst.section_keys];
string[] _doc_html_segnames = da[docAbst.segnames];
string[] _doc_epub_segnames_0_4 = da[docAbst.segnames_0_4];
auto _images = da[docAbst.images];
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step3 complete");
}
#+END_SRC

** 4. _document matters_ (doc info gathered, various sources)

- prepare document_matters, miscellany about processing and the document of use
  in downstream processing

#+NAME: sdp_each_file_do_document_matters
#+BEGIN_SRC d
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step4 commence → (doc_matters)");
}
struct DocumentMatters {
  auto keys_seq() {
    /+ contains .seg & .scroll sequences +/
    auto _k = _document_section_keys_sequenced;
    return _k;
  }
  string[] segnames() {
    string[] _k = _doc_html_segnames;
    return _k;
  }
  string[] segnames_lv_0_to_4() {
    string[] _k = _doc_epub_segnames_0_4;
    return _k;
  }
  auto conf_make_meta() {
    auto _k = _make_and_meta_struct;
    return _k;
  }
  auto source_filename() {
    string _k = fn_src;
    return _k;
  }
  auto src_path_info() {
    string _pwd = env["pwd"];
    auto _k = SiSUpathsSRC!()(_pwd, fn_src);
    return _k;
  }
  auto opt_action() {
    bool[string] _k = opts;
    return _k;
  }
  auto environment() {
    auto _k = env;
    return _k;
  }
  auto language() {
    string _k;
    if (auto m = fn_src.match(rgx.language_code_and_filename)) {
      _k = m.captures[1];
    } else {
      _k = "en";
    }
    return _k;
  }
  auto file_insert_list() {
    string[] _k = _header_body_inserts[headBody.insert_filelist];
    return _k;
  }
  auto image_list() {
    auto _k = _images;
    return _k;
  }
}
auto doc_matters = DocumentMatters();
debug(steps) {
  writeln(__LINE__, ":", __FILE__, ": step4 complete");
}
#+END_SRC

* 3. document abstraction _summary_                :module:sdp:metadoc_summary:
** 0. module template

#+BEGIN_SRC d :tangle ../src/sdp/meta/metadoc_summary.d
module sdp.meta.metadoc_summary;
template SiSUabstractionSummary() {
  auto SiSUabstractionSummary(S,T)(
    auto return ref const S  doc_abstraction,
    auto return ref T        doc_matters,
  ) {
    <<metadoc_summary_imports>>
    mixin InternalMarkup;
    <<metadoc_summary_initialize>>
    if (doc_matters.opt_action["verbose"]) {
      <<meta_metadoc_summary>>
    }
  }
}
#+END_SRC

** init
*** imports

#+name: metadoc_summary_imports
#+BEGIN_SRC d
import
  sdp.meta.defaults,
  sdp.meta.rgx;
import
  std.array,
  std.exception,
  std.stdio,
  std.regex,
  std.string,
  std.traits,
  std.typecons,
  std.uni,
  std.utf,
  std.conv : to;
#+END_SRC

*** initialize                                                     :report:

#+name: metadoc_summary_initialize
#+BEGIN_SRC d
auto markup = InlineMarkup();
#+END_SRC

** (last ocn)

#+name: meta_metadoc_summary
#+BEGIN_SRC d
string[string] check = [
  "last_obj_cite_number" : "NA [debug \"checkdoc\" not run]",
  "last_obj_cite_number_body"  : "0",
  "last_obj_cite_number_bkidx" : "0",
];
foreach (k; doc_matters.keys_seq.seg) {
  foreach (obj; doc_abstraction[k]) {
    if (obj.of_part != "empty") {
      if (!empty(obj.obj_cite_number)) {
        if (k == "body") {
          check["last_obj_cite_number_body"] = obj.obj_cite_number;
        }
        if (!(obj.obj_cite_number.empty)) {
          check["last_obj_cite_number"] = obj.obj_cite_number;
        }
      }
      if (k == "bookindex_seg") {
        if (obj.obj_cite_number_type == 2) {
          check["last_obj_cite_number_bkidx"] = obj.obj_cite_number_bkidx;
        }
      }
    }
  }
}
#+END_SRC

** summary

#+name: meta_metadoc_summary
#+BEGIN_SRC d
auto min_repeat_number = 66;
auto char_repeat_number = (doc_matters.conf_make_meta.meta.title_full.length
  + doc_matters.conf_make_meta.meta.creator_author.length + 4);
char_repeat_number = (char_repeat_number > min_repeat_number)
? char_repeat_number
: min_repeat_number;
writefln(
  "%s\n\"%s\", %s\n%s\n%s\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n%30-s%10-d\n(%s: %s)\n%s",
  markup.repeat_character_by_number_provided("-", char_repeat_number),
  doc_matters.conf_make_meta.meta.title_full,
  doc_matters.conf_make_meta.meta.creator_author,
  doc_matters.source_filename,
  markup.repeat_character_by_number_provided("-", char_repeat_number),
  "length toc arr:",
  to!int(doc_abstraction["toc_seg"].length),
  "length doc_abstraction arr:",
  to!int(doc_abstraction["body"].length),
  "last doc body ocn:",
  to!int(check["last_obj_cite_number_body"]),
  "last obj_cite_number:",
  to!int(check["last_obj_cite_number"]),
  "length endnotes:",                                // subtract headings
  (doc_abstraction["endnotes"].length > 2)
  ? (to!int(doc_abstraction["endnotes"].length - 2))
  : 0,
  "length glossary:",
  (doc_abstraction["glossary"].length > 1)
  ? (to!int(doc_abstraction["glossary"].length))
  : 0,
  "length biblio:",
  (doc_abstraction["bibliography"].length > 1)
  ? (to!int(doc_abstraction["bibliography"].length))
  : 0,
  "length bookindex:",
  (doc_abstraction["bookindex_seg"].length > 1)
  ? (to!int(doc_abstraction["bookindex_seg"].length))
  : 0,
  "  last book idx ocn:",
  to!int(check["last_obj_cite_number_bkidx"]),
  "length blurb:",
  (doc_abstraction["blurb"].length > 1)
  ? (to!int(doc_abstraction["blurb"].length))
  : 0,
  __FILE__,
  __LINE__,
  markup.repeat_character_by_number_provided("-", min_repeat_number),
);
#+END_SRC

* __END__
dev notes

** sdp glossary / terms

|------+-------------------------------------|
| sdp  | sisu document parser                |
|------+-------------------------------------|
| dmso | document markup, structure, objects |
|------+-------------------------------------|
| meta | meta document, document abstraction |
| mda  | meta, meta document abstraction     |
| adr  | abstract document representation    |
| dar  | document abstract representation    |
| (da) | (document abstraction)              |
|      | (code representation of document)   |
|------+-------------------------------------|
| ao   | abstract objects                    |
|      | (code representation of objects)    |
|------+-------------------------------------|

consider
|-------+----------------------------------------------|
| dao   | document abstraction, objects                |
|-------+----------------------------------------------|
| daso  | document abstraction, structure, objects     |
|-------+----------------------------------------------|
| drso  | document representation, structure, objects  |
|-------+----------------------------------------------|
| daows | document abstraction, objects with structure |
|-------+----------------------------------------------|

** the document notes
*** document sections
**** summary

|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
| section      | part         | opt. |   | objects                                          | ocn                            |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
| front matter | head         | *    |   |                                                  | no                             |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
| toc          | toc          |      |   | generated from headings                          | no                             |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
| body         | body         | *    |   | default section                                  | yes                            |   |   |
|              |              |      |   | - headings                                       |                                |   |   |
|              |              |      |   | - paras                                          |                                |   |   |
|              |              |      |   | - code                                           |                                |   |   |
|              |              |      |   | - poem                                           |                                |   |   |
|              |              |      |   | - group                                          |                                |   |   |
|              |              |      |   | - block                                          |                                |   |   |
|              |              |      |   | - quote                                          |                                |   |   |
|              |              |      |   | - table                                          |                                |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
| back matter  | endnote      |      |   | generated from inline note markup                | no (each endnote belongs to    |   |   |
|              |              |      |   |                                                  | a (body) object)               |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
|              | glossary     |      |   | identified section, limited markup               | possibly, to make searchable   |   |   |
|              |              |      |   | - heading                                        | hidden                         |   |   |
|              |              |      |   | - paras                                          |                                |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
|              | bibliography |      |   | generated from inline special markup             | possibly, to make searchable   |   |   |
|              |              |      |   | appended to paragraphs contained in body section | hidden                         |   |   |
|              |              |      |   | - heading                                        |                                |   |   |
|              |              |      |   | - paras                                          |                                |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
|              | book index   |      |   | generated from inline special markup             | possibly, special numbering or |   |   |
|              |              |      |   | - heading                                        | could use term as anchor?      |   |   |
|              |              |      |   | - paras                                          | to make searchable             |   |   |
|              |              |      |   |                                                  | hidden                         |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|
|              | blurb        |      |   | identified section, limited markup               | no (unless non-substantive     |   |   |
|              |              |      |   | - heading                                        | given special numbering)       |   |   |
|              |              |      |   | - paras                                          |                                |   |   |
|--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---|

**** on abstraction

- abstract for downstream processing
  - identify document structure and objects
    - identify document structure (headings/levels/sections)
    - identify objects (headings, paragraphs, tables, code blocks, verse ...)
  - set document, generate common abstraction for downstream parsing
    - set different _document sections_:
      - _head_, toc, _body_, endnotes, glossary, bibliography, book index, blurb
    - _object numbers_, heading/ chapter numbering etc, endnote numbers
      - _regular ocn_
        - body objects
        - glossary objects
        - bibliography objects
      - _special ocn_
        - non substantive text (provide special numbers)
          - blurb objects
        - book index
      - special (_exceptions_)
        - endnotes
  - unify object representations
    - multiple markups for same object type given single representation
  - extract object attributes
  - unify inline markup on objects
    - inline markup made easier to identify

- simplify downstream parsing

*** objects
**** summary

|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
|       |              | identified by              | object notes             | attributes     | inline          | embedded       | special    |
|       |              |                            |                          |                |                 | appended       | characters |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
| para  | heading      | level markers              |                          |                | - italics       | - endnotes     |            |
|       |              | at start of line           |                          |                |                 | - bibliography |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
|       | paragraph    | delimited by two new lines | default object           | - indent       | - bold          | - endnotes     |            |
|       |              |                            | [discard leading &       | - bullet       | - italics       | - bibliography |            |
|       |              |                            | newline whitespace]      |                | - underscore    |                |            |
|       |              |                            |                          |                | - strikethrough |                |            |
|       |              |                            |                          |                | - superscript   |                |            |
|       |              |                            |                          |                | - subscript     |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
| block |              | open and close tags        |                          |                |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
| TODO  | quote        |                            |                          | - language?    |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
| TODO  | group        |                            | - inline markup applied  | - language?    | as paragraph    | - endnotes     |            |
|       |              |                            | - [discard leading &     |                |                 | - bibliography |            |
|       |              |                            | newline whitespace]      |                |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
| TODO  | block        |                            | - inline markup applied  |                | as paragraph    | - endnotes     |            |
|       |              |                            | - whitespace indentation |                |                 | - bibliography |            |
|       |              |                            | & newlines               |                |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
|       | poem / verse | open and close tags        | verse is the object      |                |                 | - endnotes     |            |
|       |              |                            | - inline markup applied  |                |                 | - bibliography |            |
|       |              | (for poem)                 | - whitespace indentation |                |                 |                |            |
|       |              |                            | & newlines               |                |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
|       | code         |                            | - contents untouched     | - syntax       |                 |                |            |
|       |              |                            | - whitespace indentation | - numbered     |                 |                |            |
|       |              |                            | & newlines               |                |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|
|       | table        |                            |                          | - column width |                 |                |            |
|       |              |                            |                          | - heading row  |                 |                |            |
|-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------|

**** ocn

|-------------+-----------------------+-----------------------+----------------+------|
| *objects     | section / part        | ocn described         | how used*       | type |
|-------------+-----------------------+-----------------------+----------------+------|
| regular ocn |                       |                       |                |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | body objects          | seq. digit            | anchor         | ocn  |
|             |                       | [0-9]+                | visible        |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | glossary objects      | seq. digit            | anchor         | ocn  |
|             |                       | [0-9]+                | not-visible    |      |
|             |                       |                       | (for search)   |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | bibliography objects  | seq. digit            | anchor         | ocn  |
|             |                       | [0-9]+                | not-visible    |      |
|             |                       |                       | (for search)   |      |
|-------------+-----------------------+-----------------------+----------------+------|
| special ocn |                       |                       |                |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | non-substantive text  | x char + seq. digit   | anchor         | non  |
|             | (within body & blurb) | x[0-9]+               | not-visible    |      |
|             |                       |                       | (for search)   |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | book index            | i char + seq. digit   | anchor         | idx  |
|             |                       | i[0-9]+               | not-visible    |      |
|             |                       |                       | (for search)   |      |
|-------------+-----------------------+-----------------------+----------------+------|
| without ocn |                       |                       |                |      |
|-------------+-----------------------+-----------------------+----------------+------|
|             | endnotes              | ocn of parent object  | no ocn         | fn   |
|             |                       | + footnote seq. digit | anchor visible |      |
|-------------+-----------------------+-----------------------+----------------+------|

** make config - _composite make_

work on composite make a unification of make instructions for each document run

extract instructions from all config files, unify the make instructions and
provide the result as a single set of make instructions for each document parsed

- 1. general, document_make config file (to be applied to all documents unless
  overridden by document or command line instruction)
- 2. local, site specific (site local instructions such as the site's url, cgi
  location etc.)
- 3. each document header, make (the document header contains metadata and may
  include make instructions for that document)
  - make
  - meta
- 4. command line instruction, make (some make instructions may be passed
  through the command line)

*** instruction sources

|----+---------------------------------+----------------------------------------+---------------------+---|
|    | make instruction source         |                                        | varies (applies to) |   |
|----+---------------------------------+----------------------------------------+---------------------+---|
| 0. | unify the following as a single | take into account all the instructions |                     |   |
|    | set of make instructions        | provided below, provide interface      |                     |   |
|----+---------------------------------+----------------------------------------+---------------------+---|
| 1. | document_make file              | to be applied to all documents         | per directory       |   |
|    | "config_share"                  | (unless subsequently overridden)       | (all docs within)   |   |
|----+---------------------------------+----------------------------------------+---------------------+---|
| 2. | config file                     | local site specific                    | per directory       |   |
|    | "config_local"                  |                                        | (all docs within)   |   |
|----+---------------------------------+----------------------------------------+---------------------+---|
| 3. | document header make            | make instructions contained            | per document        |   |
|    |                                 | in document header                     | (single doc)        |   |
|----+---------------------------------+----------------------------------------+---------------------+---|
| 4. | command line instruction        | make instruction passed                | each command        |   |
|    |                                 |                                        | (all docs within)   |   |
|----+---------------------------------+----------------------------------------+---------------------+---|

*** config & metadata (from instruction sources)

|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     | 1. document make file | 2. config file          | 3. document header     | 4. command line instruction |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
| comment, fixed:     | per dir (sisupod)     | per dir                 | per document (sisupod) | per command instruction     |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     | sdl_root_config_share | sdl_root_config_local   |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
| local site specific |                       | *                       |                        | *?                          |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | webserv                 |                        |                             |
|                     |                       | - url_root              |                        |                             |
|                     |                       | - path                  |                        |                             |
|                     |                       | - images                |                        |                             |
|                     |                       | - cgi                   |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | webserv_cgi             |                        |                             |
|                     |                       | - host                  |                        |                             |
|                     |                       | - base_path             |                        |                             |
|                     |                       | - port                  |                        |                             |
|                     |                       | - user                  |                        |                             |
|                     |                       | - file_links            |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | processing              |                        |                             |
|                     |                       | - path                  |                        |                             |
|                     |                       | - dir                   |                        |                             |
|                     |                       | - concord_max           |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | flag (configure)        |                        | (call)                      |
|                     |                       | - act0                  |                        | act0                        |
|                     |                       | - act1                  |                        | act1                        |
|                     |                       | - act2                  |                        | act2                        |
|                     |                       | - act3                  |                        | act3                        |
|                     |                       | - act4                  |                        | act4                        |
|                     |                       | - act5                  |                        | act5                        |
|                     |                       | - act6                  |                        | act6                        |
|                     |                       | - act7                  |                        | act7                        |
|                     |                       | - act8                  |                        | act8                        |
|                     |                       | - act9                  |                        | act9                        |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | default                 |                        |                             |
|                     |                       | - papersize             |                        |                             |
|                     |                       | - text_wrap             |                        |                             |
|                     |                       | - emphasis              |                        |                             |
|                     |                       | - language              |                        |                             |
|                     |                       | - digest                |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | permission              |                        |                             |
|                     |                       | - share_source          |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | program_select          |                        |                             |
|                     |                       | - editor                |                        |                             |
|                     |                       | - epub_viewer           |                        |                             |
|                     |                       | - html_viewer           |                        |                             |
|                     |                       | - odf_viewer            |                        |                             |
|                     |                       | - pdf_viewer            |                        |                             |
|                     |                       | - xml_viewer            |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       | search                  |                        |                             |
|                     |                       | - flag                  |                        |                             |
|                     |                       | - action                |                        |                             |
|                     |                       | - db                    |                        |                             |
|                     |                       | - title                 |                        |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
| make instruction    | **                    | omit or override share? | **                     | *?                          |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     | make                  | make                    | make                   |                             |
|                     | - bold                | - bold                  | - bold                 |                             |
|                     | - breaks              | - breaks                | - breaks               |                             |
|                     | - cover_image         | - cover_image           | - cover_image          |                             |
|                     | - css                 | - css                   | - css                  |                             |
|                     | - emphasis            | - emphasis              | - emphasis             |                             |
|                     | - footer              | - footer                | - footer               |                             |
|                     | - headings            | - headings              | - headings             |                             |
|                     | - home_button_image   | - home_button_image     | - home_button_image    |                             |
|                     | - home_button_text    | - home_button_text      | - home_button_text     |                             |
|                     | - italics             | - italics               | - italics              |                             |
|                     | - num_top             | - num_top               | - num_top              |                             |
|                     | - num_depth           | - num_depth             | - num_depth            |                             |
|                     | - substitute          | - substitute            | - substitute           |                             |
|                     | - texpdf_font         | - texpdf_font           | - texpdf_font          |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
| actions             |                       |                         |                        | *                           |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         |                        | assertions                  |
|                     |                       |                         |                        | concordance                 |
|                     |                       |                         |                        | debug                       |
|                     |                       |                         |                        | digest                      |
|                     |                       |                         |                        | docbook                     |
|                     |                       |                         |                        | epub                        |
|                     |                       |                         |                        | html                        |
|                     |                       |                         |                        | html-seg                    |
|                     |                       |                         |                        | html-scroll                 |
|                     |                       |                         |                        | manifest                    |
|                     |                       |                         |                        | ocn                         |
|                     |                       |                         |                        | odt                         |
|                     |                       |                         |                        | pdf                         |
|                     |                       |                         |                        | postgresql                  |
|                     |                       |                         |                        | qrcode                      |
|                     |                       |                         |                        | sisupod                     |
|                     |                       |                         |                        | source                      |
|                     |                       |                         |                        | sqlite                      |
|                     |                       |                         |                        | sqlite-create               |
|                     |                       |                         |                        | sqlite-drop                 |
|                     |                       |                         |                        | text                        |
|                     |                       |                         |                        | verbose                     |
|                     |                       |                         |                        | xhtml                       |
|                     |                       |                         |                        | xml-dom                     |
|                     |                       |                         |                        | xml-sax                     |
|                     |                       |                         |                        | section_toc                 |
|                     |                       |                         |                        | section_body                |
|                     |                       |                         |                        | section_endnotes            |
|                     |                       |                         |                        | section_glossary            |
|                     |                       |                         |                        | section_biblio              |
|                     |                       |                         |                        | section_bookindex           |
|                     |                       |                         |                        | section_blurb               |
|                     |                       |                         |                        | backmatter                  |
|                     |                       |                         |                        | skip-output                 |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
| metadata            |                       |                         | *                      |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | classify               |                             |
|                     |                       |                         | - dewey                |                             |
|                     |                       |                         | - keywords             |                             |
|                     |                       |                         | - loc                  |                             |
|                     |                       |                         | - subject              |                             |
|                     |                       |                         | - topic_register       |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | creator                |                             |
|                     |                       |                         | - author               |                             |
|                     |                       |                         | - author_email         |                             |
|                     |                       |                         | - illustrator          |                             |
|                     |                       |                         | - translator           |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | date                   |                             |
|                     |                       |                         | - added_to_site        |                             |
|                     |                       |                         | - available            |                             |
|                     |                       |                         | - created              |                             |
|                     |                       |                         | - issued               |                             |
|                     |                       |                         | - modified             |                             |
|                     |                       |                         | - published            |                             |
|                     |                       |                         | - valid                |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | identifier             |                             |
|                     |                       |                         | - isbn                 |                             |
|                     |                       |                         | - oclc                 |                             |
|                     |                       |                         | - pg                   |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | links                  |                             |
|                     |                       |                         | - link                 |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | notes                  |                             |
|                     |                       |                         | - abstract             |                             |
|                     |                       |                         | - description          |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | original               |                             |
|                     |                       |                         | - language             |                             |
|                     |                       |                         | - source               |                             |
|                     |                       |                         | - title                |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | publisher              |                             |
|                     |                       |                         | - name                 |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | rights                 |                             |
|                     |                       |                         | - copyright            |                             |
|                     |                       |                         | - cover                |                             |
|                     |                       |                         | - illustrations        |                             |
|                     |                       |                         | - license              |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|
|                     |                       |                         | title                  |                             |
|                     |                       |                         | - edition              |                             |
|                     |                       |                         | - full                 |                             |
|                     |                       |                         | - language             |                             |
|                     |                       |                         | - main                 |                             |
|                     |                       |                         | - note                 |                             |
|                     |                       |                         | - sub                  |                             |
|                     |                       |                         | - subtitle             |                             |
|---------------------+-----------------------+-------------------------+------------------------+-----------------------------|