path: root/src/doc_reform/doc_reform.d

#!/usr/bin/env rdmd
/+
- Name: Doc Reform
  - Description: documents, structuring, processing, publishing, search
    - static content generator

  - Author: Ralph Amissah
    [ralph.amissah@gmail.com]

  - Copyright: (C) 2015 - 2019 Ralph Amissah, All Rights
    Reserved.

  - License: AGPL 3 or later:

    Doc Reform (SiSU), a framework for document structuring, publishing and
    search

    Copyright (C) Ralph Amissah

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU AFERO General Public License as published by the
    Free Software Foundation, either version 3 of the License, or (at your
    option) any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program. If not, see [http://www.gnu.org/licenses/].

    If you have Internet connection, the latest version of the AGPL should be
    available at these locations:
    [http://www.fsf.org/licensing/licenses/agpl.html]
    [http://www.gnu.org/licenses/agpl.html]

  - Doc Reform (related to SiSU) uses standard:
    - docReform markup syntax
      - standard SiSU markup syntax with modified headers and minor modifications
    - docReform object numbering
      - standard SiSU object citation numbering & system

  - Hompages:
    [http://www.doc_reform.org]
    [http://www.sisudoc.org]

  - Git
    [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary]
    [http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/html.rb;hb=HEAD]

+/
module doc_reform.sisu_document_parser;
import
  doc_reform.conf.compile_time_info,
  doc_reform.meta.metadoc,
  doc_reform.meta.metadochead;
import
  std.datetime,
  std.getopt,
  std.file,
  std.path,
  std.process;
import
  doc_reform.meta,
  doc_reform.meta.metadoc_summary,
  doc_reform.meta.metadoc_from_src,
  doc_reform.meta.conf_make_meta_structs,
  doc_reform.meta.conf_make_meta_toml,
  doc_reform.meta.conf_make_meta_json,
  doc_reform.meta.defaults,
  doc_reform.meta.doc_debugs,
  doc_reform.meta.rgx,
  doc_reform.source.paths_source,
  doc_reform.source.read_config_files,
  doc_reform.source.read_source_files,
  doc_reform.output.hub;
import std.algorithm;
import std.parallelism;
mixin(import("version.txt"));
mixin CompileTimeInfo;
string program_name = "doc-reform";
/++
name        "doc_reform"
description "A SiSU inspired document parser writen in D."
homepage    "http://sisudoc.org"
+/
void main(string[] args) {
  mixin DocReformRgxInit;
  mixin contentJSONtoDocReformStruct;
  mixin DocReformBiblio;
  mixin DocReformRgxInitFlags;
  mixin outputHub;
  string flag_action;
  string arg_unrecognized;
  enum dAM { abstraction, matters }
  static auto rgx = Rgx();
  scope(success) {
    writefln(
      "~ run complete, ok ~ (%s-%s.%s.%s, %s D:%s, %s %s)",
      program_name,
      _ver.major, _ver.minor, _ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
  scope(failure) {
    debug(checkdoc) {
      stderr.writefln(
        "run failure",
      );
    }
  }
  bool[string] opts = [
    "abstraction"        : false,
    "assertions"         : false,
    "concordance"        : false,
    "dark"               : false,
    "debug"              : false,
    "digest"             : false,
    "epub"               : false,
    "html"               : false,
    "html-seg"           : false,
    "html-scroll"        : false,
    "light"              : false,
    "manifest"           : false,
    "ocn"                : true,
    "parallel"           : false,
    "parallel-subprocesses" : false,
    "quiet"              : false,
    "pod"                : false,
    "serial"             : false,
    "source"             : false,
    "sqlite-discrete"    : false,
    "sqlite-db-create"   : false,
    "sqlite-db-drop"     : false,
    "sqlite-db-recreate" : false,
    "sqlite-delete"      : false,
    "sqlite-insert"      : false,
    "sqlite-update"      : false,
    "text"               : false,
    "verbose"            : false,
    "very-verbose"       : false,
    "xhtml"              : false,
    "section_toc"        : true,
    "section_body"       : true,
    "section_endnotes"   : true,
    "section_glossary"   : true,
    "section_biblio"     : true,
    "section_bookindex"  : true,
    "section_blurb"      : true,
    "backmatter"         : true,
    "skip-output"        : false,
    "theme-dark"         : false,
    "theme-light"        : false,
    "workon"             : false,
  ];
  string[string] settings = [
    "output-dir"         : "",
    "site-config-dir"    : "",
    "lang"               : "all",
    "sqlite-filename"    : "documents",
  ];
  auto helpInfo = getopt(args,
    std.getopt.config.passThrough,
    "abstraction",        "--abstraction document abstraction ",                                      &opts["abstraction"],
    "assert",             "--assert set optional assertions on",                                      &opts["assertions"],
    "concordance",        "--concordance file for document",                                          &opts["concordance"],
    "dark",               "--dark alternative dark theme",                                            &opts["dark"],
    "debug",              "--debug",                                                                  &opts["debug"],
    "digest",             "--digest hash digest for each object",                                     &opts["digest"],
    "epub",               "--epub process epub output",                                               &opts["epub"],
    "html",               "--html process html output",                                               &opts["html"],
    "html-seg",           "--html-seg process html output",                                           &opts["html-seg"],
    "html-scroll",        "--html-seg process html output",                                           &opts["html-scroll"],
    "light",              "--light default light theme",                                              &opts["light"],
    "manifest",           "--manifest process manifest output",                                       &opts["manifest"],
    "ocn",                "--ocn object cite numbers (default)",                                      &opts["ocn"],
    "parallel",           "--parallel parallelisation",                                               &opts["parallel"],
    "parallel-subprocesses", "--parallel-subprocesses nested parallelisation",                        &opts["parallel-subprocesses"],
    "quiet|q",            "--quiet output to terminal",                                               &opts["quiet"],
    "pod",                "--pod doc reform pod source content bundled",                              &opts["pod"],
    "serial",             "--serial serial processing",                                               &opts["serial"],
    "source",             "--source markup source text content",                                      &opts["source"],
    "sqlite-discrete",    "--sqlite process discrete sqlite output",                                  &opts["sqlite-discrete"],
    "sqlite-db-create",   "--sqlite-db-create create db, create tables",                              &opts["sqlite-db-create"],
    "sqlite-db-drop",     "--sqlite-db-drop drop tables & db",                                        &opts["sqlite-db-drop"],
    "sqlite-db-recreate", "--sqlite-db-recreate create db, create tables",                            &opts["sqlite-db-recreate"],
    "sqlite-delete",      "--sqlite-delete process sqlite output",                                    &opts["sqlite-delete"],
    "sqlite-insert",      "--sqlite-insert process sqlite output",                                    &opts["sqlite-insert"],
    "sqlite-update",      "--sqlite-update process sqlite output",                                    &opts["sqlite-update"],
    "text",               "--text process text output",                                               &opts["text"],
    "txt",                "--txt process text output",                                                &opts["text"],
    "verbose|v",          "--verbose output to terminal",                                             &opts["verbose"],
    "very-verbose",       "--very-verbose output to terminal",                                        &opts["very-verbose"],
    "xhtml",              "--xhtml process xhtml output",                                             &opts["xhtml"],
    "section-toc",        "--section-toc process table of contents (default)",                        &opts["section_toc"],
    "section-body",       "--section-body process document body (default)",                           &opts["section_body"],
    "section-endnotes",   "--section-endnotes process document endnotes (default)",                   &opts["section_endnotes"],
    "section-glossary",   "--section-glossary process document glossary (default)",                   &opts["section_glossary"],
    "section-biblio",     "--section-biblio process document biblio (default)",                       &opts["section_biblio"],
    "section-bookindex",  "--section-bookindex process document bookindex (default)",                 &opts["section_bookindex"],
    "section-blurb",      "--section-blurb process document blurb (default)",                         &opts["section_blurb"],
    "backmatter",         "--section-backmatter process document backmatter (default)",               &opts["backmatter"],
    "skip-output",        "--skip-output",                                                            &opts["skip-output"],
    "theme-dark",         "--theme-dark alternative dark theme",                                      &opts["theme-dark"],
    "theme-light",        "--theme-light default light theme",                                        &opts["theme-light"],
    "workon",             "--workon (reserved for some matters under development & testing)",         &opts["workon"],
    "output-dir",         "--output-dir=[dir path]",                                                  &settings["output-dir"],
    "site-config-dir",    "--site-config-dir=[dir path]",                                             &settings["site-config-dir"],
    "sqlite-filename",    "--sqlite-filename=[filename].sqlite",                                      &settings["sqlite-filename"],
    "lang",               "--lang=[lang code e.g. =en or =en,es]",                                    &settings["lang"],
  );
  if (helpInfo.helpWanted) {
    defaultGetoptPrinter("Some information about the program.", helpInfo.options);
  }
  enum outTask { pod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff }
  struct OptActions {
    bool assertions() {
      return opts["assertions"];
    }
    bool concordance() {
      return opts["concordance"];
    }
    bool css_theme_default() {
      bool _is_light;
      if (opts["light"] || opts["theme-light"]) {
        _is_light = true;
      } else if (opts["dark"] || opts["theme-dark"]) {
        _is_light = false;
      } else {
        _is_light = true;
      }
      return _is_light;
    }
    bool debug_do() {
      return opts["debug"];
    }
    bool digest() {
      return opts["digest"];
    }
    bool epub() {
      return opts["epub"];
    }
    bool html() {
      bool _is;
      if ( opts["html"] || opts["html-seg"] || opts["html-scroll"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool html_seg() {
      bool _is;
      if ( opts["html"] || opts["html-seg"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool html_scroll() {
      bool _is;
      if ( opts["html"] || opts["html-scroll"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool html_stuff() {
      bool _is;
      if (opts["html"] || opts["html-scroll"] || opts["html-seg"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool manifest() {
      return opts["manifest"];
    }
    bool ocn() {
      return opts["ocn"];
    }
    bool quiet() {
      return opts["quiet"];
    }
    bool pod() {
      return opts["pod"];
    }
    bool source() {
      return opts["source"];
    }
    bool sqlite_discrete() {
      return opts["sqlite-discrete"];
    }
    bool sqlite_db_drop() {
      bool _is;
      if ( opts["sqlite-db-recreate"] || opts["sqlite-db-drop"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool sqlite_db_create() {
      bool _is;
      if ( opts["sqlite-db-recreate"] || opts["sqlite-db-create"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool sqlite_delete() {
      return opts["sqlite-delete"];
    }
    bool sqlite_update() {
      bool _is;
      if (opts["sqlite-update"] || opts["sqlite-insert"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool sqlite_shared_db_action() {
      bool _is;
      if (opts["sqlite-db-recreate"]
        || opts["sqlite-db-create"]
        || opts["sqlite-delete"]
        || opts["sqlite-insert"]
        || opts["sqlite-update"]
      ) { _is = true; } else { _is = false; }
      return _is;
    }
    bool text() {
      return opts["text"];
    }
    bool verbose() {
      bool _is;
      if (opts["verbose"] || opts["very-verbose"])
        { _is = true; } else { _is = false; }
      return _is;
    }
    bool very_verbose() {
      return opts["very-verbose"];
    }
    bool xhtml() {
      return opts["xhtml"];
    }
    bool section_toc() {
      return opts["section_toc"];
    }
    bool section_body() {
      return opts["section_body"];
    }
    bool section_endnotes() {
      return opts["section_endnotes"];
    }
    bool section_glossary() {
      return opts["section_glossary"];
    }
    bool section_biblio() {
      return opts["section_biblio"];
    }
    bool section_bookindex() {
      return opts["section_bookindex"];
    }
    bool section_blurb() {
      return opts["section_blurb"];
    }
    bool backmatter() {
      return opts["backmatter"];
    }
    bool skip_output() {
      return opts["skip-output"];
    }
    bool workon() {
      return opts["workon"];
    }
    auto languages_set() {
      return settings["lang"].split(",");
    }
    auto output_dir_set() {
      return settings["output-dir"];
    }
    auto sqlite_filename() {
      return settings["sqlite-filename"];
    }
    bool parallelise() {
      bool _is;
      if (opts["parallel"] == true) {
        _is = true;
        if (sqlite_shared_db_action) { _is = false; }
      } else if (opts["parallel"] == false
      && opts["serial"] == true) {
        _is = false;
      } else if (opts["abstraction"]
        || concordance
        || epub
        || html
        || manifest
        || pod
        || source
        || sqlite_discrete
      ) {
        _is = true;
      } else { _is = false; }
      return _is;
    }
    bool parallelise_subprocesses() {
      return opts["parallel-subprocesses"];
    }
    auto output_task_scheduler() {
      int[] schedule;
      if (pod) {
        schedule ~= outTask.pod;
      }
      if (source) {
        schedule ~= outTask.source;
      }
      if (sqlite_discrete) {
        schedule ~= outTask.sqlite;
      }
      if (epub) {
        schedule ~= outTask.epub;
      }
      if (html_scroll) {
        schedule ~= outTask.html_scroll;
      }
      if (html_seg) {
        schedule ~= outTask.html_seg;
      }
      if (html_stuff) {
        schedule ~= outTask.html_stuff;
      }
      return schedule.sort().uniq;
    }
    bool abstraction() {
      bool _is;
      if (opts["abstraction"]
        || concordance
        || epub
        || html
        || manifest
        || pod
        || source
        || sqlite_discrete
        || sqlite_delete
        || sqlite_update
      ) { _is = true; } else { _is = false; }
      return _is;
    }
  }
  auto _opt_action = OptActions();
  auto program_info() {
    struct ProgramInfo {
      string name() {
        return program_name;
      }
      string ver() {
        string ver_ = format(
          "%s.%s.%s",
          _ver.major, _ver.minor, _ver.patch,
        );
        return ver_;
      }
    }
    return ProgramInfo();
  }
  auto _env = [
    "pwd" :     environment["PWD"],
    "home" :    environment["HOME"],
  ];
  auto _manifest_start = PodManifest!()("");
  auto _manifest_matter = PathMatters!()(_opt_action, _env, "");
  auto _manifests = [ _manifest_matter ];
  foreach(arg; args[1..$]) {
    _manifest_start = PodManifest!()(arg);
    if (arg.match(rgx.flag_action)) {
      flag_action ~= " " ~ arg;   // flags not taken by getopt
    } else if (
      !(arg.match(rgx.src_pth_sst_or_ssm))
      && _manifest_start.pod_manifest_file_with_path
      && _opt_action.abstraction
    ) {
      string contents_location_raw_;
      string contents_location_;
      string sisudoc_txt_ = _manifest_start.pod_manifest_file_with_path;
      enforce(
        exists(sisudoc_txt_)!=0,
        "file not found: «" ~
        sisudoc_txt_ ~ "»"
      );
      if (exists(sisudoc_txt_)) {
        try {
          if (exists(sisudoc_txt_)) {
            contents_location_raw_ = sisudoc_txt_.readText;
          }
        } catch (ErrnoException ex) {
        } catch (FileException ex) {
          // Handle errors
        }
        if (contents_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+
          foreach (m; contents_location_raw_.matchAll(rgx.pod_content_location)) {
            foreach (n; m.captures[2].matchAll(rgx.language_codes)) {
              contents_location_ ~= "media/text/" ~ n.captures[1].to!string ~ "/" ~ m.captures[1].to!string ~ "\n";
            }
          }
        } else {
          contents_location_ = contents_location_raw_;
        }
      } else {
        writeln("manifest not found: ", sisudoc_txt_);
      }
      auto contents_locations_arr
        = (cast(char[]) contents_location_).split;
      auto tmp_dir_ = (sisudoc_txt_).dirName.array;
      foreach (contents_location; contents_locations_arr) {
        assert(contents_location.match(rgx.src_pth_sst_or_ssm),
          "not a recognised file: «" ~
          contents_location ~ "»"
        );
        auto contents_location_pth_ = (contents_location).to!string;
        Regex!(char) lang_rgx_ = regex(r"/(" ~ _opt_action.languages_set.join("|") ~ ")/");
        if (_opt_action.languages_set[0] == "all"
          || (contents_location_pth_).match(lang_rgx_)
        ) {
          auto _fns = (((tmp_dir_).chainPath(contents_location_pth_)).array).to!string;
          _manifest_matter = PathMatters!()(_opt_action, _env, arg, _fns, contents_locations_arr);
          _manifests ~= _manifest_matter;
        }
      }
    } else if (arg.match(rgx.src_pth_sst_or_ssm)) {
      if (exists(arg)==0) {
        writeln("ERROR >> Processing Skipped! File not found: ", arg);
      } else {
        _manifest_matter = PathMatters!()(_opt_action, _env, arg, arg);
        _manifests ~= _manifest_matter;
      }
    } else if (arg.match(rgx.src_pth_zip)) {
      // fns_src ~= arg;          // gather input markup source file names for processing
    } else {                      // anything remaining, unused
      arg_unrecognized ~= " " ~ arg;
    }
  }
  if (!(_opt_action.skip_output)) {
    if ((_opt_action.debug_do)
    || (_opt_action.very_verbose)
    ) {
      writeln("step0 commence → (without processing files)");
    }
    outputHubOp!()(_env, _opt_action);
    if ((_opt_action.debug_do)
    || (_opt_action.very_verbose)
    ) {
      writeln("- step0 complete");
    }
  }
  if (_manifests.length > 1                            // _manifests[0] initialized dummy element
  && _opt_action.abstraction) {
    if (_opt_action.parallelise) {                     // note cannot parallelise sqlite shared db
      foreach(manifest; parallel(_manifests[1..$])) {
        if (!empty(manifest.src.filename)) {
          scope(success) {
            if (!(_opt_action.quiet)) {
              writefln(
                "%s",
                "-- ~ document complete, ok ~ ------------------------------------",
              );
            }
          }
          scope(failure) {
            debug(checkdoc) {
              stderr.writefln(
                "~ document run failure ~ (%s  v%s)\n\t%s\n%s",
                __VENDOR__, __VERSION__,
                manifest.src.filename,
                "------------------------------------------------------------------",
              );
            }
          }
          enforce(
            manifest.src.filename.match(rgx.src_pth_types),
            "not a sisu markup filename: «" ~
            manifest.src.filename ~ "»"
          );
          if ((_opt_action.debug_do)
          || (_opt_action.very_verbose)
          ) {
            writeln("--->\nstepX commence → (document abstraction)");
          }
          auto t = DocReformAbstraction!()(_env, program_info, _opt_action, manifest);
          static assert(!isTypeTuple!(t));
          static assert(t.length==2);
          auto doc_abstraction = t[dAM.abstraction];
          auto doc_matters = t[dAM.matters];
          if ((doc_matters.opt.action.debug_do)
          || (doc_matters.opt.action.very_verbose)
          ) {
            writeln("- stepX complete");
          }
          /+ ↓ debugs +/
          if (doc_matters.opt.action.verbose) {
            DocReformAbstractionSummary!()(doc_abstraction, doc_matters);
          }
          /+ ↓ debugs +/
          if (doc_matters.opt.action.debug_do) {
            DocReformDebugs!()(doc_abstraction, doc_matters);
          }
          /+ ↓ output hub +/
          if (!(doc_matters.opt.action.skip_output)) {
            if ((_opt_action.debug_do)
            || (_opt_action.very_verbose)
            ) {
              writeln("step5 commence → (process outputs)");
            }
            doc_abstraction.outputHub!()(doc_matters);
            if ((_opt_action.debug_do)
            || (_opt_action.very_verbose)
            ) {
              writeln("- step5 complete");
            }
          }
          scope(exit) {
            if (!(_opt_action.quiet)) {
              writefln(
                "processed file: %s (%s)",
                manifest.src.filename,
                manifest.src.language
              );
            }
            destroy(manifest);
          }
        } else {
          /+ no recognized filename provided +/
          writeln("no recognized filename");
          break; // terminate, stop
        }
      }
    } else {
      foreach(manifest; _manifests[1..$]) {
        writeln("parallelisation off: actions include sqlite shared db");
        if (!empty(manifest.src.filename)) {
          scope(success) {
            if (!(_opt_action.quiet)) {
              writefln(
                "%s",
                "-- ~ document complete, ok ~ ------------------------------------",
              );
            }
          }
          scope(failure) {
            debug(checkdoc) {
              stderr.writefln(
                "~ document run failure ~ (%s  v%s)\n\t%s\n%s",
                __VENDOR__, __VERSION__,
                manifest.src.filename,
                "------------------------------------------------------------------",
              );
            }
          }
          enforce(
            manifest.src.filename.match(rgx.src_pth_types),
            "not a sisu markup filename: «" ~
            manifest.src.filename ~ "»"
          );
          if ((_opt_action.debug_do)
          || (_opt_action.very_verbose)
          ) {
            writeln("--->\nstepX commence → (document abstraction)");
          }
          auto t = DocReformAbstraction!()(_env, program_info, _opt_action, manifest);
          static assert(!isTypeTuple!(t));
          static assert(t.length==2);
          auto doc_abstraction = t[dAM.abstraction];
          auto doc_matters = t[dAM.matters];
          if ((doc_matters.opt.action.debug_do)
          || (doc_matters.opt.action.very_verbose)
          ) {
            writeln("- stepX complete");
          }
          /+ ↓ debugs +/
          if (doc_matters.opt.action.verbose) {
            DocReformAbstractionSummary!()(doc_abstraction, doc_matters);
          }
          /+ ↓ debugs +/
          if (doc_matters.opt.action.debug_do) {
            DocReformDebugs!()(doc_abstraction, doc_matters);
          }
          /+ ↓ output hub +/
          if (!(doc_matters.opt.action.skip_output)) {
            if ((_opt_action.debug_do)
            || (_opt_action.very_verbose)
            ) {
              writeln("step5 commence → (process outputs)");
            }
            doc_abstraction.outputHub!()(doc_matters);
            if ((_opt_action.debug_do)
            || (_opt_action.very_verbose)
            ) {
              writeln("- step5 complete");
            }
          }
          scope(exit) {
            if (!(_opt_action.quiet)) {
              writefln(
                "processed file: %s (%s)",
                manifest.src.filename,
                manifest.src.language
              );
            }
            destroy(manifest);
          }
        } else {
          /+ no recognized filename provided +/
          writeln("no recognized filename");
          break; // terminate, stop
        }
      }
    }
  }
}