path: root/src/doc_reform/io_out/source_pod.d

/+
- Name: Spine, Doc Reform [a part of]
  - Description: documents, structuring, processing, publishing, search
    - static content generator

  - Author: Ralph Amissah
    [ralph.amissah@gmail.com]

  - Copyright: (C) 2015 - 2021 Ralph Amissah, All Rights
    Reserved.

  - License: AGPL 3 or later:

    Spine (SiSU), a framework for document structuring, publishing and
    search

    Copyright (C) Ralph Amissah

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU AFERO General Public License as published by the
    Free Software Foundation, either version 3 of the License, or (at your
    option) any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program. If not, see [https://www.gnu.org/licenses/].

    If you have Internet connection, the latest version of the AGPL should be
    available at these locations:
    [https://www.fsf.org/licensing/licenses/agpl.html]
    [https://www.gnu.org/licenses/agpl.html]

  - Spine (by Doc Reform, related to SiSU) uses standard:
    - docReform markup syntax
      - standard SiSU markup syntax with modified headers and minor modifications
    - docReform object numbering
      - standard SiSU object citation numbering & system

  - Homepages:
    [https://www.doc_reform.org]
    [https://www.sisudoc.org]

  - Git
    [https://git.sisudoc.org/projects/?p=software/spine.git;a=summary]

+/
module doc_reform.io_out.source_pod;
template spinePod() {
  import doc_reform.io_out;
  import
    std.digest.sha,
    std.file,
    std.outbuffer,
    std.zip,
    std.conv : to;
  import
    doc_reform.io_out.create_zip_file,
    doc_reform.io_out.xmls;
  void spinePod(T)(T doc_matters) {
    debug(asserts) {
      // static assert(is(typeof(doc_matters) == tuple));
    }
    mixin spineRgxOut;
    string pwd = doc_matters.env.pwd;
    auto src_path_info = doc_matters.src_path_info;
    auto pth_dr_doc_src = doc_matters.src_path_info;
    auto pths_pod = spinePathsPods!()(doc_matters);
    mixin spineLanguageCodes;
    auto lang = Lang();
    static auto rgx = RgxO();
    assert (doc_matters.src.filename.match(rgx.src_fn));
    @system auto pod_archive(Z)(
      string _source_type,
      string _data_in,
      string _pth_out,
      Z zip
    ) {
      auto zip_arc_member_file = new ArchiveMember();
      zip_arc_member_file.name = _pth_out;
      auto zip_data = new OutBuffer();
      switch (_source_type) {
      case "file_path_bin":
        zip_data.write(cast(char[]) ((_data_in).read));
        goto default;
      case "file_path_text":
        zip_data.write((_data_in).readText);
        goto default;
      case "string":
        zip_data.write(_data_in);
        goto default;
      default:
        zip_arc_member_file.expandedData = zip_data.toBytes();
        zip.addMember(zip_arc_member_file);
      }
      return zip;
    }
    try {
      /+ create directory structure +/
      if (!exists(pths_pod.pod_dir_())) {
        // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
        pths_pod.pod_dir_().mkdirRecurse;
      }
      if (doc_matters.opt.action.source_or_pod) {
        if (!(doc_matters.opt.action.quiet)) {
          writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod);
        }
        if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) {
          pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
        }
        if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) {
          pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
        }
        if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) {
          pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
        }
        if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) {
          pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
        }
        if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) {
          pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
        }
        if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) {
          pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse;
        }
      }
      debug(pod) {
        writeln(__LINE__, ": ",
          doc_matters.src.filename, " -> ",
          pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod
        );
      }
      auto zip = new ZipArchive();
      auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
      { /+ bundle images +/
        foreach (image; doc_matters.srcs.image_list) {
          debug(podimages) {
            writeln(
              pth_dr_doc_src.image_root.to!string, "/", image, " -> ",
              pths_pod.image_root(doc_matters.src.filename).zpod, "/", image
            );
          }
          auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image;
          auto fn_src_out_pod_zip_base
            = pths_pod.image_root(doc_matters.src.filename).zpod.to!string
            ~ "/" ~ image;
          auto fn_src_out_filesystem
            = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string
            ~ "/" ~ image;
          if (exists(fn_src_in)) {
            debug(io) {
              writeln("(io debug) src out found: ", fn_src_in);
            }
            if (doc_matters.opt.action.source_or_pod) {
              fn_src_in.copy(fn_src_out_filesystem);
            }
            if (doc_matters.opt.action.pod) {
              zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
            }
          } else {
            if (doc_matters.opt.action.debug_do
            && doc_matters.opt.action.verbose) {
              writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
            }
          }
        }
      } { /+ bundle dr_document_make +/
        auto fn_src_in = ((doc_matters.src.is_pod)
          ? doc_matters.src.conf_dir_path
          : pth_dr_doc_src.conf_root).to!string
          ~ "/" ~ "dr_document_make";
        auto fn_src_out_pod_zip_base
          = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "dr_document_make";
        auto fn_src_out_filesystem
          = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string
          ~ "/" ~ "dr_document_make";
        if (exists(fn_src_in)) {
          debug(io) {
            writeln("(io debug) src out found: ", fn_src_in);
          }
          if (doc_matters.opt.action.source_or_pod) {
            fn_src_in.copy(fn_src_out_filesystem);
          }
          if (doc_matters.opt.action.pod) {
            zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
          }
        } else {
          if (doc_matters.opt.action.debug_do
          && doc_matters.opt.action.verbose) {
            writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
          }
        }
      } { /+ pod manifest +/
        auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
        auto fn_src_out_pod_zip_base
          = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string;
        auto fn_src_out_filesystem
          = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path
        auto fn_src_out_inside_pod
          = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path
        string[] filelist_src_out_pod_arr;
        string[] filelist_src_zpod_arr;
        if (exists(fn_src_in)) {
          debug(io) {
            writeln("(io debug) src in found: ", fn_src_in);
          }
          filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
          filelist_src_zpod_arr ~= fn_src_out_inside_pod;
          {
            import dyaml;
            auto pod_filelist_yaml_string
              = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
            Node _pmy;
            string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
            if (doc_matters.opt.action.debug_do
            && doc_matters.opt.action.verbose) {
              try {
                _pmy = Loader.fromString(_pm).load();
              } catch (ErrnoException ex) {
              } catch (Throwable) {
                writeln("ERROR failed to read config file content, not parsed as yaml");
              }
              writeln("pod filename:  ", _pmy["doc"]["filename"].get!string);
              writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string);
              writeln("pod languages: ", doc_matters.src.language);
              // foreach(string _l; _pmy["doc"]["language"]) {
              //   writeln("language:      ", _l);
              // }
            }
            if (doc_matters.opt.action.source_or_pod) {
              pod_filelist_yaml_string.writeln(_pm);
            }
            if (doc_matters.opt.action.pod) {
              zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip);
            }
          }
        }
      } { /+ bundle primary file (.ssm/.sst) +/
        auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
        auto fn_src_out_pod_zip_base
          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
        auto fn_src_out_filesystem
          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path:
        auto fn_src_out_inside_pod
          = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
        string[] filelist_src_out_pod_arr;
        string[] filelist_src_zpod_arr;
        if (exists(fn_src_in)) {
          debug(io) {
            writeln("(io debug) src in found: ", fn_src_in);
          }
          filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
          filelist_src_zpod_arr ~= fn_src_out_inside_pod;
          string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename;
          if (doc_matters.opt.action.source_or_pod) {
            fn_src_in.copy(fn_src_out_filesystem);
          }
          if (doc_matters.opt.action.pod) {
            auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
            if (auto _x = fn_src_in.match(_rgx)){
              if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
                string _path_to_pod = _x.captures["path_to_pod"];
                string _podname = _x.captures["podname"];
                string _root_to_lang = _x.captures["from_root"];
                string _language = _x.captures["language"];
                string _filename = _x.captures["filename"];
                foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
                  string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
                  string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
                  zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
                }
              }
            } else {
              zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
            }
          }
        } else {
          if (doc_matters.opt.action.debug_do
          && doc_matters.opt.action.verbose) {
            writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
          }
        }
      } { /+ bundle insert files (.ssi) +/
        if (doc_matters.srcs.file_insert_list.length > 0) {
          auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
          foreach (insert_file; doc_matters.srcs.file_insert_list) {
            debug(pod) {
              writeln(
                insert_file, " -> ",
                pths_pod.fn_doc_insert(
                  doc_matters.src.filename,
                  insert_file,
                  doc_matters.src.language,
                ).zpod
              );
            }
            if (auto _x = insert_file.match(_rgx)){
              if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
                string _path_to_pod = _x.captures["path_to_pod"];
                string _podname = _x.captures["podname"];
                string _root_to_lang = _x.captures["from_root"];
                string _language = _x.captures["language"];
                string _filename = _x.captures["filename"];
                foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
                  string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
                  string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
                  if (exists(_pth_mkup_src_in)) {
                    if (doc_matters.opt.action.source_or_pod) {
                      auto fn_src_out_filesystem // you need to change language sources
                        = pths_pod.fn_doc_insert(
                          doc_matters.src.filename, // doc_matters.src.filename
                          _pth_mkup_src_in, // insert_file
                          _lang,
                        ).filesystem_open_zpod.to!string;
                      _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere
                    }
                    if (doc_matters.opt.action.pod) {
                       zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
                    }
                  } else {
                    if (doc_matters.opt.action.debug_do
                    && doc_matters.opt.action.verbose) {
                      writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in);
                    }
                  }
                }
              }
            } else {
              auto fn_src_in = insert_file;
              auto fn_src_out_pod_zip_base
                = pths_pod.fn_doc_insert(
                  doc_matters.src.filename,
                  insert_file,
                  doc_matters.src.language,
                ).zpod.to!string;
              auto fn_src_out_filesystem
                = pths_pod.fn_doc_insert(
                  doc_matters.src.filename,
                  insert_file,
                  doc_matters.src.language,
                ).filesystem_open_zpod.to!string;
              if (exists(fn_src_in)) {
                debug(io) {
                  writeln("(io debug) src out found: ", fn_src_in);
                }
                if (doc_matters.opt.action.source_or_pod) {
                  fn_src_in.copy(fn_src_out_filesystem);
                }
                if (doc_matters.opt.action.pod) {
                  zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
                }
              } else {
                if (doc_matters.opt.action.debug_do
                && doc_matters.opt.action.verbose) {
                  writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
                }
              }
            }
          }
        }
      } {
        auto fn_src_in = doc_matters.src.filename;
        if (doc_matters.opt.action.pod) {
          if (exists(doc_matters.src.file_with_absolute_path)) {
            createZipFile!()(fn_pod, zip.build());
          } else {
            writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod);
          }
          if (!(exists(fn_pod))) {
            writeln("WARNING failed to create pod zip archive: ", fn_pod);
          }
        }
      }
      if (exists(fn_pod)) {
        try {
          if (!(doc_matters.opt.action.quiet)
          && doc_matters.opt.action.pod) {
            auto data = (cast(byte[]) (fn_pod).read);
            if (doc_matters.opt.action.verbose) {
              writeln(" ", doc_matters.src.filename, " > ");
            }
            if (doc_matters.opt.action.pod) {
              writefln(" %s\n   %s %-(%02x%)", fn_pod, "sha256", data.sha256Of);
            }
          }
          debug(pod) {
            try {
              auto zipped = new ZipArchive((fn_pod).read);
              foreach (filename, member; zipped.directory) {
                auto data = zipped.expand(member);
                writeln("> ", filename, " length ", data.length);
              }
            } catch (ZipException ex) {
              // Handle errors
            }
            if (doc_matters.src.filename == "dr_doc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") {
              assert(
                ((data).sha256Of).toHexString
                == "626F83A31ED82F42CF528E922C1643498A137ABA3F2E5AFF8A379EA79EA22A1E",
                "\npod: sha256 value for "
                ~ doc_matters.src.filename
                ~ " has changed, is now: "
                ~ ((data).sha256Of).toHexString
              );
            }
            if (doc_matters.src.filename == "dr_doc/media/text/en/sisu_markup_stress_test.sst") {
              assert(
                ((data).sha256Of).toHexString
                == "AAE0C87AB3F6D5F7385AEEA6EE661F56D40475CFE87AD930C78C9FE07FFB0D91",
                "\npod: sha256 value for "
                ~ doc_matters.src.filename
                ~ " has changed, is now: "
                ~ ((data).sha256Of).toHexString
              );
            }
          }
        } catch (ErrnoException ex) {
          // Handle errors
        }
      }
      // source pod zip
    } catch (ErrnoException ex) {
      // Handle error
    }
  }
}