path: root/org/sdp.org

#+TITLE: sdp hub
#+AUTHOR: Ralph Amissah
#+EMAIL: ralph.amissah@gmail.com
#+STARTUP: indent
#+LANGUAGE: en
#+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
#+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc
#+OPTIONS: author:nil email:nil creator:nil timestamp:nil
#+PROPERTY: header-args :padline no :exports code :noweb yes
#+EXPORT_SELECT_TAGS: export
#+EXPORT_EXCLUDE_TAGS: noexport
#+FILETAGS: :sdp:rel:hub:
#+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n)

[[../maker.org][maker.org makefile]]  [[./][org/]]

* TODO version.txt: set version (sisu document parser)              :version:

#+NAME: version_txt
#+BEGIN_SRC d  :tangle ../views/version.txt
/+ obt - org generated file +/
struct Version {
  int major;
  int minor;
  int patch;
}
enum ver = Version(0, 7, 2);
#+END_SRC

* sdp.d   sisu document parser                                       :sdp.d:
- deal with imports
- get options
  - get command line instructions
  - read config instructions
- process files as instructed by options
  - read in file
  - proess file
  - output

** pre loop init                                                       :init:
*** imports                                                          :import:
**** sdp                                                         :import:sdp:

#+NAME: imports_sdp
#+BEGIN_SRC d
/+ sdp: sisu document parser, see http://sisudoc.org +/
import
  compile_time_info,          // sdp/compile_time_info.d
  ao_abstract_doc_source,     // sdp/ao_abstract_doc_source.d
  ao_conf_make_meta,          // sdp/ao_conf_make_meta.d
  ao_conf_make_meta_native,   // sdp/ao_conf_make_meta_native.d
  ao_conf_make_meta_sdlang,   // sdp/ao_conf_make_meta_sdlang.d
  ao_defaults,                // sdp/ao_defaults.d
  ao_output_debugs,           // sdp/ao_output_debugs.d
  ao_read_config_files,       // sdp/ao_read_config_files.d
  ao_read_source_files,       // sdp/ao_read_source_files.d
  ao_rgx,                     // sdp/ao_rgx.d
  output_hub,                 // output_hub.d
  output_html;                // output_html.d
#+END_SRC

***** notes
├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
└── views
    └── version.txt

[[./compile_time_info.org][compile time info]]
[[./ao_abstract_doc_source.org][ao_abstract_doc_source]]
[[./ao_conf_make_meta.org][ao_onf_make_meta]]
[[./ao_conf_make_meta_native.org][ao_onf_make_meta_native]]
[[./ao_conf_make_meta_sdlang.org][ao_onf_make_meta_sdlang]]
[[./ao_defaults.org][ao_defaults]]
[[./ao_output_debugs.org][ao_output_debugs]]
[[./ao_read_config_files.org][ao_read_config_files]]
[[./ao_read_source_files.org][ao_read_source_files]]
[[./ao_read_source_files.org][ao_read_source_files]]
[[./ao_rgx.org][ao_rgx]]
[[./output_hub.org][output hub]]

**** TODO sdlang                                              :import:sdlang:
keep up to date, configuration in ../maker.org
check:
- http://github.com/Abscissa/SDLang-D
- https://github.com/abscissa/libInputVisitor

#+NAME: imports_sdlang
#+BEGIN_SRC d
/+ sdlang http://sdlang.org +/
import sdlang;                            // sdlang.d
#+END_SRC

***** notes
sdlang.parser,                       // sdlang/parser.d
sdlang.exceptions;
// std.conv;

**** std                                                         :import:std:

#+NAME: imports_std
#+BEGIN_SRC d
/+ std +/
private import
  std.algorithm,
  std.array,
  std.container,
  std.exception,
  std.getopt,
  std.json,
  std.process,
  std.stdio,
  std.file,
  std.range,
  std.regex,
  std.string,
  std.traits,
  std.typecons,
  std.utf,
  std.conv : to;
#+END_SRC

***** notes
std.conv,
std.path,
std.stdio,
std.variant,

**** mixins
***** version.txt                                                   :version:

#+NAME: sdp_version_mixin
#+BEGIN_SRC d
mixin(import("version.txt"));
#+END_SRC

***** std mixins
#+NAME: std_mixins
#+BEGIN_SRC d
mixin CompileTimeInfo;
mixin RgxInit;
#+END_SRC

*** mixin                                                             :mixin:

#+NAME: sdp_mixin
#+BEGIN_SRC d
mixin SiSUregisters;
mixin SiSUheaderExtractHub;
mixin SiSUheaderExtractSDLang;
mixin SiSUbiblio;
mixin SiSUrgxInitFlags;
mixin SiSUconfigSDLangHub;
mixin SiSUmarkupRaw;
mixin SiSUdocAbstraction;
mixin SiSUoutputDebugs;
mixin SiSUoutputHub;
#+END_SRC

*** init                                                               :init:

#+NAME: sdp_args
#+BEGIN_SRC d
auto raw = MarkupRaw();
auto head = HeaderDocMetadataAndMake();
auto abs = Abstraction();
auto dbg = SDPoutputDebugs();
auto output = SDPoutput();
/+
struct DocumentParts {
  string[string][] contents;
  string[string][string] meta_aa;
  string[string][string] make_aa;
  string[][string][string] bookindex_unordered_hashes;
  JSONValue[] biblio;
}
+/
string[] fns_src;
string flag_action;
string arg_unrecognized;
auto rgx = Rgx();
#+END_SRC

*** scope (run complete)                                              :scope:

#+NAME: sdp_args
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)",
      ver.major, ver.minor, ver.patch,
      __VENDOR__, __VERSION__,
      bits, os,
    );
  }
  // stderr.writeln("0");
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "run failure",
    );
  }
}
#+END_SRC

*** getopt args for loop                                        :args:getopt:

look into using getopt
[[http://dlang.org/phobos/std_getopt.html][getopt]]
[[http://dlang.org/library/std/getopt.html][getopt]]

#+NAME: sdp_args
#+BEGIN_SRC d
bool[string] opt_action_bool = [
  "assertions"      : false,
  "concordance"     : false,
  "digest"          : false,
  "docbook"         : false,
  "epub"            : false,
  "html"            : false,
  "manifest"        : false,
  "no_ocn"          : false,
  "odt"             : false,
  "pdf"             : false,
  "postgresql"      : false,
  "qrcode"          : false,
  "sisupod"         : false,
  "source"          : false,
  "sqlite"          : false,
  "text"            : false,
  "verbose"         : false,
  "xhtml"           : false,
  "xml_dom"         : false,
  "xml_sax"         : false,
];
auto helpInfo = getopt(args,
  std.getopt.config.passThrough,
  "assert",         "--assert set optional assertions on",       &opt_action_bool["assertions"],
  "concordance",    "--concordance file for document",           &opt_action_bool["concordance"],
  "digest",         "--digest hash digest for each object",      &opt_action_bool["digest"],
  "docbook",        "--docbook process docbook output",          &opt_action_bool["docbook"],
  "epub",           "--epub process epub output",                &opt_action_bool["epub"],
  "html",           "--html process html output",                &opt_action_bool["html"],
  "manifest",       "--manifest process manifest output",        &opt_action_bool["manifest"],
  "no-ocn",         "--no-ocn suppress object cite numbers",     &opt_action_bool["no_ocn"],
  "odf",            "--odf process odf:odt output",              &opt_action_bool["odt"],
  "odt",            "--odt process odf:odt output",              &opt_action_bool["odt"],
  "pdf",            "--pdf process pdf output",                  &opt_action_bool["pdf"],
  "pg",             "--pg process postgresql output",            &opt_action_bool["postgresql"],
  "postgresql",     "--postgresql process postgresql output",    &opt_action_bool["postgresql"],
  "qrcode",         "--qrcode with document metadata",           &opt_action_bool["qrcode"],
  "sisupod",        "--sisupod sisupod source content bundled",  &opt_action_bool["sisupod"],
  "source",         "--source markup source text content",       &opt_action_bool["source"],
  "sqlite",         "--sqlite process sqlite output",            &opt_action_bool["sqlite"],
  "text",           "--text process text output",                &opt_action_bool["text"],
  "txt",            "--txt process text output",                 &opt_action_bool["text"],
  "verbose|v",      "--verbose output to terminal",              &opt_action_bool["verbose"],
  "xhtml",          "--xhtml process xhtml output",              &opt_action_bool["xhtml"],
  "xml-dom",        "--xml-dom process xml dom output",          &opt_action_bool["xml_dom"],
  "xml-sax",        "--xml-sax process xml sax output",          &opt_action_bool["xml_sax"],
);
if (helpInfo.helpWanted) {
  defaultGetoptPrinter("Some information about the program.", helpInfo.options);
}
foreach(arg; args) {
  if (match(arg, rgx.flag_action)) {
    flag_action ~= " " ~ arg;   // flags not taken by getopt
  } else if (match(arg, rgx.src_pth)) {
    fns_src ~= arg;             // gather input markup source file names for processing
  } else {                      // anything remaining, unused
    arg_unrecognized ~= " " ~ arg;
  }
}
#+END_SRC

*** TODO config files (load & read) (so far only SDLang)       :config:files:

#+NAME: sdp_conf_files
#+BEGIN_SRC d
auto conf = ConfigHub();
auto sdl_root_configuration = conf.configSDLang("conf.sdl");
auto sdl_root_doc_make = conf.configSDLang("sisu_document_make");
auto confsdl = HeaderExtractSDL();
auto conf_settings_aa = confsdl.configSettingsSDLangToAAmake(sdl_root_configuration);
auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make);
#+END_SRC

** each file (loop) [+2]                                         :loop:files:
*** filename provided [+1]                                     :file:process:
**** scope (loop)                                                     :scope:

#+NAME: sdp_each_file_do_scope
#+BEGIN_SRC d
scope(success) {
  debug(checkdoc) {
    writefln(
      "%s\n%s",
      "~ document complete, ok ~",
      "-------------------------------",
    );
  }
  // stderr.writeln("0");
}
scope(failure) {
  debug(checkdoc) {
    stderr.writefln(
      "~ document run failure ~ (%s  v%s)\n\t%s",
      __VENDOR__, __VERSION__,
      fn_src
    );
  }
  // stderr.writeln("1");
}
enforce(
  match(fn_src, rgx.src_pth),
  "not a sisu markup filename"
);
#+END_SRC

**** [#A] read in raw file (using filename with path)             :file:read:
[[./ao_markup_source_raw.org][ao_markup_source_raw]]

#+NAME: sdp_each_file_do_read_sisu_markup_file
#+BEGIN_SRC d
/+ ↓ read file +/
auto read_in_file_string = raw.sourceContent(fn_src);
#+END_SRC

**** [#A] split raw content file into: doc header & doc content   :file:read:

#+NAME: sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body
#+BEGIN_SRC d
/+ ↓ file tuple of header and content +/
auto header_and_body_tuple = raw.sourceContentSplitIntoHeaderAndBody(read_in_file_string, fn_src);
auto header = header_and_body_tuple[0];
auto content_body = header_and_body_tuple[1];
debug(header_and_body) {
  writeln(header);
  writeln(header_and_body_tuple.length);
  writeln(content_body[0]);
}
#+END_SRC

**** [#A] split doc header into: metadata & make   :doc:header:metadata:make:

#+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta
#+BEGIN_SRC d
/+ ↓ split header into make and meta +/
auto header_make_and_meta_tuple = head.headerContentAA(header, conf_doc_make_aa);
static assert(!isTypeTuple!(header_make_and_meta_tuple));
string[string][string] dochead_make = header_make_and_meta_tuple[0];
string[string][string] dochead_meta = header_make_and_meta_tuple[1];
#+END_SRC

**** [#A] processing: document abstraction, tuple                :processing:

#+NAME: sdp_each_file_do_document_abstraction
#+BEGIN_SRC d
/+ ↓ document abstraction: process document, return abstraction as tuple +/
auto t = abs.abstract_doc_source(content_body, dochead_make, dochead_meta);
static assert(!isTypeTuple!(t));
auto doc_ao_contents = t[0]; // contents ~ endnotes ~ bookindex;
// static assert(!isIterable!(doc_ao_contents));
auto doc_ao_bookindex_unordered_hashes = t[1];
auto doc_ao_biblio = t[2];
// destroy(t);
#+END_SRC

**** TODO debug (document parts, checkdoc)                   :debug:checkdoc:

#+NAME: sdp_each_file_do_debugs_checkdoc
#+BEGIN_SRC d
/+ ↓ debugs +/
debug(checkdoc) { // checkbook & dumpdoc
  dbg.abstract_doc_source_debugs(
    doc_ao_contents,
    doc_ao_bookindex_unordered_hashes,
    doc_ao_biblio,
    dochead_make,
    dochead_meta,
    fn_src,
    opt_action_bool
  );
}
#+END_SRC

**** TODO process outputs                                           :outputs:

#+NAME: sdp_each_file_do_selected_output
#+BEGIN_SRC d
/+ ↓ output hub +/
output.hub(
  doc_ao_contents,
  doc_ao_bookindex_unordered_hashes,
  doc_ao_biblio,
  dochead_make,
  dochead_meta,
  fn_src,
  opt_action_bool
);
#+END_SRC

**** scope (on loop exit)                                        :scope:exit:

#+NAME: sdp_each_file_do_scope_exit
#+BEGIN_SRC d
scope(exit) {
  debug(checkdoc) {
    writefln(
      "processed file: %s",
      fn_src
    );
  }
  destroy(content_body);
  destroy(t);
  destroy(doc_ao_contents);
  // destroy(doc_ao_make_aa);
  // destroy(doc_ao_meta_aa);
  // destroy(doc_ao_bookindex_unordered_hashes);
  destroy(doc_ao_biblio);
  destroy(fn_src);
}
#+END_SRC

*** no filename provided
#+NAME: sdp_no_filename_provided
#+BEGIN_SRC d
/+ no recognized filename provided +/
writeln("no recognized filename");
break;
// terminate, stop
#+END_SRC

* tangles (code structure)                                           :tangle:
** TODO sdp src/sdp.d                                               :sdp.d:

#+BEGIN_SRC d  :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd
/+
  sdp
+/
<<imports_sdp>>
<<imports_sdlang>>
<<imports_std>>
<<sdp_output_selection>>
<<sdp_version_mixin>>
<<std_mixins>>
/++ A SiSU document parser writen in D. +/
void main(string[] args) {
  <<sdp_mixin>>
  <<sdp_args>>
  <<sdp_conf_files>>
  foreach(fn_src; fns_src) {
    if (!empty(fn_src)) {
      <<sdp_each_file_do_scope>>
      <<sdp_each_file_do_read_sisu_markup_file>>
      <<sdp_each_file_do_split_sisu_markup_file_content_into_header_and_body>>
      <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>>
      <<sdp_each_file_do_document_abstraction>>
      <<sdp_each_file_do_debugs_checkdoc>>
      <<sdp_each_file_do_selected_output>>
      <<sdp_each_file_do_scope_exit>>
    } else {
      <<sdp_no_filename_provided>>
    }
  }
}
/// sdp sisu document parser
unittest {
  /++
	name        "sdp"
	description "A SiSU document parser writen in D."
	homepage    "http://sisudoc.org"
  +/
}
#+END_SRC

├── src
│   ├── sdp.d
│   └── sdp
│       ├── ao_abstract_doc_source.d
│       ├── ...
│       └── compile_time_info.d
├── views
│   └── version.txt

├── src
│   ├── sdp
│   │   ├── ao_abstract_doc_source.d
│   │   ├── ...
│   │   └── compile_time_info.d
│   └── sdp.d
├── views
│   └── version.txt

* TODO work on
** program dir structure
figure out best program dir structure for dub and compilers, issue with rdmd

** sisu document structure

|---------------------+------------------------------------------+------------------------+------------------------+--------|
|                     |                                          |                        | links                  |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| header              | sisu /header markup/                       | markup                 |                        |        |
| - metadata          |                                          |                        |                        |        |
| - make instructions |                                          |                        |                        |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| table of contents   | markup of headings                       | (regular content)      | scroll  #[ocn]         | output |
|                     |                                          |                        | seg     ../[fn]/#[ocn] |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| substantive content | sisu /content markup/                      | markup                 | internal               | output |
|                     | headings (providing document structure), |                        | scroll  #[ocn]         |        |
|                     | paragraphs, blocks                       | (regular content)      | seg     ../[fn]/#[ocn] |        |
|                     | blocks (code, poem, group, table)        |                        | footnotes              |        |
|                     |                                          |                        | seg     #[ocn]         |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| endnotes            | markup within substantive content        | markup                 | scroll  #[ocn]         | output |
|                     | (extracted from sisu /content markup/)     | (from regular content) | seg     ../[fn]/#[ocn] |        |
|                     |                                          |                        | crosslinks             |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| glossary            | identify special section                 | markup                 |                        | output |
|                     | regular /content markup/                   |                        |                        |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| bibliography        | identify section,                        | markup (special)       |                        | output |
|                     | special /bibliography markup/              |                        |                        |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| book index          | extracted from markup attached to        | markup                 | scroll  #[ocn]         | output |
|                     | related substantive content objects      |                        | seg     ../[fn]/#[ocn] |        |
|                     | (special tags in sisu /content markup/)    | (from regular content) |                        |        |
|---------------------+------------------------------------------+------------------------+------------------------+--------|
| metadata            |                                          | (from regular header)  |                        | output |
|---------------------+------------------------------------------+------------------------+------------------------+--------|

- header

- table of contents
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- substantive contents
  contents
  footnotes
  - seg               #[ocn]
  internal links
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- endnotes
  - scroll
  - seg               ../[fn]/#[ocn]
- glossary
- bibliography
- book index
  - scroll            #[ocn]
  - seg               ../[fn]/#[ocn]
- metadata

** config                                                            :config:
using sdlang in sdp

*** sdp config and header? file format? sdl ? yml ? json ?       :sdl:sdlang:

[[https://sdlang.org/][SDL: Simple Declarative Language]]  [[http://sdl4r.rubyforge.org/syntaxhighlighter_brush.html][highlighter]]
https://github.com/Abscissa/SDLang-D
https://github.com/Abscissa/SDLang-D/blob/master/HOWTO.md

**** build/ compile

The recommended way to use SDLang-D is via DUB. Just add a dependency to
sdlang-d in your project's dub.json or dub.sdl file as shown here. Then simply
build your project with DUB as usual. dub dependency
http://code.dlang.org/packages/sdlang-d


Alternatively, you can git clone both SDLang-D and the latest version of
libInputVisitor,

#+BEGIN_SRC d :tangle no
dub fetch sdlang-d
git clone https://github.com/Abscissa/SDLang-D
git clone https://github.com/abscissa/libInputVisitor
#+END_SRC

 and when running the compiler include:

#+BEGIN_SRC d :tangle no
 -I{path to SDLang-D}/src -I{path to libInputVisitor}/src
#+END_SRC

**** Importing

To use SDL, first import the module sdlang:

#+BEGIN_SRC d :tangle no
import sdlang;
#+END_SRC

If you're not using DUB, then you must also include the path the SDLang-D sources when you compile:

#+BEGIN_SRC d :tangle no
rdmd --build-only -I{path to sdlang}/src -I{path to libInputVisitor}/src {other flags} yourProgram.d
#+END_SRC

**** misc
http://forum.dlang.org/thread/hphtqkkmrfnlcipnxzai@forum.dlang.org
http://forum.dlang.org/thread/gnfctbuhiemidetngrzi@forum.dlang.org?page=23#post-rlxlfveyyzgewhkxhhta:40forum.dlang.org

*** other links
http://semitwist.com/sdlang-d-docs/v0.9.3/sdlang.html  http://semitwist.com/sdlang-d-docs/

** markup
*** code

#+BEGIN_SRC txt :tangle no
code.ruby{

}code

code.python{

}code

code.sh{

}code

code.txt{

}code

code.d{

}code
#+END_SRC

** src dir structure & files
#+BEGIN_SRC txt :tangle no
tree  /home/ralph/sisu_www/current/src/democratizing_innovation.eric_von_hippel.sst

/home/ralph/sisu_www/current/src/
democratizing_innovation.eric_von_hippel.sst
└── sisupod
    ├── doc
    │   ├── en
    │   │   └── democratizing_innovation.eric_von_hippel.sst
    │   └── _sisu
    │       └── sisu_document_make  // [interesting as part of larger conf.sdl]
    └── image
        ├── di_evh_f10-1.png
        ├── di_evh_f11-1.png
        ├── di_evh_f11-2.png
        ├── di_evh_f1-1.png
        ├── di_evh_f5-1.png
        └── di_evh.png

#+END_SRC

** read markup files
*** regular .sst
relatively straight forward
*** master .ssm
master files have been able to read in inser files .ssi and regular files .sst
**** reading in .ssi files is straightforward
**** reading in .sst files is more problematic
.sst files have their own root (structure)
either
- the root needs to be disabled - not used
or
- the root tree needs to be demoted, which is only possible if markup from
  heading D is not reached then A - C could be demoted to B - D
- the other issue there is that it is common not to write out heading level A
  text but to rely on the metadata for title and author, the issue there is that
  at present the header for .sst files that are imported is just lopped off and
  thrown away. At least the title and author information for each imported .sst
  file would have to read and available for use in its header A that is demoted
  to B

** processing files, currently using utf8
** check

#+BEGIN_SRC text
./.sisu ./_sisu ~/.sisu /etc/.sisu
#+END_SRC

#+BEGIN_SRC d
// string[string] envVars = environment.toAA();
// writeln(envVars);
/+
writefln(
  "pwd: %s; home: %s",
  environment["PWD"],
  environment["HOME"]
);
+/
#+END_SRC