path: root/dlang/src/write/markdown.d

module write.markdown;

/+ ↓ the markdown writer: a document as CommonMark, for reading anywhere.
     .
     The fifth format, and the one asked for because markdown is what a
     document is passed around as. The question was how much of spine's spirit
     survives the translation, and the answer is: the object numbers do, which
     is the part that matters.
     .
     **The dialect is CommonMark**, plus pipe tables, which every renderer
     worth using understands. Raw inline html is used in exactly two places,
     both of them for the object number, because markdown has no anchor
     syntax and nothing else will do.
     .
     What an object looks like:
     .
       <a id="12"></a>The text of the paragraph.<sup>[12](#12)</sup>
     .
     The anchor is at the head of the object so that a link to it lands on the
     object and not past it; the number is at the end, as a link to itself, so
     that a reader can take the citation out of the rendered page. That is
     what spine's own html does with an object number, and it is the nearest
     markdown comes to a number in the margin.
     .
     **Markdown's own footnote syntax is not used.** It is an extension rather
     than CommonMark, and, more to the point, a renderer numbers footnotes
     itself, in the order it meets them. The document's note numbers are part
     of its citation and are not a renderer's to choose. So a note is written
     where it is referred to, with the document's own number, linking back to
     the object - the same decision the typst and the text writers made, for
     the same reason.
     .
     Set against the plain text writer: everything the text file carries, and
     in addition a reader that renders it gets working links for every
     citation, every note, every index entry and every line of the table of
     contents, and the headings become headings. The text file has the
     numbers; the markdown has the numbers *and* the graph between them.
+/

import std.algorithm : canFind, max, min, startsWith;
import std.array : appender, array, join, replace;
import std.array : asplit = split;
import std.conv : to;
import std.regex : matchAll, matchFirst, replaceAll, split;
import std.string : strip;
import ocda.model;
import write.rgx;

enum newline  = "\n";
enum newlines = "\n\n";

static Rgx rgx_m;

/+ ↓ the table of contents depth of each indent_hang the document uses.
     spine reserves 1 to 3 for levels above the body, so a toc jumps from 1 to
     4; ranking the values the document uses gives 0, 1, 2, 3, and a markdown
     list nests by two spaces a level and cannot take a jump. +/
private size_t[int] toc_depth;

/+ ↓ the document as markdown.
     .
     image_pth is where the images are, relative to wherever this output is put.
     Given rather than assumed: the model names an image and does not say where
     the file naming it will sit. spine, whose markdown goes two directories down
     under a language, passes "../../image".
+/
string writeMarkdown(ref OcdaDocument doc, string image_pth = "image") {
  toc_depth = null;
  auto toc = doc.sectionByName("toc");
  if (toc !is null) {
    import std.algorithm : sort;
    bool[int] seen;
    foreach (obj; *toc) { if (obj.renders) { seen[obj.indent_hang] = true; } }
    foreach (i, level; seen.keys.sort.array) { toc_depth[level] = i; }
  }
  auto out_ = appender!string;
  out_ ~= frontMatter(doc);
  foreach (name; markdownSections(doc)) {
    foreach (obj; *doc.sectionByName(name)) {
      if (!obj.renders || obj.dummy_heading) { continue; }
      if (obj.text.strip.length == 0) { continue; }
      out_ ~= objectOut(obj);
    }
  }
  return out_.data.replace(image_dir_mark, image_pth);
}

/+ ↓ head, toc, body, then each trailing section with more than one object.
     Not endnotes: a note is written where it is referred to.
+/
string[] markdownSections(ref OcdaDocument doc) {
  string[] seq = ["head", "toc", "body"];
  foreach (name; ["glossary", "bibliography", "bookindex", "blurb"]) {
    auto sec = doc.sectionByName(name);
    if (sec !is null && (*sec).length > 1) { seq ~= name; }
  }
  return seq;
}

/+ ↓ the document's metadata as yaml front matter, which is how a markdown
     file carries metadata and what every static site generator reads.
+/
private string frontMatter(ref OcdaDocument doc) {
  auto out_ = appender!string;
  out_ ~= "---" ~ newline;
  void put(string key, string value) {
    if (value.length > 0) { out_ ~= key ~ ": " ~ yaml(value) ~ newline; }
  }
  put("title",     doc.meta("title.full"));
  put("author",    doc.meta("creator.author"));
  put("date",      doc.meta("date.published"));
  put("language",  ("language" in doc.source) ? doc.source["language"] : "");
  put("copyright", doc.meta("rights.copyright"));
  put("license",   doc.meta("rights.license"));
  out_ ~= "---" ~ newlines;
  return out_.data;
}

/+ ↓ a yaml scalar. Quoted always, so that a colon or a leading marker in a
     title cannot change the shape of the document's own metadata. +/
private string yaml(string s) {
  return "\"" ~ s.replace("\\", "\\\\").replace("\"", "\\\"")
    .replace("\n", " ") ~ "\"";
}

/+ ↓ one object, and then the notes it referred to.
     .
     The notes are gathered here rather than inside each kind, because a
     reference can be anywhere - the markup manual has one in a heading - and a
     reference whose note was never written is a link to nothing.
+/
private string objectOut(ref const OcdaObject obj) {
  return objectBody(obj) ~ notes(obj);
}

private string objectBody(ref const OcdaObject obj) {
  switch (obj.is_a) {
  case "heading":   return heading(obj);
  case "toc":       return tocEntry(obj);
  case "table":     return table(obj);
  case "code":      return code(obj);
  case "quote":     return quote(obj);
  case "verse": case "poem": case "group": case "block":
                    return lines(obj);
  case "bookindex": return indexEntry(obj);
  default:          return para(obj);
  }
}

/+ ↓ six levels of heading, which is all markdown has. A document may be
     eight deep; the two deepest take the sixth, and the object number, the
     anchor and the table of contents carry the true depth.
+/
private int headingLevel(string level) {
  switch (level) {
  case "A": return 1;
  case "B": return 2;
  case "C": return 3;
  case "D": return 4;
  case "1": return 5;
  case "2": case "3": case "4": return 6;
  default:  return 1;
  }
}

private string heading(ref const OcdaObject obj) {
  string hashes;
  foreach (_; 0 .. headingLevel(obj.heading_level)) { hashes ~= "#"; }
  return hashes ~ " " ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines;
}

/+ ↓ a table of contents entry: a list item at its depth, the heading as a
     link, and the object number as the locator.
     .
     The number and not a page: a page is a fact about one typesetting, and
     the object number is the same reference in every output of the document.
+/
private string tocEntry(ref const OcdaObject obj) {
  string indent;
  if (auto d = obj.indent_hang in toc_depth) {
    foreach (_; 0 .. *d) { indent ~= "  "; }
  }
  string n = internalTarget(obj);
  string locator = (n.length > 0) ? (" <sup>[" ~ n ~ "](#" ~ n ~ ")</sup>") : "";
  return indent ~ "- " ~ inline(obj) ~ locator ~ newline;
}

private string para(ref const OcdaObject obj) {
  string bullet = obj.bullet ? "- " : "";
  string indent;
  foreach (_; 0 .. obj.indent_base) { indent ~= "  "; }
  return indent ~ bullet ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines;
}

private string quote(ref const OcdaObject obj) {
  return "> " ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines;
}

/+ ↓ verse, a poem, a group or a block: every line kept, with CommonMark's
     own hard line break - a backslash at the end of a line - rather than the
     two trailing spaces an editor will strip.
+/
private string lines(ref const OcdaObject obj) {
  string txt = inline(obj);
  auto out_ = appender!string;
  out_ ~= anchor(obj);
  auto rows = txt.asplit(newline);
  foreach (i, row; rows) {
    out_ ~= row;
    if (i + 1 < rows.length) { out_ ~= "\\" ~ newline; }
  }
  return out_.data ~ ocnMark(obj) ~ newlines;
}

/+ ↓ a book index entry: a list item, and every object it names a link +/
private string indexEntry(ref const OcdaObject obj) {
  string txt = obj.text.replaceAll(rgx_m.trailing_backslash, "");
  return "- " ~ anchor(obj) ~ inlineText(txt, obj) ~ ocnMark(obj) ~ newlines;
}

/+ ↓ a fenced code block, with the language where the document named one.
     Nothing inside a fence is interpreted, which is the whole point.
+/
private string code(ref const OcdaObject obj) {
  string txt = obj.text.replaceAll(rgx_m.nbsp_char, " ");
  /+ ↓ a fence long enough that the code cannot close it +/
  string fence = "```";
  while (txt.canFind(fence)) { fence ~= "`"; }
  return anchor(obj) ~ newline ~ fence ~ obj.code_syntax ~ newline
    ~ txt ~ newline ~ fence ~ newline ~ ocnMark(obj) ~ newlines;
}

/+ ↓ a pipe table, which needs a header row: markdown has no table without
     one, so a table the document did not give a header gets an empty row and
     keeps all of its own.
+/
private string table(ref const OcdaObject obj) {
  auto rows = obj.text.split(rgx_m.table_delimiter_row);
  auto aligns = obj.table_aligns.asplit(" ");
  int cols = max(obj.table_cols, 1);
  string[][] body_;
  foreach (row; rows) {
    if (row.strip.length == 0) { continue; }
    string[] cells;
    foreach (cell; row.split(rgx_m.table_delimiter_col)) {
      cells ~= inlineText(cell, obj).replace("|", "\\|");
    }
    body_ ~= cells;
  }
  auto out_ = appender!string;
  out_ ~= anchor(obj) ~ newline;
  string rule() {
    auto r = appender!string;
    r ~= "|";
    foreach (i; 0 .. cols) {
      string a = (i < aligns.length) ? aligns[i] : "l";
      r ~= (a == "c") ? " :---: |" : (a == "r") ? " ---: |" : " :--- |";
    }
    return r.data ~ newline;
  }
  string line(string[] cells) {
    auto l = appender!string;
    l ~= "|";
    foreach (i; 0 .. cols) {
      l ~= " " ~ ((i < cells.length) ? cells[i] : "") ~ " |";
    }
    return l.data ~ newline;
  }
  if (body_.length > 0 && obj.table_header) {
    out_ ~= line(body_[0]) ~ rule();
    foreach (row; body_[1 .. $]) { out_ ~= line(row); }
  } else {
    out_ ~= line([]) ~ rule();
    foreach (row; body_) { out_ ~= line(row); }
  }
  return out_.data ~ ocnMark(obj) ~ newlines;
}

/+ ↓ the object's anchor, so that a citation can point at it +/
private string anchor(ref const OcdaObject obj) {
  return (obj.ocn == 0) ? "" : "<a id=\"" ~ obj.ocn.to!string ~ "\"></a>";
}

/+ ↓ the object's number, at the end of it, as a link to itself +/
private string ocnMark(ref const OcdaObject obj) {
  if (obj.ocn == 0) { return ""; }
  string n = obj.ocn.to!string;
  return "<sup>[" ~ n ~ "](#" ~ n ~ ")</sup>";
}

/+ ↓ the notes the object referred to, each where it was referred to, with the
     document's own number and a link back to the object +/
private string notes(ref const OcdaObject obj) {
  auto out_ = appender!string;
  foreach (m; obj.text.matchAll(rgx_m.inline_notes_al_all_note)) {
    string mark = m["num"].to!string;
    string back = (obj.ocn == 0)
      ? "" : " [\u21a9](#" ~ obj.ocn.to!string ~ ")";
    out_ ~= newlines ~ "<a id=\"note-" ~ mark ~ "\"></a>"
      ~ "<sup>" ~ inert(mark) ~ ".</sup> "
      ~ inlineText(m["note"].to!string, obj) ~ back;
  }
  /+ ↓ and a blank line after the last of them, or the next object begins on
       the same line and markdown reads the two as one paragraph +/
  return (out_.data.length > 0) ? out_.data ~ newlines : "";
}

private string inline(ref const OcdaObject obj) {
  return inlineText(obj.text, obj);
}

/+ ↓ an object's text as markdown.
     .
     The order is the one every writer here uses: images before links,
     because an image sits inside a link; then the note references, the
     faces, the anchors and the breaks.
+/
private string inlineText(string txt, ref const OcdaObject obj) {
  txt = images(txt);
  txt = links(txt, obj);
  txt = noteRefs(txt);
  txt = faces(txt);
  txt = txt.replaceAll(rgx_m.inline_link_anchor, "");
  txt = breaks(txt, obj);
  return guardLineStarts(unprotect(escape(txt)));
}

/+ ↓ a "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a
     list item, an enumeration item, a blockquote. The writer uses the first of
     them itself, for a list, and a line of the document's that begins the same
     way would be read as one - which is what a verse or a group can do.
     .
     Guarded rather than escaped everywhere, because these are special only in
     that position and a backslash in every hyphen would be absurd.
+/
private string guardLineStarts(string s) {
  auto out_ = appender!string;
  bool at_start = true;
  for (size_t i = 0; i < s.length; i++) {
    if (at_start) {
      size_t j = i;
      while (j < s.length && s[j] == ' ') { j++; }
      if (j < s.length) {
        char c = s[j];
        bool marker = (c == '-' || c == '+' || c == '>')
          && j + 1 < s.length && (s[j + 1] == ' ' || s[j + 1] == '\t');
        size_t k = j;
        while (k < s.length && s[k] >= '0' && s[k] <= '9') { k++; }
        bool enumerated = k > j && k + 1 < s.length
          && (s[k] == '.' || s[k] == ')') && s[k + 1] == ' ';
        if (marker || enumerated) {
          out_ ~= s[i .. (enumerated ? k : j)];
          out_ ~= '\\';
          i = (enumerated ? k : j) - 1;
          at_start = false;
          continue;
        }
      }
      at_start = false;
    }
    out_ ~= s[i];
    if (s[i] == '\n') { at_start = true; }
  }
  return out_.data;
}

/+ ↓ the markers that protect the markdown this writer emits from the escaping
     that follows.
     .
     The escaping has to come *last*, and this is why: the abstraction's own
     markers are built from the characters markdown reserves - "⑆*┨" for
     emphasis, "⑆_┨" for an underscore face - so escaping first turns them
     into "⑆\*┨" and no face is ever recognised. It cost 70 stray hashes and
     every face in one document before it was seen.
     .
     A private-use pair, because they must not occur in a document, and the
     protection nests, so the escaper counts it rather than flagging it.
+/
private enum keep_open  = "";
private enum keep_close = "";

/+ ↓ a third marker of the same kind, standing for the directory the images are
     in.
     .
     An image reference is written while an object is written, and where the
     file will be put is not something the model knows: it is told. So the
     reference carries a marker and writeMarkdown substitutes it once, at the
     end, out of the one argument that says.
+/
private enum image_dir_mark = "";

private string keep(string markdown_source) {
  return keep_open ~ markdown_source ~ keep_close;
}

private string unprotect(string txt) {
  return txt.replace(keep_open, "").replace(keep_close, "");
}

/+ ↓ the document's own text, escaped for a protected run: the same rule, run
     early, and then flattened so that the result is inert +/
private string inert(string txt) {
  return unprotect(escape(txt));
}

/+ ↓ the characters markdown reads as syntax, escaped wherever they occur,
     and only outside a protected run.
     .
     The same blunt rule the typst writer arrived at, and for the same reason:
     a closed set is checkable and a rule about positions is not. The three
     html characters become entities rather than backslash escapes, because
     markdown passes raw html through and "<b" would open a tag.
     .
     Last in the pipeline, because the abstraction's markers are built from
     the characters markdown reserves and escaping them first destroys every
     one. The protection nests, so it is counted, not flagged.
+/
private string escape(string txt) {
  auto out_ = appender!string;
  int depth = 0;
  foreach (dchar c; txt) {
    if (c == keep_open.to!dchar)  { depth++; continue; }
    if (c == keep_close.to!dchar) { if (depth > 0) { depth--; } continue; }
    if (depth > 0) { out_ ~= c; continue; }
    switch (c) {
    case '&':  out_ ~= "&amp;"; break;
    case '<':  out_ ~= "&lt;";  break;
    case '>':  out_ ~= "&gt;";  break;
    case '\\': case '`': case '*': case '_': case '[': case ']':
    case '#':  case '|': case '~':
      out_ ~= '\\'; out_ ~= c; break;
    default:   out_ ~= c; break;
    }
  }
  return out_.data;
}

/+ ↓ an image, with its alt text where the document gave one +/
private string images(string txt) {
  return replaceAll!((m) {
    string rest = m["post"].to!string;
    string alt;
    if (auto a = rest.matchFirst(rgx_m.inline_image_alt)) {
      alt  = a["alt"].to!string;
      rest = a.post.to!string;
    }
    return m["pre"].to!string
      ~ keep("![" ~ inert(alt) ~ "](" ~ image_dir_mark ~ "/" ~ m["img"].to!string ~ ")")
      ~ rest;
  })(txt, rgx_m.inline_image);
}

/+ ↓ a link. An internal target becomes a fragment, which is the object
     number, and so resolves against the anchor every object carries.
     .
     And *only* an object number: those are the anchors this writer defines, so
     a target that names something else - the endnotes section, an anchor tag
     the markup placed - would be a link to nothing. The text stands on its
     own instead, which is what a reader sees anyway.
+/
private string links(string txt, ref const OcdaObject obj) {
  auto stow = obj.stow_link;
  txt = replaceAll!((m) {
    size_t num = m["num"].to!size_t;
    string url = (num < stow.length) ? stow[num] : "";
    return m["linked_text"].to!string ~ "┤" ~ url ~ "├";
  })(txt, rgx_m.inline_link_number_only);
  return replaceAll!((m) {
    string text_ = m.captures[1].to!string;
    string target = m.captures[2].to!string;
    if (target.startsWith("¤")) {
      auto im = target.matchFirst(rgx_m.internal_target);
      if (!im.empty) { target = im["frag"].to!string; }
    }
    string shown = inert(faces(text_));
    if (target.length == 0) { return keep(shown); }
    if (target.startsWith("#") && !isObjectNumber(target[1 .. $])) {
      return keep(shown);
    }
    return keep("[" ~ shown ~ "](" ~ target ~ ")");
  })(txt, rgx_m.inline_link);
}

/+ ↓ a link target is a url or a fragment and not prose, so the escaping the
     text went through comes off it again. A "#" that stayed escaped made every
     table of contents line point at nothing. +/
private string unescapeTarget(string target) {
  auto out_ = appender!string;
  for (size_t i = 0; i < target.length; i++) {
    if (target[i] == 0x5c && i + 1 < target.length) { out_ ~= target[++i]; continue; }
    out_ ~= target[i];
  }
  return out_.data.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">");
}

/+ ↓ a fragment that names an object of this document +/
private bool isObjectNumber(string s) {
  if (s.length == 0 || s == "0") { return false; }
  foreach (c; s) { if (c < '0' || c > '9') { return false; } }
  return true;
}

/+ ↓ a note reference: a superscript link to the note, which sits under the
     object that refers to it +/
private string noteRefs(string txt) {
  return replaceAll!((m) {
    string mark = m["num"].to!string;
    /+ ↓ the mark is shown as well as named in the target, and a star note`s mark
         is an asterisk, which markdown would pair with the next one +/
    return keep("<sup>[" ~ inert(mark) ~ "](#note-" ~ mark ~ ")</sup>");
  })(txt, rgx_m.inline_notes_al_all_note);
}

/+ ↓ the font faces. Emphasis and strong are markdown's own; the rest have no
     markdown and take the html element markdown itself would produce.
     .
     The spaces move *outside* the markers. The abstraction's markers include
     the spaces that surrounded the run - "⑆/┨ lex mercatoria ┣/" - and
     markdown's emphasis will not open on a space: "* lex *" is three words and
     two asterisks, not an italic.
+/
private string faces(string txt) {
  static struct Face { typeof(Rgx.inline_bold) pattern; string open, close; }
  string wrap(string open, string close, string inner) {
    size_t a = 0, b = inner.length;
    while (a < b && inner[a] == ' ') { a++; }
    while (b > a && inner[b - 1] == ' ') { b--; }
    /+ ↓ the spaces are dropped rather than moved: the abstraction`s marker
         takes in the space that was beside the run, and the text before it
         already ends in one +/
    return keep(open) ~ inner[a .. b] ~ keep(close);
  }
  string apply(alias pattern)(string s, string open, string close) {
    return replaceAll!((m) => wrap(open, close, m.captures[1].to!string))
      (s, pattern);
  }
  txt = apply!(rgx_m.inline_emphasis)(txt, "**", "**");
  txt = apply!(rgx_m.inline_bold)(txt, "**", "**");
  txt = apply!(rgx_m.inline_italics)(txt, "*", "*");
  txt = apply!(rgx_m.inline_underscore)(txt, "<u>", "</u>");
  txt = apply!(rgx_m.inline_superscript)(txt, "<sup>", "</sup>");
  txt = apply!(rgx_m.inline_subscript)(txt, "<sub>", "</sub>");
  txt = apply!(rgx_m.inline_mono)(txt, "`", "`");
  txt = apply!(rgx_m.inline_strike)(txt, "~~", "~~");
  txt = apply!(rgx_m.inline_insert)(txt, "<ins>", "</ins>");
  txt = apply!(rgx_m.inline_cite)(txt, "<cite>", "</cite>");
  return txt;
}

/+ ↓ the line breaks the markup asked for, and the grouped indents +/
private string breaks(string txt, ref const OcdaObject obj) {
  if (obj.is_a == "code") { return txt; }
  if (obj.is_a == "group" || obj.is_a == "block") {
    txt = txt
      .replaceAll(rgx_m.grouped_para_indent_hang, "$2$2")
      .replaceAll(rgx_m.grouped_para_bullet_at, "$1\u25cf ")
      .replaceAll(rgx_m.grouped_para_bullet, "\u25cf ")
      .replaceAll(rgx_m.grouped_para_indent, "$1$1");
  }
  return txt
    .replaceAll(rgx_m.nbsp_char,      " ")
    .replaceAll(rgx_m.br_line,        newline)
    .replaceAll(rgx_m.br_line_inline, newline)
    .replaceAll(rgx_m.br_line_spaced, newlines)
    .replaceAll(rgx_m.line_break,     newline)
    .replaceAll(rgx_m.mark_internal_site_lnk, "");
}

/+ ↓ the object a table of contents entry points at, where that is an object
     number +/
private string internalTarget(ref const OcdaObject obj) {
  foreach (m; obj.text.matchAll(rgx_m.any_internal_target)) {
    string frag = m["frag"].to!string;
    bool digits = frag.length > 0;
    foreach (c; frag) { if (c < '0' || c > '9') { digits = false; break; } }
    if (digits) { return frag; }
  }
  return "";
}