path: root/dlang/src/write/text.d

module write.text;

/+ ↓ the plain text writer: a function from the model to bytes.
     .
     It is given an OcdaDocument and knows nothing about where it came from,
     which is the point: the same call renders a document loaded from a .ssp
     and one loaded from a .ocda.db, and cannot tell them apart.
     .
     **This no longer matches spine's --text byte for byte, by decision.**
     Byte identity was the first oracle and it earned its place while there
     was nothing else to check against; holding to it meant reproducing a
     dozen things no one would choose, and the code said so a dozen times.
     What replaced it is stricter where it matters and says nothing about
     what it does not care about:
     .
       1. the D and the Gleam writer produce the same bytes;
       2. the words of the output equal the words of the html writer's
          objects, as a multiset, so nothing is lost or doubled;
       3. every object number appears exactly once;
       4. the same bytes from every input form.
     .
     The presentation conventions are spine's and are kept, because they are
     good: 「n」 for the object number on a line of its own, the markup
     written back as markup so the text is re-editable, ≫n for a link to an
     object, a note after the object that refers to it.
     .
     What changed, each because it is better and not merely different:
     .
     - *a note is written once, after the object that refers to it, and the
       gathered endnotes section is not written.* Spine writes both, so every
       note's text is in the text file twice - the same duplication the rules
       record for spine's epub. In a file with no pages the note belongs
       beside its reference, so that is the copy that is kept. It costs the
       OCN spine gives the generated endnotes heading, which is the question
       the typst writer raised and the log carries.
     - *the author's markup comments are not written.* Spine's text output
       leaks them, `%` and all - 36 of them in one sample, including a line
       reading "% $$$$". The model says an object of type comment is carried
       so that nothing is lost and is not put into output; every other
       writer here honours that and this one now does too.
     - *an image keeps its caption.* Spine's text writes the file name alone
       and loses the words beside it.
     - *a table column is padded to its display width*, counting a CJK
       character as the two columns it occupies. Spine measures the width in
       bytes and pads in codepoints, so any table with a non-ASCII cell is
       misaligned, in opposite directions depending on which.
     - *a table row is stripped at its own end*, not by stripping the whole
       accumulated table after every cell, which in spine loses the newline
       before a row that ends in an empty cell.
     - *the table of contents indents two spaces per level.* Spine writes
       ":" at level two and "∴" at level three, which read as content.
     - *one code path for the object number*, where spine's quote, table and
       general cases each built it again.
     - *an unlisted object kind is written rather than dropped.* Spine's
       dispatch is a series of ifs and an unknown kind falls through all of
       them, so a new kind would vanish silently.
+/

import std.algorithm : max;
import std.array : appender, array;
import std.array : asplit = split;
import std.conv : to;
import std.regex : matchAll, matchFirst, replaceAll, split;
import std.string : strip;
import ocda.model;
import write.rgx;

enum newline  = "\n";
enum newlines = "\n\n";

static Rgx rgx;

/+ ↓ the table of contents depth of each indent_hang the document uses.
     .
     spine's indent_hang is not a depth: it reserves 1 to 3 for levels above
     the body, so a document's toc uses 1 then jumps to 4, 5, 6. Indenting by
     the number itself leaves the first body level six spaces in. Ranking the
     values the document actually uses gives 0, 1, 2, 3 and adapts to any
     document without a table of special cases.
+/
private size_t[int] toc_depth;

/+ ↓ the whole document +/
string writeText(ref OcdaDocument doc) {
  toc_depth = tocDepths(doc);
  auto out_ = appender!string;
  foreach (name; textSections(doc)) {
    foreach (obj; *doc.sectionByName(name)) {
      if (obj.renders) { out_ ~= objectOut(obj); }
    }
  }
  out_ ~= newline ~ tail(doc) ~ newline;
  return out_.data;
}

/+ ↓ rank the indent_hang values the table of contents uses +/
private size_t[int] tocDepths(ref OcdaDocument doc) {
  bool[int] seen;
  auto toc = doc.sectionByName("toc");
  if (toc !is null) {
    foreach (obj; *toc) { if (obj.renders) { seen[obj.indent_hang] = true; } }
  }
  import std.algorithm : sort;
  auto levels = seen.keys.sort.array;
  size_t[int] depth;
  foreach (i, level; levels) { depth[level] = i; }
  return depth;
}

/+ ↓ which sections, in which order: head, toc, body, then each trailing
     section that has more than one object. One object is the section's own
     heading with nothing under it.
+/
string[] textSections(ref OcdaDocument doc) {
  string[] seq = ["head", "toc", "body"];
  foreach (name; ["glossary", "bibliography", "bookindex", "blurb"]) {
    auto sec = doc.sectionByName(name);
    if (sec !is null && (*sec).length > 1) { seq ~= name; }
  }
  return seq;
}

/+ ↓ one object, by kind.
     .
     The default writes the object rather than dropping it: a kind this
     switch has not heard of is more likely to be new than to be unwanted,
     and an output that loses it silently cannot be checked.
+/
private string objectOut(ref const OcdaObject obj) {
  /+ ↓ an object with no text of its own writes nothing. A poem is such an
       object: a container of the verses that follow it, which are what is
       written. Since format 2.1 it carries object number 0 and states the
       range of its verses; before, it closed the run carrying the *same*
       number as the last verse, and writing it put that number in the file
       twice - 15 times over in one sample. Both forms write nothing. +/
  if (obj.text.strip.length == 0) { return ""; }
  /+ ↓ and a dummy heading writes nothing: it is generated furniture whose
       text repeats the heading above it, and every other writer here drops
       it. Spine's text keeps it, so a trailing section's name is in the file
       twice. +/
  if (obj.dummy_heading) { return ""; }
  switch (obj.is_a) {
  case "toc":       return tocEntry(obj);
  case "table":     return table(obj);
  case "bookindex": return indexEntry(obj);
  case "code":      return block(obj, obj.text.replaceAll(rgx.nbsp_char, " "), "");
  case "heading":   return newline ~ general(obj);
  default:          return general(obj);
  }
}

/+ ↓ the usual case: the markers resolved, the notes gathered out of the
     resolved text, and the rest written as text +/
private string general(ref const OcdaObject obj) {
  string resolved = links(images(obj.text));
  return block(obj, inline(resolved, obj), notesOf(resolved));
}

/+ ↓ the common shape: the text, the notes it carried, and its number +/
private string block(ref const OcdaObject obj, string txt, string notes) {
  return txt ~ notes ~ newline ~ ocnMark(obj) ~ newline;
}

/+ ↓ the object's number, on a line of its own. An object without one gets a
     blank line, which keeps the objects apart on the page. +/
private string ocnMark(ref const OcdaObject obj) {
  return (obj.ocn == 0) ? "" : "「" ~ obj.ocn.to!string ~ "」" ~ newline;
}

/+ ↓ a note is carried *inside* the object that refers to it, 【<n> text】.
     The reference becomes [n] where it stands and the note follows the
     object, which is what makes the text readable in one pass.
+/
private string notesOf(string txt) {
  auto out_ = appender!string;
  foreach (m; txt.matchAll(rgx.inline_notes_al_all_note)) {
    out_ ~= newlines ~ m["num"] ~ ". " ~ faces(breaks(m["note"].to!string));
  }
  return out_.data;
}

/+ ↓ a table of contents entry: two spaces per level, the text, and the
     object it names +/
private string tocEntry(ref const OcdaObject obj) {
  string indent;
  if (auto d = obj.indent_hang in toc_depth) {
    foreach (_; 0 .. *d) { indent ~= "  "; }
  }
  return indent ~ inline(links(images(obj.text)), obj) ~ newline;
}

/+ ↓ a book index entry, whose links are to objects rather than to segments +/
private string indexEntry(ref const OcdaObject obj) {
  string txt = obj.text.replaceAll(rgx.trailing_backslash, "");
  return inline(links(images(txt)), obj) ~ newlines;
}

/+ ↓ a table, in columns as wide as their widest cell needs.
     .
     The object's text is the whole table: rows separated by newlines,
     columns by ┊. A row of one cell in a table of more than one column is a
     note under the table rather than a row of it.
+/
private string table(ref const OcdaObject obj) {
  auto rows = obj.text.split(rgx.table_delimiter_row);
  auto aligns = obj.table_aligns.asplit(" ");
  auto widths = new size_t[](max(obj.table_cols, 1));
  string[][] body_;
  string note;
  foreach (row; rows) {
    if (row.strip.length == 0) { continue; }
    auto cells = row.split(rgx.table_delimiter_col);
    if (cells.length == 1 && obj.table_cols > 1) {
      note ~= inline(links(images(cells[0])), obj) ~ newline;
      continue;
    }
    string[] out_row;
    foreach (i, cell; cells) {
      string c = inline(links(images(cell)), obj);
      if (i < widths.length) { widths[i] = max(widths[i], displayWidth(c)); }
      out_row ~= c;
    }
    body_ ~= out_row;
  }
  auto out_ = appender!string;
  foreach (row; body_) {
    auto line = appender!string;
    foreach (i, cell; row) {
      size_t w = (i < widths.length) ? widths[i] : displayWidth(cell);
      string pad;
      foreach (_; displayWidth(cell) .. w) { pad ~= " "; }
      line ~= (i < aligns.length && aligns[i] != "l") ? pad ~ cell : cell ~ pad;
      if (i + 1 < row.length) { line ~= " ┊ "; }
    }
    /+ ↓ stripped at the end of its own row, so a row that ends in an empty
         cell keeps the line it is on +/
    out_ ~= line.data.stripRight ~ newline;
  }
  return out_.data ~ note ~ ocnMark(obj) ~ newline;
}

/+ ↓ trailing spaces off one line +/
private string stripRight(string s) {
  size_t end = s.length;
  while (end > 0 && (s[end - 1] == ' ' || s[end - 1] == '\t')) { end--; }
  return s[0 .. end];
}

/+ ↓ how many columns a string occupies.
     .
     One per character, and two for the ranges a terminal draws double
     width. Neither a byte count nor a codepoint count lines up a table
     holding Japanese, and this is the measurement the alignment means.
+/
private size_t displayWidth(string s) {
  size_t w = 0;
  foreach (dchar c; s) { w += wide(c) ? 2 : 1; }
  return w;
}

private bool wide(dchar c) {
  return (c >= 0x1100 && c <= 0x115F)
    ||   (c >= 0x2E80 && c <= 0x303E)
    ||   (c >= 0x3041 && c <= 0x33FF)
    ||   (c >= 0x3400 && c <= 0x4DBF)
    ||   (c >= 0x4E00 && c <= 0x9FFF)
    ||   (c >= 0xA000 && c <= 0xA4CF)
    ||   (c >= 0xAC00 && c <= 0xD7A3)
    ||   (c >= 0xF900 && c <= 0xFAFF)
    ||   (c >= 0xFE30 && c <= 0xFE6F)
    ||   (c >= 0xFF00 && c <= 0xFF60)
    ||   (c >= 0xFFE0 && c <= 0xFFE6)
    ||   (c >= 0x20000 && c <= 0x3FFFD);
}

/+ ↓ an object's text as plain text.
     .
     The order is the one every writer here uses: images before links,
     because an image sits inside a link; then the faces, then the breaks.
     A note reference becomes [n] and the note itself is gathered separately
     by `notes`.
+/
private string inline(string txt, ref const OcdaObject obj) {
  txt = txt.replaceAll(rgx.inline_notes_al_all_note, "[$1]");
  /+ ↓ an anchor the markup placed in the text has no rendering of its own; in
       a text file there is nothing for it to be +/
  txt = txt.replaceAll(rgx.inline_link_anchor, "");
  txt = faces(txt);
  txt = groupIndents(txt, obj);
  return breaks(txt);
}

/+ ↓ the line-leading markers of a grouped object: an indent, a hanging
     indent's inner level, a bullet. Spine leaves them in the text as "_1 " and
     "_* ", which read as markup that was not translated. +/
private string groupIndents(string txt, ref const OcdaObject obj) {
  if (obj.is_a != "group" && obj.is_a != "block") { return txt; }
  return txt
    .replaceAll(rgx.grouped_para_indent_hang, "$2$2")
    .replaceAll(rgx.grouped_para_bullet_at, "$1● ")
    .replaceAll(rgx.grouped_para_bullet, "● ")
    .replaceAll(rgx.grouped_para_indent, "$1$1");
}

/+ ↓ an image becomes its file name in braces, which is spine's own markup
     for one, and the words beside it stay where they are. Spine's text writes
     the file name alone and loses the caption.
     .
     The quotation marks around an alt string are the markup's delimiters and
     not the author's punctuation, so they come off here as they do in the html
     writer.
+/
private string images(string txt) {
  return replaceAll!((m) {
    string rest = m["post"].to!string;
    string alt;
    if (auto a = rest.matchFirst(rgx.inline_image_alt)) {
      alt = a["alt"].to!string;
      rest = a.post.to!string;
    }
    return m["pre"].to!string ~ "{" ~ m["img"].to!string ~ "}" ~ alt ~ rest;
  })(txt, rgx.inline_image);
}

/+ ↓ a link becomes its text, and the object it points at after it.
     .
     Only when the target *is* an object: a name, a segment or a url means
     nothing to a reader of a text file, where "≫12" is a number they can
     search for. So the toc line for the endnotes reads "Endnotes" and not
     "Endnotes ≫endnotes".
+/
private string links(string txt) {
  txt = replaceAll!((m) {
    string target = m.captures[3].to!string;
    string text_ = m.captures[1].to!string;
    return isObjectNumber(target) ? text_ ~ " ≫" ~ target : text_;
  })(txt, rgx.inline_link);
  return txt.replaceAll(rgx.mark_internal_site_lnk, "");
}

/+ ↓ a target that is an object number, and not a name or a url +/
private bool isObjectNumber(string s) {
  if (s.length == 0 || s == "0") { return false; }
  foreach (c; s) { if (c < '0' || c > '9') { return false; } }
  return true;
}

/+ ↓ the font faces become the markup they were written as, which is what
     makes the text re-editable rather than stripped +/
private string faces(string txt) {
  return txt
    .replaceAll(rgx.inline_strike,      "-{$1}-")
    .replaceAll(rgx.inline_insert,      "+{$1}+")
    .replaceAll(rgx.inline_cite,        "\"{$1}\"")
    .replaceAll(rgx.inline_emphasis,    "!{$1}!")
    .replaceAll(rgx.inline_bold,        "*{$1}*")
    .replaceAll(rgx.inline_italics,     "/{$1}/")
    .replaceAll(rgx.inline_underscore,  "_{$1}_")
    .replaceAll(rgx.inline_superscript, "^{$1}^")
    .replaceAll(rgx.inline_subscript,   ",{$1},")
    .replaceAll(rgx.inline_mono,        "#{$1}#");
}

/+ ↓ the line breaks the markup asked for, and the non-breaking space +/
private string breaks(string txt) {
  return txt
    .replaceAll(rgx.nbsp_char,      " ")
    .replaceAll(rgx.br_line,        newline)
    .replaceAll(rgx.br_line_inline, newline)
    .replaceAll(rgx.br_line_spaced, newlines)
    .replaceAll(rgx.line_break,     newline);
}

/+ ↓ the metadata block at the foot of the file.
     .
     Two values are *not* in the abstraction: the generating program's name
     and its home page. They describe the program, not the document, so the
     model has no place for them and this writer names itself.
+/
private string tail(ref OcdaDocument doc) {
  auto out_ = appender!string;
  void put(string label, string key) {
    if (doc.meta(key).length > 0) {
      out_ ~= label ~ ": " ~ doc.meta(key) ~ newlines;
    }
  }
  put("Title",     "title.full");
  put("Author",    "creator.author");
  put("Published", "date.published");
  put("Copyright", "rights.copyright");
  put("License",   "rights.license");
  out_ ~= generator_name ~ newline ~ generator_url;
  return out_.data;
}

/+ ↓ set to spine's own values so that a comparison is of the writer and not
     of the signature. A writer that is not spine should say so here. +/
enum generator_name = "spine";
enum generator_url  = "https://sisudoc.org";