1 files changed, 397 insertions, 0 deletions
diff --git a/dlang/src/write/text.d b/dlang/src/write/text.d
new file mode 100644
index 0000000..9853a7e
--- /dev/null
+++ b/dlang/src/write/text.d
@@ -0,0 +1,397 @@
+module write.text;
+
+/+ ↓ the plain text writer: a function from the model to bytes.
+     .
+     It is given an OcdaDocument and knows nothing about where it came from,
+     which is the point: the same call renders a document loaded from a .ssp
+     and one loaded from a .ocda.db, and cannot tell them apart.
+     .
+     **This no longer matches spine's --text byte for byte, by decision.**
+     Byte identity was the first oracle and it earned its place while there
+     was nothing else to check against; holding to it meant reproducing a
+     dozen things no one would choose, and the code said so a dozen times.
+     What replaced it is stricter where it matters and says nothing about
+     what it does not care about:
+     .
+       1. the D and the Gleam writer produce the same bytes;
+       2. the words of the output equal the words of the html writer's
+          objects, as a multiset, so nothing is lost or doubled;
+       3. every object number appears exactly once;
+       4. the same bytes from every input form.
+     .
+     The presentation conventions are spine's and are kept, because they are
+     good: 「n」 for the object number on a line of its own, the markup
+     written back as markup so the text is re-editable, ≫n for a link to an
+     object, a note after the object that refers to it.
+     .
+     What changed, each because it is better and not merely different:
+     .
+     - *a note is written once, after the object that refers to it, and the
+       gathered endnotes section is not written.* Spine writes both, so every
+       note's text is in the text file twice - the same duplication the rules
+       record for spine's epub. In a file with no pages the note belongs
+       beside its reference, so that is the copy that is kept. It costs the
+       OCN spine gives the generated endnotes heading, which is the question
+       the typst writer raised and the log carries.
+     - *the author's markup comments are not written.* Spine's text output
+       leaks them, `%` and all - 36 of them in one sample, including a line
+       reading "% $$$$". The model says an object of type comment is carried
+       so that nothing is lost and is not put into output; every other
+       writer here honours that and this one now does too.
+     - *an image keeps its caption.* Spine's text writes the file name alone
+       and loses the words beside it.
+     - *a table column is padded to its display width*, counting a CJK
+       character as the two columns it occupies. Spine measures the width in
+       bytes and pads in codepoints, so any table with a non-ASCII cell is
+       misaligned, in opposite directions depending on which.
+     - *a table row is stripped at its own end*, not by stripping the whole
+       accumulated table after every cell, which in spine loses the newline
+       before a row that ends in an empty cell.
+     - *the table of contents indents two spaces per level.* Spine writes
+       ":" at level two and "∴" at level three, which read as content.
+     - *one code path for the object number*, where spine's quote, table and
+       general cases each built it again.
+     - *an unlisted object kind is written rather than dropped.* Spine's
+       dispatch is a series of ifs and an unknown kind falls through all of
+       them, so a new kind would vanish silently.
++/
+
+import std.algorithm : max;
+import std.array : appender, array;
+import std.array : asplit = split;
+import std.conv : to;
+import std.regex : matchAll, matchFirst, replaceAll, split;
+import std.string : strip;
+import ocda.model;
+import write.rgx;
+
+enum newline  = "\n";
+enum newlines = "\n\n";
+
+static Rgx rgx;
+
+/+ ↓ the table of contents depth of each indent_hang the document uses.
+     .
+     spine's indent_hang is not a depth: it reserves 1 to 3 for levels above
+     the body, so a document's toc uses 1 then jumps to 4, 5, 6. Indenting by
+     the number itself leaves the first body level six spaces in. Ranking the
+     values the document actually uses gives 0, 1, 2, 3 and adapts to any
+     document without a table of special cases.
++/
+private size_t[int] toc_depth;
+
+/+ ↓ the whole document +/
+string writeText(ref OcdaDocument doc) {
+  toc_depth = tocDepths(doc);
+  auto out_ = appender!string;
+  foreach (name; textSections(doc)) {
+    foreach (obj; *doc.sectionByName(name)) {
+      if (obj.renders) { out_ ~= objectOut(obj); }
+    }
+  }
+  out_ ~= newline ~ tail(doc) ~ newline;
+  return out_.data;
+}
+
+/+ ↓ rank the indent_hang values the table of contents uses +/
+private size_t[int] tocDepths(ref OcdaDocument doc) {
+  bool[int] seen;
+  auto toc = doc.sectionByName("toc");
+  if (toc !is null) {
+    foreach (obj; *toc) { if (obj.renders) { seen[obj.indent_hang] = true; } }
+  }
+  import std.algorithm : sort;
+  auto levels = seen.keys.sort.array;
+  size_t[int] depth;
+  foreach (i, level; levels) { depth[level] = i; }
+  return depth;
+}
+
+/+ ↓ which sections, in which order: head, toc, body, then each trailing
+     section that has more than one object. One object is the section's own
+     heading with nothing under it.
++/
+string[] textSections(ref OcdaDocument doc) {
+  string[] seq = ["head", "toc", "body"];
+  foreach (name; ["glossary", "bibliography", "bookindex", "blurb"]) {
+    auto sec = doc.sectionByName(name);
+    if (sec !is null && (*sec).length > 1) { seq ~= name; }
+  }
+  return seq;
+}
+
+/+ ↓ one object, by kind.
+     .
+     The default writes the object rather than dropping it: a kind this
+     switch has not heard of is more likely to be new than to be unwanted,
+     and an output that loses it silently cannot be checked.
++/
+private string objectOut(ref const OcdaObject obj) {
+  /+ ↓ an object with no text of its own writes nothing. A poem is such an
+       object: it is a container that closes a run of verses and carries the
+       *same* object number as the last of them, so writing it would put that
+       number in the file twice - 15 times over in one sample. +/
+  if (obj.text.strip.length == 0) { return ""; }
+  /+ ↓ and a dummy heading writes nothing: it is generated furniture whose
+       text repeats the heading above it, and every other writer here drops
+       it. Spine's text keeps it, so a trailing section's name is in the file
+       twice. +/
+  if (obj.dummy_heading) { return ""; }
+  switch (obj.is_a) {
+  case "toc":       return tocEntry(obj);
+  case "table":     return table(obj);
+  case "bookindex": return indexEntry(obj);
+  case "code":      return block(obj, obj.text.replaceAll(rgx.nbsp_char, " "), "");
+  case "heading":   return newline ~ general(obj);
+  default:          return general(obj);
+  }
+}
+
+/+ ↓ the usual case: the markers resolved, the notes gathered out of the
+     resolved text, and the rest written as text +/
+private string general(ref const OcdaObject obj) {
+  string resolved = links(images(obj.text));
+  return block(obj, inline(resolved, obj), notesOf(resolved));
+}
+
+/+ ↓ the common shape: the text, the notes it carried, and its number +/
+private string block(ref const OcdaObject obj, string txt, string notes) {
+  return txt ~ notes ~ newline ~ ocnMark(obj) ~ newline;
+}
+
+/+ ↓ the object's number, on a line of its own. An object without one gets a
+     blank line, which keeps the objects apart on the page. +/
+private string ocnMark(ref const OcdaObject obj) {
+  return (obj.ocn == 0) ? "" : "「" ~ obj.ocn.to!string ~ "」" ~ newline;
+}
+
+/+ ↓ a note is carried *inside* the object that refers to it, 【<n> text】.
+     The reference becomes [n] where it stands and the note follows the
+     object, which is what makes the text readable in one pass.
++/
+private string notesOf(string txt) {
+  auto out_ = appender!string;
+  foreach (m; txt.matchAll(rgx.inline_notes_al_all_note)) {
+    out_ ~= newlines ~ m["num"] ~ ". " ~ faces(breaks(m["note"].to!string));
+  }
+  return out_.data;
+}
+
+/+ ↓ a table of contents entry: two spaces per level, the text, and the
+     object it names +/
+private string tocEntry(ref const OcdaObject obj) {
+  string indent;
+  if (auto d = obj.indent_hang in toc_depth) {
+    foreach (_; 0 .. *d) { indent ~= "  "; }
+  }
+  return indent ~ inline(links(images(obj.text)), obj) ~ newline;
+}
+
+/+ ↓ a book index entry, whose links are to objects rather than to segments +/
+private string indexEntry(ref const OcdaObject obj) {
+  string txt = obj.text.replaceAll(rgx.trailing_backslash, "");
+  return inline(links(images(txt)), obj) ~ newlines;
+}
+
+/+ ↓ a table, in columns as wide as their widest cell needs.
+     .
+     The object's text is the whole table: rows separated by newlines,
+     columns by ┊. A row of one cell in a table of more than one column is a
+     note under the table rather than a row of it.
++/
+private string table(ref const OcdaObject obj) {
+  auto rows = obj.text.split(rgx.table_delimiter_row);
+  auto aligns = obj.table_aligns.asplit(" ");
+  auto widths = new size_t[](max(obj.table_cols, 1));
+  string[][] body_;
+  string note;
+  foreach (row; rows) {
+    if (row.strip.length == 0) { continue; }
+    auto cells = row.split(rgx.table_delimiter_col);
+    if (cells.length == 1 && obj.table_cols > 1) {
+      note ~= inline(links(images(cells[0])), obj) ~ newline;
+      continue;
+    }
+    string[] out_row;
+    foreach (i, cell; cells) {
+      string c = inline(links(images(cell)), obj);
+      if (i < widths.length) { widths[i] = max(widths[i], displayWidth(c)); }
+      out_row ~= c;
+    }
+    body_ ~= out_row;
+  }
+  auto out_ = appender!string;
+  foreach (row; body_) {
+    auto line = appender!string;
+    foreach (i, cell; row) {
+      size_t w = (i < widths.length) ? widths[i] : displayWidth(cell);
+      string pad;
+      foreach (_; displayWidth(cell) .. w) { pad ~= " "; }
+      line ~= (i < aligns.length && aligns[i] != "l") ? pad ~ cell : cell ~ pad;
+      if (i + 1 < row.length) { line ~= " ┊ "; }
+    }
+    /+ ↓ stripped at the end of its own row, so a row that ends in an empty
+         cell keeps the line it is on +/
+    out_ ~= line.data.stripRight ~ newline;
+  }
+  return out_.data ~ note ~ ocnMark(obj) ~ newline;
+}
+
+/+ ↓ trailing spaces off one line +/
+private string stripRight(string s) {
+  size_t end = s.length;
+  while (end > 0 && (s[end - 1] == ' ' || s[end - 1] == '\t')) { end--; }
+  return s[0 .. end];
+}
+
+/+ ↓ how many columns a string occupies.
+     .
+     One per character, and two for the ranges a terminal draws double
+     width. Neither a byte count nor a codepoint count lines up a table
+     holding Japanese, and this is the measurement the alignment means.
++/
+private size_t displayWidth(string s) {
+  size_t w = 0;
+  foreach (dchar c; s) { w += wide(c) ? 2 : 1; }
+  return w;
+}
+
+private bool wide(dchar c) {
+  return (c >= 0x1100 && c <= 0x115F)
+    ||   (c >= 0x2E80 && c <= 0x303E)
+    ||   (c >= 0x3041 && c <= 0x33FF)
+    ||   (c >= 0x3400 && c <= 0x4DBF)
+    ||   (c >= 0x4E00 && c <= 0x9FFF)
+    ||   (c >= 0xA000 && c <= 0xA4CF)
+    ||   (c >= 0xAC00 && c <= 0xD7A3)
+    ||   (c >= 0xF900 && c <= 0xFAFF)
+    ||   (c >= 0xFE30 && c <= 0xFE6F)
+    ||   (c >= 0xFF00 && c <= 0xFF60)
+    ||   (c >= 0xFFE0 && c <= 0xFFE6)
+    ||   (c >= 0x20000 && c <= 0x3FFFD);
+}
+
+/+ ↓ an object's text as plain text.
+     .
+     The order is the one every writer here uses: images before links,
+     because an image sits inside a link; then the faces, then the breaks.
+     A note reference becomes [n] and the note itself is gathered separately
+     by `notes`.
++/
+private string inline(string txt, ref const OcdaObject obj) {
+  txt = txt.replaceAll(rgx.inline_notes_al_all_note, "[$1]");
+  /+ ↓ an anchor the markup placed in the text has no rendering of its own; in
+       a text file there is nothing for it to be +/
+  txt = txt.replaceAll(rgx.inline_link_anchor, "");
+  txt = faces(txt);
+  txt = groupIndents(txt, obj);
+  return breaks(txt);
+}
+
+/+ ↓ the line-leading markers of a grouped object: an indent, a hanging
+     indent's inner level, a bullet. Spine leaves them in the text as "_1 " and
+     "_* ", which read as markup that was not translated. +/
+private string groupIndents(string txt, ref const OcdaObject obj) {
+  if (obj.is_a != "group" && obj.is_a != "block") { return txt; }
+  return txt
+    .replaceAll(rgx.grouped_para_indent_hang, "$2$2")
+    .replaceAll(rgx.grouped_para_bullet_at, "$1● ")
+    .replaceAll(rgx.grouped_para_bullet, "● ")
+    .replaceAll(rgx.grouped_para_indent, "$1$1");
+}
+
+/+ ↓ an image becomes its file name in braces, which is spine's own markup
+     for one, and the words beside it stay where they are. Spine's text writes
+     the file name alone and loses the caption.
+     .
+     The quotation marks around an alt string are the markup's delimiters and
+     not the author's punctuation, so they come off here as they do in the html
+     writer.
++/
+private string images(string txt) {
+  return replaceAll!((m) {
+    string rest = m["post"].to!string;
+    string alt;
+    if (auto a = rest.matchFirst(rgx.inline_image_alt)) {
+      alt = a["alt"].to!string;
+      rest = a.post.to!string;
+    }
+    return m["pre"].to!string ~ "{" ~ m["img"].to!string ~ "}" ~ alt ~ rest;
+  })(txt, rgx.inline_image);
+}
+
+/+ ↓ a link becomes its text, and the object it points at after it.
+     .
+     Only when the target *is* an object: a name, a segment or a url means
+     nothing to a reader of a text file, where "≫12" is a number they can
+     search for. So the toc line for the endnotes reads "Endnotes" and not
+     "Endnotes ≫endnotes".
++/
+private string links(string txt) {
+  txt = replaceAll!((m) {
+    string target = m.captures[3].to!string;
+    string text_ = m.captures[1].to!string;
+    return isObjectNumber(target) ? text_ ~ " ≫" ~ target : text_;
+  })(txt, rgx.inline_link);
+  return txt.replaceAll(rgx.mark_internal_site_lnk, "");
+}
+
+/+ ↓ a target that is an object number, and not a name or a url +/
+private bool isObjectNumber(string s) {
+  if (s.length == 0 || s == "0") { return false; }
+  foreach (c; s) { if (c < '0' || c > '9') { return false; } }
+  return true;
+}
+
+/+ ↓ the font faces become the markup they were written as, which is what
+     makes the text re-editable rather than stripped +/
+private string faces(string txt) {
+  return txt
+    .replaceAll(rgx.inline_strike,      "-{$1}-")
+    .replaceAll(rgx.inline_insert,      "+{$1}+")
+    .replaceAll(rgx.inline_cite,        "\"{$1}\"")
+    .replaceAll(rgx.inline_emphasis,    "!{$1}!")
+    .replaceAll(rgx.inline_bold,        "*{$1}*")
+    .replaceAll(rgx.inline_italics,     "/{$1}/")
+    .replaceAll(rgx.inline_underscore,  "_{$1}_")
+    .replaceAll(rgx.inline_superscript, "^{$1}^")
+    .replaceAll(rgx.inline_subscript,   ",{$1},")
+    .replaceAll(rgx.inline_mono,        "#{$1}#");
+}
+
+/+ ↓ the line breaks the markup asked for, and the non-breaking space +/
+private string breaks(string txt) {
+  return txt
+    .replaceAll(rgx.nbsp_char,      " ")
+    .replaceAll(rgx.br_line,        newline)
+    .replaceAll(rgx.br_line_inline, newline)
+    .replaceAll(rgx.br_line_spaced, newlines)
+    .replaceAll(rgx.line_break,     newline);
+}
+
+/+ ↓ the metadata block at the foot of the file.
+     .
+     Two values are *not* in the abstraction: the generating program's name
+     and its home page. They describe the program, not the document, so the
+     model has no place for them and this writer names itself.
++/
+private string tail(ref OcdaDocument doc) {
+  auto out_ = appender!string;
+  void put(string label, string key) {
+    if (doc.meta(key).length > 0) {
+      out_ ~= label ~ ": " ~ doc.meta(key) ~ newlines;
+    }
+  }
+  put("Title",     "title.full");
+  put("Author",    "creator.author");
+  put("Published", "date.published");
+  put("Copyright", "rights.copyright");
+  put("License",   "rights.license");
+  out_ ~= generator_name ~ newline ~ generator_url;
+  return out_.data;
+}
+
+/+ ↓ set to spine's own values so that a comparison is of the writer and not
+     of the signature. A writer that is not spine should say so here. +/
+enum generator_name = "spine";
+enum generator_url  = "https://sisudoc.org";