diff options
Diffstat (limited to 'dlang/src/write/markdown.d')
| -rw-r--r-- | dlang/src/write/markdown.d | 579 |
1 files changed, 579 insertions, 0 deletions
diff --git a/dlang/src/write/markdown.d b/dlang/src/write/markdown.d new file mode 100644 index 0000000..d9f8f64 --- /dev/null +++ b/dlang/src/write/markdown.d @@ -0,0 +1,579 @@ +module write.markdown; + +/+ ↓ the markdown writer: a document as CommonMark, for reading anywhere. + . + The fifth format, and the one asked for because markdown is what a + document is passed around as. The question was how much of spine's spirit + survives the translation, and the answer is: the object numbers do, which + is the part that matters. + . + **The dialect is CommonMark**, plus pipe tables, which every renderer + worth using understands. Raw inline html is used in exactly two places, + both of them for the object number, because markdown has no anchor + syntax and nothing else will do. + . + What an object looks like: + . + <a id="12"></a>The text of the paragraph.<sup>[12](#12)</sup> + . + The anchor is at the head of the object so that a link to it lands on the + object and not past it; the number is at the end, as a link to itself, so + that a reader can take the citation out of the rendered page. That is + what spine's own html does with an object number, and it is the nearest + markdown comes to a number in the margin. + . + **Markdown's own footnote syntax is not used.** It is an extension rather + than CommonMark, and, more to the point, a renderer numbers footnotes + itself, in the order it meets them. The document's note numbers are part + of its citation and are not a renderer's to choose. So a note is written + where it is referred to, with the document's own number, linking back to + the object - the same decision the typst and the text writers made, for + the same reason. + . + Set against the plain text writer: everything the text file carries, and + in addition a reader that renders it gets working links for every + citation, every note, every index entry and every line of the table of + contents, and the headings become headings. The text file has the + numbers; the markdown has the numbers *and* the graph between them. ++/ + +import std.algorithm : canFind, max, min, startsWith; +import std.array : appender, array, join, replace; +import std.array : asplit = split; +import std.conv : to; +import std.regex : matchAll, matchFirst, replaceAll, split; +import std.string : strip; +import ocda.model; +import write.rgx; + +enum newline = "\n"; +enum newlines = "\n\n"; + +static Rgx rgx_m; + +/+ ↓ the table of contents depth of each indent_hang the document uses. + spine reserves 1 to 3 for levels above the body, so a toc jumps from 1 to + 4; ranking the values the document uses gives 0, 1, 2, 3, and a markdown + list nests by two spaces a level and cannot take a jump. +/ +private size_t[int] toc_depth; + +/+ ↓ the document as markdown. + . + image_pth is where the images are, relative to wherever this output is put. + Given rather than assumed: the model names an image and does not say where + the file naming it will sit. spine, whose markdown goes two directories down + under a language, passes "../../image". ++/ +string writeMarkdown(ref OcdaDocument doc, string image_pth = "image") { + toc_depth = null; + auto toc = doc.sectionByName("toc"); + if (toc !is null) { + import std.algorithm : sort; + bool[int] seen; + foreach (obj; *toc) { if (obj.renders) { seen[obj.indent_hang] = true; } } + foreach (i, level; seen.keys.sort.array) { toc_depth[level] = i; } + } + auto out_ = appender!string; + out_ ~= frontMatter(doc); + foreach (name; markdownSections(doc)) { + foreach (obj; *doc.sectionByName(name)) { + if (!obj.renders || obj.dummy_heading) { continue; } + if (obj.text.strip.length == 0) { continue; } + out_ ~= objectOut(obj); + } + } + return out_.data.replace(image_dir_mark, image_pth); +} + +/+ ↓ head, toc, body, then each trailing section with more than one object. + Not endnotes: a note is written where it is referred to. ++/ +string[] markdownSections(ref OcdaDocument doc) { + string[] seq = ["head", "toc", "body"]; + foreach (name; ["glossary", "bibliography", "bookindex", "blurb"]) { + auto sec = doc.sectionByName(name); + if (sec !is null && (*sec).length > 1) { seq ~= name; } + } + return seq; +} + +/+ ↓ the document's metadata as yaml front matter, which is how a markdown + file carries metadata and what every static site generator reads. ++/ +private string frontMatter(ref OcdaDocument doc) { + auto out_ = appender!string; + out_ ~= "---" ~ newline; + void put(string key, string value) { + if (value.length > 0) { out_ ~= key ~ ": " ~ yaml(value) ~ newline; } + } + put("title", doc.meta("title.full")); + put("author", doc.meta("creator.author")); + put("date", doc.meta("date.published")); + put("language", ("language" in doc.source) ? doc.source["language"] : ""); + put("copyright", doc.meta("rights.copyright")); + put("license", doc.meta("rights.license")); + out_ ~= "---" ~ newlines; + return out_.data; +} + +/+ ↓ a yaml scalar. Quoted always, so that a colon or a leading marker in a + title cannot change the shape of the document's own metadata. +/ +private string yaml(string s) { + return "\"" ~ s.replace("\\", "\\\\").replace("\"", "\\\"") + .replace("\n", " ") ~ "\""; +} + +/+ ↓ one object, and then the notes it referred to. + . + The notes are gathered here rather than inside each kind, because a + reference can be anywhere - the markup manual has one in a heading - and a + reference whose note was never written is a link to nothing. ++/ +private string objectOut(ref const OcdaObject obj) { + return objectBody(obj) ~ notes(obj); +} + +private string objectBody(ref const OcdaObject obj) { + switch (obj.is_a) { + case "heading": return heading(obj); + case "toc": return tocEntry(obj); + case "table": return table(obj); + case "code": return code(obj); + case "quote": return quote(obj); + case "verse": case "poem": case "group": case "block": + return lines(obj); + case "bookindex": return indexEntry(obj); + default: return para(obj); + } +} + +/+ ↓ six levels of heading, which is all markdown has. A document may be + eight deep; the two deepest take the sixth, and the object number, the + anchor and the table of contents carry the true depth. ++/ +private int headingLevel(string level) { + switch (level) { + case "A": return 1; + case "B": return 2; + case "C": return 3; + case "D": return 4; + case "1": return 5; + case "2": case "3": case "4": return 6; + default: return 1; + } +} + +private string heading(ref const OcdaObject obj) { + string hashes; + foreach (_; 0 .. headingLevel(obj.heading_level)) { hashes ~= "#"; } + return hashes ~ " " ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ a table of contents entry: a list item at its depth, the heading as a + link, and the object number as the locator. + . + The number and not a page: a page is a fact about one typesetting, and + the object number is the same reference in every output of the document. ++/ +private string tocEntry(ref const OcdaObject obj) { + string indent; + if (auto d = obj.indent_hang in toc_depth) { + foreach (_; 0 .. *d) { indent ~= " "; } + } + string n = internalTarget(obj); + string locator = (n.length > 0) ? (" <sup>[" ~ n ~ "](#" ~ n ~ ")</sup>") : ""; + return indent ~ "- " ~ inline(obj) ~ locator ~ newline; +} + +private string para(ref const OcdaObject obj) { + string bullet = obj.bullet ? "- " : ""; + string indent; + foreach (_; 0 .. obj.indent_base) { indent ~= " "; } + return indent ~ bullet ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines; +} + +private string quote(ref const OcdaObject obj) { + return "> " ~ anchor(obj) ~ inline(obj) ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ verse, a poem, a group or a block: every line kept, with CommonMark's + own hard line break - a backslash at the end of a line - rather than the + two trailing spaces an editor will strip. ++/ +private string lines(ref const OcdaObject obj) { + string txt = inline(obj); + auto out_ = appender!string; + out_ ~= anchor(obj); + auto rows = txt.asplit(newline); + foreach (i, row; rows) { + out_ ~= row; + if (i + 1 < rows.length) { out_ ~= "\\" ~ newline; } + } + return out_.data ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ a book index entry: a list item, and every object it names a link +/ +private string indexEntry(ref const OcdaObject obj) { + string txt = obj.text.replaceAll(rgx_m.trailing_backslash, ""); + return "- " ~ anchor(obj) ~ inlineText(txt, obj) ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ a fenced code block, with the language where the document named one. + Nothing inside a fence is interpreted, which is the whole point. ++/ +private string code(ref const OcdaObject obj) { + string txt = obj.text.replaceAll(rgx_m.nbsp_char, " "); + /+ ↓ a fence long enough that the code cannot close it +/ + string fence = "```"; + while (txt.canFind(fence)) { fence ~= "`"; } + return anchor(obj) ~ newline ~ fence ~ obj.code_syntax ~ newline + ~ txt ~ newline ~ fence ~ newline ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ a pipe table, which needs a header row: markdown has no table without + one, so a table the document did not give a header gets an empty row and + keeps all of its own. ++/ +private string table(ref const OcdaObject obj) { + auto rows = obj.text.split(rgx_m.table_delimiter_row); + auto aligns = obj.table_aligns.asplit(" "); + int cols = max(obj.table_cols, 1); + string[][] body_; + foreach (row; rows) { + if (row.strip.length == 0) { continue; } + string[] cells; + foreach (cell; row.split(rgx_m.table_delimiter_col)) { + cells ~= inlineText(cell, obj).replace("|", "\\|"); + } + body_ ~= cells; + } + auto out_ = appender!string; + out_ ~= anchor(obj) ~ newline; + string rule() { + auto r = appender!string; + r ~= "|"; + foreach (i; 0 .. cols) { + string a = (i < aligns.length) ? aligns[i] : "l"; + r ~= (a == "c") ? " :---: |" : (a == "r") ? " ---: |" : " :--- |"; + } + return r.data ~ newline; + } + string line(string[] cells) { + auto l = appender!string; + l ~= "|"; + foreach (i; 0 .. cols) { + l ~= " " ~ ((i < cells.length) ? cells[i] : "") ~ " |"; + } + return l.data ~ newline; + } + if (body_.length > 0 && obj.table_header) { + out_ ~= line(body_[0]) ~ rule(); + foreach (row; body_[1 .. $]) { out_ ~= line(row); } + } else { + out_ ~= line([]) ~ rule(); + foreach (row; body_) { out_ ~= line(row); } + } + return out_.data ~ ocnMark(obj) ~ newlines; +} + +/+ ↓ the object's anchor, so that a citation can point at it +/ +private string anchor(ref const OcdaObject obj) { + return (obj.ocn == 0) ? "" : "<a id=\"" ~ obj.ocn.to!string ~ "\"></a>"; +} + +/+ ↓ the object's number, at the end of it, as a link to itself +/ +private string ocnMark(ref const OcdaObject obj) { + if (obj.ocn == 0) { return ""; } + string n = obj.ocn.to!string; + return "<sup>[" ~ n ~ "](#" ~ n ~ ")</sup>"; +} + +/+ ↓ the notes the object referred to, each where it was referred to, with the + document's own number and a link back to the object +/ +private string notes(ref const OcdaObject obj) { + auto out_ = appender!string; + foreach (m; obj.text.matchAll(rgx_m.inline_notes_al_all_note)) { + string mark = m["num"].to!string; + string back = (obj.ocn == 0) + ? "" : " [\u21a9](#" ~ obj.ocn.to!string ~ ")"; + out_ ~= newlines ~ "<a id=\"note-" ~ mark ~ "\"></a>" + ~ "<sup>" ~ inert(mark) ~ ".</sup> " + ~ inlineText(m["note"].to!string, obj) ~ back; + } + /+ ↓ and a blank line after the last of them, or the next object begins on + the same line and markdown reads the two as one paragraph +/ + return (out_.data.length > 0) ? out_.data ~ newlines : ""; +} + +private string inline(ref const OcdaObject obj) { + return inlineText(obj.text, obj); +} + +/+ ↓ an object's text as markdown. + . + The order is the one every writer here uses: images before links, + because an image sits inside a link; then the note references, the + faces, the anchors and the breaks. ++/ +private string inlineText(string txt, ref const OcdaObject obj) { + txt = images(txt); + txt = links(txt, obj); + txt = noteRefs(txt); + txt = faces(txt); + txt = txt.replaceAll(rgx_m.inline_link_anchor, ""); + txt = breaks(txt, obj); + return guardLineStarts(unprotect(escape(txt))); +} + +/+ ↓ a "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a + list item, an enumeration item, a blockquote. The writer uses the first of + them itself, for a list, and a line of the document's that begins the same + way would be read as one - which is what a verse or a group can do. + . + Guarded rather than escaped everywhere, because these are special only in + that position and a backslash in every hyphen would be absurd. ++/ +private string guardLineStarts(string s) { + auto out_ = appender!string; + bool at_start = true; + for (size_t i = 0; i < s.length; i++) { + if (at_start) { + size_t j = i; + while (j < s.length && s[j] == ' ') { j++; } + if (j < s.length) { + char c = s[j]; + bool marker = (c == '-' || c == '+' || c == '>') + && j + 1 < s.length && (s[j + 1] == ' ' || s[j + 1] == '\t'); + size_t k = j; + while (k < s.length && s[k] >= '0' && s[k] <= '9') { k++; } + bool enumerated = k > j && k + 1 < s.length + && (s[k] == '.' || s[k] == ')') && s[k + 1] == ' '; + if (marker || enumerated) { + out_ ~= s[i .. (enumerated ? k : j)]; + out_ ~= '\\'; + i = (enumerated ? k : j) - 1; + at_start = false; + continue; + } + } + at_start = false; + } + out_ ~= s[i]; + if (s[i] == '\n') { at_start = true; } + } + return out_.data; +} + +/+ ↓ the markers that protect the markdown this writer emits from the escaping + that follows. + . + The escaping has to come *last*, and this is why: the abstraction's own + markers are built from the characters markdown reserves - "⑆*┨" for + emphasis, "⑆_┨" for an underscore face - so escaping first turns them + into "⑆\*┨" and no face is ever recognised. It cost 70 stray hashes and + every face in one document before it was seen. + . + A private-use pair, because they must not occur in a document, and the + protection nests, so the escaper counts it rather than flagging it. ++/ +private enum keep_open = ""; +private enum keep_close = ""; + +/+ ↓ a third marker of the same kind, standing for the directory the images are + in. + . + An image reference is written while an object is written, and where the + file will be put is not something the model knows: it is told. So the + reference carries a marker and writeMarkdown substitutes it once, at the + end, out of the one argument that says. ++/ +private enum image_dir_mark = ""; + +private string keep(string markdown_source) { + return keep_open ~ markdown_source ~ keep_close; +} + +private string unprotect(string txt) { + return txt.replace(keep_open, "").replace(keep_close, ""); +} + +/+ ↓ the document's own text, escaped for a protected run: the same rule, run + early, and then flattened so that the result is inert +/ +private string inert(string txt) { + return unprotect(escape(txt)); +} + +/+ ↓ the characters markdown reads as syntax, escaped wherever they occur, + and only outside a protected run. + . + The same blunt rule the typst writer arrived at, and for the same reason: + a closed set is checkable and a rule about positions is not. The three + html characters become entities rather than backslash escapes, because + markdown passes raw html through and "<b" would open a tag. + . + Last in the pipeline, because the abstraction's markers are built from + the characters markdown reserves and escaping them first destroys every + one. The protection nests, so it is counted, not flagged. ++/ +private string escape(string txt) { + auto out_ = appender!string; + int depth = 0; + foreach (dchar c; txt) { + if (c == keep_open.to!dchar) { depth++; continue; } + if (c == keep_close.to!dchar) { if (depth > 0) { depth--; } continue; } + if (depth > 0) { out_ ~= c; continue; } + switch (c) { + case '&': out_ ~= "&"; break; + case '<': out_ ~= "<"; break; + case '>': out_ ~= ">"; break; + case '\\': case '`': case '*': case '_': case '[': case ']': + case '#': case '|': case '~': + out_ ~= '\\'; out_ ~= c; break; + default: out_ ~= c; break; + } + } + return out_.data; +} + +/+ ↓ an image, with its alt text where the document gave one +/ +private string images(string txt) { + return replaceAll!((m) { + string rest = m["post"].to!string; + string alt; + if (auto a = rest.matchFirst(rgx_m.inline_image_alt)) { + alt = a["alt"].to!string; + rest = a.post.to!string; + } + return m["pre"].to!string + ~ keep("![" ~ inert(alt) ~ "](" ~ image_dir_mark ~ "/" ~ m["img"].to!string ~ ")") + ~ rest; + })(txt, rgx_m.inline_image); +} + +/+ ↓ a link. An internal target becomes a fragment, which is the object + number, and so resolves against the anchor every object carries. + . + And *only* an object number: those are the anchors this writer defines, so + a target that names something else - the endnotes section, an anchor tag + the markup placed - would be a link to nothing. The text stands on its + own instead, which is what a reader sees anyway. ++/ +private string links(string txt, ref const OcdaObject obj) { + auto stow = obj.stow_link; + txt = replaceAll!((m) { + size_t num = m["num"].to!size_t; + string url = (num < stow.length) ? stow[num] : ""; + return m["linked_text"].to!string ~ "┤" ~ url ~ "├"; + })(txt, rgx_m.inline_link_number_only); + return replaceAll!((m) { + string text_ = m.captures[1].to!string; + string target = m.captures[2].to!string; + if (target.startsWith("¤")) { + auto im = target.matchFirst(rgx_m.internal_target); + if (!im.empty) { target = im["frag"].to!string; } + } + string shown = inert(faces(text_)); + if (target.length == 0) { return keep(shown); } + if (target.startsWith("#") && !isObjectNumber(target[1 .. $])) { + return keep(shown); + } + return keep("[" ~ shown ~ "](" ~ target ~ ")"); + })(txt, rgx_m.inline_link); +} + +/+ ↓ a link target is a url or a fragment and not prose, so the escaping the + text went through comes off it again. A "#" that stayed escaped made every + table of contents line point at nothing. +/ +private string unescapeTarget(string target) { + auto out_ = appender!string; + for (size_t i = 0; i < target.length; i++) { + if (target[i] == 0x5c && i + 1 < target.length) { out_ ~= target[++i]; continue; } + out_ ~= target[i]; + } + return out_.data.replace("&", "&").replace("<", "<").replace(">", ">"); +} + +/+ ↓ a fragment that names an object of this document +/ +private bool isObjectNumber(string s) { + if (s.length == 0 || s == "0") { return false; } + foreach (c; s) { if (c < '0' || c > '9') { return false; } } + return true; +} + +/+ ↓ a note reference: a superscript link to the note, which sits under the + object that refers to it +/ +private string noteRefs(string txt) { + return replaceAll!((m) { + string mark = m["num"].to!string; + /+ ↓ the mark is shown as well as named in the target, and a star note`s mark + is an asterisk, which markdown would pair with the next one +/ + return keep("<sup>[" ~ inert(mark) ~ "](#note-" ~ mark ~ ")</sup>"); + })(txt, rgx_m.inline_notes_al_all_note); +} + +/+ ↓ the font faces. Emphasis and strong are markdown's own; the rest have no + markdown and take the html element markdown itself would produce. + . + The spaces move *outside* the markers. The abstraction's markers include + the spaces that surrounded the run - "⑆/┨ lex mercatoria ┣/" - and + markdown's emphasis will not open on a space: "* lex *" is three words and + two asterisks, not an italic. ++/ +private string faces(string txt) { + static struct Face { typeof(Rgx.inline_bold) pattern; string open, close; } + string wrap(string open, string close, string inner) { + size_t a = 0, b = inner.length; + while (a < b && inner[a] == ' ') { a++; } + while (b > a && inner[b - 1] == ' ') { b--; } + /+ ↓ the spaces are dropped rather than moved: the abstraction`s marker + takes in the space that was beside the run, and the text before it + already ends in one +/ + return keep(open) ~ inner[a .. b] ~ keep(close); + } + string apply(alias pattern)(string s, string open, string close) { + return replaceAll!((m) => wrap(open, close, m.captures[1].to!string)) + (s, pattern); + } + txt = apply!(rgx_m.inline_emphasis)(txt, "**", "**"); + txt = apply!(rgx_m.inline_bold)(txt, "**", "**"); + txt = apply!(rgx_m.inline_italics)(txt, "*", "*"); + txt = apply!(rgx_m.inline_underscore)(txt, "<u>", "</u>"); + txt = apply!(rgx_m.inline_superscript)(txt, "<sup>", "</sup>"); + txt = apply!(rgx_m.inline_subscript)(txt, "<sub>", "</sub>"); + txt = apply!(rgx_m.inline_mono)(txt, "`", "`"); + txt = apply!(rgx_m.inline_strike)(txt, "~~", "~~"); + txt = apply!(rgx_m.inline_insert)(txt, "<ins>", "</ins>"); + txt = apply!(rgx_m.inline_cite)(txt, "<cite>", "</cite>"); + return txt; +} + +/+ ↓ the line breaks the markup asked for, and the grouped indents +/ +private string breaks(string txt, ref const OcdaObject obj) { + if (obj.is_a == "code") { return txt; } + if (obj.is_a == "group" || obj.is_a == "block") { + txt = txt + .replaceAll(rgx_m.grouped_para_indent_hang, "$2$2") + .replaceAll(rgx_m.grouped_para_bullet_at, "$1\u25cf ") + .replaceAll(rgx_m.grouped_para_bullet, "\u25cf ") + .replaceAll(rgx_m.grouped_para_indent, "$1$1"); + } + return txt + .replaceAll(rgx_m.nbsp_char, " ") + .replaceAll(rgx_m.br_line, newline) + .replaceAll(rgx_m.br_line_inline, newline) + .replaceAll(rgx_m.br_line_spaced, newlines) + .replaceAll(rgx_m.line_break, newline) + .replaceAll(rgx_m.mark_internal_site_lnk, ""); +} + +/+ ↓ the object a table of contents entry points at, where that is an object + number +/ +private string internalTarget(ref const OcdaObject obj) { + foreach (m; obj.text.matchAll(rgx_m.any_internal_target)) { + string frag = m["frag"].to!string; + bool digits = frag.length > 0; + foreach (c; frag) { if (c < '0' || c > '9') { digits = false; break; } } + if (digits) { return frag; } + } + return ""; +} |