//// The markdown writer: a document as CommonMark, for reading anywhere.
////
//// The fifth format, and the one asked for because markdown is what a
//// document is passed around as. The question was how much of spine's spirit
//// survives the translation, and the answer is: the object numbers do, which
//// is the part that matters.
////
//// The dialect is CommonMark, plus pipe tables. Raw inline html is used in
//// exactly two places, both for the object number, because markdown has no
//// anchor syntax and nothing else will do:
////
//// The text of the paragraph.[12](#12)
////
//// Markdown's own footnote syntax is not used: a renderer numbers footnotes
//// itself, in the order it meets them, and the document's note numbers are
//// part of its citation. So a note is written where it is referred to, with
//// the document's own number and a link back to the object - the same
//// decision the typst and the text writers made.
////
//// Byte for byte the same as the D writer.
import gleam/dict.{type Dict}
import gleam/int
import gleam/list
import gleam/regexp
import gleam/string
import ocda/model.{type Document, type Object}
import write/rgx
const newline = "\n"
const newlines = "\n\n"
/// The document as markdown.
///
/// image_pth is where the images are, relative to wherever this output is put.
/// Given rather than assumed: the model names an image and does not say where the
/// file naming it will sit. spine, whose markdown goes two directories down under
/// a language, passes "../../image".
pub fn write(doc: Document, image_pth: String) -> String {
let depths = toc_depths(doc)
{
front_matter(doc)
<> {
sections(doc)
|> list.flat_map(fn(name) { model.section(doc, name) })
|> list.filter(fn(obj) {
model.renders(obj) && !obj.dummy_heading && string.trim(obj.text) != ""
})
|> list.map(fn(obj) { object(obj, depths) })
|> string.concat
}
}
|> string.replace(cp(image_dir_mark), image_pth)
}
/// head, toc, body, then each trailing section with more than one object. Not
/// endnotes: a note is written where it is referred to.
pub fn sections(doc: Document) -> List(String) {
let trailing =
["glossary", "bibliography", "bookindex", "blurb"]
|> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 })
list.append(["head", "toc", "body"], trailing)
}
/// spine reserves indent_hang 1 to 3 for levels above the body, so a toc jumps
/// from 1 to 4; a markdown list nests by two spaces a level and cannot take a
/// jump, so the values the document uses are ranked.
fn toc_depths(doc: Document) -> Dict(Int, Int) {
model.section(doc, "toc")
|> list.filter(model.renders)
|> list.map(fn(obj) { obj.indent_hang })
|> list.unique
|> list.sort(int.compare)
|> list.index_map(fn(level, i) { #(level, i) })
|> dict.from_list
}
/// The document's metadata as yaml front matter, which is how a markdown file
/// carries metadata and what every static site generator reads.
fn front_matter(doc: Document) -> String {
let language = case dict.get(doc.source, "language") {
Ok(l) -> l
Error(_) -> ""
}
"---"
<> newline
<> {
[
#("title", model.meta(doc, "title.full")),
#("author", model.meta(doc, "creator.author")),
#("date", model.meta(doc, "date.published")),
#("language", language),
#("copyright", model.meta(doc, "rights.copyright")),
#("license", model.meta(doc, "rights.license")),
]
|> list.map(fn(pair) {
let #(key, value) = pair
case value {
"" -> ""
_ -> key <> ": " <> yaml(value) <> newline
}
})
|> string.concat
}
<> "---"
<> newlines
}
/// A yaml scalar. Quoted always, so that a colon or a leading marker in a
/// title cannot change the shape of the document's own metadata.
fn yaml(s: String) -> String {
"\""
<> {
s
|> string.replace("\\", "\\\\")
|> string.replace("\"", "\\\"")
|> string.replace("\n", " ")
}
<> "\""
}
/// One object, and then the notes it referred to.
///
/// The notes are gathered here rather than inside each kind, because a
/// reference can be anywhere - the markup manual has one in a heading - and a
/// reference whose note was never written is a link to nothing.
fn object(obj: Object, depths: Dict(Int, Int)) -> String {
object_body(obj, depths) <> notes(obj)
}
fn object_body(obj: Object, depths: Dict(Int, Int)) -> String {
case obj.is_a {
"heading" -> heading(obj)
"toc" -> toc_entry(obj, depths)
"table" -> table(obj)
"code" -> code(obj)
"quote" -> quote(obj)
"verse" | "poem" | "group" | "block" -> lines(obj)
"bookindex" -> index_entry(obj)
_ -> para(obj)
}
}
/// Six levels of heading, which is all markdown has. A document may be eight
/// deep; the two deepest take the sixth, and the object number, the anchor and
/// the table of contents carry the true depth.
fn heading_level(level: String) -> Int {
case level {
"A" -> 1
"B" -> 2
"C" -> 3
"D" -> 4
"1" -> 5
"2" | "3" | "4" -> 6
_ -> 1
}
}
fn heading(obj: Object) -> String {
string.repeat("#", heading_level(obj.heading_level))
<> " "
<> anchor(obj)
<> inline(obj)
<> ocn_mark(obj)
<> newlines
}
/// A table of contents entry: a list item at its depth, the heading as a link,
/// and the object number as the locator. The number and not a page: a page is
/// a fact about one typesetting, and the object number is the same reference
/// in every output of the document.
fn toc_entry(obj: Object, depths: Dict(Int, Int)) -> String {
let indent = case dict.get(depths, obj.indent_hang) {
Ok(d) -> string.repeat(" ", d)
Error(_) -> ""
}
let locator = case internal_target(obj) {
"" -> ""
n -> " [" <> n <> "](#" <> n <> ")"
}
indent <> "- " <> inline(obj) <> locator <> newline
}
fn para(obj: Object) -> String {
let bullet = case obj.bullet {
True -> "- "
False -> ""
}
string.repeat(" ", obj.indent_base)
<> bullet
<> anchor(obj)
<> inline(obj)
<> ocn_mark(obj)
<> newlines
}
fn quote(obj: Object) -> String {
"> " <> anchor(obj) <> inline(obj) <> ocn_mark(obj) <> newlines
}
/// Verse, a poem, a group or a block: every line kept, with CommonMark's own
/// hard line break - a backslash at the end of a line - rather than the two
/// trailing spaces an editor will strip.
fn lines(obj: Object) -> String {
anchor(obj)
<> {
string.split(inline(obj), newline)
|> string.join("\\" <> newline)
}
<> ocn_mark(obj)
<> newlines
}
/// A book index entry: a list item, and every object it names a link.
fn index_entry(obj: Object) -> String {
let txt = regexp.replace(rgx.trailing_backslash(), obj.text, "")
"- " <> anchor(obj) <> inline_text(txt, obj) <> ocn_mark(obj) <> newlines
}
/// A fenced code block, with the language where the document named one.
/// Nothing inside a fence is interpreted, which is the whole point.
fn code(obj: Object) -> String {
let txt = regexp.replace(rgx.nbsp_char(), obj.text, " ")
let fence = long_enough(txt, "```")
anchor(obj)
<> newline
<> fence
<> obj.code_syntax
<> newline
<> txt
<> newline
<> fence
<> newline
<> ocn_mark(obj)
<> newlines
}
fn long_enough(txt: String, fence: String) -> String {
case string.contains(txt, fence) {
True -> long_enough(txt, fence <> "`")
False -> fence
}
}
/// A pipe table, which needs a header row: markdown has no table without one,
/// so a table the document did not give a header gets an empty row and keeps
/// all of its own.
fn table(obj: Object) -> String {
let aligns = string.split(obj.table_aligns, on: " ")
let cols = int.max(obj.table_cols, 1)
let body =
regexp.split(rgx.table_delimiter_row(), obj.text)
|> list.filter(fn(r) { string.trim(r) != "" })
|> list.map(fn(row) {
regexp.split(rgx.table_delimiter_col(), row)
|> list.map(fn(cell) { string.replace(inline_text(cell, obj), "|", "\\|") })
})
let rule =
"|"
<> {
list.repeat("", cols)
|> list.index_map(fn(_, i) {
case at(aligns, i, "l") {
"c" -> " :---: |"
"r" -> " ---: |"
_ -> " :--- |"
}
})
|> string.concat
}
<> newline
let line = fn(cells: List(String)) {
"|"
<> {
list.repeat("", cols)
|> list.index_map(fn(_, i) { " " <> at(cells, i, "") <> " |" })
|> string.concat
}
<> newline
}
let rows = case body, obj.table_header {
[first, ..rest], True ->
line(first) <> rule <> { rest |> list.map(line) |> string.concat }
_, _ -> line([]) <> rule <> { body |> list.map(line) |> string.concat }
}
anchor(obj) <> newline <> rows <> ocn_mark(obj) <> newlines
}
/// The object's anchor, so that a citation can point at it.
fn anchor(obj: Object) -> String {
case obj.ocn {
0 -> ""
n -> " int.to_string(n) <> "\">"
}
}
/// The object's number, at the end of it, as a link to itself.
fn ocn_mark(obj: Object) -> String {
case obj.ocn {
0 -> ""
n -> {
let s = int.to_string(n)
"[" <> s <> "](#" <> s <> ")"
}
}
}
/// The notes the object referred to, each where it was referred to, with the
/// document's own number and a link back to the object.
fn notes(obj: Object) -> String {
let back = case obj.ocn {
0 -> ""
n -> " [\u{21a9}](#" <> int.to_string(n) <> ")"
}
regexp.scan(rgx.inline_notes_any_mark(), obj.text)
|> list.map(fn(m) {
let mark = rgx.group(m.submatches, 0)
newlines
<> " mark
<> "\">"
<> inert(mark)
<> ". "
<> inline_text(rgx.group(m.submatches, 1), obj)
<> back
})
|> string.concat
|> fn(s) {
// and a blank line after the last of them, or the next object begins on
// the same line and markdown reads the two as one paragraph
case s {
"" -> ""
_ -> s <> newlines
}
}
}
fn inline(obj: Object) -> String {
inline_text(obj.text, obj)
}
/// An object's text as markdown.
///
/// The order is the one every writer here uses: images before links, because
/// an image sits inside a link; then the note references, the faces, the
/// anchors and the breaks.
fn inline_text(txt: String, obj: Object) -> String {
txt
|> images
|> links(obj)
|> note_refs
|> faces
|> regexp.replace(rgx.inline_link_anchor(), _, "")
|> breaks(obj)
|> escape
|> unprotect
|> guard_line_starts
}
/// A "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a list
/// item, an enumeration item, a blockquote. The writer uses the first of them
/// itself, and a line of the document's that begins the same way would be read
/// as one - which is what a verse or a group can do.
///
/// Guarded rather than escaped everywhere, because these are special only in
/// that position.
fn guard_line_starts(s: String) -> String {
string.split(s, "\n")
|> list.map(guard_one_line)
|> string.join("\n")
}
fn guard_one_line(line: String) -> String {
let spaces = leading_spaces(line, 0)
let rest = string.drop_start(line, spaces)
let pad = string.repeat(" ", spaces)
case marker_at(rest) {
True -> pad <> "\\" <> rest
False ->
case enumerated_at(rest, 0) {
0 -> line
n -> pad <> string.slice(rest, 0, n) <> "\\" <> string.drop_start(rest, n)
}
}
}
fn leading_spaces(s: String, n: Int) -> Int {
case string.slice(s, n, 1) {
" " -> leading_spaces(s, n + 1)
_ -> n
}
}
fn marker_at(s: String) -> Bool {
case string.slice(s, 0, 1), string.slice(s, 1, 1) {
"-", " " | "+", " " | ">", " " -> True
"-", "\t" | "+", "\t" | ">", "\t" -> True
_, _ -> False
}
}
/// How many digits precede a "." or ")" and a space, or 0 for none.
fn enumerated_at(s: String, n: Int) -> Int {
case string.slice(s, n, 1) {
d if d == "0" || d == "1" || d == "2" || d == "3" || d == "4" -> enumerated_at(s, n + 1)
d if d == "5" || d == "6" || d == "7" || d == "8" || d == "9" -> enumerated_at(s, n + 1)
"." | ")" ->
case n > 0 && string.slice(s, n + 1, 1) == " " {
True -> n
False -> 0
}
_ -> 0
}
}
// The markers that protect the markdown this writer emits from the escaping
// that follows.
//
// The escaping has to come *last*, and this is why: the abstraction's own
// markers are built from the characters markdown reserves - "⑆*┨" for emphasis,
// "⑆_┨" for an underscore face - so escaping first turns them into "⑆\*┨" and
// no face is ever recognised.
const keep_open = 0xe000
const keep_close = 0xe001
// A third marker of the same kind, standing for the directory the images are in.
//
// An image reference is written while an object is written, and where the file
// will be put is not something the model knows: it is told. So the reference
// carries a marker and write substitutes it once, at the end, out of the one
// argument that says.
const image_dir_mark = 0xe002
fn keep(markdown_source: String) -> String {
cp(keep_open) <> markdown_source <> cp(keep_close)
}
fn unprotect(txt: String) -> String {
txt |> string.replace(cp(keep_open), "") |> string.replace(cp(keep_close), "")
}
fn cp(n: Int) -> String {
let assert Ok(c) = string.utf_codepoint(n)
string.from_utf_codepoints([c])
}
/// The document's own text, escaped for a protected run: the same rule, run
/// early, and then flattened so that the result is inert.
fn inert(txt: String) -> String {
unprotect(escape(txt))
}
/// The characters markdown reads as syntax, escaped wherever they occur.
///
/// The same rule the typst writer arrived at, and for the same reason: a closed
/// set is checkable and a rule about positions is not. The three html
/// characters become entities rather than backslash escapes, because markdown
/// passes raw html through and " String {
let #(out, _) =
string.to_utf_codepoints(txt)
|> list.fold(#([], 0), fn(state, c) {
let #(acc, depth) = state
let n = string.utf_codepoint_to_int(c)
case n == keep_open, n == keep_close, depth > 0 {
True, _, _ -> #(acc, depth + 1)
_, True, _ -> #(acc, int.max(depth - 1, 0))
_, _, True -> #([string.from_utf_codepoints([c]), ..acc], depth)
_, _, False -> #([escaped(n, c), ..acc], depth)
}
})
string.concat(list.reverse(out))
}
fn escaped(n: Int, c: UtfCodepoint) -> String {
case n {
0x26 -> "&"
0x3c -> "<"
0x3e -> ">"
// \ ` * _ [ ] # | ~
0x5c | 0x60 | 0x2a | 0x5f | 0x5b | 0x5d | 0x23 | 0x7c | 0x7e ->
"\\" <> string.from_utf_codepoints([c])
_ -> string.from_utf_codepoints([c])
}
}
/// An image, with its alt text where the document gave one.
fn images(txt: String) -> String {
rgx.replace_with(rgx.inline_image(), txt, fn(g) {
let post = g(5)
let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) {
[m, ..] -> #(rgx.group(m.submatches, 0), case
string.split_once(post, m.content)
{
Ok(#(_, after)) -> after
Error(_) -> post
})
[] -> #("", post)
}
g(0)
<> keep(" <> "/" <> g(2) <> ")")
<> rest
})
}
/// A link. An internal target becomes a fragment, which is the object number,
/// and so resolves against the anchor every object carries.
///
/// And *only* an object number: those are the anchors this writer defines, so a
/// target that names something else - the endnotes section, an anchor tag the
/// markup placed - would be a link to nothing. The text stands on its own
/// instead, which is what a reader sees anyway.
fn links(txt: String, obj: Object) -> String {
rgx.replace_with(rgx.inline_link_number_only(), txt, fn(g) {
let url = case int.parse(g(1)) {
Ok(n) -> at(obj.stow_link, n, "")
Error(_) -> ""
}
g(0) <> "┤" <> url <> "├"
})
|> rgx.replace_with(rgx.inline_link(), _, fn(g) {
let target = case string.starts_with(g(1), "¤") {
False -> g(1)
True ->
case regexp.scan(rgx.internal_target(), g(1)) {
[m, ..] -> rgx.group(m.submatches, 1)
[] -> g(1)
}
}
let shown = inert(faces(g(0)))
case target, string.starts_with(target, "#") && !is_object_number(target) {
"", _ -> keep(shown)
_, True -> keep(shown)
_, False -> keep("[" <> shown <> "](" <> target <> ")")
}
})
}
/// A link target is a url or a fragment and not prose, so the escaping the
/// text went through comes off it again. A "#" that stayed escaped made every
/// table of contents line point at nothing.
fn unescape_target(target: String) -> String {
target
|> string.replace("\\", "")
|> string.replace("&", "&")
|> string.replace("<", "<")
|> string.replace(">", ">")
}
/// A fragment that names an object of this document.
fn is_object_number(clean: String) -> Bool {
case string.drop_start(clean, 1) {
"" -> False
"0" -> False
rest ->
case int.parse(rest) {
Ok(_) -> True
Error(_) -> False
}
}
}
/// A note reference: a superscript link to the note, which sits under the
/// object that refers to it.
fn note_refs(txt: String) -> String {
rgx.replace_with(rgx.inline_notes_any_mark(), txt, fn(g) {
// the mark is shown as well as named in the target, and a star note's mark
// is an asterisk, which markdown would pair with the next one
keep("[" <> inert(g(0)) <> "](#note-" <> g(0) <> ")")
})
}
/// The font faces. Emphasis and strong are markdown's own; the rest have no
/// markdown and take the html element markdown itself would produce.
///
/// The spaces inside a run are dropped rather than kept: the abstraction's
/// marker takes in the space that was beside the run, the text before it
/// already ends in one, and markdown's emphasis will not open on a space.
fn faces(txt: String) -> String {
[
#(rgx.inline_emphasis(), "**", "**"),
#(rgx.inline_bold(), "**", "**"),
#(rgx.inline_italics(), "*", "*"),
#(rgx.inline_underscore(), "", ""),
#(rgx.inline_superscript(), "", ""),
#(rgx.inline_subscript(), "", ""),
#(rgx.inline_mono(), "`", "`"),
#(rgx.inline_strike(), "~~", "~~"),
#(rgx.inline_insert(), "", ""),
#(rgx.inline_cite(), "", ""),
]
|> list.fold(txt, fn(acc, face) {
let #(pattern, open, close) = face
rgx.replace_with(pattern, acc, fn(g) {
keep(open) <> trim_spaces(g(0)) <> keep(close)
})
})
}
fn trim_spaces(s: String) -> String {
case string.starts_with(s, " "), string.ends_with(s, " ") {
True, _ -> trim_spaces(string.drop_start(s, 1))
_, True -> trim_spaces(string.drop_end(s, 1))
_, _ -> s
}
}
/// The line breaks the markup asked for, and the grouped indents.
fn breaks(txt: String, obj: Object) -> String {
case obj.is_a {
"code" -> txt
_ -> {
let txt = case obj.is_a == "group" || obj.is_a == "block" {
False -> txt
True ->
txt
|> rgx.replace_with(rgx.grouped_para_indent_hang(), _, fn(g) {
g(1) <> g(1)
})
|> rgx.replace_with(rgx.grouped_para_bullet_at(), _, fn(g) {
g(0) <> g(0) <> "\u{25cf} "
})
|> rgx.replace_with(rgx.grouped_para_bullet(), _, fn(_) {
"\u{25cf} "
})
|> rgx.replace_with(rgx.grouped_para_indent(), _, fn(g) {
g(0) <> g(0)
})
}
txt
|> regexp.replace(rgx.nbsp_char(), _, " ")
|> regexp.replace(rgx.br_line(), _, newline)
|> regexp.replace(rgx.br_line_inline(), _, newline)
|> regexp.replace(rgx.br_line_spaced(), _, newlines)
|> regexp.replace(rgx.line_break(), _, newline)
|> regexp.replace(rgx.mark_internal_site_lnk(), _, "")
}
}
}
/// The object a table of contents entry points at, where that is an object
/// number.
fn internal_target(obj: Object) -> String {
regexp.scan(rgx.any_internal_target(), obj.text)
|> list.fold("", fn(found, m) {
case found {
"" -> {
let frag = rgx.group(m.submatches, 0)
case int.parse(frag) {
Ok(_) -> frag
Error(_) -> ""
}
}
_ -> found
}
})
}
fn at(items: List(String), index: Int, fallback: String) -> String {
case items, index {
[], _ -> fallback
[first, ..], 0 -> first
[_, ..rest], n -> at(rest, n - 1, fallback)
}
}