//// The markdown writer: a document as CommonMark, for reading anywhere. //// //// The fifth format, and the one asked for because markdown is what a //// document is passed around as. The question was how much of spine's spirit //// survives the translation, and the answer is: the object numbers do, which //// is the part that matters. //// //// The dialect is CommonMark, plus pipe tables. Raw inline html is used in //// exactly two places, both for the object number, because markdown has no //// anchor syntax and nothing else will do: //// //// The text of the paragraph.[12](#12) //// //// Markdown's own footnote syntax is not used: a renderer numbers footnotes //// itself, in the order it meets them, and the document's note numbers are //// part of its citation. So a note is written where it is referred to, with //// the document's own number and a link back to the object - the same //// decision the typst and the text writers made. //// //// Byte for byte the same as the D writer. import gleam/dict.{type Dict} import gleam/int import gleam/list import gleam/regexp import gleam/string import ocda/model.{type Document, type Object} import write/rgx const newline = "\n" const newlines = "\n\n" /// The document as markdown. /// /// image_pth is where the images are, relative to wherever this output is put. /// Given rather than assumed: the model names an image and does not say where the /// file naming it will sit. spine, whose markdown goes two directories down under /// a language, passes "../../image". pub fn write(doc: Document, image_pth: String) -> String { let depths = toc_depths(doc) { front_matter(doc) <> { sections(doc) |> list.flat_map(fn(name) { model.section(doc, name) }) |> list.filter(fn(obj) { model.renders(obj) && !obj.dummy_heading && string.trim(obj.text) != "" }) |> list.map(fn(obj) { object(obj, depths) }) |> string.concat } } |> string.replace(cp(image_dir_mark), image_pth) } /// head, toc, body, then each trailing section with more than one object. Not /// endnotes: a note is written where it is referred to. pub fn sections(doc: Document) -> List(String) { let trailing = ["glossary", "bibliography", "bookindex", "blurb"] |> list.filter(fn(name) { list.length(model.section(doc, name)) > 1 }) list.append(["head", "toc", "body"], trailing) } /// spine reserves indent_hang 1 to 3 for levels above the body, so a toc jumps /// from 1 to 4; a markdown list nests by two spaces a level and cannot take a /// jump, so the values the document uses are ranked. fn toc_depths(doc: Document) -> Dict(Int, Int) { model.section(doc, "toc") |> list.filter(model.renders) |> list.map(fn(obj) { obj.indent_hang }) |> list.unique |> list.sort(int.compare) |> list.index_map(fn(level, i) { #(level, i) }) |> dict.from_list } /// The document's metadata as yaml front matter, which is how a markdown file /// carries metadata and what every static site generator reads. fn front_matter(doc: Document) -> String { let language = case dict.get(doc.source, "language") { Ok(l) -> l Error(_) -> "" } "---" <> newline <> { [ #("title", model.meta(doc, "title.full")), #("author", model.meta(doc, "creator.author")), #("date", model.meta(doc, "date.published")), #("language", language), #("copyright", model.meta(doc, "rights.copyright")), #("license", model.meta(doc, "rights.license")), ] |> list.map(fn(pair) { let #(key, value) = pair case value { "" -> "" _ -> key <> ": " <> yaml(value) <> newline } }) |> string.concat } <> "---" <> newlines } /// A yaml scalar. Quoted always, so that a colon or a leading marker in a /// title cannot change the shape of the document's own metadata. fn yaml(s: String) -> String { "\"" <> { s |> string.replace("\\", "\\\\") |> string.replace("\"", "\\\"") |> string.replace("\n", " ") } <> "\"" } /// One object, and then the notes it referred to. /// /// The notes are gathered here rather than inside each kind, because a /// reference can be anywhere - the markup manual has one in a heading - and a /// reference whose note was never written is a link to nothing. fn object(obj: Object, depths: Dict(Int, Int)) -> String { object_body(obj, depths) <> notes(obj) } fn object_body(obj: Object, depths: Dict(Int, Int)) -> String { case obj.is_a { "heading" -> heading(obj) "toc" -> toc_entry(obj, depths) "table" -> table(obj) "code" -> code(obj) "quote" -> quote(obj) "verse" | "poem" | "group" | "block" -> lines(obj) "bookindex" -> index_entry(obj) _ -> para(obj) } } /// Six levels of heading, which is all markdown has. A document may be eight /// deep; the two deepest take the sixth, and the object number, the anchor and /// the table of contents carry the true depth. fn heading_level(level: String) -> Int { case level { "A" -> 1 "B" -> 2 "C" -> 3 "D" -> 4 "1" -> 5 "2" | "3" | "4" -> 6 _ -> 1 } } fn heading(obj: Object) -> String { string.repeat("#", heading_level(obj.heading_level)) <> " " <> anchor(obj) <> inline(obj) <> ocn_mark(obj) <> newlines } /// A table of contents entry: a list item at its depth, the heading as a link, /// and the object number as the locator. The number and not a page: a page is /// a fact about one typesetting, and the object number is the same reference /// in every output of the document. fn toc_entry(obj: Object, depths: Dict(Int, Int)) -> String { let indent = case dict.get(depths, obj.indent_hang) { Ok(d) -> string.repeat(" ", d) Error(_) -> "" } let locator = case internal_target(obj) { "" -> "" n -> " [" <> n <> "](#" <> n <> ")" } indent <> "- " <> inline(obj) <> locator <> newline } fn para(obj: Object) -> String { let bullet = case obj.bullet { True -> "- " False -> "" } string.repeat(" ", obj.indent_base) <> bullet <> anchor(obj) <> inline(obj) <> ocn_mark(obj) <> newlines } fn quote(obj: Object) -> String { "> " <> anchor(obj) <> inline(obj) <> ocn_mark(obj) <> newlines } /// Verse, a poem, a group or a block: every line kept, with CommonMark's own /// hard line break - a backslash at the end of a line - rather than the two /// trailing spaces an editor will strip. fn lines(obj: Object) -> String { anchor(obj) <> { string.split(inline(obj), newline) |> string.join("\\" <> newline) } <> ocn_mark(obj) <> newlines } /// A book index entry: a list item, and every object it names a link. fn index_entry(obj: Object) -> String { let txt = regexp.replace(rgx.trailing_backslash(), obj.text, "") "- " <> anchor(obj) <> inline_text(txt, obj) <> ocn_mark(obj) <> newlines } /// A fenced code block, with the language where the document named one. /// Nothing inside a fence is interpreted, which is the whole point. fn code(obj: Object) -> String { let txt = regexp.replace(rgx.nbsp_char(), obj.text, " ") let fence = long_enough(txt, "```") anchor(obj) <> newline <> fence <> obj.code_syntax <> newline <> txt <> newline <> fence <> newline <> ocn_mark(obj) <> newlines } fn long_enough(txt: String, fence: String) -> String { case string.contains(txt, fence) { True -> long_enough(txt, fence <> "`") False -> fence } } /// A pipe table, which needs a header row: markdown has no table without one, /// so a table the document did not give a header gets an empty row and keeps /// all of its own. fn table(obj: Object) -> String { let aligns = string.split(obj.table_aligns, on: " ") let cols = int.max(obj.table_cols, 1) let body = regexp.split(rgx.table_delimiter_row(), obj.text) |> list.filter(fn(r) { string.trim(r) != "" }) |> list.map(fn(row) { regexp.split(rgx.table_delimiter_col(), row) |> list.map(fn(cell) { string.replace(inline_text(cell, obj), "|", "\\|") }) }) let rule = "|" <> { list.repeat("", cols) |> list.index_map(fn(_, i) { case at(aligns, i, "l") { "c" -> " :---: |" "r" -> " ---: |" _ -> " :--- |" } }) |> string.concat } <> newline let line = fn(cells: List(String)) { "|" <> { list.repeat("", cols) |> list.index_map(fn(_, i) { " " <> at(cells, i, "") <> " |" }) |> string.concat } <> newline } let rows = case body, obj.table_header { [first, ..rest], True -> line(first) <> rule <> { rest |> list.map(line) |> string.concat } _, _ -> line([]) <> rule <> { body |> list.map(line) |> string.concat } } anchor(obj) <> newline <> rows <> ocn_mark(obj) <> newlines } /// The object's anchor, so that a citation can point at it. fn anchor(obj: Object) -> String { case obj.ocn { 0 -> "" n -> " int.to_string(n) <> "\">" } } /// The object's number, at the end of it, as a link to itself. fn ocn_mark(obj: Object) -> String { case obj.ocn { 0 -> "" n -> { let s = int.to_string(n) "[" <> s <> "](#" <> s <> ")" } } } /// The notes the object referred to, each where it was referred to, with the /// document's own number and a link back to the object. fn notes(obj: Object) -> String { let back = case obj.ocn { 0 -> "" n -> " [\u{21a9}](#" <> int.to_string(n) <> ")" } regexp.scan(rgx.inline_notes_any_mark(), obj.text) |> list.map(fn(m) { let mark = rgx.group(m.submatches, 0) newlines <> " mark <> "\">" <> inert(mark) <> ". " <> inline_text(rgx.group(m.submatches, 1), obj) <> back }) |> string.concat |> fn(s) { // and a blank line after the last of them, or the next object begins on // the same line and markdown reads the two as one paragraph case s { "" -> "" _ -> s <> newlines } } } fn inline(obj: Object) -> String { inline_text(obj.text, obj) } /// An object's text as markdown. /// /// The order is the one every writer here uses: images before links, because /// an image sits inside a link; then the note references, the faces, the /// anchors and the breaks. fn inline_text(txt: String, obj: Object) -> String { txt |> images |> links(obj) |> note_refs |> faces |> regexp.replace(rgx.inline_link_anchor(), _, "") |> breaks(obj) |> escape |> unprotect |> guard_line_starts } /// A "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a list /// item, an enumeration item, a blockquote. The writer uses the first of them /// itself, and a line of the document's that begins the same way would be read /// as one - which is what a verse or a group can do. /// /// Guarded rather than escaped everywhere, because these are special only in /// that position. fn guard_line_starts(s: String) -> String { string.split(s, "\n") |> list.map(guard_one_line) |> string.join("\n") } fn guard_one_line(line: String) -> String { let spaces = leading_spaces(line, 0) let rest = string.drop_start(line, spaces) let pad = string.repeat(" ", spaces) case marker_at(rest) { True -> pad <> "\\" <> rest False -> case enumerated_at(rest, 0) { 0 -> line n -> pad <> string.slice(rest, 0, n) <> "\\" <> string.drop_start(rest, n) } } } fn leading_spaces(s: String, n: Int) -> Int { case string.slice(s, n, 1) { " " -> leading_spaces(s, n + 1) _ -> n } } fn marker_at(s: String) -> Bool { case string.slice(s, 0, 1), string.slice(s, 1, 1) { "-", " " | "+", " " | ">", " " -> True "-", "\t" | "+", "\t" | ">", "\t" -> True _, _ -> False } } /// How many digits precede a "." or ")" and a space, or 0 for none. fn enumerated_at(s: String, n: Int) -> Int { case string.slice(s, n, 1) { d if d == "0" || d == "1" || d == "2" || d == "3" || d == "4" -> enumerated_at(s, n + 1) d if d == "5" || d == "6" || d == "7" || d == "8" || d == "9" -> enumerated_at(s, n + 1) "." | ")" -> case n > 0 && string.slice(s, n + 1, 1) == " " { True -> n False -> 0 } _ -> 0 } } // The markers that protect the markdown this writer emits from the escaping // that follows. // // The escaping has to come *last*, and this is why: the abstraction's own // markers are built from the characters markdown reserves - "⑆*┨" for emphasis, // "⑆_┨" for an underscore face - so escaping first turns them into "⑆\*┨" and // no face is ever recognised. const keep_open = 0xe000 const keep_close = 0xe001 // A third marker of the same kind, standing for the directory the images are in. // // An image reference is written while an object is written, and where the file // will be put is not something the model knows: it is told. So the reference // carries a marker and write substitutes it once, at the end, out of the one // argument that says. const image_dir_mark = 0xe002 fn keep(markdown_source: String) -> String { cp(keep_open) <> markdown_source <> cp(keep_close) } fn unprotect(txt: String) -> String { txt |> string.replace(cp(keep_open), "") |> string.replace(cp(keep_close), "") } fn cp(n: Int) -> String { let assert Ok(c) = string.utf_codepoint(n) string.from_utf_codepoints([c]) } /// The document's own text, escaped for a protected run: the same rule, run /// early, and then flattened so that the result is inert. fn inert(txt: String) -> String { unprotect(escape(txt)) } /// The characters markdown reads as syntax, escaped wherever they occur. /// /// The same rule the typst writer arrived at, and for the same reason: a closed /// set is checkable and a rule about positions is not. The three html /// characters become entities rather than backslash escapes, because markdown /// passes raw html through and " String { let #(out, _) = string.to_utf_codepoints(txt) |> list.fold(#([], 0), fn(state, c) { let #(acc, depth) = state let n = string.utf_codepoint_to_int(c) case n == keep_open, n == keep_close, depth > 0 { True, _, _ -> #(acc, depth + 1) _, True, _ -> #(acc, int.max(depth - 1, 0)) _, _, True -> #([string.from_utf_codepoints([c]), ..acc], depth) _, _, False -> #([escaped(n, c), ..acc], depth) } }) string.concat(list.reverse(out)) } fn escaped(n: Int, c: UtfCodepoint) -> String { case n { 0x26 -> "&" 0x3c -> "<" 0x3e -> ">" // \ ` * _ [ ] # | ~ 0x5c | 0x60 | 0x2a | 0x5f | 0x5b | 0x5d | 0x23 | 0x7c | 0x7e -> "\\" <> string.from_utf_codepoints([c]) _ -> string.from_utf_codepoints([c]) } } /// An image, with its alt text where the document gave one. fn images(txt: String) -> String { rgx.replace_with(rgx.inline_image(), txt, fn(g) { let post = g(5) let #(alt, rest) = case regexp.scan(rgx.inline_image_alt(), post) { [m, ..] -> #(rgx.group(m.submatches, 0), case string.split_once(post, m.content) { Ok(#(_, after)) -> after Error(_) -> post }) [] -> #("", post) } g(0) <> keep("![" <> inert(alt) <> "](" <> cp(image_dir_mark) <> "/" <> g(2) <> ")") <> rest }) } /// A link. An internal target becomes a fragment, which is the object number, /// and so resolves against the anchor every object carries. /// /// And *only* an object number: those are the anchors this writer defines, so a /// target that names something else - the endnotes section, an anchor tag the /// markup placed - would be a link to nothing. The text stands on its own /// instead, which is what a reader sees anyway. fn links(txt: String, obj: Object) -> String { rgx.replace_with(rgx.inline_link_number_only(), txt, fn(g) { let url = case int.parse(g(1)) { Ok(n) -> at(obj.stow_link, n, "") Error(_) -> "" } g(0) <> "┤" <> url <> "├" }) |> rgx.replace_with(rgx.inline_link(), _, fn(g) { let target = case string.starts_with(g(1), "¤") { False -> g(1) True -> case regexp.scan(rgx.internal_target(), g(1)) { [m, ..] -> rgx.group(m.submatches, 1) [] -> g(1) } } let shown = inert(faces(g(0))) case target, string.starts_with(target, "#") && !is_object_number(target) { "", _ -> keep(shown) _, True -> keep(shown) _, False -> keep("[" <> shown <> "](" <> target <> ")") } }) } /// A link target is a url or a fragment and not prose, so the escaping the /// text went through comes off it again. A "#" that stayed escaped made every /// table of contents line point at nothing. fn unescape_target(target: String) -> String { target |> string.replace("\\", "") |> string.replace("&", "&") |> string.replace("<", "<") |> string.replace(">", ">") } /// A fragment that names an object of this document. fn is_object_number(clean: String) -> Bool { case string.drop_start(clean, 1) { "" -> False "0" -> False rest -> case int.parse(rest) { Ok(_) -> True Error(_) -> False } } } /// A note reference: a superscript link to the note, which sits under the /// object that refers to it. fn note_refs(txt: String) -> String { rgx.replace_with(rgx.inline_notes_any_mark(), txt, fn(g) { // the mark is shown as well as named in the target, and a star note's mark // is an asterisk, which markdown would pair with the next one keep("[" <> inert(g(0)) <> "](#note-" <> g(0) <> ")") }) } /// The font faces. Emphasis and strong are markdown's own; the rest have no /// markdown and take the html element markdown itself would produce. /// /// The spaces inside a run are dropped rather than kept: the abstraction's /// marker takes in the space that was beside the run, the text before it /// already ends in one, and markdown's emphasis will not open on a space. fn faces(txt: String) -> String { [ #(rgx.inline_emphasis(), "**", "**"), #(rgx.inline_bold(), "**", "**"), #(rgx.inline_italics(), "*", "*"), #(rgx.inline_underscore(), "", ""), #(rgx.inline_superscript(), "", ""), #(rgx.inline_subscript(), "", ""), #(rgx.inline_mono(), "`", "`"), #(rgx.inline_strike(), "~~", "~~"), #(rgx.inline_insert(), "", ""), #(rgx.inline_cite(), "", ""), ] |> list.fold(txt, fn(acc, face) { let #(pattern, open, close) = face rgx.replace_with(pattern, acc, fn(g) { keep(open) <> trim_spaces(g(0)) <> keep(close) }) }) } fn trim_spaces(s: String) -> String { case string.starts_with(s, " "), string.ends_with(s, " ") { True, _ -> trim_spaces(string.drop_start(s, 1)) _, True -> trim_spaces(string.drop_end(s, 1)) _, _ -> s } } /// The line breaks the markup asked for, and the grouped indents. fn breaks(txt: String, obj: Object) -> String { case obj.is_a { "code" -> txt _ -> { let txt = case obj.is_a == "group" || obj.is_a == "block" { False -> txt True -> txt |> rgx.replace_with(rgx.grouped_para_indent_hang(), _, fn(g) { g(1) <> g(1) }) |> rgx.replace_with(rgx.grouped_para_bullet_at(), _, fn(g) { g(0) <> g(0) <> "\u{25cf} " }) |> rgx.replace_with(rgx.grouped_para_bullet(), _, fn(_) { "\u{25cf} " }) |> rgx.replace_with(rgx.grouped_para_indent(), _, fn(g) { g(0) <> g(0) }) } txt |> regexp.replace(rgx.nbsp_char(), _, " ") |> regexp.replace(rgx.br_line(), _, newline) |> regexp.replace(rgx.br_line_inline(), _, newline) |> regexp.replace(rgx.br_line_spaced(), _, newlines) |> regexp.replace(rgx.line_break(), _, newline) |> regexp.replace(rgx.mark_internal_site_lnk(), _, "") } } } /// The object a table of contents entry points at, where that is an object /// number. fn internal_target(obj: Object) -> String { regexp.scan(rgx.any_internal_target(), obj.text) |> list.fold("", fn(found, m) { case found { "" -> { let frag = rgx.group(m.submatches, 0) case int.parse(frag) { Ok(_) -> frag Error(_) -> "" } } _ -> found } }) } fn at(items: List(String), index: Int, fallback: String) -> String { case items, index { [], _ -> fallback [first, ..], 0 -> first [_, ..rest], n -> at(rest, n - 1, fallback) } }