path: root/gleam/src/write/epub.gleam

//// The epub3 writer: a zip of xhtml, one file per segment.
////
//// This is the step the scroll html put off. A scroll is one file, so an
//// internal link is a fragment and a segment is a notion the writer can
//// ignore; an epub is *the segmented output plus a container*, so the writer
//// has to answer two questions the scroll writer did not: which file does
//// this object belong to, and what is the link from one file to another.
////
//// The model answers both: `segment_epub` says which segment a heading opens,
//// `segment_epub_is` which segment an object sits in, and an internal link
//// names its own segment.
////
//// The judge is *epubcheck*, which is better than a diff: it reads the
//// specification so that this writer does not have to be trusted to have read
//// it. And, as with the html, the two languages' writers must produce the
//// same bytes.

import gleam/bit_array
import gleam/dict.{type Dict}
import gleam/int
import gleam/list
import gleam/regexp
import gleam/string
import ocda/images
import ocda/model.{type Document, type Object}
import write/html
import write/zip

const generator_name = "spine"

/// One segment: its file name and the objects in it, in document order.
pub type Segment {
  Segment(file: String, title: String, objects: List(Object))
}

/// The epub, as bytes. `source` is the path the document was loaded from,
/// which is how the images are found: the model names them and does not hold
/// them.
pub fn write(doc: Document, source: String) -> BitArray {
  let segs = segments(doc)
  let targets = target_map(segs)
  let files = list.map(segs, fn(s) { s.file })
  let image_names = case images.can_give_images(source) {
    False -> []
    True -> images_named(doc)
  }
  let #(image_entries, images_kept) =
    list.fold(image_names, #([], []), fn(state, name) {
      let #(entries, kept) = state
      case images.image_bytes(source, name) {
        Ok(data) ->
          case bit_array.byte_size(data) > 0 {
            True -> #(
              [zip.Entry("OEBPS/image/" <> name, data), ..entries],
              [name, ..kept],
            )
            False -> #(entries, kept)
          }
        Error(_) -> #(entries, kept)
      }
    })
  let image_entries = list.reverse(image_entries)
  let images_kept = list.reverse(images_kept)

  // The claimed ids are threaded *across* segments, not restarted in each.
  //
  // An id must be unique in the publication, not merely in a file, because a
  // link carries a file and a fragment and an anchor that appears in two files
  // has two destinations. Restarting the set per segment emitted an anchor
  // again in every segment that carried it, which the D writer does not, and
  // twelve documents differed between the two writers until this threaded.
  let #(segment_entries, _) =
    list.fold(segs, #([], html.reserved(doc)), fn(state, seg) {
      let #(entries, ids) = state
      let ctx = html.Segmented(in_file: targets, files: files, current: seg.file)
      let #(body, ids) = html.render_with(seg.objects, doc, ctx, ids)
      #(
        [
          zip.Entry("OEBPS/" <> seg.file <> ".xhtml", bytes(xhtml(doc, seg, body))),
          ..entries
        ],
        ids,
      )
    })
  let segment_entries = list.reverse(segment_entries)

  zip.write(
    list.flatten([
      // "mimetype" first, stored, no extra field. That is the one ordering the
      // format insists on, so that a reading system can identify the file from
      // its first bytes.
      [zip.Entry("mimetype", bytes("application/epub+zip"))],
      [zip.Entry("META-INF/container.xml", bytes(container))],
      segment_entries,
      image_entries,
      [zip.Entry("OEBPS/style.css", bytes(html.style_sheet()))],
      [zip.Entry("OEBPS/nav.xhtml", bytes(nav(doc, segs)))],
      [zip.Entry("OEBPS/content.opf", bytes(opf(doc, files, images_kept)))],
    ]),
  )
}

/// The segments, in document order.
///
/// An object says which segment it is in, and the first object of a segment is
/// where that segment begins. Objects with no segment at all - the document
/// title, the table of contents - go into a segment named for the section they
/// sit in, because an epub has no place for an object outside the reading
/// order.
pub fn segments(doc: Document) -> List(Segment) {
  // Grouped by name, in order of first appearance, *not* by run.
  //
  // A segment's objects are not always contiguous: a document may put an
  // object with no segment of its own after a segment has begun, and the
  // fallback then names it for its section, so "body" recurs. Grouping by run
  // made two zip entries of one name, which is an epubcheck error (OPF-060)
  // and was one in nine of the collection.
  let pairs =
    html.sections(doc)
    |> list.flat_map(fn(name) {
      model.section(doc, name)
      |> list.filter(model.renders)
      |> list.map(fn(obj) { #(segment_of(obj, name), obj) })
    })
  let #(order, by_name) =
    list.fold(pairs, #([], dict.new()), fn(state, pair) {
      let #(order, by_name) = state
      let #(seg, obj) = pair
      let title = heading_text(obj)
      case dict.get(by_name, seg) {
        Ok(Segment(file, have, objects)) -> #(
          order,
          dict.insert(
            by_name,
            seg,
            Segment(
              file,
              case have, title {
                "", t -> t
                h, _ -> h
              },
              [obj, ..objects],
            ),
          ),
        )
        Error(_) -> #(
          [seg, ..order],
          dict.insert(by_name, seg, Segment(seg, title, [obj])),
        )
      }
    })
  order
  |> list.reverse
  |> list.filter_map(fn(seg) {
    case dict.get(by_name, seg) {
      Ok(s) -> Ok(Segment(s.file, s.title, list.reverse(s.objects)))
      Error(_) -> Error(Nil)
    }
  })
}

fn segment_of(obj: Object, section: String) -> String {
  case obj.segment_epub_is, obj.segment {
    "", "" -> safe_name(section)
    "", seg -> safe_name(seg)
    seg, _ -> safe_name(seg)
  }
}

/// A file name a zip entry and an href can both carry. A segment name comes
/// from the markup, so it may hold anything; these are file names inside the
/// container and a reading system has to be able to resolve them.
fn safe_name(s: String) -> String {
  let out =
    s
    |> string.to_graphemes
    |> list.map(fn(c) {
      case is_safe(c) {
        True -> c
        False -> "_"
      }
    })
    |> string.concat
  case out {
    "" -> "segment"
    _ -> out
  }
}

fn is_safe(c: String) -> Bool {
  string.contains(
    does: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.",
    contain: c,
  )
}

fn heading_text(obj: Object) -> String {
  case obj.is_a {
    "heading" | "toc" -> plain(obj.text)
    _ -> ""
  }
}

/// Every target an object carries, and the file it is in. This is what a
/// cross-segment link needs and what only the whole document can say.
pub fn target_map(segs: List(Segment)) -> Dict(String, String) {
  list.fold(segs, dict.new(), fn(acc, seg) {
    list.fold(seg.objects, acc, fn(acc, obj) {
      let put = fn(d, key) {
        case key {
          "" -> d
          k ->
            case dict.has_key(d, k) {
              True -> d
              False -> dict.insert(d, k, seg.file)
            }
        }
      }
      let acc = case obj.ocn {
        0 -> acc
        n -> put(acc, int.to_string(n))
      }
      let acc = put(acc, obj.identifier)
      let acc = put(acc, obj.heading_lev_anchor)
      let acc = list.fold(obj.anchor_tags, acc, put)
      // and the note references this object's own text will create: a note
      // reference is an id the *writer* makes, and the gathered endnote links
      // back to it from another segment
      let acc =
        list.fold(regexp.scan(note_mark(), obj.text), acc, fn(d, m) {
          put(d, "noteref_" <> html.submatch(m, 0))
        })
      // and an anchor the markup placed *in the text*, ┃name┃. Those are ids
      // too, and a link to one is a bare fragment rather than an internal
      // link naming its segment, so only this map can say which file it is in.
      list.fold(regexp.scan(inline_anchor(), obj.text), acc, fn(d, m) {
        put(d, html.submatch(m, 0))
      })
    })
  })
}

fn inline_anchor() -> regexp.Regexp {
  let assert Ok(r) =
    regexp.compile(
      "┃(\\S+?)┃",
      regexp.Options(case_insensitive: False, multi_line: True),
    )
  r
}

fn note_mark() -> regexp.Regexp {
  let assert Ok(r) =
    regexp.compile(
      "【(\\d+|(?:[*]|[+])+)\\s",
      regexp.Options(case_insensitive: False, multi_line: True),
    )
  r
}

/// The images the document names, each once, in the order met.
pub fn images_named(doc: Document) -> List(String) {
  html.sections(doc)
  |> list.flat_map(fn(name) { model.section(doc, name) })
  |> list.flat_map(fn(obj) { obj.images })
  |> list.filter(fn(img) { img.name != "" && !img.missing })
  |> list.map(fn(img) { img.name })
  |> list.fold([], fn(acc, name) {
    case list.contains(acc, name) {
      True -> acc
      False -> [name, ..acc]
    }
  })
  |> list.reverse
}

const container = "<?xml version=\"1.0\" encoding=\"utf-8\"?>
<container version=\"1.0\" xmlns=\"urn:oasis:names:tc:opendocument:xmlns:container\">
  <rootfiles>
    <rootfile full-path=\"OEBPS/content.opf\" media-type=\"application/oebps-package+xml\" />
  </rootfiles>
</container>
"

/// One segment as xhtml. xhtml rather than html: an epub's content documents
/// are xml, so the namespace is declared and every element is closed.
fn xhtml(doc: Document, seg: Segment, body: String) -> String {
  let lang = language(doc)
  let title = case seg.title {
    "" -> model.meta(doc, "title.full")
    t -> t
  }
  "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
  <> "<html xmlns=\"http://www.w3.org/1999/xhtml\""
  <> " xmlns:epub=\"http://www.idpf.org/2007/ops\""
  <> " lang=\""
  <> html.attribute(lang)
  <> "\" xml:lang=\""
  <> html.attribute(lang)
  <> "\">\n"
  <> "<head>\n<title>"
  <> html.escape_text(title)
  <> "</title>\n"
  <> "<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" />\n"
  <> "</head>\n<body>\n"
  <> body
  <> "</body>\n</html>\n"
}

/// The navigation document, which epub3 requires and which is the document's
/// own table of contents rather than a second one.
///
/// Built from the segments, not from the toc objects: a toc object's link may
/// point inside a segment, and a navigation entry has to name the file. One
/// level, because the segment list is already flat.
fn nav(doc: Document, segs: List(Segment)) -> String {
  let lang = language(doc)
  let title = model.meta(doc, "title.full")
  "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
  <> "<html xmlns=\"http://www.w3.org/1999/xhtml\""
  <> " xmlns:epub=\"http://www.idpf.org/2007/ops\""
  <> " lang=\""
  <> html.attribute(lang)
  <> "\" xml:lang=\""
  <> html.attribute(lang)
  <> "\">\n"
  <> "<head>\n<title>"
  <> html.escape_text(title)
  <> "</title>\n"
  <> "<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" />\n"
  <> "</head>\n<body>\n"
  <> "<nav epub:type=\"toc\" id=\"toc\" role=\"doc-toc\">\n"
  <> "<h1>"
  <> html.escape_text(title)
  <> "</h1>\n<ol>\n"
  <> {
    segs
    |> list.map(fn(seg) {
      let label = case seg.title {
        "" -> seg.file
        t -> t
      }
      "<li><a href=\""
      <> html.attribute(seg.file)
      <> ".xhtml\">"
      <> html.escape_text(label)
      <> "</a></li>\n"
    })
    |> string.concat
  }
  <> "</ol>\n</nav>\n</body>\n</html>\n"
}

/// The package document.
///
/// dcterms:modified is required and must be to the second in UTC. It is taken
/// from the document's own publication date rather than from the clock, so
/// that the same document gives the same epub twice: a writer that stamps the
/// time of its run cannot be compared with itself.
fn opf(doc: Document, files: List(String), image_names: List(String)) -> String {
  let lang = language(doc)
  let m = fn(key) { model.meta(doc, key) }
  let title = case m("title.full") {
    "" -> m("title.main")
    t -> t
  }
  let element = fn(name, value) {
    case value {
      "" -> ""
      v -> "<" <> name <> ">" <> html.escape_text(v) <> "</" <> name <> ">\n"
    }
  }
  "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
  <> "<package version=\"3.0\" xmlns=\"http://www.idpf.org/2007/opf\""
  <> " unique-identifier=\"bookid\" xml:lang=\""
  <> html.attribute(lang)
  <> "\">\n"
  <> "<metadata xmlns:dc=\"http://purl.org/dc/elements/1.1/\">\n"
  <> "<dc:identifier id=\"bookid\">urn:uuid:"
  <> uuid(doc)
  <> "</dc:identifier>\n"
  <> element("dc:title", title)
  <> element("dc:creator", m("creator.author"))
  <> element("dc:language", lang)
  <> element("dc:date", m("date.published"))
  <> element("dc:rights", m("rights.copyright"))
  <> "<meta property=\"dcterms:modified\">"
  <> modified(doc)
  <> "</meta>\n"
  <> "</metadata>\n<manifest>\n"
  <> "<item id=\"nav\" href=\"nav.xhtml\" media-type=\"application/xhtml+xml\" properties=\"nav\" />\n"
  <> "<item id=\"css\" href=\"style.css\" media-type=\"text/css\" />\n"
  <> {
    files
    |> list.map(fn(f) {
      "<item id=\"s_"
      <> html.attribute(f)
      <> "\" href=\""
      <> html.attribute(f)
      <> ".xhtml\" media-type=\"application/xhtml+xml\" />\n"
    })
    |> string.concat
  }
  <> {
    image_names
    |> list.index_map(fn(name, i) {
      "<item id=\"i_"
      <> int.to_string(i)
      <> "\" href=\"image/"
      <> html.attribute(name)
      <> "\" media-type=\""
      <> media_type(name)
      <> "\" />\n"
    })
    |> string.concat
  }
  <> "</manifest>\n<spine>\n"
  <> {
    files
    |> list.map(fn(f) { "<itemref idref=\"s_" <> html.attribute(f) <> "\" />\n" })
    |> string.concat
  }
  <> "</spine>\n</package>\n"
}

/// The language, as a language tag rather than as a locale name.
///
/// BCP 47 separates a language from its region with a hyphen, and one document
/// of the reference collection declares "pt_BR", which epubcheck refuses:
/// "Invalid subtag: pt_BR". The underscore is the POSIX locale spelling and
/// the abstraction carries it as the markup gave it, so the conversion belongs
/// here, in the writer that has a specification to satisfy.
fn language(doc: Document) -> String {
  case dict.get(doc.source, "language") {
    Ok(l) if l != "" -> string.replace(l, each: "_", with: "-")
    _ -> "en"
  }
}

/// A uuid from the document's own digest, so that the same document has the
/// same identifier every time it is written. A random one would make two runs
/// of the same input two different publications.
fn uuid(doc: Document) -> String {
  let seed = case dict.get(doc.source, "digest") {
    Ok(d) if d != "" -> d
    _ -> model.meta(doc, "title.full") <> doc.source_filename
  }
  let h =
    sha1(<<seed:utf8>>)
    |> bit_array.base16_encode
    |> string.lowercase
  // version 5 shape: the version nibble and the variant bits set, which is
  // what says "this uuid is a hash of a name"
  string.slice(h, 0, 8)
  <> "-"
  <> string.slice(h, 8, 4)
  <> "-5"
  <> string.slice(h, 13, 3)
  <> "-a"
  <> string.slice(h, 17, 3)
  <> "-"
  <> string.slice(h, 20, 12)
}

fn modified(doc: Document) -> String {
  let d = case model.meta(doc, "date.modified") {
    "" -> model.meta(doc, "date.published")
    v -> v
  }
  case string.length(d) >= 10 {
    True -> string.slice(d, 0, 10) <> "T00:00:00Z"
    False -> "1970-01-01T00:00:00Z"
  }
}

fn media_type(name: String) -> String {
  let n = string.lowercase(name)
  case string.ends_with(n, ".png") {
    True -> "image/png"
    False ->
      case string.ends_with(n, ".jpg") || string.ends_with(n, ".jpeg") {
        True -> "image/jpeg"
        False ->
          case string.ends_with(n, ".gif") {
            True -> "image/gif"
            False ->
              case string.ends_with(n, ".svg") {
                True -> "image/svg+xml"
                False -> "application/octet-stream"
              }
          }
      }
  }
}

/// sha1, from the runtime's own crypto rather than from a package.
@external(erlang, "crypto", "hash")
fn crypto_hash(algorithm: HashName, data: BitArray) -> BitArray

type HashName

@external(erlang, "erlang", "binary_to_atom")
fn atom(name: String) -> HashName

fn sha1(data: BitArray) -> BitArray {
  crypto_hash(atom("sha"), data)
}

/// An object's text with the internal markup taken out, for a title or a
/// navigation label, where no markup may go.
fn plain(txt: String) -> String {
  let assert Ok(markers) =
    regexp.compile(
      "⑆.┨|┣.|┥|┝|┤.*?├|【.+?】|░|┘|┙|┚|┃",
      regexp.Options(case_insensitive: False, multi_line: True),
    )
  html.escape_text(string.trim(regexp.replace(markers, txt, "")))
}

fn bytes(s: String) -> BitArray {
  <<s:utf8>>
}

/// Unused, kept so the generator can be named where a format wants it.
pub fn generator() -> String {
  generator_name
}