path: root/dlang/src/write/epub.d

module write.epub;

/+ ↓ the epub3 writer: a zip of xhtml, one file per segment.
     .
     This is the step the scroll html put off. A scroll is one file, so an
     internal link is a fragment and a segment is a notion the writer can
     ignore; an epub is *the segmented output plus a container*, so the writer
     has to answer two questions the scroll writer did not:
     .
       which file does this object belong to, and
       what is the link from one file to another.
     .
     The model answers both: .segment_epub says which segment a heading
     opens, .segment_epub_is which segment an object sits in, and the target
     of a link is an identifier or an anchor tag that some object carries. So
     the writer builds one map from target to file and hands it to the same
     object rendering the scroll writer uses.
     .
     The judge is *epubcheck*, which is better than a diff: it reads the
     specification so that this writer does not have to be trusted to have
     read it. And, as with the html, the two languages' writers must produce
     the same bytes.
     .
     Not byte identical with spine's epub, and not trying to be: a spine epub
     is a zip whose entry order and compression would have to match, and
     matching them would prove nothing that epubcheck does not prove better.
+/

import std.algorithm : canFind, endsWith, sort, startsWith;
import std.array : appender, array, join, replace;
import std.conv : to;
import std.regex : ctRegex, matchAll;
import std.digest.sha : sha1Of, toHexString;
import std.string : representation, strip, toLower;
import ocda.images;
import ocda.model;
import write.html;
import write.zip;

/+ ↓ a note's mark, a number or a run of stars or pluses +/
private static note_mark = ctRegex!(`【(?P<num>\d+|[*+]+[0-9]*)\s`, "mg");

/+ ↓ an anchor the markup placed in the text +/
private static inline_anchor = ctRegex!(`┃(?P<anchor>\S+?)┃`, "mg");

enum generator_name = "spine";
enum generator_url  = "https://sisudoc.org";

/+ ↓ the epub, as bytes.
     source is the path the document was loaded from, which is how the images
     are found: the model names them and does not hold them.
+/
ubyte[] writeEpub(ref OcdaDocument doc, string source) {
  auto segs = segments(doc);
  auto targets = targetMap(doc, segs);
  string[string] in_file;   /+ target -> file, for the resolver +/
  foreach (k, v; targets) { in_file[k] = v; }
  bool[string] files_written;
  foreach (s; segs) { files_written[s.file] = true; }
  string current;           /+ the file being written, for a same-file link +/

  /+ ↓ a target in this file is a fragment; one in another file is that file
       and the fragment. A target no object carries is left as it stands,
       which an epubcheck run then reports rather than this writer hiding.
  +/
  useLinkResolver(
    /+ ↓ a bare fragment: which file holds it +/
    (string target) {
      if (!target.startsWith("#")) { return target; }
      string name = target[1 .. $];
      if (auto p = name in in_file) {
        return (*p == current) ? target : (*p ~ ".xhtml" ~ target);
      }
      return target;
    },
    /+ ↓ and an internal link, which already names its segment. Trust the
         abstraction's answer over the map: the markup said which segment,
         and the map is only an index of where targets ended up.
    +/
    (string seg, string frag) {
      string file = safeName(seg);
      if (!(file in files_written)) {
        /+ ↓ a segment this writer did not write is one it cannot link to,
             so the fragment alone is better than a file that is not there
        +/
        return frag;
      }
      return (file == current) ? frag : (file ~ ".xhtml" ~ frag);
    });

  auto ids = AnchorIds();
  reserveIdentifiers(doc, ids);

  ZipEntryOut[] entries;
  /+ ↓ "mimetype" first, stored, no extra field. That is the one ordering the
       format insists on, so that a reading system can identify the file from
       its first bytes.
  +/
  entries ~= ZipEntryOut("mimetype", bytesOf("application/epub+zip"));
  entries ~= ZipEntryOut("META-INF/container.xml", bytesOf(container));

  string[] spine_files;
  foreach (seg; segs) {
    current = seg.file;
    auto body_ = appender!string;
    foreach (obj; seg.objects) {
      body_ ~= writeObj(obj, doc, ids);
    }
    entries ~= ZipEntryOut("OEBPS/" ~ seg.file ~ ".xhtml",
      bytesOf(xhtml(doc, seg, body_.data)));
    spine_files ~= seg.file;
  }
  current = "";

  /+ ↓ the images, and only the ones the document names.
       A manifest item whose file is absent is an epubcheck error, and so is
       an <img> whose file is not in the manifest, so the two lists are built
       from the same walk.
  +/
  string[] image_names;
  if (canGiveImages(source)) {
    foreach (name; imagesNamed(doc)) {
      auto data = imageBytes(source, name);
      if (data.length > 0) {
        entries ~= ZipEntryOut("OEBPS/image/" ~ name, data);
        image_names ~= name;
      }
    }
  }

  entries ~= ZipEntryOut("OEBPS/style.css", bytesOf(stylesheet));
  entries ~= ZipEntryOut("OEBPS/nav.xhtml", bytesOf(nav(doc, segs)));
  entries ~= ZipEntryOut("OEBPS/content.opf",
    bytesOf(opf(doc, spine_files, image_names)));
  return zipWrite(entries);
}

/+ ↓ one segment: its file name and the objects in it, in document order +/
struct Segment {
  string        file;
  string        title;
  OcdaObject[]  objects;
}

/+ ↓ the segments, in document order.
     .
     An object says which segment it is in, and the first object of a segment
     is where that segment begins. Objects with no segment at all - the
     document title, the table of contents - go into a segment named for
     themselves, because an epub has no place for an object outside the
     reading order.
+/
Segment[] segments(ref OcdaDocument doc) {
  /+ ↓ grouped by name, in order of first appearance, *not* by run.
       .
       A segment's objects are not always contiguous: a document may put an
       object with no segment of its own after a segment has begun, and the
       fallback then names it for its section, so "body" recurs. Grouping by
       run made two zip entries of one name, which is an epubcheck error
       (OPF-060) and was one in nine of the collection.
  +/
  string[] order;
  Segment[string] by_name;
  foreach (name; htmlSections(doc)) {
    auto sec = doc.sectionByName(name);
    if (sec is null) { continue; }
    foreach (obj; *sec) {
      if (!obj.renders) { continue; }
      string seg = segmentOf(obj, name);
      if (!(seg in by_name)) {
        by_name[seg] = Segment(seg, "", []);
        order ~= seg;
      }
      by_name[seg].objects ~= obj;
      if (by_name[seg].title.length == 0) {
        string title = headingTextOf(obj);
        if (title.length > 0) { by_name[seg].title = title; }
      }
    }
  }
  Segment[] out_;
  foreach (seg; order) { out_ ~= by_name[seg]; }
  return out_;
}

/+ ↓ which segment an object is in.
     .segment_epub_is is the answer where there is one. Where there is none
     the object is outside the segmented body - the title, the table of
     contents - and the section it sits in is the segment.
+/
private string segmentOf(ref const OcdaObject obj, string section) {
  if (obj.segment_epub_is.length > 0) { return safeName(obj.segment_epub_is); }
  if (obj.segment.length > 0)         { return safeName(obj.segment); }
  return safeName(section);
}

/+ ↓ a file name a zip entry and an href can both carry.
     A segment name comes from the markup, so it may hold anything; these are
     file names inside the container and a reading system has to be able to
     resolve them.
+/
private string safeName(string s) {
  string out_;
  foreach (c; s) {
    bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
      || (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.';
    out_ ~= ok ? [c] : "_";
  }
  return (out_.length > 0) ? out_ : "segment";
}

/+ ↓ the plain text of a heading, for a title and a navigation label +/
private string headingTextOf(ref const OcdaObject obj) {
  if (obj.is_a != "heading" && obj.is_a != "toc") { return ""; }
  return plain(obj.text);
}

/+ ↓ every target an object carries, and the file it is in.
     This is what a cross-segment link needs and what only the whole document
     can say.
+/
string[string] targetMap(ref OcdaDocument doc, Segment[] segs) {
  string[string] out_;
  foreach (seg; segs) {
    foreach (obj; seg.objects) {
      void put(string key) {
        if (key.length > 0 && !(key in out_)) { out_[key] = seg.file; }
      }
      if (obj.ocn != 0)        { put(obj.ocn.to!string); }
      put(obj.identifier);
      put(obj.heading_lev_anchor);
      foreach (a; obj.anchor_tags) { put(a); }
      /+ ↓ and the note references this object's own text will create.
           .
           A note reference is an id the *writer* makes, not one the object
           carries: the object's text holds 【<n> the note】 and the writer
           turns the mark into an anchor named noteref_<n>. The gathered
           endnote then links back to it, from another segment, so the map
           has to know where it is. Without this every backlink in the
           endnotes points at a fragment in a file that does not declare it:
           306 of them in one document, which is what epubcheck said.
      +/
      foreach (m; obj.text.matchAll(note_mark)) {
        put("noteref_" ~ m["num"].to!string);
      }
      /+ ↓ and an anchor the markup placed *in the text*, ┃name┃.
           .
           Those are ids too, and a link to one is written as a bare
           fragment rather than as an internal link naming its segment, so
           only this map can say which file it is in. Two links in the
           reference collection point at one, and both dangled until the map
           knew.
      +/
      foreach (m; obj.text.matchAll(inline_anchor)) {
        put(m["anchor"].to!string);
      }
    }
  }
  return out_;
}

/+ ↓ the images the document names, each once, in the order met +/
string[] imagesNamed(ref OcdaDocument doc) {
  string[] out_;
  bool[string] seen;
  foreach (name; htmlSections(doc)) {
    auto sec = doc.sectionByName(name);
    if (sec is null) { continue; }
    foreach (obj; *sec) {
      foreach (img; obj.images) {
        if (img.name.length > 0 && !(img.name in seen) && !img.missing) {
          seen[img.name] = true;
          out_ ~= img.name;
        }
      }
    }
  }
  return out_;
}

private enum container = `<?xml version="1.0" encoding="utf-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml" />
  </rootfiles>
</container>
`;

/+ ↓ one segment as xhtml.
     xhtml rather than html: an epub's content documents are xml, so the
     namespace is declared and every element is closed.
+/
private string xhtml(ref OcdaDocument doc, Segment seg, string body_) {
  string lang = languageTag(doc);
  string title = seg.title.length > 0 ? seg.title : doc.meta("title.full");
  auto out_ = appender!string;
  out_ ~= `<?xml version="1.0" encoding="utf-8"?>` ~ "\n";
  out_ ~= `<html xmlns="http://www.w3.org/1999/xhtml"`
    ~ ` xmlns:epub="http://www.idpf.org/2007/ops"`
    ~ ` lang="` ~ attr(lang) ~ `" xml:lang="` ~ attr(lang) ~ `">` ~ "\n";
  out_ ~= "<head>\n<title>" ~ escape(title) ~ "</title>\n";
  out_ ~= `<link rel="stylesheet" type="text/css" href="style.css" />` ~ "\n";
  out_ ~= "</head>\n<body>\n";
  out_ ~= body_;
  out_ ~= "</body>\n</html>\n";
  return out_.data;
}

/+ ↓ the navigation document, which epub3 requires and which is the
     document's own table of contents rather than a second one.
     .
     Built from the segments, not from the toc objects: a toc object's link
     may point inside a segment, and a navigation entry has to name the file.
     One level, because the segment list is already flat; a nested nav would
     need the heading tree, which is a refinement and not a requirement.
+/
private string nav(ref OcdaDocument doc, Segment[] segs) {
  string lang = languageTag(doc);
  auto out_ = appender!string;
  out_ ~= `<?xml version="1.0" encoding="utf-8"?>` ~ "\n";
  out_ ~= `<html xmlns="http://www.w3.org/1999/xhtml"`
    ~ ` xmlns:epub="http://www.idpf.org/2007/ops"`
    ~ ` lang="` ~ attr(lang) ~ `" xml:lang="` ~ attr(lang) ~ `">` ~ "\n";
  out_ ~= "<head>\n<title>" ~ escape(doc.meta("title.full")) ~ "</title>\n";
  out_ ~= `<link rel="stylesheet" type="text/css" href="style.css" />` ~ "\n";
  out_ ~= "</head>\n<body>\n";
  out_ ~= `<nav epub:type="toc" id="toc" role="doc-toc">` ~ "\n";
  out_ ~= "<h1>" ~ escape(doc.meta("title.full")) ~ "</h1>\n<ol>\n";
  foreach (seg; segs) {
    string label = seg.title.length > 0 ? seg.title : seg.file;
    out_ ~= `<li><a href="` ~ attr(seg.file) ~ `.xhtml">`
      ~ escape(label) ~ "</a></li>\n";
  }
  out_ ~= "</ol>\n</nav>\n</body>\n</html>\n";
  return out_.data;
}

/+ ↓ the package document.
     .
     dcterms:modified is required and must be to the second in UTC. It is
     taken from the document's own publication date rather than from the
     clock, so that the same document gives the same epub twice: a writer
     that stamps the time of its run cannot be compared with itself.
+/
private string opf(ref OcdaDocument doc, string[] files, string[] images) {
  string lang = languageTag(doc);
  string title = doc.meta("title.full");
  if (title.length == 0) { title = doc.meta("title.main"); }
  auto out_ = appender!string;
  out_ ~= `<?xml version="1.0" encoding="utf-8"?>` ~ "\n";
  out_ ~= `<package version="3.0" xmlns="http://www.idpf.org/2007/opf"`
    ~ ` unique-identifier="bookid" xml:lang="` ~ attr(lang) ~ `">` ~ "\n";
  out_ ~= `<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">` ~ "\n";
  out_ ~= `<dc:identifier id="bookid">urn:uuid:` ~ uuid(doc) ~ "</dc:identifier>\n";
  out_ ~= "<dc:title>" ~ escape(title) ~ "</dc:title>\n";
  if (doc.meta("creator.author").length > 0) {
    out_ ~= "<dc:creator>" ~ escape(doc.meta("creator.author")) ~ "</dc:creator>\n";
  }
  out_ ~= "<dc:language>" ~ escape(lang) ~ "</dc:language>\n";
  if (doc.meta("date.published").length > 0) {
    out_ ~= "<dc:date>" ~ escape(doc.meta("date.published")) ~ "</dc:date>\n";
  }
  if (doc.meta("rights.copyright").length > 0) {
    out_ ~= "<dc:rights>" ~ escape(doc.meta("rights.copyright")) ~ "</dc:rights>\n";
  }
  out_ ~= `<meta property="dcterms:modified">` ~ modified(doc) ~ "</meta>\n";
  out_ ~= "</metadata>\n<manifest>\n";
  out_ ~= `<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav" />` ~ "\n";
  out_ ~= `<item id="css" href="style.css" media-type="text/css" />` ~ "\n";
  foreach (f; files) {
    out_ ~= `<item id="s_` ~ attr(f) ~ `" href="` ~ attr(f)
      ~ `.xhtml" media-type="application/xhtml+xml" />` ~ "\n";
  }
  foreach (i, name; images) {
    out_ ~= `<item id="i_` ~ i.to!string ~ `" href="image/` ~ attr(name)
      ~ `" media-type="` ~ mediaType(name) ~ `" />` ~ "\n";
  }
  out_ ~= "</manifest>\n<spine>\n";
  foreach (f; files) {
    out_ ~= `<itemref idref="s_` ~ attr(f) ~ `" />` ~ "\n";
  }
  out_ ~= "</spine>\n</package>\n";
  return out_.data;
}

/+ ↓ a uuid from the document's own digest, so that the same document has the
     same identifier every time it is written. A random one would make two
     runs of the same input two different publications.
+/
private string uuid(ref OcdaDocument doc) {
  string seed = ("digest" in doc.source) ? doc.source["digest"] : "";
  if (seed.length == 0) { seed = doc.meta("title.full") ~ doc.source_filename; }
  auto h = sha1Of(seed.representation).toHexString.idup.toLower;
  /+ ↓ version 5 shape: the version nibble and the variant bits set, which is
       what says "this uuid is a hash of a name" +/
  return h[0 .. 8] ~ "-" ~ h[8 .. 12] ~ "-5" ~ h[13 .. 16]
    ~ "-a" ~ h[17 .. 20] ~ "-" ~ h[20 .. 32];
}

/+ ↓ the publication date as a timestamp to the second in UTC +/
private string modified(ref OcdaDocument doc) {
  string d = doc.meta("date.modified");
  if (d.length == 0) { d = doc.meta("date.published"); }
  if (d.length < 10) { return "1970-01-01T00:00:00Z"; }
  return d[0 .. 10] ~ "T00:00:00Z";
}

/+ ↓ the language, as a language tag rather than as a locale name.
     .
     BCP 47 separates a language from its region with a hyphen, and one
     document of the reference collection declares "pt_BR", which epubcheck
     refuses: "Invalid subtag: pt_BR". The underscore is the POSIX locale
     spelling and the abstraction carries it as the markup gave it, so the
     conversion belongs here, in the writer that has a specification to
     satisfy.
     .
     A finding for the markup as much as for the writer: "pt-BR" is what the
     document means, and every output that declares a language is affected.
+/
private string languageTag(ref OcdaDocument doc) {
  string lang = ("language" in doc.source) ? doc.source["language"] : "en";
  return lang.replace("_", "-");
}

private string mediaType(string name) {
  string n = name.toLower;
  if (n.endsWith(".png"))  { return "image/png"; }
  if (n.endsWith(".jpg") || n.endsWith(".jpeg")) { return "image/jpeg"; }
  if (n.endsWith(".gif"))  { return "image/gif"; }
  if (n.endsWith(".svg"))  { return "image/svg+xml"; }
  return "application/octet-stream";
}

/+ ↓ an object's text with the internal markup taken out, for a title or a
     navigation label, where no markup may go +/
private string plain(string txt) {
  import std.regex : regex, replaceAll;
  static auto markers = regex(`⑆.┨|┣.|┥|┝|┤.*?├|【.+?】|░|┘|┙|┚|┃`, "g");
  return escape(txt.replaceAll(markers, "")).strip;
}

private ubyte[] bytesOf(string s) {
  return cast(ubyte[]) s.representation.dup;
}