-*- mode: org -*-
#+TITLE:       sisudoc spine (doc_reform) output zip
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+FILETAGS:    :spine:zip:
#+AUTHOR:      Ralph Amissah
#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT:   Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE:    en
#+STARTUP:     content hideblocks hidestars noindent entitiespretty
#+PROPERTY:    header-args+ :eval never-export :exports code
#+PROPERTY:    header-args+ :noweb yes :padline no
#+PROPERTY:    header-args+ :results silent :cache no
#+PROPERTY:    header-args+ :mkdirp yes
#+OPTIONS:     H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
- magic single double-quote → " ← FIX changes hilighting behavior (occuring
  after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX

- [[./doc-reform.org][doc-reform.org]]  [[./][org/]]
- [[./output_hub.org][output_hub]]

* _zstd_ :module:spine:compression:

#+HEADER: :tangle "../src/sisudoc/ocda/zstd.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/+
  module zstd;<BR>
  - the few libzstd entry points spine uses, and a buffer in / buffer out
    wrapper over each direction<BR>
  - one frame per call: what is compressed is a whole pod archive, not a
    stream
+/
module sisudoc.ocda.zstd;
@safe:
/+ ↓ libzstd, declared rather than bound by a generated header
     Provides the fifteen declarations that spine needs (there is nothing
     here to generate and no vendored tree to track): the C library is linked
     (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these
     prototypes say what is being called. That is the same arrangement as
     sqlite3, which is declared in the vendored d2sqlite3 and linked from the
     system.
     .
     It sits under ocda/ rather than outputs/ because the pod *reader* needs it,
     and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part,
     which the outputs depend on and not the other way round.
     A compression primitive is not an abstraction concern; the dependency
     direction is what puts it here.
+/
extern (C) @nogc nothrow private {
  size_t ZSTD_compressBound(size_t srcSize);
  size_t ZSTD_compress(void* dst, size_t dstCapacity,
    const(void)* src, size_t srcSize, int compressionLevel);
  size_t ZSTD_decompress(void* dst, size_t dstCapacity,
    const(void)* src, size_t compressedSize);
  ulong  ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
  uint   ZSTD_isError(size_t code);
  const(char)* ZSTD_getErrorName(size_t code);
  int    ZSTD_minCLevel();
  int    ZSTD_maxCLevel();
  uint   ZSTD_versionNumber();
}
/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/
private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max;       /+ (0ULL - 1) +/
private enum ulong ZSTD_CONTENTSIZE_ERROR   = ulong.max - 1;   /+ (0ULL - 2) +/
/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian.
     A pod is recognised by this rather than by its name, so that a pod
     already published as a plain zip keeps being read whatever it is called.
+/
enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD];
/+ ↓ the ceiling on what one frame may be allowed to become.
     A frame declares its uncompressed size in its header and that number is
     attacker-controlled: a .sisupod fetched over https could claim a size no
     machine can allocate. The declared size is checked against this before a
     buffer is asked for, so a hostile declaration is refused rather than
     attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what
     the archive inside may extract to.
+/
enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024;
/+ ↓ thrown rather than returned: every caller here wants the whole buffer or
     nothing, and there is no partial result worth handing back
+/
class ZstdException : Exception {
  this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow {
    super(_msg, _file, _line);
  }
}
/+ ↓ does this begin a zstd frame? +/
bool zstdIsFrame(const(ubyte)[] _bytes) {
  if (_bytes.length < zstd_frame_magic.length) {
    return false;
  }
  foreach (_i, _b; zstd_frame_magic) {
    if (_bytes[_i] != _b) {
      return false;
    }
  }
  return true;
}
/+ ↓ the library's version, for a run that wants to record what compressed a
     pod. The bytes of a frame are a function of the library and the level, so
     two zstd versions may compress the same input differently; nothing is
     promised about archive bytes
+/
@trusted uint zstdVersion() {
  return ZSTD_versionNumber();
}
/+ ↓ the level clamped to what this library actually supports +/
@trusted int zstdLevelClamped(int _level) {
  int _min = ZSTD_minCLevel();
  int _max = ZSTD_maxCLevel();
  if (_level < _min) { return _min; }
  if (_level > _max) { return _max; }
  return _level;
}
private @trusted string _zstdError(size_t _code) {
  import std.string : fromStringz;
  return ZSTD_getErrorName(_code).fromStringz.idup;
}
/+ ↓ one buffer to one frame +/
@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) {
  size_t _bound = ZSTD_compressBound(_src.length);
  auto _dst = new ubyte[_bound];
  size_t _got = ZSTD_compress(
    _dst.ptr, _dst.length,
    _src.ptr, _src.length,
    zstdLevelClamped(_level),
  );
  if (ZSTD_isError(_got)) {
    throw new ZstdException("zstd could not compress: " ~ _zstdError(_got));
  }
  return _dst[0 .. _got];
}
/+ ↓ one frame back to one buffer, refusing a frame that will not say how
     large it is or that says something absurd
+/
@trusted ubyte[] zstdDecompress(const(ubyte)[] _src,
  size_t _max = ZSTD_MAX_FRAME_CONTENT
) {
  import std.conv : to;
  if (!(zstdIsFrame(_src))) {
    throw new ZstdException("not a zstd frame");
  }
  ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length);
  if (_declared == ZSTD_CONTENTSIZE_ERROR) {
    throw new ZstdException("zstd frame header will not read");
  }
  if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) {
    /+ every frame spine writes is one whole buffer, so its size is always
       declared; a frame without one was made by something else
    +/
    throw new ZstdException("zstd frame does not declare its size");
  }
  if (_declared > _max) {
    throw new ZstdException("zstd frame declares "
      ~ _declared.to!string ~ " bytes, over the "
      ~ _max.to!string ~ " allowed");
  }
  auto _dst = new ubyte[_declared.to!size_t];
  size_t _got = ZSTD_decompress(
    _dst.ptr, _dst.length,
    _src.ptr, _src.length,
  );
  if (ZSTD_isError(_got)) {
    throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got));
  }
  if (_got != _dst.length) {
    throw new ZstdException("zstd frame gave " ~ _got.to!string
      ~ " bytes where its header declared " ~ _dst.length.to!string);
  }
  return _dst;
}
#+END_SRC

* org includes
** project version

#+NAME: spine_version
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
#+END_SRC

** year

#+NAME: year
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
#+END_SRC

** document header including copyright & license

#+NAME: doc_header_including_copyright_and_license
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
#+END_SRC

* __END__