/+
- Name: SisuDoc Spine, Doc Reform [a part of]
  - Description: documents, structuring, processing, publishing, search
    - static content generator

  - Author: Ralph Amissah
    [ralph.amissah@gmail.com]

  - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.

  - License: AGPL 3 or later:

    Spine (SiSU), a framework for document structuring, publishing and
    search

    Copyright (C) Ralph Amissah

    This program is free software: you can redistribute it and/or modify it
    under the terms of the GNU AFERO General Public License as published by the
    Free Software Foundation, either version 3 of the License, or (at your
    option) any later version.

    This program is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
    more details.

    You should have received a copy of the GNU General Public License along with
    this program. If not, see [https://www.gnu.org/licenses/].

    If you have Internet connection, the latest version of the AGPL should be
    available at these locations:
    [https://www.fsf.org/licensing/licenses/agpl.html]
    [https://www.gnu.org/licenses/agpl.html]

  - Spine (by Doc Reform, related to SiSU) uses standard:
    - docReform markup syntax
      - standard SiSU markup syntax with modified headers and minor modifications
    - docReform object numbering
      - standard SiSU object citation numbering & system

  - Homepages:
    [https://www.sisudoc.org]
    [https://www.doc-reform.org]

  - Git
    [https://git.sisudoc.org/]

+/
module read.llm_jsonl;

import std.array : split, join, replace;
import std.conv : to;
import std.stdio : writeln, File;
import std.string : strip, indexOf, stripRight;
import ocda.model;
import read.llm_text : stripMarkup;

/+ ↓ JSONL output with embeddings-ready chunks for RAG pipelines
   one JSON object per line, each containing:
     doc, ocn, path (breadcrumb), type, text (clean),
     images (only where the object embeds one, so that a chunk whose
     text reads around a figure carries the figure's identity),
     index_terms (from bookindex entries associated with this object)
+/
void outputLlmJsonl(OcdaDocument doc, string output_file) {
  string[] lines;
  string doc_id = doc.meta("title.main");
  string doc_author = doc.meta("creator.author");

  /+ ↓ build bookindex -> OCN mapping for index_terms
     bookindex objects in the .ssp have parent_ocn linking to body objects
     but the index terms are in the text of bookindex objects, not
     directly attached to body objects. We use a simpler approach:
     headings carry heading_ancestors_text which gives context. +/

  /+ ↓ track current heading path +/
  string current_path = "";

  foreach (obj; doc.body) {
    if (obj.is_a == "heading") {
      current_path = buildPath(doc_id, obj.heading_ancestors_text, obj);
      /+ ↓ emit heading as a chunk too +/
      string heading_text = stripMarkup(obj.text);
      if (heading_text.strip.length > 0) {
        lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, "heading", heading_text, obj.images);
      }
      continue;
    }

    /+ ↓ markup commentary is not content, and least of all is it a chunk
         to embed: it would be indexed as though the author had written it
         into the document
    +/
    if (!obj.renders) { continue; }

    string text = stripMarkup(obj.text);
    if (text.strip.length == 0) continue;

    lines ~= jsonLine(doc_id, doc_author, obj.ocn, current_path, obj.is_a, text, obj.images);
  }

  /+ ↓ write output +/
  if (output_file.length > 0) {
    auto f = File(output_file, "w");
    foreach (line; lines) f.writeln(line);
  } else {
    foreach (line; lines) writeln(line);
  }
}

private string jsonLine(string doc_id, string doc_author, int ocn, string path, string type, string text, OcdaImage[] images) {
  return `{"doc":` ~ jsonStr(doc_id)
    ~ `,"author":` ~ jsonStr(doc_author)
    ~ `,"ocn":` ~ ocn.to!string
    ~ `,"path":` ~ jsonStr(path)
    ~ `,"type":` ~ jsonStr(type)
    ~ `,"text":` ~ jsonStr(text)
    ~ jsonImages(images)
    ~ `}`;
}

private string jsonImages(OcdaImage[] images) {
  /+ ↓ omitted entirely where the object embeds no image +/
  if (images.length == 0) return "";
  string[] items;
  foreach (img; images) {
    string item = `{"name":` ~ jsonStr(img.name);
    if (img.missing) {
      item ~= `,"missing":true`;
    } else {
      item ~= `,"sha256":` ~ jsonStr(img.sha256)
        ~ `,"bytes":` ~ img.bytes.to!string;
      if (img.width > 0 && img.height > 0) {
        item ~= `,"width":` ~ img.width.to!string
          ~ `,"height":` ~ img.height.to!string;
      }
    }
    items ~= item ~ `}`;
  }
  return `,"images":[` ~ items.join(",") ~ `]`;
}

private string buildPath(string doc_title, string ancestors_text, OcdaObject obj) {
  string[] parts;
  if (ancestors_text.length > 0) {
    foreach (part; ancestors_text.split("|")) {
      string clean = stripMarkup(part).strip;
      if (clean.length > 0 && clean != doc_title) {
        parts ~= clean;
      }
    }
  }
  string heading_text = stripMarkup(obj.text).strip;
  if (parts.length > 0)
    return parts.join(" > ") ~ " > " ~ heading_text;
  else
    return heading_text;
}

private string jsonStr(string s) {
  /+ ↓ JSON string escaping +/
  string r = s;
  r = r.replace(`\`, `\\`);
  r = r.replace(`"`, `\"`);
  r = r.replace("\n", `\n`);
  r = r.replace("\r", `\r`);
  r = r.replace("\t", `\t`);
  return `"` ~ r ~ `"`;
}