From 479e71e90146b1067120faf489af1b13a5a0cf3d Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Sat, 14 Apr 2018 16:39:52 -0400 Subject: sqlite3 some stuff & rearrange files, dlang, org --- src/sdp/output/sqlite.d | 806 ++++++++++++++++++++++++++++-------------------- 1 file changed, 467 insertions(+), 339 deletions(-) (limited to 'src/sdp/output/sqlite.d') diff --git a/src/sdp/output/sqlite.d b/src/sdp/output/sqlite.d index 480c317..2409e3b 100644 --- a/src/sdp/output/sqlite.d +++ b/src/sdp/output/sqlite.d @@ -3,18 +3,92 @@ import sdp.output; import std.file, std.conv : to; +long _metadata_tid_lastrowid; +template SQLiteDbRun() { + void SQLiteDbRun(Db,St,O)( + auto ref Db db, + auto ref St db_statement, + auto ref O opt_action, + string note, + ) { + debug(sql_statement) { + writeln(db_statement); + } + try { + db.run( + "\nBEGIN;\n" ~ + db_statement ~ + "\nCOMMIT;\n" + ); + if (!(opt_action.sqlite_create)) { + _metadata_tid_lastrowid = db.lastInsertRowid(); + writeln("last row id: ", _metadata_tid_lastrowid); + } + } catch (ErrnoException ex) { + writeln("ERROR SQLite : ", ex); + } catch (Exception ex) { + writeln("ERROR SQLite : ", ex); + } + if (opt_action.debug_do) { + writeln(note); + if (opt_action.verbose) { + writeln(db_statement); + } + } + } +} template SQLiteBuildTablesAndPopulate() { void SQLiteBuildTablesAndPopulate(D,I)( auto ref const D doc_abstraction, - auto ref I doc_matters, + auto ref I doc_matters, ) { import d2sqlite3; import std.typecons : Nullable; mixin SiSUoutputRgxInit; static auto rgx = Rgx(); + auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path); + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); + template SQLiteDbStatementComposite() { + void SQLiteDbStatementComposite(Db,D,I)( + auto ref Db db, + auto ref const D doc_abstraction, + auto ref I doc_matters, + ) { + { + { + string _db_statement; + { + if ((doc_matters.opt.action.sqlite_create)) { + auto pth_sqlite = SiSUpathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); + pth_sqlite.base.mkdirRecurse; + _db_statement ~= SQLiteTablesReCreate!()(); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE"); + } + if ((doc_matters.opt.action.sqlite_update)) { // TODO + _db_statement ~= SQLiteInsertMetadata!()(doc_matters); + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT MetaData"); + /+ get tid (lastrowid or max) for use in doc_objects table +/ + _metadata_tid_lastrowid = db.lastInsertRowid(); + _db_statement ~= SQLiteInsertDocObjectsLoop!()(doc_matters.xml.keys_seq.sql, _metadata_tid_lastrowid); // FIX + SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table INSERT DocObjects"); + } + } + db.close; + } + } + } + } + template SQLinsertDelimiter() { + auto SQLinsertDelimiter(string _txt) { + _txt = _txt + .replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0"); + return _txt; + } + } template SQLiteFormatAndLoadObject() { auto SQLiteFormatAndLoadObject(I)( - auto ref I doc_matters, + auto ref I doc_matters, ) { mixin SiSUoutputRgxInit; struct sqlite_format_and_load_objects { @@ -23,7 +97,6 @@ template SQLiteBuildTablesAndPopulate() { ) { string _notes; string _urls; - _txt = _txt.replaceAll(rgx.inline_fontface_clean, ""); if (_txt.matchFirst(rgx.inline_notes_al_gen)) { foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) { _notes ~= "\n" ~ m["text"]; @@ -239,9 +312,17 @@ template SQLiteBuildTablesAndPopulate() { string _align = ("style=\"text-align:" ~ ((obj.table_column_aligns[col_idx] == "l") ? "left\"" : "right\"")); - _table ~= "<" ~ _col_is ~ " width=\"" ~ obj.table_column_widths[col_idx].to!string ~ "%\" " ~ _align ~ ">"; + _table ~= "<" + ~ _col_is + ~ " width=\"" + ~ obj.table_column_widths[col_idx].to!string + ~ "%\" " + ~ _align + ~ ">"; _table ~= cell; - _table ~= ""; + _table ~= ""; } } _table ~= ""; @@ -452,31 +533,41 @@ template SQLiteBuildTablesAndPopulate() { return sqlite_format_and_load_objects(); } } - template SQLiteInstruct() { - Statement SQLiteInstruct(I)( - auto ref I doc_matters, - ) { - auto pth_sqlite = SiSUpathsSQLite!()(doc_matters.output_path, doc_matters.src.language); - pth_sqlite.base.mkdirRecurse; - auto db = Database(pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); - // auto db = Database(":memory:"); // open database in memory - if (doc_matters.opt.action.sqlite_create) { - if ((doc_matters.opt.action.verbose)) { writeln("sqlite create tables... "); } - db.run(" + template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"¶ + DROP INDEX IF EXISTS ocn; + DROP INDEX IF EXISTS digest_clean; + DROP INDEX IF EXISTS digest_all; + DROP INDEX IF EXISTS clean; + DROP INDEX IF EXISTS lev0; + DROP INDEX IF EXISTS lev1; + DROP INDEX IF EXISTS lev2; + DROP INDEX IF EXISTS lev3; + DROP INDEX IF EXISTS lev4; + DROP INDEX IF EXISTS lev5; + DROP INDEX IF EXISTS lev6; + DROP INDEX IF EXISTS title; + DROP INDEX IF EXISTS creator_author; + DROP INDEX IF EXISTS src_filename; + DROP INDEX IF EXISTS language_document_char; + DROP INDEX IF EXISTS classify_topic_register; DROP TABLE IF EXISTS metadata_and_text; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE metadata_and_text ( - tid BIGINT PRIMARY KEY, + tid INTEGER PRIMARY KEY, + /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ title VARCHAR(800) NOT NULL, - title_main VARCHAR(400) NULL, /*NOT*/ + title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, title_note VARCHAR(2500) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, - creator_author VARCHAR(600) NULL, + creator_author VARCHAR(600) NOT NULL, creator_author_email VARCHAR(100) NULL, creator_author_hon VARCHAR(100) NULL, creator_author_nationality VARCHAR(100) NULL, @@ -490,7 +581,7 @@ template SQLiteBuildTablesAndPopulate() { creator_audio VARCHAR(600) NULL, creator_video VARCHAR(600) NULL, language_document VARCHAR(100) NULL, - language_document_char VARCHAR(6) NULL, /*NOT*/ + language_document_char VARCHAR(6) NOT NULL, language_original VARCHAR(100) NULL, language_original_char VARCHAR(6) NULL, date_added_to_site VARCHAR(10) NULL, @@ -539,18 +630,20 @@ template SQLiteBuildTablesAndPopulate() { notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, - src_filename VARCHAR(256) NULL, + src_filename VARCHAR(256) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, + pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ src_text TEXT NULL, fulltext TEXT NULL, links TEXT NULL ); - /* */ CREATE TABLE doc_objects ( - lid BIGINT PRIMARY KEY, - metadata_tid BIGINT REFERENCES metadata_and_text, + lid INTEGER PRIMARY KEY, + metadata_tid INTEGER REFERENCES metadata_and_text, ocn SMALLINT, ocnd VARCHAR(6), ocns VARCHAR(6), @@ -581,37 +674,50 @@ template SQLiteBuildTablesAndPopulate() { digest_clean CHAR(256), digest_all CHAR(256), types CHAR(1) NULL - ) - "); - } - Statement insert_metadata = db.prepare(" + ); + CREATE INDEX idx_ocn ON doc_objects(ocn); + CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); + CREATE INDEX idx_digest_all ON doc_objects(digest_all); + CREATE INDEX idx_clean ON doc_objects(clean); + CREATE INDEX idx_lev0 ON doc_objects(lev0); + CREATE INDEX idx_lev1 ON doc_objects(lev1); + CREATE INDEX idx_lev2 ON doc_objects(lev2); + CREATE INDEX idx_lev3 ON doc_objects(lev3); + CREATE INDEX idx_lev4 ON doc_objects(lev4); + CREATE INDEX idx_lev5 ON doc_objects(lev5); + CREATE INDEX idx_lev6 ON doc_objects(lev6); + CREATE INDEX idx_title ON metadata_and_text(title); + CREATE INDEX idx_author ON metadata_and_text(creator_author); + CREATE INDEX idx_filename ON metadata_and_text(src_filename); + CREATE INDEX idx_language ON metadata_and_text(language_document_char); + CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); + ¶",); + return _sql_instruct; + } + } + template SQLiteInsertMetadata() { + string SQLiteInsertMetadata(I)( + auto ref I doc_matters, + ) { + string _insert_metadata; + _insert_metadata = format(q"¶ INSERT INTO metadata_and_text ( - tid, + src_filename, title, title_main, title_sub, title_short, title_edition, - title_note, title_language, - title_language_char, + classify_dewey, + classify_keywords, + classify_loc, + classify_subject, + classify_topic_register, creator_author, creator_author_email, - creator_author_hon, - creator_author_nationality, - creator_editor, - creator_contributor, creator_illustrator, - creator_photographer, creator_translator, - creator_prepared_by, - creator_digitized_by, - creator_audio, - creator_video, - language_document, - language_document_char, - language_original, - language_original_char, date_added_to_site, date_available, date_created, @@ -619,17 +725,18 @@ template SQLiteBuildTablesAndPopulate() { date_modified, date_published, date_valid, - date_translated, - date_original_publication, - date_generated, - publisher, + identifier_isbn, + identifier_oclc, + language_document, + language_document_char, + notes_abstract, + notes_description, original_publisher, original_language, original_language_char, original_source, - original_institution, - original_nationality, original_title, + publisher, rights_copyright, rights_copyright_audio, rights_copyright_cover, @@ -638,272 +745,144 @@ template SQLiteBuildTablesAndPopulate() { rights_copyright_text, rights_copyright_translation, rights_copyright_video, - rights_license, - identifier_oclc, - identifier_isbn, - classify_topic_register, - classify_subject, - classify_loc, - classify_dewey, - classify_keywords, - notes_abstract, - notes_description, - notes_comment, - notes_coverage, - notes_relation, - notes_history, - notes_type, - notes_format, - notes_prefix, - notes_prefix_a, - notes_prefix_b, - notes_suffix, - src_filename, - src_fingerprint, - src_filesize, - src_wordcount, - src_text, - fulltext, - links + rights_license ) VALUES ( - :tid, - :title, - :title_main, - :title_sub, - :title_short, - :title_edition, - :title_note, - :title_language, - :title_language_char, - :creator_author, - :creator_author_email, - :creator_author_hon, - :creator_author_nationality, - :creator_editor, - :creator_contributor, - :creator_illustrator, - :creator_photographer, - :creator_translator, - :creator_prepared_by, - :creator_digitized_by, - :creator_audio, - :creator_video, - :language_document, - :language_document_char, - :language_original, - :language_original_char, - :date_added_to_site, - :date_available, - :date_created, - :date_issued, - :date_modified, - :date_published, - :date_valid, - :date_translated, - :date_original_publication, - :date_generated, - :publisher, - :original_publisher, - :original_language, - :original_language_char, - :original_source, - :original_institution, - :original_nationality, - :original_title, - :rights_copyright, - :rights_copyright_audio, - :rights_copyright_cover, - :rights_copyright_illustrations, - :rights_copyright_photographs, - :rights_copyright_text, - :rights_copyright_translation, - :rights_copyright_video, - :rights_license, - :identifier_oclc, - :identifier_isbn, - :classify_topic_register, - :classify_subject, - :classify_loc, - :classify_dewey, - :classify_keywords, - :notes_abstract, - :notes_description, - :notes_comment, - :notes_coverage, - :notes_relation, - :notes_history, - :notes_type, - :notes_format, - :notes_prefix, - :notes_prefix_a, - :notes_prefix_b, - :notes_suffix, - :src_filename, - :src_fingerprint, - :src_filesize, - :src_wordcount, - :src_text, - :fulltext, - :links - ) - "); - // insert_metadata.bind(":tid", ); // unique identifier you could try use sha of title author & language? or auto increment?? - insert_metadata.bind(":title", doc_matters.conf_make_meta.meta.title_full); - insert_metadata.bind(":title_main", doc_matters.conf_make_meta.meta.title_main); - insert_metadata.bind(":title_sub", doc_matters.conf_make_meta.meta.title_subtitle); - insert_metadata.bind(":title_short", doc_matters.conf_make_meta.meta.title_short); // - insert_metadata.bind(":title_edition", doc_matters.conf_make_meta.meta.title_edition); - insert_metadata.bind(":title_language", doc_matters.conf_make_meta.meta.title_language); - // insert_metadata.bind(":title_language_char", doc_matters.conf_make_meta.meta.title_language_char); - insert_metadata.bind(":classify_dewey", doc_matters.conf_make_meta.meta.classify_dewey); - insert_metadata.bind(":classify_keywords", doc_matters.conf_make_meta.meta.classify_keywords); - insert_metadata.bind(":classify_loc", doc_matters.conf_make_meta.meta.classify_loc); - insert_metadata.bind(":classify_subject", doc_matters.conf_make_meta.meta.classify_subject); - insert_metadata.bind(":classify_topic_register", doc_matters.conf_make_meta.meta.classify_topic_register); - insert_metadata.bind(":creator_author", doc_matters.conf_make_meta.meta.creator_author); - insert_metadata.bind(":creator_author_email", doc_matters.conf_make_meta.meta.creator_author_email); - // insert_metadata.bind(":creator_editor", doc_matters.conf_make_meta.meta.creator_editor); - // insert_metadata.bind(":creator_contributor", doc_matters.conf_make_meta.meta.creator_contributor); - insert_metadata.bind(":creator_illustrator", doc_matters.conf_make_meta.meta.creator_illustrator); - // insert_metadata.bind(":creator_photographer", doc_matters.conf_make_meta.meta.creator_photographer); - insert_metadata.bind(":creator_translator", doc_matters.conf_make_meta.meta.creator_translator); - // insert_metadata.bind(":creator_audio", doc_matters.conf_make_meta.meta.creator_audio); - // insert_metadata.bind(":creator_video", doc_matters.conf_make_meta.meta.creator_video); - insert_metadata.bind(":date_added_to_site", doc_matters.conf_make_meta.meta.date_added_to_site); - insert_metadata.bind(":date_available", doc_matters.conf_make_meta.meta.date_available); - insert_metadata.bind(":date_created", doc_matters.conf_make_meta.meta.date_created); - insert_metadata.bind(":date_issued", doc_matters.conf_make_meta.meta.date_issued); - insert_metadata.bind(":date_modified", doc_matters.conf_make_meta.meta.date_modified); - insert_metadata.bind(":date_published", doc_matters.conf_make_meta.meta.date_published); - insert_metadata.bind(":date_valid", doc_matters.conf_make_meta.meta.date_valid); - // insert_metadata.bind(":date_translated", doc_matters.conf_make_meta.meta.date_translated); - // insert_metadata.bind(":date_original_publication", doc_matters.conf_make_meta.meta.date_original_publication); - // insert_metadata.bind(":date_generated", doc_matters.conf_make_meta.meta.date_generated); - insert_metadata.bind(":identifier_isbn", doc_matters.conf_make_meta.meta.identifier_isbn); - insert_metadata.bind(":identifier_oclc", doc_matters.conf_make_meta.meta.identifier_oclc); - insert_metadata.bind(":language_document", doc_matters.conf_make_meta.meta.language_document); - insert_metadata.bind(":language_document_char", doc_matters.conf_make_meta.meta.language_document_char); - // insert_metadata.bind(":language_original", doc_matters.conf_make_meta.meta.language_original); - // insert_metadata.bind(":language_original_char", doc_matters.conf_make_meta.meta.language_original_char); - insert_metadata.bind(":notes_abstract", doc_matters.conf_make_meta.meta.notes_abstract); - insert_metadata.bind(":notes_description", doc_matters.conf_make_meta.meta.notes_description); - insert_metadata.bind(":original_publisher", doc_matters.conf_make_meta.meta.original_publisher); - insert_metadata.bind(":original_language", doc_matters.conf_make_meta.meta.original_language); - insert_metadata.bind(":original_language_char", doc_matters.conf_make_meta.meta.original_language_char); - insert_metadata.bind(":original_source", doc_matters.conf_make_meta.meta.original_source); - insert_metadata.bind(":original_title", doc_matters.conf_make_meta.meta.original_title); - insert_metadata.bind(":publisher", doc_matters.conf_make_meta.meta.publisher); - // insert_metadata.bind(":rights", doc_matters.conf_make_meta.meta.rights); - insert_metadata.bind(":rights_copyright", doc_matters.conf_make_meta.meta.rights_copyright); - insert_metadata.bind(":rights_copyright_audio", doc_matters.conf_make_meta.meta.rights_copyright_audio); - insert_metadata.bind(":rights_copyright_cover", doc_matters.conf_make_meta.meta.rights_copyright_cover); - insert_metadata.bind(":rights_copyright_illustrations", doc_matters.conf_make_meta.meta.rights_copyright_illustrations); - insert_metadata.bind(":rights_copyright_photographs", doc_matters.conf_make_meta.meta.rights_copyright_photographs); - // insert_metadata.bind(":rights_copyright_preparation", doc_matters.conf_make_meta.meta.rights_preparation); - insert_metadata.bind(":rights_copyright_text", doc_matters.conf_make_meta.meta.rights_copyright_text); - insert_metadata.bind(":rights_copyright_translation", doc_matters.conf_make_meta.meta.rights_copyright_translation); - insert_metadata.bind(":rights_copyright_video", doc_matters.conf_make_meta.meta.rights_copyright_video); - // insert_metadata.bind(":rights_copyright_digitization", doc_matters.conf_make_meta.meta.rights_digitization); - // insert_metadata.bind(":rights_copyright_audio", doc_matters.conf_make_meta.meta.rights_audio); - // insert_metadata.bind(":rights_copyright_video", doc_matters.conf_make_meta.meta.rights_video); - insert_metadata.bind(":rights_license", doc_matters.conf_make_meta.meta.rights_license); - // insert_metadata.bind(":src_filename", doc_matters.conf_make_meta.meta.src_filename); - // insert_metadata.bind(":src_fingerprint", doc_matters.conf_make_meta.meta.src_fingerprint); - // insert_metadata.bind(":src_filesize", doc_matters.conf_make_meta.meta.src_filesize); - // insert_metadata.bind(":src_wordcount", doc_matters.conf_make_meta.meta.src_wordcount); - // insert_metadata.bind(":src_text", doc_matters.conf_make_meta.meta.src_text); - // insert_metadata.bind(":fulltext", doc_matters.conf_make_meta.meta.fulltext); - // insert_metadata.bind(":links", doc_matters.conf_make_meta.meta.links); - insert_metadata.execute(); insert_metadata.reset(); - /+ watch +/ - writeln(" ", pth_sqlite.sqlite_file(doc_matters.env.pwd.baseName)); - if ((doc_matters.opt.action.verbose)) { - writeln("sql statement executed"); - } - assert(db.totalChanges == 1); - // - Statement insert_doc_objects = db.prepare(" - INSERT INTO doc_objects ( - lid, - metadata_tid, - ocn, - ocnd, - ocns, - clean, - body, - book_idx, - seg, - lev_an, - lev, - lev0, - lev1, - lev2, - lev3, - lev4, - lev5, - lev6, - lev7, - en_a, - en_z, - en_a_asterisk, - en_z_asterisk, - en_a_plus, - en_z_plus, - t_of, - t_is, - node, - parent, - digest_clean, - digest_all, - types - ) - VALUES ( - :lid, - :metadata_tid, - :ocn, - :ocnd, - :ocns, - :clean, - :body, - :book_idx, - :seg, - :lev_an, - :lev, - :lev0, - :lev1, - :lev2, - :lev3, - :lev4, - :lev5, - :lev6, - :lev7, - :en_a, - :en_z, - :en_a_asterisk, - :en_z_asterisk, - :en_a_plus, - :en_z_plus, - :t_of, - :t_is, - :node, - :parent, - :digest_clean, - :digest_all, - :types - ) - "); - return insert_doc_objects; + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s', + '%s' + ); + ¶", + SQLinsertDelimiter!()(doc_matters.src.filename), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), + SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license) + ); + return _insert_metadata; } } - template SQLiteObjectsLoop() { - void SQLiteObjectsLoop(P)( - auto ref P doc_parts, + template SQLiteInsertDocObjectsLoop() { + string SQLiteInsertDocObjectsLoop(P,I)( + auto ref P doc_parts, + auto ref I _metadata_tid, ) { - Statement insert_doc_objects = SQLiteInstruct!()(doc_matters); + string insertDocObjectsRow(O)(O obj) { + string _insert_doc_objects_row; + _insert_doc_objects_row = format(q"¶ + INSERT INTO doc_objects ( + metadata_tid, + ocn, + clean, + body, + lev, + t_of, + t_is + ) + VALUES ( + %d, + %s, + '%s', + '%s', + %s, + '%s', + '%s' + ); + ¶", + _metadata_tid, + obj.ocn, + SQLinsertDelimiter!()(obj_txt["text"]), + SQLinsertDelimiter!()(obj_txt["html"]), + obj.heading_lev_markup, + obj.is_of, + obj.is_a, + ); + return _insert_doc_objects_row; + } auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters); string[string] obj_txt; string doc_text; + string[] _insert_doc_objects; foreach (part; doc_parts) { foreach (obj; doc_abstraction[part]) { switch (obj.of_part) { @@ -1048,43 +1027,192 @@ template SQLiteBuildTablesAndPopulate() { ); } } - insert_doc_objects.bind(":t_of", obj.is_of); - insert_doc_objects.bind(":t_is", obj.is_a); - insert_doc_objects.bind(":ocn", obj.ocn); - insert_doc_objects.bind(":clean", obj_txt["text"]); // consider whether book index info should be made available within clear text for search - insert_doc_objects.bind(":body", obj_txt["html"]); - // insert_doc_objects.bind(":book_idx", ""); // not needed, but, consider whether should be made available within object for clear text search - insert_doc_objects.bind(":lev", obj.heading_lev_markup); - // // insert_doc_objects.bind(":dom_markedup", ""); // should make lev sequence below obsolete - // // insert_doc_objects.bind(":dom_collapsed", ""); // should add info - // insert_doc_objects.bind(":lev0", ""); - // insert_doc_objects.bind(":lev1", ""); - // insert_doc_objects.bind(":lev2", ""); - // insert_doc_objects.bind(":lev3", ""); - // insert_doc_objects.bind(":lev4", ""); - // insert_doc_objects.bind(":lev5", ""); - // insert_doc_objects.bind(":lev6", ""); - // insert_doc_objects.bind(":lev7", ""); - // insert_doc_objects.bind(":node", ""); - // insert_doc_objects.bind(":type", ""); - // insert_doc_objects.bind(":parent_ocn", ""); - // insert_doc_objects.bind(":ancestors", ""); - // insert_doc_objects.bind(":heading_lev_markup", ""); - // insert_doc_objects.bind(":heading_lev_collapsed", ""); - // insert_doc_objects.bind(":parent_lev_markup", ""); - // insert_doc_objects.bind(":heading_ancestors", ""); - // insert_doc_objects.bind(":node", ""); - insert_doc_objects.execute(); insert_doc_objects.reset(); - } + if (!(obj.is_a == "comment")) { + _insert_doc_objects ~= insertDocObjectsRow(obj); + } + } // loop closes } + return _insert_doc_objects.join.to!(char[]).toUTF8; } } - SQLiteObjectsLoop!()(doc_matters.xml.keys_seq.sql); + SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters); } } template SQLiteTablesCreate() { - void SQLiteTablesCreate()() { - writeln("table create"); + void SQLiteTablesCreate(E,O)(E env, O opt_action) { + import d2sqlite3; + template SQLiteTablesReCreate() { + string SQLiteTablesReCreate()() { + string _sql_instruct; + _sql_instruct = format(q"¶ + DROP INDEX IF EXISTS ocn; + DROP INDEX IF EXISTS digest_clean; + DROP INDEX IF EXISTS digest_all; + DROP INDEX IF EXISTS clean; + DROP INDEX IF EXISTS lev0; + DROP INDEX IF EXISTS lev1; + DROP INDEX IF EXISTS lev2; + DROP INDEX IF EXISTS lev3; + DROP INDEX IF EXISTS lev4; + DROP INDEX IF EXISTS lev5; + DROP INDEX IF EXISTS lev6; + DROP INDEX IF EXISTS title; + DROP INDEX IF EXISTS creator_author; + DROP INDEX IF EXISTS src_filename; + DROP INDEX IF EXISTS language_document_char; + DROP INDEX IF EXISTS classify_topic_register; + DROP TABLE IF EXISTS metadata_and_text; + DROP TABLE IF EXISTS doc_objects; + DROP TABLE IF EXISTS urls; + CREATE TABLE metadata_and_text ( + tid INTEGER PRIMARY KEY, + /* src_filename_composite VARCHAR(256) NOT NULL UNIQUE, /* z pod name if any + src filename + language code */ + title VARCHAR(800) NOT NULL, + title_main VARCHAR(400) NOT NULL, + title_sub VARCHAR(400) NULL, + title_short VARCHAR(400) NULL, + title_edition VARCHAR(10) NULL, + title_note VARCHAR(2500) NULL, + title_language VARCHAR(100) NULL, + title_language_char VARCHAR(6) NULL, + creator_author VARCHAR(600) NOT NULL, + creator_author_email VARCHAR(100) NULL, + creator_author_hon VARCHAR(100) NULL, + creator_author_nationality VARCHAR(100) NULL, + creator_editor VARCHAR(600) NULL, + creator_contributor VARCHAR(600) NULL, + creator_illustrator VARCHAR(600) NULL, + creator_photographer VARCHAR(600) NULL, + creator_translator VARCHAR(600) NULL, + creator_prepared_by VARCHAR(600) NULL, + creator_digitized_by VARCHAR(600) NULL, + creator_audio VARCHAR(600) NULL, + creator_video VARCHAR(600) NULL, + language_document VARCHAR(100) NULL, + language_document_char VARCHAR(6) NOT NULL, + language_original VARCHAR(100) NULL, + language_original_char VARCHAR(6) NULL, + date_added_to_site VARCHAR(10) NULL, + date_available VARCHAR(10) NULL, + date_created VARCHAR(10) NULL, + date_issued VARCHAR(10) NULL, + date_modified VARCHAR(10) NULL, + date_published VARCHAR(10) NULL, + date_valid VARCHAR(10) NULL, + date_translated VARCHAR(10) NULL, + date_original_publication VARCHAR(10) NULL, + date_generated VARCHAR(10) NULL, + publisher VARCHAR(600) NULL, + original_publisher VARCHAR(600) NULL, + original_language VARCHAR(100) NULL, + original_language_char VARCHAR(6) NULL, + original_source VARCHAR(600) NULL, + original_institution VARCHAR(600) NULL, + original_nationality VARCHAR(100) NULL, + original_title VARCHAR(800) NULL, + rights_copyright VARCHAR(2500) NULL, + rights_copyright_audio VARCHAR(2500) NULL, + rights_copyright_cover VARCHAR(2500) NULL, + rights_copyright_illustrations VARCHAR(2500) NULL, + rights_copyright_photographs VARCHAR(2500) NULL, + rights_copyright_text VARCHAR(2500) NULL, + rights_copyright_translation VARCHAR(2500) NULL, + rights_copyright_video VARCHAR(2500) NULL, + rights_license VARCHAR(2500) NULL, + identifier_oclc VARCHAR(30) NULL, + identifier_isbn VARCHAR(16) NULL, + classify_topic_register VARCHAR(2500) NULL, + classify_subject VARCHAR(600) NULL, + classify_loc VARCHAR(30) NULL, + classify_dewey VARCHAR(30) NULL, + classify_keywords VARCHAR(600) NULL, + notes_abstract TEXT NULL, + notes_description TEXT NULL, + notes_comment TEXT NULL, + notes_coverage VARCHAR(200) NULL, + notes_relation VARCHAR(200) NULL, + notes_history VARCHAR(600) NULL, + notes_type VARCHAR(600) NULL, + notes_format VARCHAR(600) NULL, + notes_prefix TEXT NULL, + notes_prefix_a TEXT NULL, + notes_prefix_b TEXT NULL, + notes_suffix TEXT NULL, + src_filename VARCHAR(256) NOT NULL, + src_fingerprint VARCHAR(256) NULL, + src_filesize VARCHAR(10) NULL, + src_wordcount VARCHAR(10) NULL, + pod_name VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_fingerprint VARCHAR(256) NULL, /* zipped pod, work to be done here */ + pod_size VARCHAR(10) NULL, /* zipped pod, work to be done here */ + src_text TEXT NULL, + fulltext TEXT NULL, + links TEXT NULL + ); + CREATE TABLE doc_objects ( + lid INTEGER PRIMARY KEY, + metadata_tid INTEGER REFERENCES metadata_and_text, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + clean TEXT NULL, + body TEXT NULL, + book_idx TEXT NULL, + seg VARCHAR(256) NULL, + lev_an VARCHAR(1), + lev SMALLINT NULL, + lev0 SMALLINT, + lev1 SMALLINT, + lev2 SMALLINT, + lev3 SMALLINT, + lev4 SMALLINT, + lev5 SMALLINT, + lev6 SMALLINT, + lev7 SMALLINT, + en_a SMALLINT NULL, + en_z SMALLINT NULL, + en_a_asterisk SMALLINT NULL, + en_z_asterisk SMALLINT NULL, + en_a_plus SMALLINT NULL, + en_z_plus SMALLINT NULL, + t_of VARCHAR(16), + t_is VARCHAR(16), + node VARCHAR(16) NULL, + parent VARCHAR(16) NULL, + digest_clean CHAR(256), + digest_all CHAR(256), + types CHAR(1) NULL + ); + CREATE INDEX idx_ocn ON doc_objects(ocn); + CREATE INDEX idx_digest_clean ON doc_objects(digest_clean); + CREATE INDEX idx_digest_all ON doc_objects(digest_all); + CREATE INDEX idx_clean ON doc_objects(clean); + CREATE INDEX idx_lev0 ON doc_objects(lev0); + CREATE INDEX idx_lev1 ON doc_objects(lev1); + CREATE INDEX idx_lev2 ON doc_objects(lev2); + CREATE INDEX idx_lev3 ON doc_objects(lev3); + CREATE INDEX idx_lev4 ON doc_objects(lev4); + CREATE INDEX idx_lev5 ON doc_objects(lev5); + CREATE INDEX idx_lev6 ON doc_objects(lev6); + CREATE INDEX idx_title ON metadata_and_text(title); + CREATE INDEX idx_author ON metadata_and_text(creator_author); + CREATE INDEX idx_filename ON metadata_and_text(src_filename); + CREATE INDEX idx_language ON metadata_and_text(language_document_char); + CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register); + ¶",); + return _sql_instruct; + } + } + if (opt_action.sqlite_create) { + string _db_statement; + auto pth_sqlite = SiSUpathsSQLite!()(opt_action.output_dir_set); // ISSUE + pth_sqlite.base.mkdirRecurse; + auto db = Database(pth_sqlite.sqlite_file); // ISSUE + { + _db_statement ~= SQLiteTablesReCreate!()(); + } + SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); + } } } template SQLiteTablesDrop() { -- cgit v1.2.3