diff options
| -rw-r--r-- | CHANGELOG | 14 | ||||
| -rw-r--r-- | lib/sisu/v0/cgi_pgsql.rb | 6 | ||||
| -rw-r--r-- | lib/sisu/v0/cgi_sqlite.rb | 2 | ||||
| -rw-r--r-- | lib/sisu/v0/character_encoding.rb | 2 | ||||
| -rw-r--r-- | lib/sisu/v0/db_create.rb | 459 | ||||
| -rw-r--r-- | lib/sisu/v0/db_dbi.rb (renamed from lib/sisu/v0/shared_db.rb) | 8 | ||||
| -rw-r--r-- | lib/sisu/v0/db_drop.rb | 129 | ||||
| -rw-r--r-- | lib/sisu/v0/db_import.rb | 384 | ||||
| -rw-r--r-- | lib/sisu/v0/db_load_tuple.rb | 34 | ||||
| -rw-r--r-- | lib/sisu/v0/db_remove.rb | 2 | ||||
| -rw-r--r-- | lib/sisu/v0/db_select.rb | 66 | ||||
| -rw-r--r-- | lib/sisu/v0/dbi.rb | 12 | ||||
| -rw-r--r-- | lib/sisu/v0/help.rb | 16 | ||||
| -rw-r--r-- | lib/sisu/v0/param.rb | 2 | ||||
| -rw-r--r-- | lib/sisu/v0/sysenv.rb | 76 |
15 files changed, 668 insertions, 544 deletions
@@ -9,11 +9,19 @@ Reverse Chronological: %% STABLE MANIFEST +%% sisu_0.62.2.orig.tar.gz (2007-10-26:43/5) +http://www.jus.uio.no/sisu/pkg/src/sisu_0.62.2.orig.tar.gz + sisu_0.62.2.orig.tar.gz + sisu_0.62.2-1.dsc + sisu_0.62.2-1.diff.gz + + * db rearranging basic things todo (create, drop, update) + %% sisu_0.62.1.orig.tar.gz (2007-10-22:43/1) http://www.jus.uio.no/sisu/pkg/src/sisu_0.62.1.orig.tar.gz - sisu_0.62.1.orig.tar.gz - sisu_0.62.1-1.dsc - sisu_0.62.1-1.diff.gz + 5a9226473b409a8a9345781deeca12b8 1480117 sisu_0.62.1.orig.tar.gz + 59e50685bb42eedfcad1540227aa1c9c 606 sisu_0.62.1-1.dsc + dd147d5f2d978a7e2195ba3ccbe8c827 145468 sisu_0.62.1-1.diff.gz * db common and cgi, regex to match new composite file filename (.ssm.sst) diff --git a/lib/sisu/v0/cgi_pgsql.rb b/lib/sisu/v0/cgi_pgsql.rb index b5012fdd..9a6486ff 100644 --- a/lib/sisu/v0/cgi_pgsql.rb +++ b/lib/sisu/v0/cgi_pgsql.rb @@ -111,8 +111,8 @@ module SiSU_CGI_pgsql } tell=SiSU_Screen::Ansi.new(@opt.cmd,a) tell.warn - a="postgresql db used for present directory: #{@db.db_psql}" - b="\n\t(to create and populate postgresql database see 'man sisu' and in particular the -D flag)\n\t[the database to be used for this directory (#{@db.db_psql}) will have to be created manually if it does not exist,\n\tsee 'sisu --help sql'\n\tif you have permission to create databases:\n\t'sisu -d --createdb'\n\tor using postgresql tools directly:\n\t'createdb #{@db.db_psql}'\n\tfor a list of existing databases try 'psql --list']" + a="postgresql db used for present directory: #{@db.psql.db}" + b="\n\t(to create and populate postgresql database see 'man sisu' and in particular the -D flag)\n\t[the database to be used for this directory (#{@db.psql.db}) will have to be created manually if it does not exist,\n\tsee 'sisu --help sql'\n\tif you have permission to create databases:\n\t'sisu -d --createdb'\n\tor using postgresql tools directly:\n\t'createdb #{@db.psql.db}'\n\tfor a list of existing databases try 'psql --list']" tell=SiSU_Screen::Ansi.new(@opt.cmd,a,b) tell.txt_cyan else puts 'failed in attempt to write sisu_pgsql.cgi to present directory, is directory writable?' @@ -133,7 +133,7 @@ module SiSU_CGI_pgsql @image_src="#{@env.url.webserv_cgi}/_sisu/image_sys" @hosturl_db="#{@env.url.webserv_base_cgi}" @hosturl_files="#{@env.url.webserv_files_from_db}" - @port="#{@db.port_psql}" + @port="#{@db.psql.port}" user='' # '#{@env.user}' WOK_SQL end diff --git a/lib/sisu/v0/cgi_sqlite.rb b/lib/sisu/v0/cgi_sqlite.rb index ef62cd35..3a76e529 100644 --- a/lib/sisu/v0/cgi_sqlite.rb +++ b/lib/sisu/v0/cgi_sqlite.rb @@ -207,7 +207,7 @@ module SiSU_CGI_sqlite end def dbi_connect <<-'WOK_SQL' - @dbi="DBI:SQLite:#{db_sqlite}" #sqlite3 ? + @dbi="DBI:SQLite:#{sqlite.db}" #sqlite3 ? @conn=DBI.connect(@dbi) WOK_SQL end diff --git a/lib/sisu/v0/character_encoding.rb b/lib/sisu/v0/character_encoding.rb index e25e8b8e..e0eb542f 100644 --- a/lib/sisu/v0/character_encoding.rb +++ b/lib/sisu/v0/character_encoding.rb @@ -71,7 +71,7 @@ module SiSU_character_encode #env=@env.path.encoding #@db=KirbyBase.new(:local,nil,nil,env) @db=KirbyBase.new #default database location, pwd - @ce_tbl=@db.create_table(:character_encoding, :glyph,:String, :dec,:Integer, :hex,:String, :rb,:String, :html,:String, :html_name,:String, :tex,:String, :tex_licr,:String, :tex_alt,:String, :name,:String) + @ce_tbl=@db.create_table.documents(:character_encoding, :glyph,:String, :dec,:Integer, :hex,:String, :rb,:String, :html,:String, :html_name,:String, :tex,:String, :tex_licr,:String, :tex_alt,:String, :name,:String) end def db x=Character_encoding.new.table diff --git a/lib/sisu/v0/db_create.rb b/lib/sisu/v0/db_create.rb index 633c92fa..61505cb3 100644 --- a/lib/sisu/v0/db_create.rb +++ b/lib/sisu/v0/db_create.rb @@ -63,7 +63,7 @@ module SiSU_DB_create def initialize(opt,conn='',sql_type='pg') @opt,@conn,@sql_type=opt,conn,sql_type @cX=SiSU_Screen::Ansi.new(@opt.cmd).cX - @comment=comment + @comment=Comment.new(@sql_type) @@dl ||=SiSU_Env::Info_env.new.digest.length end def available @@ -80,11 +80,222 @@ module SiSU_DB_create tell.colorize unless @opt.cmd =~/q/ SiSU_Env::System_call.new.create_pg_db(@env.path.stub_pwd) #watch use of path.stub_pwd instead of stub end - def comment - @comment=Hash.new('') - case @sql_type - when /pg/ - @comment['metadata'] =%{ + def output_dir? + dir=SiSU_Env::Info_env.new('') + if @opt.cmd =~/d/; dir.path.webserv_stub_ensure + end + end + def create_table + def metadata + print %{ + currently using sisu dbi module + to be populated from documents files + create tables metadata + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE metadata ( + tid INT4 PRIMARY KEY, + title VARCHAR(#{lt_title}) NULL, + subtitle VARCHAR(#{lt_subtitle}) NULL, + creator VARCHAR(#{lt_creator}) NULL, + illustrator VARCHAR(#{lt_illustrator}) NULL, + translator VARCHAR(#{lt_translator}) NULL, + subject VARCHAR(#{lt_subject}) NULL, + date VARCHAR(#{lt_date}) NULL, + date_created VARCHAR(#{lt_date_created}) NULL, + date_issued VARCHAR(#{lt_date_issued}) NULL, + date_available VARCHAR(#{lt_date_available}) NULL, + date_valid VARCHAR(#{lt_date_valid}) NULL, + date_modified VARCHAR(#{lt_date_modified}) NULL, +/* date DATE, */ +/* date_created DATE, */ +/* date_issued DATE, */ +/* date_available DATE, */ +/* date_valid DATE, */ +/* date_modified DATE, */ + type VARCHAR(#{lt_type}) NULL, + description VARCHAR(#{lt_description}) NULL, + publisher VARCHAR(#{lt_publisher}) NULL, + contributor VARCHAR(#{lt_contributor}) NULL, + prepared_by VARCHAR(#{lt_prepared_by}) NULL, + digitized_by VARCHAR(#{lt_digitized_by}) NULL, + format VARCHAR(#{lt_format}) NULL, + identifier VARCHAR(#{lt_identifier}) NULL, + source VARCHAR(#{lt_source}) NULL, + language VARCHAR(#{lt_language}) NULL, + language_original VARCHAR(#{lt_language_original}) NULL, + relation VARCHAR(#{lt_relation}) NULL, + coverage VARCHAR(#{lt_coverage}) NULL, + rights VARCHAR(#{lt_rights}) NULL, + copyright VARCHAR(#{lt_copyright}) NULL, + owner VARCHAR(#{lt_owner}) NULL, + keywords VARCHAR(#{lt_keywords}) NULL, + comment VARCHAR(#{lt_comment}) NULL, + loc VARCHAR(#{lt_loc}) NULL, + dewey VARCHAR(#{lt_dewey}) NULL, + isbn VARCHAR(#{lt_isbn}) NULL, + pg VARCHAR(#{lt_pg}) NULL, + abstract VARCHAR(#{lt_abstract}) NULL, + prefix_a TEXT NULL, + prefix_b TEXT NULL, + skin VARCHAR(#{lt_skin}) NULL, + markup VARCHAR(#{lt_markup}) NULL, + links VARCHAR(#{lt_links}) NULL, + information VARCHAR(#{lt_information}) NULL, + contact VARCHAR(#{lt_contact}) NULL, + suffix VARCHAR(#{lt_suffix}) NULL, + filename VARCHAR(#{lt_filename}) NULL UNIQUE, + types CHAR(#{lt_types}) NULL, + subj VARCHAR(#{lt_subj}) NULL + ); + #{@comment.metadata} + }) + end + def documents # create documents base + print %{ + to be populated from documents files + create tables documents document_trade document_env + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE documents ( + lid INT4 PRIMARY KEY, + metadata_tid INT4 REFERENCES metadata, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + clean TEXT NULL, + body TEXT NULL, + seg VARCHAR(#{document_seg}) NULL, + lev SMALLINT NULL, + lev1 SMALLINT, + lev2 SMALLINT, + lev3 SMALLINT, + lev4 SMALLINT, + lev5 SMALLINT, + lev6 SMALLINT, + en_a SMALLINT NULL, + en_z SMALLINT NULL, + en_a_asterisk SMALLINT NULL, + en_z_asterisk SMALLINT NULL, + en_a_plus SMALLINT NULL, + en_z_plus SMALLINT NULL, + digest_clean CHAR(#{@@dl}), + digest_all CHAR(#{@@dl}), + types CHAR(1) NULL + ); + #{@comment.documents} + }) + end + def endnotes + print %{ + to be populated from documents files + create tables endnotes + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE endnotes ( + nid INT4 PRIMARY KEY, + document_lid INT4 REFERENCES documents, + nr SMALLINT, + clean TEXT NULL, + body TEXT NULL, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + digest_clean CHAR(#{@@dl}), + metadata_tid INT4 REFERENCES metadata + ); + #{@comment.endnotes} + }) + end + def endnotes_asterisk + print %{ + to be populated from documents files + create tables endnotes_asterisk + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE endnotes_asterisk ( + nid INT4 PRIMARY KEY, + document_lid INT4 REFERENCES documents, + nr SMALLINT, + clean TEXT NULL, + body TEXT NULL, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + digest_clean CHAR(#{@@dl}), + metadata_tid INT4 REFERENCES metadata + ); + #{@comment.endnotes_asterisk} + }) + end + def endnotes_plus + print %{ + to be populated from documents files + create tables endnotes_plus + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE endnotes_plus ( + nid INT4 PRIMARY KEY, + document_lid INT4 REFERENCES documents, + nr SMALLINT, + clean TEXT NULL, + body TEXT NULL, + ocn SMALLINT, + ocnd VARCHAR(6), + ocns VARCHAR(6), + digest_clean CHAR(#{@@dl}), + metadata_tid INT4 REFERENCES metadata + ); + #{@comment.endnotes_plus} + }) + end + def urls # create documents file links mapping + print %{ + currently using sisu dbi module + to be populated from documents files + create tables urls + data import through ruby transfer + } unless @opt.cmd =~/q/ + @conn.execute(%{ + CREATE TABLE urls ( + metadata_tid INT4 REFERENCES metadata, + plaintext varchar(512), + html_toc varchar(512), + html_doc varchar(512), + xhtml varchar(512), + xml_sax varchar(512), + xml_dom varchar(512), + odf varchar(512), + pdf_p varchar(512), + pdf_l varchar(512), + concordance varchar(512), + latex_p varchar(512), + latex_l varchar(512), + digest varchar(512), + manifest varchar(512), + markup varchar(512), + sisupod varchar(512) + ); + #{@comment.urls} + }) + end + self + end + end + class Comment + def initialize(sql_type='pg') + if sql_type =~ /pg/; psql + else none + end + end + def psql + def metadata + %{ COMMENT ON Table metadata IS 'contains SiSU documents metadata with metadata'; COMMENT ON COLUMN metadata.tid @@ -196,7 +407,9 @@ module SiSU_DB_create PROCEDURE fileremoval(); */ } - @comment['documents'] =%{ + end + def documents + %{ COMMENT ON Table documents IS 'contains searchable text of SiSU documents'; COMMENT ON COLUMN documents.lid @@ -240,7 +453,9 @@ module SiSU_DB_create COMMENT ON COLUMN documents.lev6 IS 'document structure, level 6'; } - @comment['endnotes'] =%{ + end + def endnotes + %{ COMMENT ON Table endnotes IS 'contains searchable text of SiSU documents endnotes'; COMMENT ON COLUMN endnotes.nid @@ -258,7 +473,9 @@ module SiSU_DB_create COMMENT ON COLUMN documents.metadata_tid IS 'tie to title in metadata - unique for each document'; } - @comment['endnotes_asterisk'] =%{ + end + def endnotes_asterisk + %{ COMMENT ON Table endnotes_asterisk IS 'contains searchable text of SiSU documents endnotes asterisk'; COMMENT ON COLUMN endnotes_asterisk.nid @@ -276,7 +493,9 @@ module SiSU_DB_create COMMENT ON COLUMN documents.metadata_tid IS 'tie to title in metadata - unique for each document'; } - @comment['endnotes_plus'] =%{ + end + def endnotes_plus + %{ COMMENT ON Table endnotes_plus IS 'contains searchable text of SiSU documents endnotes'; COMMENT ON COLUMN endnotes_plus.nid @@ -294,7 +513,9 @@ module SiSU_DB_create COMMENT ON COLUMN documents.metadata_tid IS 'tie to title in metadata - unique for each document'; } - @comment['urls'] =%{ + end + def urls + %{ COMMENT ON Table urls IS 'contains base url links to different SiSU output'; COMMENT ON COLUMN documents.metadata_tid @@ -329,210 +550,22 @@ module SiSU_DB_create IS 'SiSU document format .tgz (all SiSU information on document)'; } end - @comment + self end - def output_dir? - dir=SiSU_Env::Info_env.new('') - if @opt.cmd =~/d/; dir.path.webserv_stub_ensure + def none + def metadata end - end - def create_table_metadata - print %{ - currently using sisu dbi module - to be populated from documents files - create tables metadata - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE metadata ( - tid INT4 PRIMARY KEY, - title VARCHAR(#{lt_title}) NULL, - subtitle VARCHAR(#{lt_subtitle}) NULL, - creator VARCHAR(#{lt_creator}) NULL, - illustrator VARCHAR(#{lt_illustrator}) NULL, - translator VARCHAR(#{lt_translator}) NULL, - subject VARCHAR(#{lt_subject}) NULL, - date VARCHAR(#{lt_date}) NULL, - date_created VARCHAR(#{lt_date_created}) NULL, - date_issued VARCHAR(#{lt_date_issued}) NULL, - date_available VARCHAR(#{lt_date_available}) NULL, - date_valid VARCHAR(#{lt_date_valid}) NULL, - date_modified VARCHAR(#{lt_date_modified}) NULL, -/* date DATE, */ -/* date_created DATE, */ -/* date_issued DATE, */ -/* date_available DATE, */ -/* date_valid DATE, */ -/* date_modified DATE, */ - type VARCHAR(#{lt_type}) NULL, - description VARCHAR(#{lt_description}) NULL, - publisher VARCHAR(#{lt_publisher}) NULL, - contributor VARCHAR(#{lt_contributor}) NULL, - prepared_by VARCHAR(#{lt_prepared_by}) NULL, - digitized_by VARCHAR(#{lt_digitized_by}) NULL, - format VARCHAR(#{lt_format}) NULL, - identifier VARCHAR(#{lt_identifier}) NULL, - source VARCHAR(#{lt_source}) NULL, - language VARCHAR(#{lt_language}) NULL, - language_original VARCHAR(#{lt_language_original}) NULL, - relation VARCHAR(#{lt_relation}) NULL, - coverage VARCHAR(#{lt_coverage}) NULL, - rights VARCHAR(#{lt_rights}) NULL, - copyright VARCHAR(#{lt_copyright}) NULL, - owner VARCHAR(#{lt_owner}) NULL, - keywords VARCHAR(#{lt_keywords}) NULL, - comment VARCHAR(#{lt_comment}) NULL, - loc VARCHAR(#{lt_loc}) NULL, - dewey VARCHAR(#{lt_dewey}) NULL, - isbn VARCHAR(#{lt_isbn}) NULL, - pg VARCHAR(#{lt_pg}) NULL, - abstract VARCHAR(#{lt_abstract}) NULL, - prefix_a TEXT NULL, - prefix_b TEXT NULL, - skin VARCHAR(#{lt_skin}) NULL, - markup VARCHAR(#{lt_markup}) NULL, - links VARCHAR(#{lt_links}) NULL, - information VARCHAR(#{lt_information}) NULL, - contact VARCHAR(#{lt_contact}) NULL, - suffix VARCHAR(#{lt_suffix}) NULL, - filename VARCHAR(#{lt_filename}) NULL UNIQUE, - types CHAR(#{lt_types}) NULL, - subj VARCHAR(#{lt_subj}) NULL - ); - #{@comment['metadata']} - }) - end - def create_table # create documents base - print %{ - to be populated from documents files - create tables documents document_trade document_env - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE documents ( - lid INT4 PRIMARY KEY, - metadata_tid INT4 REFERENCES metadata, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - clean TEXT NULL, - body TEXT NULL, - seg VARCHAR(#{document_seg}) NULL, - lev SMALLINT NULL, - lev1 SMALLINT, - lev2 SMALLINT, - lev3 SMALLINT, - lev4 SMALLINT, - lev5 SMALLINT, - lev6 SMALLINT, - en_a SMALLINT NULL, - en_z SMALLINT NULL, - en_a_asterisk SMALLINT NULL, - en_z_asterisk SMALLINT NULL, - en_a_plus SMALLINT NULL, - en_z_plus SMALLINT NULL, - digest_clean CHAR(#{@@dl}), - digest_all CHAR(#{@@dl}), - types CHAR(1) NULL - ); - #{@comment['documents']} - }) - end - def create_table_endnotes - print %{ - to be populated from documents files - create tables endnotes - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE endnotes ( - nid INT4 PRIMARY KEY, - document_lid INT4 REFERENCES documents, - nr SMALLINT, - clean TEXT NULL, - body TEXT NULL, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - digest_clean CHAR(#{@@dl}), - metadata_tid INT4 REFERENCES metadata - ); - #{@comment['endnotes']} - }) - end - def create_table_endnotes_asterisk - print %{ - to be populated from documents files - create tables endnotes_asterisk - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE endnotes_asterisk ( - nid INT4 PRIMARY KEY, - document_lid INT4 REFERENCES documents, - nr SMALLINT, - clean TEXT NULL, - body TEXT NULL, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - digest_clean CHAR(#{@@dl}), - metadata_tid INT4 REFERENCES metadata - ); - #{@comment['endnotes_asterisk']} - }) - end - def create_table_endnotes_plus - print %{ - to be populated from documents files - create tables endnotes_plus - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE endnotes_plus ( - nid INT4 PRIMARY KEY, - document_lid INT4 REFERENCES documents, - nr SMALLINT, - clean TEXT NULL, - body TEXT NULL, - ocn SMALLINT, - ocnd VARCHAR(6), - ocns VARCHAR(6), - digest_clean CHAR(#{@@dl}), - metadata_tid INT4 REFERENCES metadata - ); - #{@comment['endnotes_plus']} - }) - end - def create_table_urls # create documents file links mapping - print %{ - currently using sisu dbi module - to be populated from documents files - create tables urls - data import through ruby transfer - } unless @opt.cmd =~/q/ - @conn.execute(%{ - CREATE TABLE urls ( - metadata_tid INT4 REFERENCES metadata, - plaintext varchar(512), - html_toc varchar(512), - html_doc varchar(512), - xhtml varchar(512), - xml_sax varchar(512), - xml_dom varchar(512), - odf varchar(512), - pdf_p varchar(512), - pdf_l varchar(512), - concordance varchar(512), - latex_p varchar(512), - latex_l varchar(512), - digest varchar(512), - manifest varchar(512), - markup varchar(512), - sisupod varchar(512) - ); - #{@comment['urls']} - }) + def documents + end + def endnotes + end + def endnotes_asterisk + end + def endnotes_plus + end + def urls + end + self end end end diff --git a/lib/sisu/v0/shared_db.rb b/lib/sisu/v0/db_dbi.rb index 8d225e75..6bf21309 100644 --- a/lib/sisu/v0/shared_db.rb +++ b/lib/sisu/v0/db_dbi.rb @@ -55,7 +55,7 @@ sqlite =end -module SiSU_DB +module SiSU_DB_DBI require "#{SiSU_lib}/db_columns" require "#{SiSU_lib}/db_tests" require "#{SiSU_lib}/db_create" @@ -79,7 +79,11 @@ module SiSU_DB end class Remove <SiSU_DB_remove::Remove end - class Load_tuple <SiSU_DB_tuple::Load_tuple #% main database populate + class Load_documents <SiSU_DB_tuple::Load_documents #% main database populate/update + end + class Load_metadata <SiSU_DB_tuple::Load_metadata #% update metadata + end + class Load_urls <SiSU_DB_tuple::Load_urls #% update urls end class Import <SiSU_DB_import::Import #<SiSU_DB::Column_size #% main end diff --git a/lib/sisu/v0/db_drop.rb b/lib/sisu/v0/db_drop.rb index f6bc5022..294e531f 100644 --- a/lib/sisu/v0/db_drop.rb +++ b/lib/sisu/v0/db_drop.rb @@ -60,73 +60,72 @@ module SiSU_DB_drop def initialize(opt,conn='',sql_type='') @opt,@conn,@sql_type=opt,conn,sql_type end - def drop_tables #% drop all tables - begin - case @sql_type - when /sqlite/ - cascade='' - commit=@conn.commit - else - cascade='CASCADE' - commit='' + def drop + def tables #% drop all tables + begin + case @sql_type + when /sqlite/ + cascade='' + commit=@conn.commit + else + cascade='CASCADE' + commit='' + end + @conn.do(%{ + DROP TABLE metadata #{cascade}; + DROP TABLE documents #{cascade}; + DROP TABLE urls #{cascade}; + DROP TABLE endnotes #{cascade}; + DROP TABLE endnotes_asterisk #{cascade}; + DROP TABLE endnotes_plus #{cascade}; + }) + commit + rescue + @conn.do(%{ + DROP TABLE endnotes; + DROP TABLE endnotes_asterisk; + DROP TABLE endnotes_plus; + DROP TABLE urls #{cascade}; + DROP TABLE documents #{cascade}; + DROP TABLE metadata #{cascade}; + }) + ensure + commit end - @conn.do(%{ - DROP TABLE metadata #{cascade}; - DROP TABLE documents #{cascade}; - DROP TABLE endnotes #{cascade}; - DROP TABLE endnotes_asterisk #{cascade}; - DROP TABLE endnotes_plus #{cascade}; - DROP TABLE urls #{cascade}; - }) - commit - rescue end - end - def drop_indexes #% drop all indexes - #@conn.do(%{ - # DROP INDEX object_nr ON documents(ocn); - # DROP INDEX body ON documents(body); - # DROP INDEX clean ON documents(clean); - # DROP INDEX lev1 ON documents(lev1); - # DROP INDEX lev2 ON documents(lev2); - # DROP INDEX lev3 ON documents(lev3); - # DROP INDEX lev4 ON documents(lev4); - # DROP INDEX lev5 ON documents(lev5); - # DROP INDEX lev6 ON documents(lev6); - # DROP INDEX endnote_nr ON endnotes(nr); - # DROP INDEX endnote ON endnotes(body); - # DROP INDEX title ON metadata(title); - # DROP INDEX filename ON metadata(filename) - # /* - # DROP INDEX object_nr ON documents(ocn) CASCADE; - # DROP INDEX body ON documents(body) CASCADE; - # DROP INDEX clean ON documents(clean) CASCADE; - # DROP INDEX lev1 ON documents(lev1) CASCADE; - # DROP INDEX lev2 ON documents(lev2) CASCADE; - # DROP INDEX lev3 ON documents(lev3) CASCADE; - # DROP INDEX lev4 ON documents(lev4) CASCADE; - # DROP INDEX lev5 ON documents(lev5) CASCADE; - # DROP INDEX lev6 ON documents(lev6) CASCADE; - # DROP INDEX endnote_nr ON endnotes(nr) CASCADE; - # DROP INDEX endnote ON endnotes(body) CASCADE; - # DROP INDEX title ON metadata(title) CASCADE; - # DROP INDEX filename ON metadata(filename) CASCADE - # */ - #}) - end - def db_help #% help - puts %{ - not yet implemented - init createall - createtable - recreate\t(destroy and create) - createlex\t() - createmetadata\t() - import\t(import data) - index\t(create index) - droptables\t(where tables made and content imported) - dropall\t(where tables made, content imported and tables indexed) - } + def indexes #% drop all indexes + #@conn.do(%{ + # DROP INDEX object_nr ON documents(ocn); + # DROP INDEX body ON documents(body); + # DROP INDEX clean ON documents(clean); + # DROP INDEX lev1 ON documents(lev1); + # DROP INDEX lev2 ON documents(lev2); + # DROP INDEX lev3 ON documents(lev3); + # DROP INDEX lev4 ON documents(lev4); + # DROP INDEX lev5 ON documents(lev5); + # DROP INDEX lev6 ON documents(lev6); + # DROP INDEX endnote_nr ON endnotes(nr); + # DROP INDEX endnote ON endnotes(body); + # DROP INDEX title ON metadata(title); + # DROP INDEX filename ON metadata(filename) + # /* + # DROP INDEX object_nr ON documents(ocn) CASCADE; + # DROP INDEX body ON documents(body) CASCADE; + # DROP INDEX clean ON documents(clean) CASCADE; + # DROP INDEX lev1 ON documents(lev1) CASCADE; + # DROP INDEX lev2 ON documents(lev2) CASCADE; + # DROP INDEX lev3 ON documents(lev3) CASCADE; + # DROP INDEX lev4 ON documents(lev4) CASCADE; + # DROP INDEX lev5 ON documents(lev5) CASCADE; + # DROP INDEX lev6 ON documents(lev6) CASCADE; + # DROP INDEX endnote_nr ON endnotes(nr) CASCADE; + # DROP INDEX endnote ON endnotes(body) CASCADE; + # DROP INDEX title ON metadata(title) CASCADE; + # DROP INDEX filename ON metadata(filename) CASCADE + # */ + #}) + end + self end end end diff --git a/lib/sisu/v0/db_import.rb b/lib/sisu/v0/db_import.rb index 404d9437..631c3209 100644 --- a/lib/sisu/v0/db_import.rb +++ b/lib/sisu/v0/db_import.rb @@ -95,7 +95,7 @@ module SiSU_DB_import def marshal_load require "#{SiSU_lib}/dal" @dal_array=SiSU_DAL::Source.new(@opt).get # dal file drawn here - tell=SiSU_Screen::Ansi.new(@opt.cmd,"#{@db.db_psql}::#{@opt.fns}") + tell=SiSU_Screen::Ansi.new(@opt.cmd,"#{@db.psql.db}::#{@opt.fns}") tell.puts_blue unless @opt.cmd =~/q/ tell=SiSU_Screen::Ansi.new(@opt.cmd,'Marshal Load',@fnm) tell.print_grey if @opt.cmd =~/v/ @@ -115,7 +115,7 @@ module SiSU_DB_import @conn.execute('COMMIT') else @db=SiSU_Env::Info_db.new - puts "\n#{@cX.grey}file #{@cX.off} #{@cX.blue}#{@opt.fns}#{@cX.off} #{@cX.grey}already exists in database#{@cX.off} #{@cX.blue}#{@db.db_psql}#{@cX.off} #{@cX.brown}update instead?#{@cX.off}" + puts "\n#{@cX.grey}file #{@cX.off} #{@cX.blue}#{@opt.fns}#{@cX.off} #{@cX.grey}already exists in database#{@cX.off} #{@cX.blue}#{@db.psql.db}#{@cX.off} #{@cX.brown}update instead?#{@cX.off}" end end end @@ -127,9 +127,6 @@ module SiSU_DB_import string.gsub!(/\{\s*(\S+?\.(?:png|jpg))(?:\s+\d+x\d+)?(.+?)\}\S+/,'[image: \1] \2') string.gsub!(/\{\s*(.+?)\s*\}(?:https?|file|ftp):\/\/\S+?([.,!?]?(?:\s|$))/,'\1\2') end - def unicode_special_character_escape(string) - #string.gsub!(/(["';:,])/, %{\\\\\\1}) - end def strip_markup(string) #define rules, make same as in dal clean string.gsub!(/<sup>(\d+)<\/sup>/,'[\1]') string.gsub!(/<:i[12]>/,'') @@ -151,7 +148,7 @@ module SiSU_DB_import if @md.title; @tp[:title]=@md.title special_character_escape(@tp[:title]) @tp[:title_f],@tp[:title_i]='title, ',"'#{@tp[:title]}', " - sql="SELECT MAX(tid) FROM metadata" + sql='SELECT MAX(tid) FROM metadata' id_t=@conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } @@id_t=id_t if id_t @@id_t ||=0 @@ -159,191 +156,190 @@ module SiSU_DB_import puts %{\n#{@cX.grey}Processing file number#{@cX.off}: #{@cX.green}#{@@id_t}#{@@cX.off}} unless @opt.cmd =~/q/ end if @md.dc_title; @tp[:long_title]=@md.dc_title - #sql="SELECT MAX(tid) FROM metadata" + #sql='SELECT MAX(tid) FROM metadata' #id_t=@conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } #@@id_t=id_t if id_t #@@id_t ||=0 #@@id_t+=1 #bug related, needs to be performed once at start of file, but consider moving, as, placed here it means program will fail if document header lacks 0~title #puts %{\n#{@cX.grey}Processing file number#{@cX.off}: #{@cX.green}#{@@id_t}#{@@cX.off}} end - if @md.subtitle; @tp[:subtitle]=@md.subtitle - special_character_escape(@tp[:subtitle]) - @tp[:subtitle_f],@tp[:subtitle_i]='subtitle, ',"'#{@tp[:subtitle]}', " + if @md.subtitle; txt=@md.subtitle + special_character_escape(txt) + @tp[:subtitle_f],@tp[:subtitle_i]='subtitle, ',"'#{txt}', " end - if @md.dc_creator; @tp[:creator]=@md.dc_creator - special_character_escape(@tp[:creator]) - @tp[:creator_f],@tp[:creator_i]='creator, ',"'#{@tp[:creator]}', " + if @md.dc_creator; txt=@md.dc_creator + special_character_escape(txt) + @tp[:creator_f],@tp[:creator_i]='creator, ',"'#{txt}', " end - if @md.dc_contributor; @tp[:contributor]=@md.dc_contributor - special_character_escape(@tp[:contributor]) - @tp[:contributor_f],@tp[:contributor_i]='contributor, ',"'#{@tp[:contributor]}', " + if @md.dc_contributor; txt=@md.dc_contributor + special_character_escape(txt) + @tp[:contributor_f],@tp[:contributor_i]='contributor, ',"'#{txt}', " end - if @md.translator; @tp[:translator]=@md.translator - special_character_escape(@tp[:translator]) - @tp[:translator_f],@tp[:translator_i]='translator, ',"'#{@tp[:translator]}', " + if @md.translator; txt=@md.translator + special_character_escape(txt) + @tp[:translator_f],@tp[:translator_i]='translator, ',"'#{txt}', " end - if @md.illustrator; @tp[:illustrator]=@md.illustrator - special_character_escape(@tp[:illustrator]) - @tp[:illustrator_f],@tp[:illustrator_i]='illustrator, ',"'#{@tp[:illustrator]}', " + if @md.illustrator; txt=@md.illustrator + special_character_escape(txt) + @tp[:illustrator_f],@tp[:illustrator_i]='illustrator, ',"'#{txt}', " end - if @md.dc_publisher; @tp[:publisher]=@md.dc_publisher - special_character_escape(@tp[:publisher]) - @tp[:publisher_f],@tp[:publisher_i]='publisher, ',"'#{@tp[:publisher]}', " + if @md.dc_publisher; txt=@md.dc_publisher + special_character_escape(txt) + @tp[:publisher_f],@tp[:publisher_i]='publisher, ',"'#{txt}', " end - if @md.prepared_by; @tp[:prepared_by]=@md.prepared_by - special_character_escape(@tp[:prepared_by]) - @tp[:prepared_by_f],@tp[:prepared_by_i]='prepared_by, ',"'#{@tp[:prepared_by]}', " + if @md.prepared_by; txt=@md.prepared_by + special_character_escape(txt) + @tp[:prepared_by_f],@tp[:prepared_by_i]='prepared_by, ',"'#{txt}', " end - if @md.digitized_by; @tp[:digitized_by]=@md.digitized_by - special_character_escape(@tp[:digitized_by]) - @tp[:digitized_by_f],@tp[:digitized_by_i]='digitized_by, ',"'#{@tp[:digitized_by]}', " + if @md.digitized_by; txt=@md.digitized_by + special_character_escape(txt) + @tp[:digitized_by_f],@tp[:digitized_by_i]='digitized_by, ',"'#{txt}', " end - if @md.dc_subject; @tp[:subject]=@md.dc_subject - special_character_escape(@tp[:subject]) - @tp[:subject_f],@tp[:subject_i]='subject, ',"'#{@tp[:subject]}', " + if @md.dc_subject; txt=@md.dc_subject + special_character_escape(txt) + @tp[:subject_f],@tp[:subject_i]='subject, ',"'#{txt}', " end - if @md.dc_description; @tp[:description]=@md.dc_description - special_character_escape(@tp[:description]) - @tp[:description_f],@tp[:description_i]='description, ',"'#{@tp[:description]}', " + if @md.dc_description; txt=@md.dc_description + special_character_escape(txt) + @tp[:description_f],@tp[:description_i]='description, ',"'#{txt}', " end - if @md.abstract; @tp[:abstract]=@md.abstract - special_character_escape(@tp[:abstract]) - @tp[:abstract_f],@tp[:abstract_i]='abstract, ',"'#{@tp[:abstract]}', " + if @md.abstract; txt=@md.abstract + special_character_escape(txt) + @tp[:abstract_f],@tp[:abstract_i]='abstract, ',"'#{txt}', " end - if @md.dc_type; @tp[:type]=@md.dc_type - special_character_escape(@tp[:type]) - @tp[:type_f],@tp[:type_i]='type, ',"'#{@tp[:type]}', " + if @md.dc_type; txt=@md.dc_type + special_character_escape(txt) + @tp[:type_f],@tp[:type_i]='type, ',"'#{txt}', " end - #if @md.owner; @tp[:owner]=@md.owner - # special_character_escape(@tp[:owner]) - # @tp[:owner_f],@tp[:owner_i]='owner, ',"'#{@tp[:owner}', " + #if @md.owner; txt=@md.owner + # special_character_escape(txt) + # @tp[:owner_f],@tp[:owner_i]='owner, ',"'#{txt}', " #end - #if @md.copyright; @tp[:copyright]=@md.copyright - # special_character_escape(@tp[:copyright]) - # @tp[:copyright_f],@tp[:copyright_i]='copyright, ',"'#{@tp[:copyright]}', " + #if @md.copyright; txt=@md.copyright + # special_character_escape(txt) + # @tp[:copyright_f],@tp[:copyright_i]='copyright, ',"'#{txt}', " #end - if @md.dc_rights; @tp[:rights]=@md.dc_rights - special_character_escape(@tp[:rights]) - @tp[:rights_f],@tp[:rights_i]='rights, ',"'#{@tp[:rights]}', " + if @md.dc_rights; txt=@md.dc_rights + special_character_escape(txt) + @tp[:rights_f],@tp[:rights_i]='rights, ',"'#{txt}', " end - if @md.dc_date; @tp[:date]=@md.dc_date - special_character_escape(@tp[:date]) - @tp[:date_f],@tp[:date_i]='date, ',"'#{@tp[:date]}', " + if @md.dc_date; txt=@md.dc_date + special_character_escape(txt) + @tp[:date_f],@tp[:date_i]='date, ',"'#{txt}', " end - if @md.dc_date_created; @tp[:date_created]=@md.dc_date_created - special_character_escape(@tp[:date_created]) - @tp[:date_created_f],@tp[:date_created_i]='date_created, ',"'#{@tp[:date_created]}', " + if @md.dc_date_created; txt=@md.dc_date_created + special_character_escape(txt) + @tp[:date_created_f],@tp[:date_created_i]='date_created, ',"'#{txt}', " end - if @md.dc_date_issued; @tp[:date_issued]=@md.dc_date_issued - special_character_escape(@tp[:date_issued]) - @tp[:date_issued_f],@tp[:date_issued_i]='date_issued, ',"'#{@tp[:date_issued]}', " + if @md.dc_date_issued; txt=@md.dc_date_issued + special_character_escape(txt) + @tp[:date_issued_f],@tp[:date_issued_i]='date_issued, ',"'#{txt}', " end - if @md.dc_date_available; @tp[:date_available]=@md.dc_date_available - special_character_escape(@tp[:date_available]) - @tp[:date_available_f],@tp[:date_available_i]='date_available, ',"'#{@tp[:date_available]}', " + if @md.dc_date_available; txt=@md.dc_date_available + special_character_escape(txt) + @tp[:date_available_f],@tp[:date_available_i]='date_available, ',"'#{txt}', " end - if @md.dc_date_modified; @tp[:date_modified]=@md.dc_date_modified - special_character_escape(@tp[:date_modified]) - @tp[:date_modified_f],@tp[:date_modified_i]='date_modified, ',"'#{@tp[:date_modified]}', " + if @md.dc_date_modified; txt=@md.dc_date_modified + special_character_escape(txt) + @tp[:date_modified_f],@tp[:date_modified_i]='date_modified, ',"'#{txt}', " end - if @md.dc_date_valid; @tp[:date_valid]=@md.dc_date_valid - special_character_escape(@tp[:date_valid]) - @tp[:date_valid_f],@tp[:date_valid_i]='date_valid, ',"'#{@tp[:date_valid]}', " + if @md.dc_date_valid; txt=@md.dc_date_valid + special_character_escape(txt) + @tp[:date_valid_f],@tp[:date_valid_i]='date_valid, ',"'#{txt}', " end - if @md.dc_language[:name]; @tp[:language]=@md.dc_language[:name] - special_character_escape(@tp[:language]) - @tp[:language_f],@tp[:language_i]='language, ',"'#{@tp[:language]}', " + if @md.dc_language[:name]; txt=@md.dc_language[:name] + special_character_escape(txt) + @tp[:language_f],@tp[:language_i]='language, ',"'#{txt}', " end - if @md.language_original[:name]; @tp[:language_original]=@md.language_original[:name] - special_character_escape(@tp[:language_original]) - @tp[:language_original_f],@tp[:language_original_i]='language_original, ',"'#{@tp[:language_original]}', " + if @md.language_original[:name]; txt=@md.language_original[:name] + special_character_escape(txt) + @tp[:language_original_f],@tp[:language_original_i]='language_original, ',"'#{txt}', " end - if @md.dc_format; @tp[:format]=@md.dc_format - special_character_escape(@tp[:format]) - @tp[:format_f],@tp[:format_i]='format, ',"'#{@tp[:format]}', " + if @md.dc_format; txt=@md.dc_format + special_character_escape(txt) + @tp[:format_f],@tp[:format_i]='format, ',"'#{txt}', " end - if @md.dc_identifier; @tp[:identifier]=@md.dc_identifier - special_character_escape(@tp[:identifier]) - @tp[:identifier_f],@tp[:identifier_i]='identifier, ',"'#{@tp[:identifier]}', " + if @md.dc_identifier; txt=@md.dc_identifier + special_character_escape(txt) + @tp[:identifier_f],@tp[:identifier_i]='identifier, ',"'#{txt}', " end - if @md.dc_source; @tp[:source]=@md.dc_source - special_character_escape(@tp[:source]) - @tp[:source_f],@tp[:source_i]='source, ',"'#{@tp[:source]}', " + if @md.dc_source; txt=@md.dc_source + special_character_escape(txt) + @tp[:source_f],@tp[:source_i]='source, ',"'#{txt}', " end - if @md.dc_relation; @tp[:relation]=@md.dc_relation - special_character_escape(@tp[:relation]) - @tp[:relation_f],@tp[:relation_i]='relation, ',"'#{@tp[:relation]}', " + if @md.dc_relation; txt=@md.dc_relation + special_character_escape(txt) + @tp[:relation_f],@tp[:relation_i]='relation, ',"'#{txt}', " end - if @md.dc_coverage; @tp[:coverage]=@md.dc_coverage - special_character_escape(@tp[:coverage]) - @tp[:coverage_f],@tp[:coverage_i]='coverage, ',"'#{@tp[:coverage]}', " + if @md.dc_coverage; txt=@md.dc_coverage + special_character_escape(txt) + @tp[:coverage_f],@tp[:coverage_i]='coverage, ',"'#{txt}', " end - if @md.keywords; @tp[:keywords]=@md.keywords - special_character_escape(@tp[:keywords]) - @tp[:keywords_f],@tp[:keywords_i]='keywords, ',"'#{@tp[:keywords]}', " + if @md.keywords; txt=@md.keywords + special_character_escape(txt) + @tp[:keywords_f],@tp[:keywords_i]='keywords, ',"'#{txt}', " end - if @md.comments; @tp[:comments]=@md.comments - special_character_escape(@tp[:comments]) - @tp[:comments_f],@tp[:comments_i]='comments, ',"'#{@tp[:comments]}', " + if @md.comments; txt=@md.comments + special_character_escape(txt) + @tp[:comments_f],@tp[:comments_i]='comments, ',"'#{txt}', " end - if @md.cls_loc; @tp[:cls_loc]=@md.cls_loc - special_character_escape(@tp[:cls_loc]) - @tp[:cls_loc_f],@tp[:cls_loc_i]='cls_loc, ',"'#{@tp[:cls_loc]}', " + if @md.cls_loc; txt=@md.cls_loc + special_character_escape(txt) + @tp[:cls_loc_f],@tp[:cls_loc_i]='cls_loc, ',"'#{txt}', " end - if @md.cls_dewey; @tp[:cls_dewey]=@md.cls_dewey - special_character_escape(@tp[:cls_dewey]) - @tp[:cls_dewey_f],@tp[:cls_dewey_i]='cls_dewey, ',"'#{@tp[:cls_dewey]}', " + if @md.cls_dewey; txt=@md.cls_dewey + special_character_escape(txt) + @tp[:cls_dewey_f],@tp[:cls_dewey_i]='cls_dewey, ',"'#{txt}', " end - if @md.cls_pg; @tp[:cls_pg]=@md.cls_pg - special_character_escape(@tp[:cls_pg]) - @tp[:cls_pg_f],@tp[:cls_pg_i]='cls_pg, ',"'#{@tp[:cls_pg]}', " + if @md.cls_pg; txt=@md.cls_pg + special_character_escape(txt) + @tp[:cls_pg_f],@tp[:cls_pg_i]='cls_pg, ',"'#{txt}', " end - if @md.cls_isbn; @tp[:cls_isbn]=@md.cls_isbn - special_character_escape(@tp[:cls_isbn]) - @tp[:cls_isbn_f],@tp[:cls_isbn_i]='cls_isbn, ',"'#{@tp[:cls_isbn]}', " + if @md.cls_isbn; txt=@md.cls_isbn + special_character_escape(txt) + @tp[:cls_isbn_f],@tp[:cls_isbn_i]='cls_isbn, ',"'#{txt}', " end - if @md.prefix_a; @tp[:prefix_a]=@md.prefix_a - special_character_escape(@tp[:prefix_a]) - @tp[:prefix_a_f],@tp[:prefix_a_i]='prefix_a, ',"'#{@tp[:prefix_a]}', " + if @md.prefix_a; txt=@md.prefix_a + special_character_escape(txt) + @tp[:prefix_a_f],@tp[:prefix_a_i]='prefix_a, ',"'#{txt}', " end - if @md.prefix_b; @tp[:prefix_b]=@md.prefix_b - special_character_escape(@tp[:prefix_b]) - @tp[:prefix_b_f],@tp[:prefix_b_i]='prefix_b, ',"'#{@tp[:prefix_b]}', " + if @md.prefix_b; txt=@md.prefix_b + special_character_escape(txt) + @tp[:prefix_b_f],@tp[:prefix_b_i]='prefix_b, ',"'#{txt}', " end - #if @md.suffix; @tp[:suffix]=@md.suffix - # special_character_escape(@tp[:suffix]) - # @tp[:suffix_f],@tp[:suffix_i]='suffix, ',"'#{@tp[:suffix]}', " + #if @md.suffix; txt=@md.suffix + # special_character_escape(txt) + # @tp[:suffix_f],@tp[:suffix_i]='suffix, ',"'#{txt}', " #end - if @md.fns; @tp[:fns]=@md.fns - special_character_escape(@tp[:fns]) - @tp[:fns_f],@tp[:fns_i]="filename, ","'#{@tp[:fns]}', " + if @md.fns; txt=@md.fns + special_character_escape(txt) + @tp[:fns_f],@tp[:fns_i]="filename, ","'#{txt}', " end #if @md.en[:mismatch] > 0 # id,info='WARNING document error in endnote markup, number mismatch',"endnotes: #{@md.en[:note]} != endnote reference marks: #{@md.en[:mark]} (difference = #{@md.en[:mismatch]})" #end - if @md.wc_words; @tp[:wc_words]=@md.wc_words - @tp[:wc_words_f],@tp[:wc_words_i]='wc_words, ',"'#{@tp[:wc_words]}', " + if @md.wc_words; txt=@md.wc_words + @tp[:wc_words_f],@tp[:wc_words_i]='wc_words, ',"'#{txt}', " end - if @md.dgst; @tp[:dgst]=@md.dgst - @tp[:dgst_f],@tp[:dgst_i]='dgst, ',"'#{@tp[:dgst]}', " + if @md.dgst; txt=@md.dgst + @tp[:dgst_f],@tp[:dgst_i]='dgst, ',"'#{txt}', " end - if @md.sc_number; @tp[:sc_number]=@md.sc_number - @tp[:sc_number_f],@tp[:sc_number_i]='sc_number, ',"'#{@tp[:sc_number]}', " + if @md.sc_number; txt=@md.sc_number + @tp[:sc_number_f],@tp[:sc_number_i]='sc_number, ',"'#{txt}', " end - if @md.sc_date; @tp[:sc_date]=@md.sc_date - @tp[:sc_date_f],@tp[:sc_date_i]='sc_date, ',"'#{@tp[:sc_date]}', " + if @md.sc_date; txt=@md.sc_date + @tp[:sc_date_f],@tp[:sc_date_i]='sc_date, ',"'#{txt}', " end - if @md.generated; @tp[:generated]=@md.generated - @tp[:generated_f],@tp[:generated_i]='generated, ',"'#{@tp[:generated]}', " + if @md.generated; txt=@md.generated + @tp[:generated_f],@tp[:generated_i]='generated, ',"'#{@txt}', " end #if @md.sisu_version; special_character_escape(@md.sisu_version) # #id,info='Generated by',"#{@md.sisu_version[:project]} #{@md.sisu_version[:version]} #{@md.sisu_version[:date_stamp]} (#{@md.sisu_version[:date]})" #end #if @md.ruby_version; special_character_escape(@md.ruby_version) - SiSU_DB::Test.new(self,@opt).verify #% import title names, filenames (tuple) - @conn.execute(%{ - INSERT INTO metadata (#{@tp[:fns_f]} #{@tp[:suffix_f]} #{@tp[:title_f]} #{@tp[:subtitle_f]} #{@tp[:creator_f]} #{@tp[:illustrator_f]} #{@tp[:translator_f]} #{@tp[:subject_f]} #{@tp[:description_f]} #{@tp[:publisher_f]} #{@tp[:contributor_f]} #{@tp[:prepared_by_f]} #{@tp[:digitized_by_f]} #{@tp[:date_f]} #{@tp[:date_created_f]} #{@tp[:date_issued_f]} #{@tp[:date_valid_f]} #{@tp[:date_available_f]} #{@tp[:date_modified_f]} #{@tp[:type_f]} #{@tp[:format_f]} #{@tp[:identifier_f]} #{@tp[:source_f]} #{@tp[:language_f]} #{@tp[:language_original_f]} #{@tp[:relation_f]} #{@tp[:coverage_f]} #{@tp[:rights_f]} #{@tp[:copyright_f]} #{@tp[:owner_f]} #{@tp[:keywords_f]} #{@tp[:abstract_f]} #{@tp[:comment_f]} #{@tp[:loc_f]} #{@tp[:dewey_f]} #{@tp[:isbn_f]} #{@tp[:pg_f]} #{@tp[:prefix_a_f]} #{@tp[:prefix_b_f]} tid) VALUES (#{@tp[:fns_i]} #{@tp[:suffix_i]} #{@tp[:title_i]} #{@tp[:subtitle_i]} #{@tp[:creator_i]} #{@tp[:illustrator_i]} #{@tp[:translator_i]} #{@tp[:subject_i]} #{@tp[:description_i]} #{@tp[:publisher_i]} #{@tp[:contributor_i]} #{@tp[:prepared_by_i]} #{@tp[:digitized_by_i]} #{@tp[:date_i]} #{@tp[:date_created_i]} #{@tp[:date_issued_i]} #{@tp[:date_valid_i]} #{@tp[:date_available_i]} #{@tp[:date_modified_i]} #{@tp[:type_i]} #{@tp[:format_i]} #{@tp[:identifier_i]} #{@tp[:source_i]} #{@tp[:language_i]} #{@tp[:language_original_i]} #{@tp[:relation_i]} #{@tp[:coverage_i]} #{@tp[:rights_i]} #{@tp[:copyright_i]} #{@tp[:owner_i]} #{@tp[:keywords_i]} #{@tp[:abstract_i]} #{@tp[:comment_i]} #{@tp[:loc_i]} #{@tp[:dewey_i]} #{@tp[:isbn_i]} #{@tp[:pg_i]} #{@tp[:prefix_a_i]} #{@tp[:prefix_b_i]} #{@@id_t}); - }) + SiSU_DB_DBI::Test.new(self,@opt).verify #% import title names, filenames (tuple) + t=SiSU_DB_tuple::Load_metadata.new(@conn,@tp,@@id_t) + t.tuple end def import_documents(dbi_unit) #% import documents - populate main database table #% import into substantive database tables (tuple) @@ -384,7 +380,7 @@ module SiSU_DB_import end if @en_pls[0]; @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last end - t=SiSU_DB_tuple::Load_tuple.new(@conn,@col,@opt) + t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt) t.tuple case @col[:lev] when /1/; @col[:lv1]+=1 @@ -425,7 +421,7 @@ module SiSU_DB_import end if @en_pls[0]; @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last end - t=SiSU_DB_tuple::Load_tuple.new(@conn,@col,@opt) + t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt) t.tuple @col[:lev]=@col[:plaintext]=@col[:body]='' elsif data[/^5~(?:~\S+)?(.+?)<~(\d+);((?:\w|[0-6]:)\d+);(\w\d+)><([0-9a-f]{#{@@dl}}):([0-9a-f]{#{@@dl}})>/] # header lev5 seg level @@ -464,7 +460,7 @@ module SiSU_DB_import end if @en_pls[0]; @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last end - t=SiSU_DB_tuple::Load_tuple.new(@conn,@col,@opt) + t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt) t.tuple @col[:lev]=@col[:plaintext]=@col[:body]='' elsif data[/^6~(?:~\S+)?(.+?)<~(\d+);((?:\w|[0-6]:)\d+);(\w\d+)><([0-9a-f]{#{@@dl}}):([0-9a-f]{#{@@dl}})>/] # header lev6 seg level @@ -503,7 +499,7 @@ module SiSU_DB_import end if @en_pls[0]; @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last end - t=SiSU_DB_tuple::Load_tuple.new(@conn,@col,@opt) + t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt) t.tuple @col[:lev]=@col[:plaintext]=@col[:body]='' else #% regular text @@ -553,7 +549,7 @@ module SiSU_DB_import special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup strip_markup(@col[:plaintext]) - t=SiSU_DB_tuple::Load_tuple.new(@conn,@col,@opt) + t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt) t.tuple @en,@en_ast,@en_pls=[],[],[] @col[:en_a]=@col[:en_z]=nil @@ -564,10 +560,10 @@ module SiSU_DB_import endnote_array.each do |inf| if inf[/~\{\d+.+?<[0-9a-f]{#{@@dl}}>\}~/] # dal new endnotes 2003w31/1 if inf[/~\{(\d+)(.+?)<([0-9a-f]{#{@@dl}})>\}~/] # dal new endnotes 2003w31/1 - en,txt,digest_clean=$1,$2,$3 + nr,txt,digest_clean=$1,$2,$3 end @id_n+=1 - body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,en).endnote + body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,nr).endnote special_character_escape(body) special_character_escape(txt) strip_markup(txt) @@ -579,10 +575,21 @@ module SiSU_DB_import txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n} end if txt - #puts "'#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}'" #% endnotes - @conn.execute(%{ - INSERT INTO endnotes (nid, document_lid, nr, clean, body, ocn, ocnd, ocns, metadata_tid, digest_clean) VALUES ('#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}', '#{digest_clean}'); - }) + en={} + en={ :type => 'endnotes', + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean + } + t=SiSU_DB_tuple::Load_endnotes.new(@conn,en) + t.tuple end end end @@ -593,10 +600,10 @@ module SiSU_DB_import endnote_array.each do |inf| if inf[/~\[\*\d+.+?<[0-9a-f]{#{@@dl}}>\]~/] # dal new endnotes 2003w31/1 if inf[/~\[[*](\d+)(.+?)<([0-9a-f]{#{@@dl}})>\]~/] # dal new endnotes 2003w31/1 - en,txt,digest_clean=$1,$2,$3 + nr,txt,digest_clean=$1,$2,$3 end @id_n+=1 - body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,en).endnote + body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,nr).endnote special_character_escape(body) special_character_escape(txt) strip_markup(txt) @@ -609,9 +616,21 @@ module SiSU_DB_import end if txt #puts "'#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}'" #% endnotes - @conn.execute(%{ - INSERT INTO endnotes_asterisk (nid, document_lid, nr, clean, body, ocn, ocnd, ocns, metadata_tid, digest_clean) VALUES ('#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}', '#{digest_clean}'); - }) + en={} + en={ :type => 'endnotes_asterisk', + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean + } + t=SiSU_DB_tuple::Load_endnotes.new(@conn,en) + t.tuple end end end @@ -622,10 +641,10 @@ module SiSU_DB_import endnote_array.each do |inf| if inf[/~\[\+\d+.+?<[0-9a-f]{#{@@dl}}>\]~/] # dal new endnotes 2003w31/1 if inf[/~\[[+](\d+)(.+?)<([0-9a-f]{#{@@dl}})>\]~/] # dal new endnotes 2003w31/1 - en,txt,digest_clean=$1,$2,$3 + nr,txt,digest_clean=$1,$2,$3 end @id_n+=1 - body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,en).endnote + body=SiSU_Format_Shared::CSS_Format.new(@md,txt,@col,nr).endnote special_character_escape(body) special_character_escape(txt) strip_markup(txt) @@ -637,10 +656,21 @@ module SiSU_DB_import txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n} end if txt - #puts "'#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}'" #% endnotes - @conn.execute(%{ - INSERT INTO endnotes_plus (nid, document_lid, nr, clean, body, ocn, ocnd, ocns, metadata_tid, digest_clean) VALUES ('#{@id_n}', '#{@col[:lid]}', '#{en}', '#{txt}', '#{body}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@@id_t}', '#{digest_clean}'); - }) + en={} + en={ :type => 'endnotes_plus', + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean + } + t=SiSU_DB_tuple::Load_endnotes.new(@conn,en) + t.tuple end end end @@ -703,79 +733,77 @@ module SiSU_DB_import @env=SiSU_Env::Info_env.new(@opt.fns) base=@env.url.root out=@env.path.output - markup,meta,latex,plaintext,html_toc,html_doc,xml_sax,xml_dom,pdf_p,pdf_l,concordance,sisupod='','','','','','','','','','','','' - markup_li,meta_li,latex_li,plaintext_li,html_li,xml_sax_li,xml_dom_li,pdf_p_li,pdf_l_li,concordance_li,sisupod_li='','','','','','','','','','','' + f,u={},{} if @fnb.empty? \ or @fnb.nil? p 'file output path error' #remove end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:plain]}")==true) - plaintext,plaintext_li='plaintext,', "'#{base}/#@fnb/#{@md.fn[:plain]}'," + f[:txt],u[:txt]='plaintext,', "'#{base}/#@fnb/#{@md.fn[:plain]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:toc]}")==true) - html_toc,html_toc_li='html_toc,', "'#{base}/#@fnb/#{@md.fn[:toc]}'," + f[:html_toc],u[:html_toc]='html_toc,', "'#{base}/#@fnb/#{@md.fn[:toc]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:doc]}")==true) - html_doc,html_doc_li='html_doc,', "'#{base}/#@fnb/#{@md.fn[:doc]}'," + f[:html_doc],u[:html_doc]='html_doc,', "'#{base}/#@fnb/#{@md.fn[:doc]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:xhtml]}")==true) - xhtml,xhtml_li='xhtml,', "'#{base}/#@fnb/#{@md.fn[:xhtml]}'," + f[:xhtml],u[:xhtml]='xhtml,', "'#{base}/#@fnb/#{@md.fn[:xhtml]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:sax]}")==true) - xml_sax,xml_sax_li='xml_sax,', "'#{base}/#@fnb/#{@md.fn[:sax]}'," + f[:xml_sax],u[:xml_sax]='xml_sax,', "'#{base}/#@fnb/#{@md.fn[:sax]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:dom]}")==true) - xml_dom,xml_dom_li='xml_dom,', "'#{base}/#@fnb/#{@md.fn[:dom]}'," + f[:xml_dom],u[:xml_dom]='xml_dom,', "'#{base}/#@fnb/#{@md.fn[:dom]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:odf]}")==true) - odf,odf_li='odf,', "'#{base}/#@fnb/#{@md.fn[:odf]}'," + f[:odf],u[:odf]='odf,', "'#{base}/#@fnb/#{@md.fn[:odf]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:pdf_p]}")==true) - pdf_p,pdf_p_li='pdf_p,', "'#{base}/#@fnb/#{@md.fn[:pdf_p]}'," + f[:pdf_p],u[:pdf_p]='pdf_p,', "'#{base}/#@fnb/#{@md.fn[:pdf_p]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:pdf_l]}")==true) - pdf_l,pdf_l_li='pdf_l,', "'#{base}/#@fnb/#{@md.fn[:pdf_l]}'," + f[:pdf_l],u[:pdf_l]='pdf_l,', "'#{base}/#@fnb/#{@md.fn[:pdf_l]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:concordance]}")==true) - concordance,concordance_li='concordance,', "'#{base}/#@fnb/#{@md.fn[:concordance]}'," + f[:concordance],u[:concordance]='concordance,', "'#{base}/#@fnb/#{@md.fn[:concordance]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@opt.fns}.tex")==true) - latex_p,latex_p_li='latex_p,', "'#{base}/#@fnb/#{@opt.fns}.tex'," + f[:latex_p],u[:latex_p]='latex_p,', "'#{base}/#@fnb/#{@opt.fns}.tex'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@opt.fns}.landscape.tex")==true) - latex_l,latex_l_li='latex_l,', "'#{base}/#@fnb/#@opt.fns}.landscape.tex'," + f[:latex_l],u[:latex_l]='latex_l,', "'#{base}/#@fnb/#@opt.fns}.landscape.tex'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:digest]}")==true) - digest,digest_li='digest,', "'#{base}/#@fnb/#{@md.fn[:digest]}'," + f[:digest],u[:digest]='digest,', "'#{base}/#@fnb/#{@md.fn[:digest]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@md.fn[:manifest]}")==true) #revisit, was to be text, this is html - manifest,manifest_li='manifest,', "'#{base}/#@fnb/#{@md.fn[:manifest]}'," + f[:manifest],u[:manifest]='manifest,', "'#{base}/#@fnb/#{@md.fn[:manifest]}'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@opt.fns}.meta")==true) - markup,markup_li='markup,', "'#{base}/#@fnb/#{@opt.fns}.meta'," + f[:markup],u[:markup]='markup,', "'#{base}/#@fnb/#{@opt.fns}.meta'," end if @opt.cmd !~/e/ \ or (@opt.cmd=~/e/ and FileTest.file?("#{out}/#@fnb/#{@opt.fns}.tgz")==true) - sisupod,sisupod_li='sisupod,', "'#{base}/#@fnb/#{@opt.fns}.tgz'," + f[:sisupod],u[:sisupod]='sisupod,', "'#{base}/#@fnb/#{@opt.fns}.tgz'," end - @conn.execute(%{ - INSERT INTO urls (#{plaintext} #{html_toc} #{html_doc} #{xhtml} #{xml_sax} #{xml_dom} #{odf} #{pdf_p} #{pdf_l} #{concordance} #{latex_p} #{latex_l} #{manifest} #{digest} #{markup} #{sisupod} metadata_tid) VALUES (#{plaintext_li} #{html_toc_li} #{html_doc_li} #{xhtml_li} #{xml_sax_li} #{xml_dom_li} #{odf_li} #{pdf_p_li} #{pdf_l_li} #{concordance_li} #{latex_p_li} #{latex_l_li} #{manifest_li} #{digest_li} #{markup_li} #{sisupod_li} #{@@id_t}); - }) + t=SiSU_DB_tuple::Load_urls.new(@conn,f,u,@@id_t) + t.tuple rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error ensure end diff --git a/lib/sisu/v0/db_load_tuple.rb b/lib/sisu/v0/db_load_tuple.rb index 3baf2f82..510dd94e 100644 --- a/lib/sisu/v0/db_load_tuple.rb +++ b/lib/sisu/v0/db_load_tuple.rb @@ -56,7 +56,7 @@ =end module SiSU_DB_tuple - class Load_tuple #% main database populate + class Load_documents require "#{SiSU_lib}/param" include SiSU_Param def initialize(conn,col,opt) @@ -77,12 +77,10 @@ module SiSU_DB_tuple def tuple #% import line begin if @col[:en_a] - #puts "#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}', '#{@col[:en_a]}', '#{@col[:en_z]}'" @conn.execute(%{ INSERT INTO documents (lid, metadata_tid, lev, clean, body, ocn, ocnd, ocns, seg, lev1, lev2, lev3, lev4, lev5, lev6, en_a, en_z, digest_clean, digest_all) VALUES (#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}', '#{@col[:en_a]}', '#{@col[:en_z]}', '#{@col[:digest_clean]}', '#{@col[:digest_all]}'); }) else - #puts "#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}'" @conn.execute(%{ INSERT INTO documents (lid, metadata_tid, lev, clean, body, ocn, ocnd, ocns, seg, lev1, lev2, lev3, lev4, lev5, lev6, digest_clean, digest_all) VALUES (#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}', '#{@col[:digest_clean]}', '#{@col[:digest_all]}'); }) @@ -107,5 +105,35 @@ module SiSU_DB_tuple end end end + class Load_metadata + def initialize(conn,tp,id) + @conn,@tp,@id=conn,tp,id + end + def tuple + @conn.execute(%{ + INSERT INTO metadata (#{@tp[:fns_f]} #{@tp[:suffix_f]} #{@tp[:title_f]} #{@tp[:subtitle_f]} #{@tp[:creator_f]} #{@tp[:illustrator_f]} #{@tp[:translator_f]} #{@tp[:subject_f]} #{@tp[:description_f]} #{@tp[:publisher_f]} #{@tp[:contributor_f]} #{@tp[:prepared_by_f]} #{@tp[:digitized_by_f]} #{@tp[:date_f]} #{@tp[:date_created_f]} #{@tp[:date_issued_f]} #{@tp[:date_valid_f]} #{@tp[:date_available_f]} #{@tp[:date_modified_f]} #{@tp[:type_f]} #{@tp[:format_f]} #{@tp[:identifier_f]} #{@tp[:source_f]} #{@tp[:language_f]} #{@tp[:language_original_f]} #{@tp[:relation_f]} #{@tp[:coverage_f]} #{@tp[:rights_f]} #{@tp[:copyright_f]} #{@tp[:owner_f]} #{@tp[:keywords_f]} #{@tp[:abstract_f]} #{@tp[:comment_f]} #{@tp[:loc_f]} #{@tp[:dewey_f]} #{@tp[:isbn_f]} #{@tp[:pg_f]} #{@tp[:prefix_a_f]} #{@tp[:prefix_b_f]} tid) VALUES (#{@tp[:fns_i]} #{@tp[:suffix_i]} #{@tp[:title_i]} #{@tp[:subtitle_i]} #{@tp[:creator_i]} #{@tp[:illustrator_i]} #{@tp[:translator_i]} #{@tp[:subject_i]} #{@tp[:description_i]} #{@tp[:publisher_i]} #{@tp[:contributor_i]} #{@tp[:prepared_by_i]} #{@tp[:digitized_by_i]} #{@tp[:date_i]} #{@tp[:date_created_i]} #{@tp[:date_issued_i]} #{@tp[:date_valid_i]} #{@tp[:date_available_i]} #{@tp[:date_modified_i]} #{@tp[:type_i]} #{@tp[:format_i]} #{@tp[:identifier_i]} #{@tp[:source_i]} #{@tp[:language_i]} #{@tp[:language_original_i]} #{@tp[:relation_i]} #{@tp[:coverage_i]} #{@tp[:rights_i]} #{@tp[:copyright_i]} #{@tp[:owner_i]} #{@tp[:keywords_i]} #{@tp[:abstract_i]} #{@tp[:comment_i]} #{@tp[:loc_i]} #{@tp[:dewey_i]} #{@tp[:isbn_i]} #{@tp[:pg_i]} #{@tp[:prefix_a_i]} #{@tp[:prefix_b_i]} #{@id}); + }) + end + end + class Load_urls + def initialize(conn,f,u,id) + @conn,@f,@u,@id=conn,f,u,id + end + def tuple + @conn.execute(%{ + INSERT INTO urls (#{@f[:txt]} #{@f[:html_toc]} #{@f[:html_doc]} #{@f[:xhtml]} #{@f[:xml_sax]} #{@f[:xml_dom]} #{@f[:odf]} #{@f[:pdf_p]} #{@f[:pdf_l]} #{@f[:concordance]} #{@f[:latex_p]} #{@f[:latex_l]} #{@f[:manifest]} #{@f[:digest]} #{@f[:markup]} #{@f[:sisupod]} metadata_tid) VALUES (#{@u[:txt]} #{@u[:html_toc]} #{@u[:html_doc]} #{@u[:xhtml]} #{@u[:xml_sax]} #{@u[:xml_dom]} #{@u[:odf]} #{@u[:pdf_p]} #{@u[:pdf_l]} #{@u[:concordance]} #{@u[:latex_p]} #{@u[:latex_l]} #{@u[:manifest]} #{@u[:digest]} #{@u[:markup]} #{@u[:sisupod]} #{@id}); + }) + end + end + class Load_endnotes + def initialize(conn,en) + @conn,@en=conn,en + end + def tuple + @conn.execute(%{ + INSERT INTO #{@en[:type]} (nid, document_lid, nr, clean, body, ocn, ocnd, ocns, metadata_tid, digest_clean) VALUES ('#{@en[:id]}', '#{@en[:lid]}', '#{@en[:nr]}', '#{@en[:txt]}', '#{@en[:body]}', '#{@en[:ocn]}', '#{@en[:ocnd]}', '#{@en[:ocns]}', '#{@en[:id_t]}', '#{@en[:hash]}'); + }) + end + end end __END__ diff --git a/lib/sisu/v0/db_remove.rb b/lib/sisu/v0/db_remove.rb index c034c61f..75061e38 100644 --- a/lib/sisu/v0/db_remove.rb +++ b/lib/sisu/v0/db_remove.rb @@ -86,7 +86,7 @@ module SiSU_DB_remove }) #@conn.execute("COMMIT") else - tell=SiSU_Screen::Ansi.new(@opt.cmd,"no such file in database #{@db.db_psql}::#{@opt.fns}") + tell=SiSU_Screen::Ansi.new(@opt.cmd,"no such file in database #{@db.psql.db}::#{@opt.fns}") tell.puts_grey #if @opt.cmd.include? ?v end end diff --git a/lib/sisu/v0/db_select.rb b/lib/sisu/v0/db_select.rb index acf4356f..8471bde9 100644 --- a/lib/sisu/v0/db_select.rb +++ b/lib/sisu/v0/db_select.rb @@ -60,11 +60,11 @@ module SiSU_DB_select def initialize(opt,conn='',sql_type='pg') @opt,@conn,@sql_type=opt,conn,sql_type #@sdb=Create.new(@conn) - @sdb=SiSU_DB::Create.new(@opt,@conn,@sql_type) - @index=SiSU_DB::Index.new(@opt,@conn,@sql_type) - @sdb_no=SiSU_DB::Drop.new(@opt,@conn,@sql_type) - @sdb_import=SiSU_DB::Import.new(@opt,@conn,@sql_type) if @opt.mod.inspect =~/update|import/ - @remove=SiSU_DB::Remove.new(@opt,@conn) if @opt.mod.inspect =~/update|remove/ + @sdb=SiSU_DB_DBI::Create.new(@opt,@conn,@sql_type) + @index=SiSU_DB_DBI::Index.new(@opt,@conn,@sql_type) + @sdb_no=SiSU_DB_DBI::Drop.new(@opt,@conn,@sql_type) + @sdb_import=SiSU_DB_DBI::Import.new(@opt,@conn,@sql_type) if @opt.mod.inspect =~/update|import/ + @remove=SiSU_DB_DBI::Remove.new(@opt,@conn) if @opt.mod.inspect =~/update|remove/ @db=SiSU_Env::Info_db.new end def cases @@ -80,57 +80,57 @@ module SiSU_DB_select @sdb.output_dir? #@sdb.create_db begin - @sdb.create_table_metadata - @sdb.create_table - @sdb.create_table_endnotes - @sdb.create_table_endnotes_asterisk - @sdb.create_table_endnotes_plus - @sdb.create_table_urls + @sdb.create_table.metadata + @sdb.create_table.documents + @sdb.create_table.endnotes + @sdb.create_table.endnotes_asterisk + @sdb.create_table.endnotes_plus + @sdb.create_table.urls @index.create_indexes rescue; SiSU_Errors::Info_error.new($!,$@,'-D').error; @sdb.output_dir? end when /^--createtable(s)?$/ @sdb.output_dir? begin - @sdb.create_table_metadata - @sdb.create_table - @sdb.create_table_endnotes - @sdb.create_table_endnotes_asterisk - @sdb.create_table_endnotes_plus - @sdb.create_table_urls + @sdb.create_table.metadata + @sdb.create_table.documents + @sdb.create_table.endnotes + @sdb.create_table.endnotes_asterisk + @sdb.create_table.endnotes_plus + @sdb.create_table.urls @index.create_indexes rescue; @sdb.output_dir? end when /^--recreate$/ @sdb.output_dir? begin - @sdb_no.drop_tables - @sdb.create_table_metadata - @sdb.create_table - @sdb.create_table_endnotes - @sdb.create_table_endnotes_asterisk - @sdb.create_table_endnotes_plus - @sdb.create_table_urls + @sdb_no.drop.tables + @sdb.create_table.metadata + @sdb.create_table.documents + @sdb.create_table.endnotes + @sdb.create_table.endnotes_asterisk + @sdb.create_table.endnotes_plus + @sdb.create_table.urls @index.create_indexes rescue; @sdb.output_dir? end when /^--cr(eate)?lex$/ @sdb.output_dir? begin - @sdb.create_table + @sdb.create_table.documents rescue; @sdb.output_dir? end when /^--cr(eate)?metadata$/ @sdb.output_dir? begin - @sdb.create_table_metadata + @sdb.create_table.metadata rescue; @sdb.output_dir? end when /^--import$/ @sdb_import.marshal_load tell=case @sql_type - when /sqlite/; SiSU_Screen::Ansi.new(@opt.cmd,"sqlite #{@db.db_sqlite} database?") - when /pg/; SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.db_psql} database?") + when /sqlite/; SiSU_Screen::Ansi.new(@opt.cmd,"sqlite #{@db.sqlite.db} database?") + when /pg/; SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.psql.db} database?") else '???' end tell.puts_grey if @opt.cmd =~/v/ @@ -139,17 +139,17 @@ module SiSU_DB_select when /^--update$/ @remove.remove @sdb_import.marshal_load - tell=SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.db_psql} database?") + tell=SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.psql.db} database?") tell.puts_grey if @opt.cmd =~/v/ when /^--index$/ @index.create_indexes when /^droptable(s)?$/ - @sdb_no.drop_tables + @sdb_no.drop.tables when /^--dropindex(es)?$/ - @sdb_no.drop_indexes + @sdb_no.drop.indexes when /^--(?:dropall|drop)$/ - @sdb_no.drop_tables - #@sdb_no.drop_indexes + @sdb_no.drop.tables + #@sdb_no.drop.indexes when /^--(?:db=)?(?:(?:sq)?lite|pg(?:sql)?|my(?:sql)?)$/ else help=SiSU_Help::Help.new diff --git a/lib/sisu/v0/dbi.rb b/lib/sisu/v0/dbi.rb index 17568ce6..d9f73989 100644 --- a/lib/sisu/v0/dbi.rb +++ b/lib/sisu/v0/dbi.rb @@ -61,8 +61,8 @@ module SiSU_DBI include SiSU_Screen require "#{SiSU_lib}/param" include SiSU_Param - require "#{SiSU_lib}/shared_db" - include SiSU_DB + require "#{SiSU_lib}/db_dbi" + include SiSU_DB_DBI require "#{SiSU_lib}/shared_html_lite" include SiSU_Format_Shared class SiSU_SQL @@ -90,14 +90,14 @@ module SiSU_DBI end def read_psql begin - db,user,dbi=@db.db_psql,@db.user,@db.dbi_psql + db,user,dbi=@db.psql.db,@db.psql.user,@db.psql.dbi @conn=DBI.connect(dbi,user,db) rescue if @opt.mod.inspect=~/--(createall|create)/ puts %{manually create the database: "#{db}" if it does not yet exist} #sudo su -p postgres; createdb #{db}; #[createuser?] end - SiSU_DB::Case.new(@opt,@conn,@sql_type).cases + SiSU_DB_DBI::Case.new(@opt,@conn,@sql_type).cases @conn=DBI.connect(dbi,user,db) ensure end @@ -105,7 +105,7 @@ module SiSU_DBI def read_sqlite begin sql_type='sqlite' - dbi=@db.dbi_sqlite + dbi=@db.sqlite.dbi @conn=DBI.connect(dbi) rescue ensure @@ -119,7 +119,7 @@ module SiSU_DBI tell=SiSU_Screen::Ansi.new(@opt.cmd,"DBI (#@sql_type) #{@opt.mod}") tell.dbi_title unless @opt.cmd =~/q/ begin - SiSU_DB::Case.new(@opt,@conn,@sql_type).cases + SiSU_DB_DBI::Case.new(@opt,@conn,@sql_type).cases rescue; SiSU_Errors::Info_error.new($!,$@,@cf,@opt.fns).error ensure end diff --git a/lib/sisu/v0/help.rb b/lib/sisu/v0/help.rb index 84a7391d..c739a04b 100644 --- a/lib/sisu/v0/help.rb +++ b/lib/sisu/v0/help.rb @@ -1072,9 +1072,9 @@ WOK webrick url: #{@cX.blue}#{@env.url.webserv_base_cgi}#{@cX.off} (to start webrick server 'sisu -W') sqlite db for present directory: #{@cX.blue}sqlite #{@env.path.output}/sisu_sqlite.db#{@cX.off} - postgresql port set to: #{@cX.blue}#{@db.port_psql}#{@cX.off} - postgresql db for present directory: #{@cX.blue}#{@db.db_psql}#{@cX.off} - [first create manually if necessary: 'createdb #{@db.db_psql}'] + postgresql port set to: #{@cX.blue}#{@db.psql.port}#{@cX.off} + postgresql db for present directory: #{@cX.blue}#{@db.psql.db}#{@cX.off} + [first create manually if necessary: 'createdb #{@db.psql.name}'] [generated sqlite cgi search form]: #{@cX.blue}#{@env.url.webserv_base_cgi}/cgi-bin/sisu_sqlite.cgi#{@cX.off} [generated postgresql cgi search form]: #{@cX.blue}#{@env.url.webserv_base_cgi}/cgi-bin/sisu_pgsql.cgi#{@cX.off} @@ -1755,13 +1755,13 @@ WOK Postgresql user: #{@cX.blue}#{@db.user}#{@cX.off} - current db set: #{@cX.blue}#{@db.db_psql}#{@cX.off} - port: #{@cX.blue}#{@db.port_psql}#{@cX.off} - dbi connect: #{@cX.blue}#{@db.dbi_psql}#{@cX.off} + current db set: #{@cX.blue}#{@db.psql.db}#{@cX.off} + port: #{@cX.blue}#{@db.psql.port}#{@cX.off} + dbi connect: #{@cX.blue}#{@db.psql.dbi}#{@cX.off} sqlite - current db set: #{@cX.blue}#{@db.db_sqlite}#{@cX.off} - dbi connect #{@cX.blue}#{@db.dbi_sqlite} #{@cX.off} + current db set: #{@cX.blue}#{@db.sqlite.db}#{@cX.off} + dbi connect #{@cX.blue}#{@db.sqlite.dbi} #{@cX.off} Note on databases built By default, [unless otherwise specified] databases are built on a directory basis, from collections of documents within that directory. diff --git a/lib/sisu/v0/param.rb b/lib/sisu/v0/param.rb index 1147cd80..9bf100d5 100644 --- a/lib/sisu/v0/param.rb +++ b/lib/sisu/v0/param.rb @@ -116,7 +116,7 @@ module SiSU_Param @doc={ :lv=>[] } @doc[:fns],@doc[:fnb],@doc[:scr_suffix]='','','' @@publisher='SiSU scribe' - attr_accessor :cmd,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:sfx,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:dc_title,:html_title,:subtitle,:subtitle_tex,:creator_home,:dc_creator,:translator,:illustrator,:prepared_by,:digitized_by,:dc_subject,:dc_description,:dc_publisher,:dc_contributor,:dc_date,:dc_date_created,:dc_date_issued,:dc_date_available,:dc_date_valid,:dc_date_modified,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:dc_type,:dc_format,:dc_identifier,:dc_source,:dc_language,:language_original,:dc_relation,:dc_coverage,:dc_rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:flag_auto_heading_num,:markup,:markup_instruction,:markup_version,:markup_declared,:make_bold,:make_italic,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:file_size,:user,:home,:hostname,:pwd,:firstseg,:programs,:creator_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:man_section,:man_name,:man_synopsis,:ec + attr_accessor :cmd,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:sfx,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:dc_title,:html_title,:subtitle,:subtitle_tex,:creator_home,:dc_creator,:translator,:illustrator,:prepared_by,:digitized_by,:dc_subject,:dc_description,:dc_publisher,:dc_contributor,:dc_date,:dc_date_created,:dc_date_issued,:dc_date_available,:dc_date_valid,:dc_date_modified,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:dc_type,:dc_format,:dc_identifier,:dc_source,:dc_language,:language_original,:dc_relation,:dc_coverage,:dc_rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:flag_auto_heading_num,:markup,:markup_instruction,:markup_version,:markup_declared,:make_bold,:make_italic,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:file_size,:user,:home,:hostname,:pwd,:firstseg,:programs,:creator_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:man_section,:man_name,:man_synopsis,:ec,:opt def initialize(fns_array,opt) @env=@fn=@fns=@fnb=@fnn=@fnt=@fnl=@flv=@fnz=@fnstex=@ocn=@sfx_src=@sfx=@pdf=@file_type=@dir_out=@dir_tex=@dir_lout=@txt_path=@flag_endnotes=@flag_auto_endnotes=@flag_separate_endnotes=@flag_separate_endnotes_make=@site_skin=@sisu=@sisu_version=@ruby_version=@title=@dc_title=@html_title=@subtitle=@subtitle_tex=@creator_home=@dc_creator=@translator=@illustrator=@prepared_by=@digitized_by=@dc_subject=@dc_description=@dc_publisher=@dc_contributor=@dc_date=@dc_date_created=@dc_date_issued=@dc_date_available=@dc_date_valid=@dc_date_modified=@date_scheme=@date_created_scheme=@date_issued_scheme=@date_available_scheme=@date_valid_scheme=@date_modified_scheme=@dc_type=@dc_format=@dc_identifier=@dc_source=@dc_language=@language_original=@dc_relation=@dc_coverage=@dc_rights=@keywords=@comments=@abstract=@cls_loc=@cls_dewey=@cls_pg=@cls_isbn=@papersize=@toc=@lv1=@lv2=@lv3=@lv4=@lv5=@lv6=@pagenew=@pagebreak=@num_top=@toc_lev_limit=@flag_auto_heading_num=@make_bold=@make_italic=@flag_tables=@vocabulary=@doc_skin=@doc_css=@yaml=@lnk=@prefix_a=@prefix_b=@suffix=@information=@contact=@icon=@ad_url=@ad_png=@ad_alt=@ad_began=@promo=@ad_home=@stmp=@stmpd=@sc_filename=@sc_number=@sc_date=@sc_time=@sc_info=@yamladdr=@locale=@wc_lines=@wc_words=@wc_bytes=@file_encoding=@file_size=@firstseg=@programs=@creator_copymark=@lang=@en=@dgst=@dgst_skin=@generated=@heading_seg_first=@base_program=@man_synopsis=nil @man_section=1 diff --git a/lib/sisu/v0/sysenv.rb b/lib/sisu/v0/sysenv.rb index 7e94d8a3..0661aa13 100644 --- a/lib/sisu/v0/sysenv.rb +++ b/lib/sisu/v0/sysenv.rb @@ -2549,36 +2549,60 @@ p @zap @rc=@@rc ||=Get_init.instance.yamlrc @defaults=Info_env.new.defaults end - def user - @env.user - end - def db_psql - "SiSU_#@pwd_stub" - end - def db_mysql - 'lm' - end - def db_sqlite - "#{@env.path.webserv}/#@pwd_stub/sisu_sqlite.db" - end - def port_psql #PGPORT - if defined? @rc['db']['postgresql']['port'] \ - and @rc['db']['postgresql']['port'] =~/\d+/ - @rc['db']['postgresql']['port'] - else @defaults[:postgresql_port] + def engine + def default + if defined? @rc['db']['engine']['default'] \ + and @rc['db']['engine']['default']=~/postgresql|sqlite/ + @rc['db']['engine']['default'] + else 'sqlite' + end end + self end - def port_mysql - '**' - end - def dbi_psql - "DBI:Pg:database=#{db_psql};port=#{port_psql}" + def psql + def user + if defined? @rc['db']['postgresql']['user'] \ + and @rc['db']['postgresql']['user']=~/\S+/ + @rc['db']['postgresql']['user'] + else @env.user + end + end + def db #db_name + "SiSU_#@pwd_stub" + end + def port #PGPORT + if defined? @rc['db']['postgresql']['port'] \ + and ( @rc['db']['postgresql']['port'] =~/\d+/ \ + or @rc['db']['postgresql']['port'].class == Fixnum) + @rc['db']['postgresql']['port'] + else @defaults[:postgresql_port] + end + end + def dbi + "DBI:Pg:database=#{psql.db};port=#{psql.port}" + end + self end - def dbi_mysql - "dbi:Mysql:database=#{db_mysql};port=#{port_mysql}" + def mysql + def db + #"SiSU_#@pwd_stub" + end + def port + '**' + end + def dbi + "dbi:Mysql:database=#{mysql.db};port=#{mysql.port}" + end + self end - def dbi_sqlite - "DBI:SQLite:#{db_sqlite}" #sqlite3 ? + def sqlite + def db + "#{@env.path.webserv}/#@pwd_stub/sisu_sqlite.db" + end + def dbi + "DBI:SQLite:#{sqlite.db}" #sqlite3 ? + end + self end end class Info_port < Info_env |