db name, tables, columns, indexes changes, review (need another version bump 2.2.0)

* db (sql) table structure, further review and changes (hence breakage & version bump)
  * new pgsql db name prefix "sisu_v2b_"
  * new table column words in doc_objects & endnotes, VARCHAR 3000 to contain
    list of unique sorted words in object
  * increase use of VARCHAR
  * constants takes on related additions

* param, extensive db column size checks for metadata

1 files changed, 41 insertions, 16 deletions

diff --git a/lib/sisu/v2/param.rb b/lib/sisu/v2/param.rb
index ef91f10d..87dd9aab 100644
--- a/lib/sisu/v2/param.rb
+++ b/lib/sisu/v2/param.rb
@@ -162,8 +162,8 @@ module SiSU_Param
           puts "#{n} is #{s.class}: programming error, String expected #{__FILE__}:#{__LINE__}"
           s
         else 
-          tell=SiSU_Screen::Ansi.new('v',"#{n} length #{s.length} exceeds set db field length #{l}, metadata dropped")
-          tell.warn
+          tell=SiSU_Screen::Ansi.new('v',"#{n} length #{s.length} exceeds set db field length #{l}, metadata dropped",@opt.fns)
+          tell.warn unless @opt.cmd =~/q/
           nil 
         end
       end
@@ -244,7 +244,9 @@ module SiSU_Param
           validate_length(s,l,n)
         end
         def note
-          @h['note']                   #TEXT
+          s=@h['note']
+          l,n=Db[:col_info_note],'title.note'
+          validate_length(s,l,n)
         end
         def short
           s=(@h['short'] ? @h['short'] : @h['main'])
@@ -380,9 +382,9 @@ module SiSU_Param
       def rights
         a=@s.split(/[ ]*\n[ ]*/m)
         @h=build_hash(a)
-        def copyright                  # TEXT used db sql 
+        def copyright
           def text #you may wish to expand to take from all
-            r=if @h['copyright']
+            s=if @h['copyright']
               @h['copyright']
             elsif @h['text']
               @h['text']
@@ -393,26 +395,40 @@ module SiSU_Param
               tell.warn unless @opt.cmd =~/q/
               ''
             end
+            l,n=Db[:col_info_note],'rights.copyright.text'
+            validate_length(s,l,n)
           end
           def translation
-            r=(@h['translation'] ? @h['translation'] : nil)
+            s=(@h['translation'] ? @h['translation'] : nil)
+            l,n=Db[:col_info_note],'rights.copyright.translation'
+            validate_length(s,l,n)
           end
           def illustrations
-            r=(@h['illustrations'] ? @h['illustrations'] : nil)
+            s=(@h['illustrations'] ? @h['illustrations'] : nil)
+            l,n=Db[:col_info_note],'rights.copyright.illustrations'
+            validate_length(s,l,n)
           end
           def photographs
-            r=(@h['photographs'] ? @h['photographs'] : nil)
+            s=(@h['photographs'] ? @h['photographs'] : nil)
+            l,n=Db[:col_info_note],'rights.copyright.photographs'
+            validate_length(s,l,n)
           end
           def digitiztion
-            r=(@h['digitization'] ? @h['digitization'] : nil)
+            s=(@h['digitization'] ? @h['digitization'] : nil)
+            l,n=Db[:col_info_note],'rights.copyright.digitization'
+            validate_length(s,l,n)
           end
           def audio
-            r=(@h['audio'] ? @h['audio'] : nil)
+            s=(@h['audio'] ? @h['audio'] : nil)
+            l,n=Db[:col_info_note],'rights.copyright.audio'
+            validate_length(s,l,n)
           end
           self
         end
         def license
-          r=(@h['license'] ? @h['license'] : nil)
+          s=(@h['license'] ? @h['license'] : nil)
+          l,n=Db[:col_info_note],'rights.license'
+          validate_length(s,l,n)
         end
         def all
           s=if @h['all']; @h['all']
@@ -445,6 +461,9 @@ module SiSU_Param
             if s.empty?
               tell=SiSU_Screen::Ansi.new(@cmd,'WARNING Document Rights information missing; provide @rights: :copyright:')
               tell.warn unless @opt.cmd =~/q/
+            else
+              l,n=Db[:col_info_note],'rights.all'
+              validate_length(s,l,n)
             end
             s
           end
@@ -472,7 +491,7 @@ module SiSU_Param
         end
         def topic_register
           s=@h['topic_register']
-          l,n=Db[:col_classify_txt_long],'classify.topic_register'
+          l,n=Db[:col_info_note],'classify.topic_register'
           validate_length(s,l,n)
         end
         def type
@@ -743,9 +762,9 @@ module SiSU_Param
       @doc={ :lv=>[] }
       @doc[:fns],@doc[:fnb],@doc[:scr_suffix]='','',''
       @@publisher='SiSU scribe'
-      attr_accessor :cmd,:make,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:subtitle,:full_title,:html_title,:subtitle_tex,:creator,:classify,:author_home,:author,:author_title,:author_nationality,:authors,:authorship,:translator,:illustrator,:prepared_by,:digitized_by,:subject,:description,:publisher,:contributor,:date,:date_created,:date_issued,:date_available,:date_valid,:date_modified,:date_translated,:date_added_to_site,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:type,:format,:identifier,:source,:language,:language_original,:relation,:coverage,:rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:papersize_array,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:lvs,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:markup,:markup_instruction,:markup_version,:markup_declared,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:file_size,:user,:home,:hostname,:pwd,:firstseg,:programs,:author_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:ec,:opt,:sem_tag,:book_idx,:topic_register,:topic_register_array,:original_publication,:original_publication_date,:original_publication_nationality,:original_publication_institution,:writing_focus,:audio,:daisy
+      attr_accessor :cmd,:make,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:subtitle,:full_title,:html_title,:subtitle_tex,:creator,:classify,:author_home,:author,:author_title,:author_nationality,:authors,:authorship,:translator,:illustrator,:prepared_by,:digitized_by,:subject,:description,:publisher,:contributor,:date,:date_created,:date_issued,:date_available,:date_valid,:date_modified,:date_translated,:date_added_to_site,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:type,:format,:identifier,:source,:language,:language_original,:relation,:coverage,:rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:papersize_array,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:lvs,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:markup,:markup_instruction,:markup_version,:markup_declared,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:filesize,:user,:home,:hostname,:pwd,:firstseg,:programs,:author_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:ec,:opt,:sem_tag,:book_idx,:topic_register,:topic_register_array,:original_publication,:original_publication_date,:original_publication_nationality,:original_publication_institution,:writing_focus,:audio,:daisy
       def initialize(fns_array,opt)
-        @env=@fn=@fns=@fnb=@fnn=@fnt=@fnl=@flv=@fnz=@fnstex=@ocn=@sfx_src=@pdf=@file_type=@dir_out=@dir_tex=@dir_lout=@txt_path=@make=@flag_endnotes=@flag_auto_endnotes=@flag_separate_endnotes=@flag_separate_endnotes_make=@site_skin=@sisu=@sisu_version=@ruby_version=@title=@subtitle=@full_title=@html_title=@subtitle_tex=@creator=@classify=@author_home=@author=@author_title=@author_nationality=@translator=@illustrator=@prepared_by=@digitized_by=@subject=@description=@publisher=@contributor=@date=@date_created=@date_issued=@date_available=@date_valid=@date_modified=@date_translated=@date_added_to_site=@date_scheme=@date_created_scheme=@date_issued_scheme=@date_available_scheme=@date_valid_scheme=@date_modified_scheme=@type=@format=@identifier=@source=@language=@language_original=@relation=@coverage=@rights=@keywords=@comments=@abstract=@cls_loc=@cls_dewey=@cls_pg=@cls_isbn=@papersize=@toc=@lv1=@lv2=@lv3=@lv4=@lv5=@lv6=@pagenew=@pagebreak=@num_top=@toc_lev_limit=@flag_tables=@vocabulary=@doc_skin=@doc_css=@yaml=@lnk=@prefix_a=@prefix_b=@suffix=@information=@contact=@icon=@ad_url=@ad_png=@ad_alt=@ad_began=@promo=@ad_home=@stmp=@stmpd=@sc_filename=@sc_number=@sc_date=@sc_time=@sc_info=@yamladdr=@locale=@wc_lines=@wc_words=@wc_bytes=@file_encoding=@file_size=@firstseg=@programs=@author_copymark=@lang=@en=@dgst=@dgst_skin=@generated=@heading_seg_first=@base_program=@topic_register=@original_publication_details=@original_publication=@original_publication_date=@original_publication_nationality=@original_publication_institution=@writing_focus=@audio=nil
+        @env=@fn=@fns=@fnb=@fnn=@fnt=@fnl=@flv=@fnz=@fnstex=@ocn=@sfx_src=@pdf=@file_type=@dir_out=@dir_tex=@dir_lout=@txt_path=@make=@flag_endnotes=@flag_auto_endnotes=@flag_separate_endnotes=@flag_separate_endnotes_make=@site_skin=@sisu=@sisu_version=@ruby_version=@title=@subtitle=@full_title=@html_title=@subtitle_tex=@creator=@classify=@author_home=@author=@author_title=@author_nationality=@translator=@illustrator=@prepared_by=@digitized_by=@subject=@description=@publisher=@contributor=@date=@date_created=@date_issued=@date_available=@date_valid=@date_modified=@date_translated=@date_added_to_site=@date_scheme=@date_created_scheme=@date_issued_scheme=@date_available_scheme=@date_valid_scheme=@date_modified_scheme=@type=@format=@identifier=@source=@language=@language_original=@relation=@coverage=@rights=@keywords=@comments=@abstract=@cls_loc=@cls_dewey=@cls_pg=@cls_isbn=@papersize=@toc=@lv1=@lv2=@lv3=@lv4=@lv5=@lv6=@pagenew=@pagebreak=@num_top=@toc_lev_limit=@flag_tables=@vocabulary=@doc_skin=@doc_css=@yaml=@lnk=@prefix_a=@prefix_b=@suffix=@information=@contact=@icon=@ad_url=@ad_png=@ad_alt=@ad_began=@promo=@ad_home=@stmp=@stmpd=@sc_filename=@sc_number=@sc_date=@sc_time=@sc_info=@yamladdr=@locale=@wc_lines=@wc_words=@wc_bytes=@file_encoding=@filesize=@firstseg=@programs=@author_copymark=@lang=@en=@dgst=@dgst_skin=@generated=@heading_seg_first=@base_program=@topic_register=@original_publication_details=@original_publication=@original_publication_date=@original_publication_nationality=@original_publication_institution=@writing_focus=@audio=nil
         @data,@fns,@cmd,@mod,@opt=fns_array,opt.fns,opt.cmd,opt.mod,opt #@data used as data
         @flag_tables,@set_header_title,@set_heading_top,@set_heading_seg,@heading_seg_first_flag,@flag_promo,@book_idx=false,false,false,false,false,false,false
         @seg_autoname_safe=true
@@ -1107,16 +1126,22 @@ module SiSU_Param
           @papersize=determine_papersize(@mod.inspect)
         end
         @papersize_array=@papersize.scan(/(?:a4|letter|legal|book|a5|b5)/i)
+        fn=@fns=~/\.ssm\.sst$/ ? @fns.gsub(/.sst/,'') : @fns #decide what to do a filesize on .ssm tells very little about actual document size
+        @filesize=(File.size(fn)).to_s
         if @sys.openssl !=false
           skin=@doc_skin \
           ? (SiSU_Env::Info_skin.new(@opt,@doc_skin).select) \
           : SiSU_Env::Info_skin.new(@opt).select
           @dgst,@dgst_skin=[],[]
           if @env.digest.type =~/sha256/
-            @dgst=@sys.sha256(@env.source_file_with_path)
+            dgst=@sys.sha256(@env.source_file_with_path)
+            @dgst=dgst[1].length==64 ? dgst : nil
+            puts 'check document (sha256) digest' if not @dgst
             @dgst_skin=skin ? (@sys.sha256(skin)) : nil 
           else
-            @dgst=@sys.md5(@env.source_file_with_path)
+            dgst=@sys.md5(@env.source_file_with_path)
+            @dgst=dgst[1].length==32 ? dgst : nil
+            puts 'check document (md5) digest' if not @dgst
             @dgst_skin=skin ? (@sys.md5(skin)) : nil
           end
         end