From 879a4e6cbe97721bbc6cd18b324524159b4807ef Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Thu, 9 May 2013 22:21:33 -0400 Subject: v5 dev branch opened (starts as copy of v4 stable branch); v3 branch closed * 5.0.0 dev opened * 4.1.0 stable * 3.* branch gone --- lib/sisu/v5/air.rb | 85 + lib/sisu/v5/author_format.rb | 116 + lib/sisu/v5/cgi.rb | 78 + lib/sisu/v5/cgi_pgsql.rb | 258 + lib/sisu/v5/cgi_sql_common.rb | 1062 ++++ lib/sisu/v5/cgi_sqlite.rb | 219 + lib/sisu/v5/composite.rb | 254 + lib/sisu/v5/concordance.rb | 376 ++ lib/sisu/v5/conf.rb | 258 + lib/sisu/v5/constants.rb | 344 ++ lib/sisu/v5/css.rb | 3444 +++++++++++++ lib/sisu/v5/dal.rb | 510 ++ lib/sisu/v5/dal_character_check.rb | 107 + lib/sisu/v5/dal_doc_objects.rb | 452 ++ lib/sisu/v5/dal_doc_str.rb | 1051 ++++ lib/sisu/v5/dal_endnotes.rb | 128 + lib/sisu/v5/dal_expand_insertions.rb | 382 ++ lib/sisu/v5/dal_hash_digest.rb | 159 + lib/sisu/v5/dal_idx.rb | 366 ++ lib/sisu/v5/dal_images.rb | 161 + lib/sisu/v5/dal_metadata.rb | 82 + lib/sisu/v5/dal_numbering.rb | 475 ++ lib/sisu/v5/dal_substitutions_and_insertions.rb | 150 + lib/sisu/v5/dal_syntax.rb | 606 +++ lib/sisu/v5/db_columns.rb | 2001 ++++++++ lib/sisu/v5/db_create.rb | 618 +++ lib/sisu/v5/db_dbi.rb | 96 + lib/sisu/v5/db_drop.rb | 194 + lib/sisu/v5/db_import.rb | 717 +++ lib/sisu/v5/db_indexes.rb | 117 + lib/sisu/v5/db_load_tuple.rb | 326 ++ lib/sisu/v5/db_remove.rb | 124 + lib/sisu/v5/db_select.rb | 227 + lib/sisu/v5/db_sqltxt.rb | 133 + lib/sisu/v5/db_tests.rb | 120 + lib/sisu/v5/dbi.rb | 146 + lib/sisu/v5/dbi_discrete.rb | 186 + lib/sisu/v5/defaults.rb | 1212 +++++ lib/sisu/v5/digests.rb | 391 ++ lib/sisu/v5/embedded.rb | 148 + lib/sisu/v5/epub.rb | 762 +++ lib/sisu/v5/epub_concordance.rb | 327 ++ lib/sisu/v5/epub_format.rb | 2236 +++++++++ lib/sisu/v5/epub_segments.rb | 539 ++ lib/sisu/v5/epub_tune.rb | 329 ++ lib/sisu/v5/errors.rb | 89 + lib/sisu/v5/git.rb | 236 + lib/sisu/v5/harvest.rb | 116 + lib/sisu/v5/harvest_authors.rb | 407 ++ lib/sisu/v5/harvest_topics.rb | 825 +++ lib/sisu/v5/help.rb | 1881 +++++++ lib/sisu/v5/html.rb | 693 +++ lib/sisu/v5/html_format.rb | 1428 ++++++ lib/sisu/v5/html_minitoc.rb | 206 + lib/sisu/v5/html_promo.rb | 448 ++ lib/sisu/v5/html_scroll.rb | 224 + lib/sisu/v5/html_segments.rb | 628 +++ lib/sisu/v5/html_table.rb | 68 + lib/sisu/v5/html_tune.rb | 297 ++ lib/sisu/v5/hub.rb | 764 +++ lib/sisu/v5/i18n.rb | 701 +++ lib/sisu/v5/manifest.rb | 934 ++++ lib/sisu/v5/manpage.rb | 441 ++ lib/sisu/v5/manpage_format.rb | 106 + lib/sisu/v5/odf.rb | 828 +++ lib/sisu/v5/odf_format.rb | 667 +++ lib/sisu/v5/options.rb | 1069 ++++ lib/sisu/v5/param.rb | 1493 ++++++ lib/sisu/v5/param_identify_markup.rb | 171 + lib/sisu/v5/param_make.rb | 646 +++ lib/sisu/v5/particulars.rb | 235 + lib/sisu/v5/plaintext.rb | 432 ++ lib/sisu/v5/plaintext_format.rb | 116 + lib/sisu/v5/po4a.rb | 983 ++++ lib/sisu/v5/po4a_set.rb | 302 ++ lib/sisu/v5/prog_text_translation.rb | 1707 +++++++ lib/sisu/v5/qrcode.rb | 748 +++ lib/sisu/v5/relaxng.rb | 1161 +++++ lib/sisu/v5/remote.rb | 206 + lib/sisu/v5/response.rb | 78 + lib/sisu/v5/rexml.rb | 127 + lib/sisu/v5/screen_text_color.rb | 484 ++ lib/sisu/v5/share_src.rb | 95 + lib/sisu/v5/share_src_kdissert.rb | 91 + lib/sisu/v5/shared_html.rb | 68 + lib/sisu/v5/shared_html_lite.rb | 325 ++ lib/sisu/v5/shared_images.rb | 141 + lib/sisu/v5/shared_markup_alt.rb | 325 ++ lib/sisu/v5/shared_metadata.rb | 1261 +++++ lib/sisu/v5/shared_sem.rb | 162 + lib/sisu/v5/shared_sisupod_source.rb | 309 ++ lib/sisu/v5/shared_txt.rb | 234 + lib/sisu/v5/shared_xhtml.rb | 68 + lib/sisu/v5/shared_xml.rb | 722 +++ lib/sisu/v5/sisupod_make.rb | 155 + lib/sisu/v5/sitemaps.rb | 220 + lib/sisu/v5/spell.rb | 104 + lib/sisu/v5/sst_convert_markup.rb | 315 ++ lib/sisu/v5/sst_do_inline_footnotes.rb | 478 ++ lib/sisu/v5/sst_from_xml.rb | 175 + lib/sisu/v5/sst_identify_markup.rb | 487 ++ lib/sisu/v5/sst_to_s_xml_sax.rb | 462 ++ lib/sisu/v5/sysenv.rb | 6105 +++++++++++++++++++++++ lib/sisu/v5/termsheet.rb | 169 + lib/sisu/v5/texinfo.rb | 421 ++ lib/sisu/v5/texinfo_format.rb | 576 +++ lib/sisu/v5/texpdf.rb | 1074 ++++ lib/sisu/v5/texpdf_format.rb | 1651 ++++++ lib/sisu/v5/update.rb | 142 + lib/sisu/v5/urls.rb | 458 ++ lib/sisu/v5/utils.rb | 124 + lib/sisu/v5/webrick.rb | 189 + lib/sisu/v5/wikispeak.rb | 377 ++ lib/sisu/v5/xhtml.rb | 442 ++ lib/sisu/v5/xhtml_table.rb | 102 + lib/sisu/v5/xml.rb | 476 ++ lib/sisu/v5/xml_dom.rb | 552 ++ lib/sisu/v5/xml_fictionbook.rb | 306 ++ lib/sisu/v5/xml_format.rb | 1424 ++++++ lib/sisu/v5/xml_md_oai_pmh_dc.rb | 234 + lib/sisu/v5/xml_scaffold.rb | 205 + lib/sisu/v5/xml_tables.rb | 202 + lib/sisu/v5/zap.rb | 89 + 123 files changed, 65532 insertions(+) create mode 100644 lib/sisu/v5/air.rb create mode 100644 lib/sisu/v5/author_format.rb create mode 100644 lib/sisu/v5/cgi.rb create mode 100644 lib/sisu/v5/cgi_pgsql.rb create mode 100644 lib/sisu/v5/cgi_sql_common.rb create mode 100644 lib/sisu/v5/cgi_sqlite.rb create mode 100644 lib/sisu/v5/composite.rb create mode 100644 lib/sisu/v5/concordance.rb create mode 100644 lib/sisu/v5/conf.rb create mode 100644 lib/sisu/v5/constants.rb create mode 100644 lib/sisu/v5/css.rb create mode 100644 lib/sisu/v5/dal.rb create mode 100644 lib/sisu/v5/dal_character_check.rb create mode 100644 lib/sisu/v5/dal_doc_objects.rb create mode 100644 lib/sisu/v5/dal_doc_str.rb create mode 100644 lib/sisu/v5/dal_endnotes.rb create mode 100644 lib/sisu/v5/dal_expand_insertions.rb create mode 100644 lib/sisu/v5/dal_hash_digest.rb create mode 100644 lib/sisu/v5/dal_idx.rb create mode 100644 lib/sisu/v5/dal_images.rb create mode 100644 lib/sisu/v5/dal_metadata.rb create mode 100644 lib/sisu/v5/dal_numbering.rb create mode 100644 lib/sisu/v5/dal_substitutions_and_insertions.rb create mode 100644 lib/sisu/v5/dal_syntax.rb create mode 100644 lib/sisu/v5/db_columns.rb create mode 100644 lib/sisu/v5/db_create.rb create mode 100644 lib/sisu/v5/db_dbi.rb create mode 100644 lib/sisu/v5/db_drop.rb create mode 100644 lib/sisu/v5/db_import.rb create mode 100644 lib/sisu/v5/db_indexes.rb create mode 100644 lib/sisu/v5/db_load_tuple.rb create mode 100644 lib/sisu/v5/db_remove.rb create mode 100644 lib/sisu/v5/db_select.rb create mode 100644 lib/sisu/v5/db_sqltxt.rb create mode 100644 lib/sisu/v5/db_tests.rb create mode 100644 lib/sisu/v5/dbi.rb create mode 100644 lib/sisu/v5/dbi_discrete.rb create mode 100644 lib/sisu/v5/defaults.rb create mode 100644 lib/sisu/v5/digests.rb create mode 100644 lib/sisu/v5/embedded.rb create mode 100644 lib/sisu/v5/epub.rb create mode 100644 lib/sisu/v5/epub_concordance.rb create mode 100644 lib/sisu/v5/epub_format.rb create mode 100644 lib/sisu/v5/epub_segments.rb create mode 100644 lib/sisu/v5/epub_tune.rb create mode 100644 lib/sisu/v5/errors.rb create mode 100644 lib/sisu/v5/git.rb create mode 100644 lib/sisu/v5/harvest.rb create mode 100644 lib/sisu/v5/harvest_authors.rb create mode 100644 lib/sisu/v5/harvest_topics.rb create mode 100644 lib/sisu/v5/help.rb create mode 100644 lib/sisu/v5/html.rb create mode 100644 lib/sisu/v5/html_format.rb create mode 100644 lib/sisu/v5/html_minitoc.rb create mode 100644 lib/sisu/v5/html_promo.rb create mode 100644 lib/sisu/v5/html_scroll.rb create mode 100644 lib/sisu/v5/html_segments.rb create mode 100644 lib/sisu/v5/html_table.rb create mode 100644 lib/sisu/v5/html_tune.rb create mode 100644 lib/sisu/v5/hub.rb create mode 100644 lib/sisu/v5/i18n.rb create mode 100644 lib/sisu/v5/manifest.rb create mode 100644 lib/sisu/v5/manpage.rb create mode 100644 lib/sisu/v5/manpage_format.rb create mode 100644 lib/sisu/v5/odf.rb create mode 100644 lib/sisu/v5/odf_format.rb create mode 100644 lib/sisu/v5/options.rb create mode 100644 lib/sisu/v5/param.rb create mode 100644 lib/sisu/v5/param_identify_markup.rb create mode 100644 lib/sisu/v5/param_make.rb create mode 100644 lib/sisu/v5/particulars.rb create mode 100644 lib/sisu/v5/plaintext.rb create mode 100644 lib/sisu/v5/plaintext_format.rb create mode 100644 lib/sisu/v5/po4a.rb create mode 100644 lib/sisu/v5/po4a_set.rb create mode 100644 lib/sisu/v5/prog_text_translation.rb create mode 100644 lib/sisu/v5/qrcode.rb create mode 100644 lib/sisu/v5/relaxng.rb create mode 100644 lib/sisu/v5/remote.rb create mode 100644 lib/sisu/v5/response.rb create mode 100644 lib/sisu/v5/rexml.rb create mode 100644 lib/sisu/v5/screen_text_color.rb create mode 100644 lib/sisu/v5/share_src.rb create mode 100644 lib/sisu/v5/share_src_kdissert.rb create mode 100644 lib/sisu/v5/shared_html.rb create mode 100644 lib/sisu/v5/shared_html_lite.rb create mode 100644 lib/sisu/v5/shared_images.rb create mode 100644 lib/sisu/v5/shared_markup_alt.rb create mode 100644 lib/sisu/v5/shared_metadata.rb create mode 100644 lib/sisu/v5/shared_sem.rb create mode 100644 lib/sisu/v5/shared_sisupod_source.rb create mode 100644 lib/sisu/v5/shared_txt.rb create mode 100644 lib/sisu/v5/shared_xhtml.rb create mode 100644 lib/sisu/v5/shared_xml.rb create mode 100644 lib/sisu/v5/sisupod_make.rb create mode 100644 lib/sisu/v5/sitemaps.rb create mode 100644 lib/sisu/v5/spell.rb create mode 100644 lib/sisu/v5/sst_convert_markup.rb create mode 100644 lib/sisu/v5/sst_do_inline_footnotes.rb create mode 100644 lib/sisu/v5/sst_from_xml.rb create mode 100644 lib/sisu/v5/sst_identify_markup.rb create mode 100644 lib/sisu/v5/sst_to_s_xml_sax.rb create mode 100644 lib/sisu/v5/sysenv.rb create mode 100644 lib/sisu/v5/termsheet.rb create mode 100644 lib/sisu/v5/texinfo.rb create mode 100644 lib/sisu/v5/texinfo_format.rb create mode 100644 lib/sisu/v5/texpdf.rb create mode 100644 lib/sisu/v5/texpdf_format.rb create mode 100644 lib/sisu/v5/update.rb create mode 100644 lib/sisu/v5/urls.rb create mode 100644 lib/sisu/v5/utils.rb create mode 100644 lib/sisu/v5/webrick.rb create mode 100644 lib/sisu/v5/wikispeak.rb create mode 100644 lib/sisu/v5/xhtml.rb create mode 100644 lib/sisu/v5/xhtml_table.rb create mode 100644 lib/sisu/v5/xml.rb create mode 100644 lib/sisu/v5/xml_dom.rb create mode 100644 lib/sisu/v5/xml_fictionbook.rb create mode 100644 lib/sisu/v5/xml_format.rb create mode 100644 lib/sisu/v5/xml_md_oai_pmh_dc.rb create mode 100644 lib/sisu/v5/xml_scaffold.rb create mode 100644 lib/sisu/v5/xml_tables.rb create mode 100644 lib/sisu/v5/zap.rb (limited to 'lib/sisu/v5') diff --git a/lib/sisu/v5/air.rb b/lib/sisu/v5/air.rb new file mode 100644 index 00000000..cd5c2f68 --- /dev/null +++ b/lib/sisu/v5/air.rb @@ -0,0 +1,85 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +module SiSU_Air + require_relative 'particulars' # particulars.rb + class Source + @@dal_array=[] + @@fns=nil + def initialize(opt) + @opt=opt + @@fns||@opt.fns + @particulars=SiSU_Particulars::Combined.new(opt) + #@env=@particulars.env + #@md=@particulars.md + #@dal_array=@particulars.dal_array + end + def read + end + protected + def print + puts @particulars.md.inspect + puts @particulars.env.inspect + puts @particulars.dal_array + end + end +end +__END__ diff --git a/lib/sisu/v5/author_format.rb b/lib/sisu/v5/author_format.rb new file mode 100644 index 00000000..3a9d49a6 --- /dev/null +++ b/lib/sisu/v5/author_format.rb @@ -0,0 +1,116 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +module SiSU_FormatAuthor + class Author + def initialize(author_param) + @author_param=author_param + end + def author_details + @authors,@author_array=[],[] + authors=@author_param.scan(/[^;]+/) + authors.each do |a| + a=a.strip + if a =~/"(.+?)"/ + @authors << { the: $1 } + @author_array << $1.upcase + else #if a =~/,/ + x=a.scan(/[^,]+/) + x[0]=x[0].strip + x[1]=x[1].strip if x[1] + if x.length==1 + @authors << { the: x[0] } + @author_array << x[0].upcase + elsif x.length==2 + @authors << { the: x[0], others: x[1] } + @author_array << "#{x[0].upcase}, #{x[1]}" + else #p x.length + end + end + end + l = @authors.length + authors_string='' + @authors.each_with_index do |a,i| + authors_string += if a[:others] + if (l - i) > 1 + "#{a[:others]} #{a[:the]}, " + else + "#{a[:others]} #{a[:the]}" + end + else + if (l - i) > 2 + "#{a[:the]}, " + else + "#{a[:the]}" + end + end + end + { + last_first_a: authors, + last_first_format_a: @author_array, + authors_h: @authors, + authors_s: authors_string, + authors_param: @author_param + } + end + end +end +__END__ diff --git a/lib/sisu/v5/cgi.rb b/lib/sisu/v5/cgi.rb new file mode 100644 index 00000000..dd9cdf22 --- /dev/null +++ b/lib/sisu/v5/cgi.rb @@ -0,0 +1,78 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: generates naive cgi search form for search of sisu database + (pgsql sqlite) + +=end +module SiSU_CGI #% database building documents + require_relative 'sysenv' # sysenv.rb + require_relative 'cgi_pgsql' # cgi_pgsql.rb + require_relative 'cgi_sqlite' # cgi_sqlite.rb + class SearchSQL + def initialize(opt) + @opt=opt + @webserv=@opt.files[0].to_s.strip + end + def read + SiSU_CGI_SQLite::SearchSQLite.new(@opt,@webserv).sqlite + SiSU_CGI_PgSQL::SearchPgSQL.new(@opt,@webserv).pgsql + end + end +end +__END__ diff --git a/lib/sisu/v5/cgi_pgsql.rb b/lib/sisu/v5/cgi_pgsql.rb new file mode 100644 index 00000000..8d09f733 --- /dev/null +++ b/lib/sisu/v5/cgi_pgsql.rb @@ -0,0 +1,258 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: generates naive cgi search form for search of sisu database (pgsql sqlite) + +=end +module SiSU_CGI_PgSQL #% database building documents + require_relative 'sysenv' # sysenv.rb + require_relative 'cgi_sql_common' # cgi_sql_common.rb + include SiSU_CGI_SQL + class SearchPgSQL < CGI_Common + def initialize(opt,webserv) + @opt,@webserv=opt,webserv + @env=SiSU_Env::InfoEnv.new('',opt) + @sys=SiSU_Env::SystemCall.new + @db=SiSU_Env::InfoDb.new + get_init=SiSU_Env::GetInit.new + @rc=get_init.sisu_yaml.rc + @name_of={} + if defined? @rc['webserv'] \ + and defined? @rc['webserv']['url_root'] \ + and defined? @rc['webserv']['url_root'] =~/\S+/ + @name_of[:host_url_docs]=@rc['webserv']['url_root'] + end + @name_of[:output_dir_structure]=if defined? @rc['output_dir_structure_by'] \ + and defined? @rc['output_dir_structure_by'] =~/(language|filetype|filename)/ + @rc['output_dir_structure_by'] + else 'language' + end + if defined? @rc['search'] \ + and defined? @rc['search']['sisu'] \ + and defined? @rc['search']['sisu']['action'] \ + and @rc['search']['sisu']['action'] =~/https?:\/\/\S+?\.cgi/ \ + and defined? @rc['search']['sisu']['db'] \ + and @rc['search']['sisu']['db'] =~/\S+/ + @name_of[:db]=@rc['search']['sisu']['db'] + #@name_of[:cgi_script]=/https?:\/\/\S+?([^\/]+?)\.cgi$/.match(@rc['search']['sisu']['action'])[1] + @name_of[:host_url_cgi],@name_of[:cgi_script]=/(https?:\/\/\S+?)\/([^\/]+?)\.cgi$/.match(@rc['search']['sisu']['action'])[1,2] + else + @name_of[:host_url_docs]=@env.url.webserv_files_from_db + @name_of[:db]='sisu' #breaks if not present + @name_of[:host_url_cgi]=@env.url.webserv_base_cgi + @name_of[:cgi_script]='sisu_pgsql' + false + end + @cgi_file_name="#{Db[:name_prefix_db]}by_#{opt.dir_structure_by.to_s}_pgsql.cgi" + @cgi_link_name="#{@name_of[:db]}.cgi" #sisu_pgsql.cgi, + @image_src="#{@name_of[:host_url_docs]}/_sisu/image_sys" + @common=SiSU_CGI_SQL::CGI_Common.new(@webserv,@opt,@image_src,@env) + end + def pgsql + serve=[] + if @sys.psql + available_db_table=`psql --list` # system call requires psql + available_db=available_db_table.scan(/(#{Db[:name_prefix]}\S+)/) if not available_db_table.nil? + if available_db \ + and available_db.is_a?(Array) + available_db.flatten.each do |x| + serve << x.gsub(/#{Db[:name_prefix]}(\S+)/,'\1') + end + else STDERR.puts "WARNING: no postgresql database available, (have you created one?)" + end + serve=serve.sort + f1,f2,f3='','','' + serve.each do |x| + f1 << %{ \n} unless x =~/apache|sisu\/image/ #check + end + end + f2 << %q{ selected_db=%{}} + "\n" + if FileTest.writable?('.') + output=File.open(@cgi_file_name,'w') + output << header0 << header1 << header_desc << header2 << f1 << buttons1 << buttons1_pgsql << buttons2 << search_request << search_statement << search_statement_common << search_query1 << @common.pages << search_query2 << @common.tail << @common.main1 << f2 << dbi_connect << @common.main2 << @common.dir_structure << @common.main3 + a=%{ generated #{@cgi_file_name}, + BASED ON ALREADY EXISTING directories that could potentially be used to populate postgresql db, (-D) + } + SiSU_Screen::Ansi.new(@opt.cmd,a).print_grey + c=case @webserv + when /pwd/; '' + else "if necessary make the directory /usr/lib/cgi-bin : + sudo cp -vi #{Dir.pwd}/#{@cgi_file_name} /usr/lib/cgi-bin/. + sudo chmod -v 755 /usr/lib/cgi-bin/#{@cgi_file_name} + sudo ln -s /usr/lib/cgi-bin/#{@cgi_file_name} /usr/lib/cgi-bin/#{@cgi_link_name} + (copy #{@cgi_file_name} to your cgi directory) set file permissions to 755, and make symbolic link to #{@cgi_link_name}" + end + a=%{#{c} + #{@env.webserv_base_cgi}/cgi-bin/#{@cgi_link_name} + } + SiSU_Screen::Ansi.new(@opt.cmd,a).warn + a="postgresql db used for present directory: #{@db.psql.db}" + b="\n\t(to create and populate postgresql database see 'man sisu' and in particular the -D flag)\n\t[the database to be used for this directory (#{@db.psql.db}) will have to be created manually if it does not exist,\n\tsee 'sisu --help sql'\n\tif you have permission to create databases:\n\t'sisu -d --createdb'\n\tor using postgresql tools directly:\n\t'createdb #{@db.psql.db}'\n\tfor a list of existing databases try 'psql --list']" + SiSU_Screen::Ansi.new(@opt.cmd,a,b).txt_cyan + else puts 'failed in attempt to write #{@cgi_file_name} to present directory, is directory writable?' + end + end + def header0 + <<-WOK_SQL +#!/usr/bin/env ruby +=begin +#{about} + * Description: generates naive cgi search form for search of sisu database (pgsql) +#{gpl} +=end + require 'cgi' + require 'fcgi' + require 'dbi' + @stub_default="#{@name_of[:db]}" + @image_src="#{@name_of[:host_url_docs]}/_sisu/image_sys" + @hosturl_cgi="#{@name_of[:host_url_cgi]}" + @hosturl_files="#{@name_of[:host_url_docs]}" + @output_dir_structure_by="#{@name_of[:output_dir_structure]}" + @port="#{@db.psql.port}" + @db_name_prefix="#{Db[:name_prefix]}" + user='www-data' # check user name for access to pg database: e.g. www-data or '#{@env.user}' + WOK_SQL + end + def search_statement + <<-'WOK_SQL' + class DBI_SearchString + def initialize(l,t,q,cse=false) + @l,@t,@q,@c=l,t,q,cse + end + def string + search={ search: [], flag: false } + if @t =~/\S+/ or @q =~/\S+/ + if @t =~/\S+/ then unescaped_search=CGI.unescape(@t) + elsif @q =~/\S+/ then unescaped_search=CGI.unescape(@q) + end + search_construct=[] + unescaped_search=if @c + unescaped_search.gsub(/\s*(AND|OR)\s*/,"' \) \\1 #{@l}~\( '"). + gsub(/(.+)/,"#{@l}~\( '\\1' \)") + else + unescaped_search.gsub(/\s*(AND|OR)\s*/,"' \) \\1 #{@l}~*\( '"). + gsub(/(.+)/,"#{@l}~*\( '\\1' \)") + end + search_construct << unescaped_search + search_construct=search_construct.join(' ') + search[:search] << search_construct + search[:flag]=true + search + end + search + end + end + WOK_SQL + end + def search_query1 + <<-'WOK_SQL' + @search_text,@search_endnotes='','' + @search_text=search[:text].flatten.join(' AND ') + @search_text=@search_text.gsub(/(doc_objects\.clean~[*]?\(\s*'[^']+'\s*\)\s+(?:(?:AND|OR)\s+doc_objects\.clean~[*]?\(\s*'[^']+'\s*\))+)/,'(\1)') + @search_endnotes=search[:endnotes].flatten.join(' AND ') + @search_endnotes=@search_endnotes.gsub(/(endnotes\.clean~\(\s*'[^']+'\s*\)\s+(?:(?:AND|OR)\s+endnotes\.clean~\(\s*'[^']+'\s*\))+)/,'(\1)') + end + WOK_SQL + end + def search_query2 + <<-'WOK_SQL' + def sql_select_body + limit ||=@@limit + offset ||=@@offset + @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.language_document_char, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE (#{@search_text}) AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.language_document_char, metadata_and_text.title, metadata_and_text.src_filename, doc_objects.ocn} + @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} + select=@sql_statement[:body] + ' ' + @sql_statement[:range] + select + end + def sql_select_endnotes + limit ||=@@limit + offset ||=@@offset + @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.language_document_char, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE (#{@search_endnotes}) AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.language_document_char, metadata_and_text.title, metadata_and_text.src_filename, endnotes.nr} + @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} + select=@sql_statement[:endnotes] + ' ' + @sql_statement[:range] + select + end + def sql_select_body_format + %{#{sql_select_body}} + end + def sql_select_endnotes_format + %{#{sql_select_endnotes}} + end + def contents + @conn.select_all(sql_select_body) + end + def endnotes + @conn.select_all(sql_select_endnotes) + end + end + WOK_SQL + end + def buttons1_pgsql + <<-'WOK_SQL' + case sensitive + WOK_SQL + end + def dbi_connect + <<-'WOK_SQL' + dbi="dbi:Pg:database=#{@db};port=#{@port}" + @conn=DBI.connect(dbi,user) + WOK_SQL + end + end +end +__END__ diff --git a/lib/sisu/v5/cgi_sql_common.rb b/lib/sisu/v5/cgi_sql_common.rb new file mode 100644 index 00000000..5604d6cb --- /dev/null +++ b/lib/sisu/v5/cgi_sql_common.rb @@ -0,0 +1,1062 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: generates naive cgi search form for search of sisu database (pgsql sqlite) + +=end +module SiSU_CGI_SQL + class CGI_Common + def initialize(webserv,opt,image_src,dir) + @webserv,@opt,@image_src,@env=webserv,opt,image_src,dir + @cmd=opt.cmd + end + def about + <<-'WOK_SQL' + * Name: SiSU information Structuring Universe + * Author: Ralph Amissah + * http://www.jus.uio.no/sisu + * http://www.jus.uio.no/sisu/SiSU/download + WOK_SQL + end + def gpl + <<-'WOK_SQL' + * Name: SiSU generated sample cgi search form + + * Description: generated sample cgi search form for SiSU + (SiSU is a framework for document structuring, publishing and search) + + * Author: Ralph Amissah + + * Copyright: (C) 1997 - 2013, Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Ralph Amissah + + + WOK_SQL + end + def header1 + <<-'WOK_SQL' +#Common TOP + @@offset=0 + @base="#{@hosturl_cgi}/#{@stub_default}.cgi" #fix sqlite + @@canned_search_url=@base + @color_heading='#DDFFAA' + @color_match='#ffff48' + class Form + def initialize(base,search_field,selected_db,result_type,checked_sql_limit,checked_tip,checked_stats,checked_searched,checked_url,checked_case,checked_echo,checked_sql,checked_all,checked_none,checked_selected,checked_default,search_note,the_can='') + search_note='' if checked_searched !~/\S/ + the_can='' if checked_url !~/\S/ + search_field='' if checked_echo !~/\S/ + @base,@search_field,@selected_db,@result_type,@checked_sql_limit,@checked_tip,@checked_stats,@checked_searched,@checked_url,@checked_case,@checked_echo,@checked_sql,@checked_all,@checked_none,@checked_selected,@checked_default,@search_note,@the_can=base,search_field,selected_db,result_type,checked_sql_limit,checked_tip,checked_stats,checked_searched,checked_url,checked_case,checked_echo,checked_sql,checked_all,checked_none,checked_selected,checked_default,search_note,the_can + @tip=if checked_tip =~/\S/ + 'text:__; fulltxt:__; keywords:__; title:__; author:__; topic_register:__; subject:__; description:__; publisher:__; editor:__; contributor:__; date:__; type:__; format:__; identifier:__; source:__; language:__; relation:__; coverage:__; rights:__; comment:__; abstract:__; filename:__;
' + else '' + end + end + def submission_form + search_form=<<-WOK + WOK_SQL + end + def header_desc + <<-WOK_SQL + + + + + SiSU search form (sample): SiSU information Structuring Universe + + + + + + + + + + +
SiSU --> + +
+ WOK_SQL + end + def header2 + <<-'WOK_SQL' +
+ + +
+ + + #{@tip} + #{@search_note} + #{@the_can} +
+ + + + + to search: select which database to search (drop-down menu below); enter your search query (in the form above); and click on the search button (below) +
+ + + index + text / grep + WOK_SQL + end + def buttons2 + <<-'WOK_SQL' +
+ match limit: + 1,000 + 2,500 +
+ echo query + result stats + search url + searched + available fields + sql statement +
+ checks: + default + selected + all + none + + + +
+ WOK + end + end + WOK_SQL + end + def search_request + <<-'WOK_SQL' + class SearchRequest #% search_for + attr_accessor :text1,:fulltext,:keywords,:title,:author,:topic_register,:subject,:description,:publisher,:editor,:contributor,:date,:type,:format,:identifier,:source,:language,:relation,:coverage,:rights,:comment,:abstract,:owner,:date_created,:date_issued,:date_modified,:date_available,:date_valid,:filename + def initialize(search_field='',q='') + @search_field,@q=search_field,q + @text1=@fulltext=@keywords=@title=@author=@topic_register=@subject=@description=@publisher=@editor=@contributor=@date=@type=@format=@identifier=@source=@language=@relation=@coverage=@rights=@comment=@abstract=@owner=@date_created=@date_issued=@date_modified=@date_available=@date_valid=@filename='' + if @search_field=~/\S/ + @text1=text_to_match('text:') + @fulltext=text_to_match('fulltxt:') + @topic_register=text_to_match('topic_register:') + @title=text_to_match('title:') # DublinCore 1 - title + @author=text_to_match('(?:author|creator)s?:') # DublinCore 2 - creator/author + @subject=text_to_match('subj(?:ect)?:') # DublinCore 3 - subject + @description=text_to_match('description:') # DublinCore 4 - description + @publisher=text_to_match('pub(?:lisher)?:') # DublinCore 5 - publisher + @editor=text_to_match('editor:') + @contributor=text_to_match('contributor:') # DublinCore 6 - contributor + @date=text_to_match('date:') # DublinCore 7 - date dd-mm-yy + @type=text_to_match('type:') # DublinCore 8 - type + @format=text_to_match('format:') # DublinCore 9 - format + @identifier=text_to_match('identifier:') # DublinCore 10 - identifier + @source=text_to_match('source:') # DublinCore 11 - source + @language=text_to_match('language:') # DublinCore 12 - language + @relation=text_to_match('relation:') # DublinCore 13 - relation + @coverage=text_to_match('coverage:') # DublinCore 14 - coverage + @rights=text_to_match('rights:') # DublinCore 15 - rights + @keywords=text_to_match('key(?:words?)?:') + @comment=text_to_match('comment:') + @abstract=text_to_match('abs(?:tract)?:') + @owner=text_to_match('owner:') + @date_created=text_to_match('date_created:') + @date_issued=text_to_match('date_issued:') + @date_modified=text_to_match('date_modified:') + @date_available=text_to_match('date_available:') + @date_valid=text_to_match('date_valid:') + @filename=text_to_match('filename:') + @text1=text_to_match unless @keywords or @author or @title or @text1 or @fulltext or @comment or @abstract or @rights or @subject or @publisher or @date or @filename or @topic_register + else + @text1=q['s1'] if q['s1']=~/\S/ + @fulltext=q['ft'] if q['ft']=~/\S/ + @keywords=q['key'] if q['key']=~/\S/ + @title=q['ti'] if q['ti']=~/\S/ + @author=q['au'] if q['au']=~/\S/ + @topic_register=q['tr'] if q['tr']=~/\S/ + @subject=q['sj'] if q['sj']=~/\S/ + @description=q['dsc'] if q['dsc']=~/\S/ + @publisher=q['pb'] if q['pb']=~/\S/ + @editor=q['cntr'] if q['cntr']=~/\S/ + @contributor=q['cntr'] if q['cntr']=~/\S/ + @date=q['dt'] if q['dt']=~/\S/ + @type=q['ty'] if q['ty']=~/\S/ + @identifier=q['id'] if q['id']=~/\S/ + @source=q['src'] if q['src']=~/\S/ + @language=q['lang'] if q['lang']=~/\S/ + @relation=q['rel'] if q['rel']=~/\S/ + @coverage=q['cov'] if q['cov']=~/\S/ + @rights=q['cr'] if q['cr']=~/\S/ + @comment=q['co'] if q['co']=~/\S/ + @abstract=q['ab'] if q['ab']=~/\S/ + @date_created=q['dtc'] if q['dtc']=~/\S/ + @date_issued=q['dti'] if q['dti']=~/\S/ + @date_modified=q['dtm'] if q['dtm']=~/\S/ + @date_available=q['dta'] if q['dta']=~/\S/ + @date_valid=q['dtv'] if q['dtv']=~/\S/ + @filename=if q['doc'] and q['search'] !~/search db/ then q['doc'] + elsif q['fns']=~/\S/ then q['fns'] + end + @@limit=q['ltd'] if q['ltd']=~/\d+/ # 1000 + @@offset=q['off'] if q['off']=~/\d+/ # 0 + end + end + def text_to_match(identifier='') + m={ + string: /#{identifier}\s*(.+?)/, + string: /#{identifier}\s*(.+?)(?:;|\n|\r|$)/, + word: /#{identifier}[\s(]*(\S+)/ + } + search_string=if @search_field =~m[:word] + search_string=if @search_field =~m[:braces] then m[:braces].match(@search_field)[1] + elsif @search_field =~m[:string] then m[:string].match(@search_field)[1] + else + str=m[:word].match(@search_field)[1] + str=str.gsub(/[()]/,'') + str + end + search_string=search_string.strip.gsub(/\s+/,'+') + #else + # "__" + end + end + end + WOK_SQL + end + def search_statement_common + <<-'WOK_SQL' + class DBI_SearchStatement + attr_reader :text_search_flag,:sql_select_body_format,:sql_select_endnotes_format,:sql_offset,:sql_limit + def initialize(conn,search_for,q,c) + @conn=conn + @text_search_flag=false + @sql_statement={ body: '', endnotes: '', range: '' } + #@offset||=@@offset + #@offset+=@@limit + search={ text: [], endnotes: [] } + cse=(c =~/\S/) ? true : false + st=DBI_SearchString.new('doc_objects.clean',search_for.text1,q['s1'],cse).string + se=DBI_SearchString.new('endnotes.clean',search_for.text1,q['s1'],cse).string + @text_search_flag=st[:flag] + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << se[:search] + end + st=DBI_SearchString.new('metadata_and_text.fulltext',search_for.fulltext,q['ft'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.title',search_for.title,q['ti'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.creator_author',search_for.author,q['au'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.classify_topic_register',search_for.topic_register,q['tr'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.classify_subject',search_for.subject,q['sj'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.classify_keywords',search_for.keywords,q['key'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_description',search_for.description,q['dsc'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.publisher',search_for.publisher,q['pb'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.creator_editor',search_for.editor,q['cntr'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.creator_contributor',search_for.contributor,q['cntr'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.date_published',search_for.date,q['dt'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_type',search_for.type,q['ty'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.original_source',search_for.source,q['src'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.language_document_char',search_for.language,q['lang'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_relation',search_for.relation,q['rel'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_coverage',search_for.coverage,q['cov'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.rights_all',search_for.rights,q['cr'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_comment',search_for.comment,q['co'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.notes_abstract',search_for.abstract,q['ab'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + st=DBI_SearchString.new('metadata_and_text.src_filename',search_for.filename,q['fns'],cse).string + if st[:flag] + search[:text] << st[:search] + search[:endnotes] << st[:search] + end + @@limit=q['ltd'] if q['ltd']=~/\d+/ # 1000 + @@offset=q['off'] if q['off']=~/\d+/ # 0 + WOK_SQL + end + def pages + <<-'WOK_SQL' + def sql_offset + @@offset + end + def sql_match_limit + @@limit + end + def sql_canned_search + @offset_next=sql_offset.to_i + sql_match_limit.to_i + @offset_previous=sql_offset.to_i - sql_match_limit.to_i + def current + @@canned_search_url.to_s + '<d=' + sql_match_limit.to_s + '&off=' + sql_offset.to_s + end + def next + @@canned_search_url.to_s + '<d=' + sql_match_limit.to_s + '&off=' + @offset_next.to_s + end + def previous + @offset_previous >= 0 \ + ? (@@canned_search_url.to_s + '<d=' + sql_match_limit.to_s + '&off=' + @offset_previous.to_s) + : '' + end + def start + @@canned_search_url.to_s + '<d=' + sql_match_limit.to_s + '&off=' + 0.to_s + end + self + end + def pre_next(beyond_limit,img) + can=sql_canned_search + page=(sql_offset.to_i + sql_match_limit.to_i)/sql_match_limit.to_i + if beyond_limit + if page.to_s =~ /^1$/ + %{
+ pg. #{page.to_s} + +  >> + +
} + elsif page.to_s =~ /^2$/ + %{
+ + <<  + + pg. #{page.to_s} + +  >> + +
} + else + %{
+ + |<  + + + <<  + + pg. #{page.to_s} + +  >> + +
} + end + else + if page.to_s =~ /^1$/ then '' + elsif page.to_s =~ /^2$/ + %{
+ + <<  + + pg. #{page.to_s} +
} + else + %{
+ + |<  + + + <<  + + pg. #{page.to_s} +
} + end + end + end + WOK_SQL + end + def tail + v=SiSU_Env::InfoVersion.instance.get_version + <<-WOK_SQL + def tail + <<-'WOK' +

+ + + + + +
+

+ + SiSU + +

+		 + +

+ Generated by + + #{v[:project]} #{v[:version]} #{v[:date]} (#{v[:date_stamp]}) +
+ + #{v[:project]} © Ralph Amissah + 1993, current 2013. + All Rights Reserved. +
+ + #{v[:project]} + is software for document structuring, publishing and search, +
+ + www.jus.uio.no/sisu + + and + + www.sisudoc.org + +
+ w3 since October 3 1993 + + ralph@amissah.com + +
+ mailing list subscription + + http://lists.sisudoc.org/listinfo/sisu + +
+ + sisu@lists.sisudoc.org + +

+		 +

+ #{v[:project]} using: +
Standard SiSU markup syntax, +
Standard SiSU meta-markup syntax, and the +
Standard SiSU object citation numbering and system, (object/text positioning system) +
+ © Ralph Amissah 1997, current 2013. + All Rights Reserved. +

+
+

+ + GPLv3 + +

+		 +

+ SiSU is released under + GPL v3 + or later, + + http://www.gnu.org/licenses/gpl.html + +

+		 +

+ #{v[:project]}, developed using + + Ruby + + on + + Debian/Gnu/Linux + + software infrastructure, + with the usual GPL (or OSS) suspects. +

+
+ + + WOK + end + WOK_SQL + end + def main1 + <<-'WOK_SQL' + @tail=tail + @counter_txt_doc,@counter_txt_ocn,@counter_endn_doc,@counter_endn_ocn=0,0,0,0 + @counters_txt,@counters_endn,@sql_select_body,@sql_select_endnotes='','','','' + FCGI.each_cgi do |cgi| + begin # all code goes in begin section + @search={ text: [], endnotes: [] } + q=CGI.new + @db=if cgi['db'] =~ /#{@db_name_prefix}(\S+)/ + @stub=$1 + cgi['db'] + else + @stub=@stub_default + @db_name_prefix + @stub + end + checked_url,checked_stats,checked_searched,checked_tip,checked_case,checked_echo,checked_sql,checked_all,checked_none,checked_selected,checked_default,selected_db='','','','','','','','','' + result_type=(cgi['view']=~/text/) \ + ? result_type={ index: '', text: 'checked'} + : result_type={ index: 'checked', text: ''} + @@limit=if cgi['sql_match_limit'].to_s=~/2500/ + checked_sql_limit={ l1000: '', l2500: 'checked'} + '2500' + else + checked_sql_limit={ l1000: 'checked', l2500: ''} + '1000' + end + checked_echo='checked' if cgi['echo'] =~/\S/ + checked_stats='checked' if cgi['stats'] =~/\S/ + checked_url='checked' if cgi['url'] =~/\S/ or cgi['u'].to_i==1 + checked_searched='checked' if cgi['searched'] =~/\S/ + checked_tip='checked' if cgi['tip'] =~/\S/ + checked_case='checked' if cgi['casesense'] =~/\S/ + checked_sql='checked' if cgi['sql'] =~/\S/ + if cgi['checks'] =~/check_all/ or cgi['check_all'] =~/\S/ or cgi['a'].to_i==1 + checked_all='checked' + checked_echo=checked_stats=checked_url=checked_searched=checked_tip=checked_sql='checked' + checked_none='' + elsif cgi['checks'] =~/check_none/ + checked_none='checked' + checked_all=checked_url=checked_stats=checked_searched=checked_tip=checked_echo=checked_sql='' + elsif cgi['checks'] =~/check_selected/ + checked_selected='checked' + elsif cgi['checks'] =~/check_default/ + checked_default='checked' + checked_echo=checked_stats=checked_url='checked' + checked_searched=checked_tip=checked_case=checked_sql='' + else + checked_selected='checked' + checked_echo=checked_stats=checked_url='checked' + checked_searched=checked_tip=checked_case=checked_sql='' + end + WOK_SQL + end + def main2 + <<-'WOK_SQL' + search_field=cgi['find'] if cgi['find'] # =~/\S+/ + @search_for=SearchRequest.new(search_field,q) #.analyze #% search_for + #% searches + #Canned_search.new(@base,@search_for.text1,cgi) + if @search_for.text1=~/\S+/ or @search_for.fulltext=~/\S+/ or @search_for.author=~/\S+/ or @search_for.topic_register=~/\S+/ #and search_field =~/\S/ + s1='s1=' + CGI.escape(@search_for.text1) if @search_for.text1=~/\S/ + ft='&ft=' + CGI.escape(@search_for.fulltext) if @search_for.fulltext=~/\S/ + key='key=' + CGI.escape(@search_for.keywords) if @search_for.keywords=~/\S/ + ti='&ti=' + CGI.escape(@search_for.title) if @search_for.title=~/\S/ + au='&au=' + CGI.escape(@search_for.author) if @search_for.author=~/\S/ + tr='&tr=' + CGI.escape(@search_for.topic_register) if @search_for.topic_register=~/\S/ + sj='&sj=' + CGI.escape(@search_for.subject) if @search_for.subject=~/\S/ + dsc='&dsc=' + CGI.escape(@search_for.description) if @search_for.description=~/\S/ + pb='&pb=' + CGI.escape(@search_for.publisher) if @search_for.publisher=~/\S/ + edt='&edt=' + CGI.escape(@search_for.editor) if @search_for.editor=~/\S/ + cntr='&cntr=' + CGI.escape(@search_for.contributor) if @search_for.contributor=~/\S/ + dt='&dt=' + CGI.escape(@search_for.date) if @search_for.date=~/\S/ + ty='&ty=' + CGI.escape(@search_for.type) if @search_for.type=~/\S/ + id='&id=' + CGI.escape(@search_for.identifier) if @search_for.identifier=~/\S/ + src='&src=' + CGI.escape(@search_for.source) if @search_for.source=~/\S/ + lang='&lang=' + CGI.escape(@search_for.language) if @search_for.language=~/\S/ + rel='&rel=' + CGI.escape(@search_for.relation) if @search_for.relation=~/\S/ + cov='&cov=' + CGI.escape(@search_for.coverage) if @search_for.coverage=~/\S/ + cr='&cr=' + CGI.escape(@search_for.rights) if @search_for.rights=~/\S/ + co='&co=' + CGI.escape(@search_for.comment) if @search_for.comment=~/\S/ + ab='&ab=' + CGI.escape(@search_for.abstract) if @search_for.abstract=~/\S/ + dtc='&dtc=' + CGI.escape(@search_for.date_created) if @search_for.date_created=~/\S/ + dti='&dti=' + CGI.escape(@search_for.date_issued) if @search_for.date_issued=~/\S/ + dtm='&dtm=' + CGI.escape(@search_for.date_modified) if @search_for.date_modified=~/\S/ + dta='&dta=' + CGI.escape(@search_for.date_available) if @search_for.date_available=~/\S/ + dtv='&dtv=' + CGI.escape(@search_for.date_valid) if @search_for.date_valid=~/\S/ + fns='&fns=' + CGI.escape(@search_for.filename) if @search_for.filename=~/\S/ + @@canned_search_url=(checked_all =~/checked/) \ + ? "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{edt}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}&a=1" + : "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{edt}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}" + mod=ft=~/\S+/ ? (ft.gsub(/ft/,'s1')) : s1 + @canned_base_url="#{@base}?#{mod}&db=#{cgi['db']}" + if checked_case=~/\S/ + @search[:text][1]=%{doc_objects.clean~'#{@search_for.text1}'} #s1 + @search[:endnotes][1]=%{endnotes.clean~'#{@search_for.text1}'} #s1 + else + @search[:text][1]=%{doc_objects.clean~*'#{@search_for.text1}'} #s1 + @search[:endnotes][1]=%{endnotes.clean~*'#{@search_for.text1}'} #s1 + end + canned_note='search url:' + else + @@canned_search_url="#{@base}?db=#{@db}&view=index" + canned_note='search url example:' + end + if search_field =~/\S+/ + analyze_format=search_field.gsub(/\s*\n/,'; ') + elsif checked_all =~/checked/ or checked_url =~/checked/ + canned_search=@@canned_search_url.scan(/(?:s1|ft|au|ti|fns|tr)=[^&]+/) + af=canned_search.join('; ') + af=af.gsub(/s1=/,'text: '). + gsub(/ft=/,'fulltxt: '). + gsub(/au=/,'author: '). + gsub(/ti=/,'title: '). + gsub(/fns=/,'filename: '). + gsub(/tr=/,'topic_register: '). + gsub(/%2B/,' ') + analyze_format=af + st=af.split(/\s*;\s*/) + search_field=st.join("\n") + end + green=%{} + canned_search_url_txt=CGI.escapeHTML(@@canned_search_url) + the_can=%{#{canned_note} #{canned_search_url_txt}
} + p_text=p_fulltext=p_keywords=p_title=p_author=p_topic_register=p_subject=p_description=p_publisher=p_editor=p_contributor=p_date=p_type=p_format=p_identifier=p_source=p_language=p_relation=p_coverage=p_rights=p_comment=p_abstract=p_filename='' + p_filename=%{filename: #{green}#{@search_for.filename}
} if @search_for.filename =~/\S+/ + p_text=%{text: #{green}#{@search_for.text1}
} if @search_for.text1 =~/\S+/ + p_fulltext=%{fulltxt: #{green}#{@search_for.fulltext}
} if @search_for.fulltext =~/\S+/ + p_title=%{title: #{green}#{@search_for.title}
} if @search_for.title =~/\S+/ + p_author=%{author: #{green}#{@search_for.author}
} if @search_for.author =~/\S+/ + p_editor=%{editor: #{green}#{@search_for.editor}
} if @search_for.editor=~/\S+/ + p_contributor=%{contributor: #{green}#{@search_for.contributor}
} if @search_for.contributor =~/\S+/ + p_date=%{date: #{green}#{@search_for.date}
} if @search_for.date =~/\S+/ + p_rights=%{rights: #{green}#{@search_for.rights}
} if @search_for.rights =~/\S+/ + p_topic_register=%{topic_register: #{green}#{@search_for.topic_register}
} if @search_for.topic_register =~/\S+/ + p_subject=%{subject: #{green}#{@search_for.subject}
} if @search_for.subject =~/\S+/ + p_keywords=%{keywords: #{green}#{@search_for.keywords}
} if @search_for.keywords =~/\S+/ + p_identifier=%{identifier: #{green}#{@search_for.identifier}
} if @search_for.identifier =~/\S+/ + p_type=%{type: #{green}#{@search_for.type}
} if @search_for.type =~/\S+/ + p_format=%{format: #{green}#{@search_for.format}
} if @search_for.format =~/\S+/ + p_relation=%{relation: #{green}#{@search_for.relation}
} if @search_for.relation =~/\S+/ + p_coverage=%{coverage: #{green}#{@search_for.coverage}
} if @search_for.coverage =~/\S+/ + p_description=%{description: #{green}#{@search_for.description}
} if @search_for.description =~/\S+/ + p_abstract=%{abstract: #{green}#{@search_for.abstract}
} if @search_for.abstract =~/\S+/ + p_comment=%{comment: #{green}#{@search_for.comment}
} if @search_for.comment =~/\S+/ + p_publisher=%{publisher: #{green}#{@search_for.publisher}
} if @search_for.publisher =~/\S+/ + p_source=%{source: #{green}#{@search_for.source}
} if @search_for.source =~/\S+/ + p_language=%{language: #{green}#{@search_for.language}
} if @search_for.language =~/\S+/ + search_note=<<-WOK + + database: #{green}#{@db}; selected view: #{green}#{cgi['view']} + search string: "#{green}#{analyze_format}"
+ #{p_text} #{p_fulltext} #{p_keywords} #{p_title} #{p_author} #{p_topic_register} #{p_subject} #{p_description} #{p_publisher} #{p_editor} #{p_contributor} #{p_date} #{p_type} #{p_format} #{p_identifier} #{p_source} #{p_language} #{p_relation} #{p_coverage} #{p_rights} #{p_comment} #{p_abstract} #{p_filename} + + WOK + #eg = %{canned search e.g.:
#{url}
find: #{analyze}
database: #{database}} + #% dbi_canning + @header=Form.new(@base,search_field,selected_db,result_type,checked_sql_limit,checked_tip,checked_stats,checked_searched,checked_url,checked_case,checked_echo,checked_sql,checked_all,checked_none,checked_selected,checked_default,search_note,the_can).submission_form #% form + unless q['s1'] =~/\S/ or q['au'] =~/\S/ or @search[:text][1] =~/\S/ + print "Content-type: text/html\n\n" + puts (@header+@tail) + else #% searches + s1=(@search_for.text1 =~/\S/) \ + ? @search_for.text1 + : 'Unavailable' + if checked_case=~/\S/ + @search[:text]<<%{doc_objects.clean~'#{CGI.unescape(s1)}'} + @search[:endnotes]<<%{endnotes.clean~'#{CGI.unescape(s1)}'} + else + @search[:text]<<%{doc_objects.clean~*'#{CGI.unescape(s1)}'} + @search[:endnotes]<<%{endnotes.clean~*'#{CGI.unescape(s1)}'} + end + #% dbi_request + dbi_statement=DBI_SearchStatement.new(@conn,@search_for,q,checked_case) + @text_search_flag=false + @text_search_flag=dbi_statement.text_search_flag + s_contents=dbi_statement.contents + s_endnotes=dbi_statement.endnotes + @body_main,@endnotes='','' + @search_regx=nil + oldtid=0 + if @text_search_flag + if checked_sql =~/\S/ + sql_select_body=dbi_statement.sql_select_body_format + sql_select_endnotes=dbi_statement.sql_select_endnotes_format + else sql_select_body,sql_select_endnotes='','' + end + @body_main << '


Main Text:
' << sql_select_body + @endnotes << '


Endnotes:
' << sql_select_endnotes + else + end + @hostpath="#{@hosturl_files}/#{@stub}" + WOK_SQL + end + def dir_structure + case @opt.dir_structure_by + when :language + <<-'WOK_SQL' + def path_manifest(fn,ln=nil) + "#{@hostpath}/#{ln}/manifest/#{fn}.html" + end + def path_html_seg(fn,ln=nil) + "#{@hostpath}/#{ln}/html/#{fn}" + end + def path_toc(fn,ln=nil) + "#{path_html_seg(fn,ln)}/toc.html" + end + def path_filename(fn,seg,ln=nil) + "#{path_html_seg(fn,ln)}/#{seg}.html" + end + def path_endnotes(fn,ln=nil) + "#{path_html_seg(fn,ln)}/endnotes.html" + end + def path_html_doc(fn,ln=nil) + "#{@hostpath}/#{ln}/html/#{fn}.html" + end + WOK_SQL + when :filetype + <<-'WOK_SQL' + def path_manifest(fn,ln=nil) + "#{@hostpath}/manifest/#{fn}.#{ln}.html" + end + def path_html_seg(fn,ln=nil) + "#{@hostpath}/html/#{fn}" + end + def path_toc(fn,ln=nil) + "#{path_html_seg(fn,ln)}/toc.#{ln}.html" + end + def path_filename(fn,seg,ln=nil) + "#{path_html_seg(fn,ln)}/#{seg}.#{ln}.html" + end + def path_endnotes(fn,ln=nil) + "#{path_html_seg(fn,ln)}/endnotes.#{ln}.html" + end + def path_html_doc(fn,ln=nil) + "#{@hostpath}/html/#{fn}.#{ln}.html" + end + WOK_SQL + else + <<-'WOK_SQL' + def path_manifest(fn,ln=nil) + "#{@hostpath}/#{fn}/sisu_manifest.#{ln}.html" + end + def path_html_seg(fn,ln=nil) + "#{@hostpath}/#{fn}" + end + def path_toc(fn,ln=nil) + "#{path_html_seg(fn,ln)}/toc.#{ln}.html" + end + def path_filename(fn,seg,ln=nil) + "#{path_html_seg(fn,ln)}/#{seg}.#{ln}.html" + end + def path_endnotes(fn,ln=nil) + "#{path_html_seg(fn,ln)}/endnotes.#{ln}.html" + end + def path_html_doc(fn,ln=nil) + "#{path_html_seg(fn,ln)}/scroll.#{ln}.html" + end + WOK_SQL + end + end + def main3 + <<-'WOK_SQL' + #% text_objects_body + s_contents.each do |c| #% text body + location=c['src_filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] + file_suffix=c['src_filename'][/.+?\.(_?sst|ssm)$/,1] + lang=if location =~ /\S+?~(\S\S\S?)$/ + l=location[/\S+?~(\S\S\S?)$/,1] + location=location.gsub(/(\S+?)~\S\S\S?/,'\1') + l=".#{l}" + else '' + end + #% metadata_found_body + if c['tid'].to_i != oldtid.to_i + ti=c['title'] + can_txt_srch=(cgi['view']=~/index/) \ + ? %{search } + : %{search } + title=%{toc html #{ti} [#{c['language_document_char']}] by #{c['creator_author']} manifest #{can_txt_srch}
} if file_suffix=~/s/ #hmm watch file_suffix + title=@text_search_flag \ + ? '
'+title + : '
'+title + @counter_txt_doc+=1 + oldtid=c['tid'].to_i + else title='' + end + if @text_search_flag + if cgi['view']=~/text/ \ + or (cgi['view']!~/index/ and cgi['search'] !~/search db/) #% txt body + text=if c['suffix'] !~/1/ #seg + if @search_for.text1 =~/\S+/ \ + or q['s1'] =~/\S+/ #% only this branch is working !! + unescaped_search=if @search_for.text1 =~/\S+/ + CGI.unescape(@search_for.text1) + elsif q['s1'] =~/\S+/ + CGI.unescape(q['s1']) + else nil + end + @search_regx=if unescaped_search #check + search_regex=[] + build=unescaped_search.scan(/\S+/).each do |g| + (g.to_s =~/(AND|OR)/) \ + ? (search_regex << '|') + : (search_regex << %{#{g.to_s}}) + end + search_regex=search_regex.join(' ') + search_regex=search_regex.gsub(/\s*\|\s*/,'|') + Regexp.new(search_regex, Regexp::IGNORECASE) + else nil + end + else nil + end + matched_para=(@search_regx.to_s.class==String && @search_regx.to_s=~/\S\S+/) \ + ? (c['body'].gsub(/(<\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{\\1})) + : c['body'] + %{

ocn #{c['ocn']}:

#{matched_para}} + elsif c['suffix'] =~/1/ #doc + %{#{title}

ocn #{c['ocn']}:#{c['body']}} + end + @counter_txt_ocn+=1 + output=title+text + else #elsif cgi['view']=~/index/ #% idx body + if c['suffix'] !~/1/ #seg + index=%{#{c['ocn']}, } if @text_search_flag + elsif c['suffix'] =~/1/ #doc #FIX + index=%{#{c['ocn']}, } + end + if c['seg'] =~/\S+/ + if @text_search_flag + @counter_txt_ocn+=1 + output=title+index + end + else + @counter_txt_ocn+=1 + output=c['suffix'] !~/1/ \ + ? title+index + : %{#{title}#{c['ocn'].sort}, } + end + end + else output=title + end + @counters_txt=if @counter_txt_doc > 0 + if checked_stats =~/\S/ + @@lt_t=(@counter_txt_ocn==dbi_statement.sql_match_limit.to_i) ? true : false + start=(@@offset.to_i+1).to_s + range=(@@offset.to_i+@counter_txt_ocn.to_i).to_s + %{

Found #{@counter_txt_ocn} times in the main body of #{@counter_txt_doc} documents [ matches #{start} to #{range} ]
} + else '' + end + else '' + end + @body_main << output #+ details + end + #text_objects_endnote + oldtid = 0 + s_endnotes.each do |e| #% endnotes + location=e['src_filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] + file_suffix=e['src_filename'][/.+?\.(_?sst|ssm)$/,1] + lang=if location =~ /\S+?~(\S\S\S?)$/ + l=location[/\S+?~(\S\S\S?)$/,1] + location=location.gsub(/(\S+?)~\S\S\S?/,'\1') + l=".#{l}" + else '' + end + #metadata_found_endnotes + if @text_search_flag + if e['metadata_tid'].to_i != oldtid.to_i + ti=e['title'] + can_txt_srch=(cgi['view']=~/index/) \ + ? %{search } + : %{search } + title=%{
toc html #{ti} [#{e['language_document_char']}] by #{e['creator_author']} #{can_txt_srch}toc html manifest
} if file_suffix=~/s/ + @counter_endn_doc+=1 + oldtid=e['metadata_tid'].to_i + else title = '' + end + if cgi['view']=~/text/ \ + or (cgi['view']!~/index/ and cgi['search'] !~/search db/) #% txt endnotes + @counter_endn_ocn+=1 + matched_endnote=(@search_regx.to_s.is_a?(String) \ + && @search_regx.to_s=~/\S\S+/) \ + ? (matched=e['body'].gsub(/(<\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{\\1})) + : e['body'] + output=%{#{title}
note #{e['nr']} referred to from ocn #{e['ocn']}: #{matched_endnote}} + else #elsif cgi['view']=~/index/ #doc #FIX #% idx endnotes + @counter_endn_ocn+=1 + output=%{#{title}#{e['nr']} [§ #{e['ocn']}], } + end + @counters_endn=if @counter_endn_doc > 0 + if checked_stats =~/\S/ + @@lt_e=@counter_endn_ocn==dbi_statement.sql_match_limit.to_i \ + ? true + : false + start=(@@offset.to_i+1).to_s + range=(@@offset.to_i+@counter_endn_ocn.to_i).to_s + %{Found #{@counter_endn_ocn} times in the endnotes of #{@counter_endn_doc} documents [ matches #{start} to #{range} ]
} + else '' + end + end + @endnotes << output #+ details + else @endnotes='' #does not take out yet + end + end + offset=dbi_statement.sql_offset.to_s + limit=dbi_statement.sql_match_limit.to_s + @@lt_t ||=false; @@lt_e ||=false + canned=(@@lt_t or @@lt_e) \ + ? dbi_statement.pre_next(true,@image_src).to_s + : dbi_statement.pre_next(false,@image_src).to_s + limit=dbi_statement.sql_match_limit.to_s + cgi.out{@header.force_encoding("UTF-8") + @counters_txt.force_encoding("UTF-8") + @counters_endn.force_encoding("UTF-8") + canned.force_encoding("UTF-8") + @body_main.force_encoding("UTF-8") + @endnotes.force_encoding("UTF-8") + canned.force_encoding("UTF-8") + @tail.force_encoding("UTF-8")} #% print cgi_output_header+counters+body+endnotes + end + rescue Exception => e + s='
' + CGI::escapeHTML(e.backtrace.reverse.join("\n"))
+          s << CGI::escapeHTML(e.message) + '
' + cgi.out{s} + next + ensure # eg. disconnect from server + @conn.disconnect if @conn + end + end + WOK_SQL + end + end +end +__END__ diff --git a/lib/sisu/v5/cgi_sqlite.rb b/lib/sisu/v5/cgi_sqlite.rb new file mode 100644 index 00000000..45f6d1ca --- /dev/null +++ b/lib/sisu/v5/cgi_sqlite.rb @@ -0,0 +1,219 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: generates naive cgi search form for search of sisu database (pgsql sqlite) + +=end +module SiSU_CGI_SQLite #% database building documents + require_relative 'sysenv' # sysenv.rb + require_relative 'cgi_sql_common' # cgi_sql_common.rb + include SiSU_CGI_SQL + class SearchSQLite < CGI_Common + def initialize(opt,webserv) + @opt,@webserv=opt,webserv + @env=SiSU_Env::InfoEnv.new('',opt) + @image_src="#{@env.url.webserv_cgi}/_sisu/image_sys" + @common=SiSU_CGI_SQL::CGI_Common.new(@webserv,@opt,@image_src,@env) + @cgi_file_name="#{Db[:name_prefix_db]}by_#{opt.dir_structure_by.to_s}_sqlite.cgi" + end + def sqlite + serve=[] + Dir.foreach(@env.path.webserv) do |x| + if x !~/^\./ \ + and FileTest.directory?("#{@env.path.webserv}/#{x}") + if FileTest.file?("#{@env.path.webserv}/#{x}/sisu_sqlite.db") + serve << x unless x =~/^_\S+/ + end + end + end + serve=serve.sort + f1,f2,f3='','','' + serve.each do |x| + f1 << %{ \n} + end + f2 << %{ selected_db=case cgi['db']\n} + serve.each do |x| + f2 << %{ when /#{Db[:name_prefix]}#{x}/; ''\n} + end + f2 << " end\n" + f3 << %{ db_name='sisu_sqlite.db'\n} + f3 << %{ db_sqlite=case cgi['db']\n} + serve.each do |x| + f3 << %{ when /#{Db[:name_prefix]}#{x}/; "#{@env.path.webserv}/#{x}/sisu_sqlite.db"\n} + end + f3 << %{ else '#{@env.path.webserv}/#{serve[0]}/sisu_sqlite.db'\n end\n} + if FileTest.writable?('.') + output=File.open(@cgi_file_name,'w') + output << header0 << header1 << header_desc << header2 << f1 << buttons1 << buttons2 << search_request << search_statement << search_statement_common << search_query1 << @common.pages << search_query2 << @common.tail << @common.main1 << f2 << f3 << dbi_connect << @common.main2 << @common.dir_structure << @common.main3 + a=%{ generated sisu_sqlite.cgi, + BASED ON ALREADY CREATED sisu_sqlite.db OUTPUT, (-d) + } + SiSU_Screen::Ansi.new(@opt.cmd,a).print_grey + c=case @webserv + when /pwd/; '' + else "if necessary make the directory /usr/lib/cgi-bin : + sudo cp -vi #{Dir.pwd}/#{@cgi_file_name} /usr/lib/cgi-bin/. + sudo chmod -v 755 /usr/lib/cgi-bin/#{@cgi_file_name} + sudo ln -s /usr/lib/cgi-bin/#{@cgi_file_name} /usr/lib/cgi-bin/sisu_sqlite.cgi + (copy #{@cgi_file_name} to your cgi directory) set file permissions to 755, and make symbolic link to sisu_sqlite.cgi" + end + a=%{#{c} + #{@env.webserv_base_cgi}/cgi-bin/sisu_sqlite.cgi + } + b='(to create and populate sisu sqlite database see "man sisu" and in particular the -d flag)' + SiSU_Screen::Ansi.new(@opt.cmd,a,b).warn + else puts "failed in attempt to write #{@cgi_file_name} to present directory, is directory writable?" + end + end + def header0 + <<-WOK_SQL +#!/usr/bin/env ruby +=begin +#{about} + * Description: generates naive cgi search form for search of sisu database (sqlite) +#{gpl} +=end + require 'cgi' + require 'fcgi' + require 'dbi' + @stub_default="#{@name_of[:db]}" + @image_src="#{@name_of[:host_url_docs]}/_sisu/image_sys" + @hosturl_cgi="#{@name_of[:host_url_cgi]}" + @hosturl_files="#{@name_of[:host_url_docs]}" + @output_dir_structure_by="#{@name_of[:output_dir_structure]}" + @db_name_prefix="#{Db[:name_prefix]}" + WOK_SQL + end + def search_statement + <<-'WOK_SQL' + class DBI_SearchString + def initialize(l,t,q,cse=false) + @l,@t,@q=l,t,q + end + def string + search={ search: [], flag: false } + if @t =~/\S+/ or @q =~/\S+/ + if @t =~/\S+/ then unescaped_search=CGI.unescape(@t) + elsif @q =~/\S+/ then unescaped_search=CGI.unescape(@q) + end + search_construct=[] + unescaped_search=unescaped_search.gsub(/\s*(AND|OR)\s*/,"%' \) \\1 #{@l} LIKE \( '%"). + gsub(/(.+)/,"#{@l} LIKE \( '%\\1%' \)") + search_construct << unescaped_search + search_construct=search_construct.join(' ') + search[:search] << search_construct + search[:flag]=true + search + end + search + end + end + WOK_SQL + end + def search_query1 + <<-'WOK_SQL' + @search_text,@search_endnotes='','' + @search_text=search[:text].flatten.join(' AND ') + @search_text=@search_text.gsub(/(doc_objects\.clean\s+LIKE\s+\(\s*'%[^']+%'\s*\)\s+(?:(?:AND|OR)\s+doc_objects\.clean\s+LIKE\s+\(\s*'%[^']+%'\s*\))+)/,'(\1)') + @search_endnotes=search[:endnotes].flatten.join(' AND ') + @search_endnotes=@search_endnotes.gsub(/(endnotes\.clean\s+LIKE\s+\(\s*'%[^']+%'\s*\)\s+(?:(?:AND|OR)\s+endnotes\.clean\s+LIKE\s+\(\s*'%[^']+%'\s*\))+)/,'(\1)') + end + WOK_SQL + end + def search_query2 + <<-'WOK_SQL' + def sql_select_body + limit ||=@@limit + offset ||=@@offset + @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.language_document_char, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE #{@search_text} AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.language_document_char, metadata_and_text.title, metadata_and_text.src_filename, doc_objects.ocn} + @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} + select=@sql_statement[:body] + ' ' + @sql_statement[:range] + select + end + def sql_select_endnotes + limit ||=@@limit + offset ||=@@offset + @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.language_document_char, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE #{@search_endnotes} AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.language_document_char, metadata_and_text.title, metadata_and_text.src_filename, endnotes.nr} + @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} + select=@sql_statement[:endnotes] + ' ' + @sql_statement[:range] + select + end + def sql_select_body_format + %{#{sql_select_body}} + end + def sql_select_endnotes_format + %{#{sql_select_endnotes}} + end + def contents + @conn.select_all(sql_select_body) + end + def endnotes + @conn.select_all(sql_select_endnotes) + end + end + WOK_SQL + end + def dbi_connect + <<-'WOK_SQL' + @dbi="DBI:SQLite3:#{db_sqlite}" #sqlite3 ? + @conn=DBI.connect(@dbi) + WOK_SQL + end + end +end +__END__ diff --git a/lib/sisu/v5/composite.rb b/lib/sisu/v5/composite.rb new file mode 100644 index 00000000..c1cfb127 --- /dev/null +++ b/lib/sisu/v5/composite.rb @@ -0,0 +1,254 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: composite documents, assemble/build documents from other + documents or parts of marked up text + +=end +module SiSU_Assemble + require_relative 'sysenv' # sysenv.rb + class RemoteImage + def initialize + @env=SiSU_Env::InfoEnv.new + end + def image(dir) + images=[] + images[0]=dir + images + end + def download_images(images_info) + path="#{@env.processing_path.processing}/external_document/image" + FileUtils::mkdir_p(path) unless FileTest.directory?(path) + download_from=images_info.shift + images_info.each do |i| + image="#{path}/#{i}" + imagefile=File.new(image,'w+') + open("#{download_from}/#{i}") do |g| + imagefile << g.read + end + imagefile.close + end + output_path="#{@env.path.webserv}/#{@env.path.stub_pwd}/_sisu/image_external" + FileUtils::mkdir_p(output_path) unless FileTest.directory?(output_path) + SiSU_Env::SystemCall.new("#{path}/*",output_path,'q').rsync + end + end + class Composite + @@imager={} + def initialize(opt) + @opt=opt + @env=SiSU_Env::InfoEnv.new + end + def read + begin + pwd=Dir.pwd + Dir.chdir(@opt.f_pth[:pth]) + @fns_array=IO.readlines(@opt.fno,'') + assembled=insertions? + write(assembled) + Dir.chdir(pwd) + rescue + SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do + __LINE__.to_s + ':' + __FILE__ + end + ensure + end + end + def write(assembled) + assembled_file=File.new("#{@env.processing_path.composite_file}/#{@opt.fnb}.ssm.sst",'w+') + assembled.each {|a| assembled_file << a } + assembled_file.close + end + def download_images(download_from,images_array) + path="#{@env.processing_path.processing}/external_document/image" + FileUtils::mkdir_p(path) unless FileTest.directory?(path) + images_array.each do |i| + image="#{path}/#{i}" + unless FileTest.exists?(image) + imagefile=File.new(image,'w+') + open("#{download_from}/#{i}") do |g| + imagefile << g.read + end + imagefile.close + end + end + end + def insertion(fni,insert_array) + file={ prepared: [], images: [] } + rgx_image=/(?:^|[^_\\])\{\s*(\S+?\.(?:png|jpg|gif))/ + file[:prepared] << "\n% |#{fni}|@|^|>>ok\n" + @code_flag=false + insert_array.each do |i| + @code_flag=if i =~/^code\{/ then true + elsif i =~/^\}code/ then false + else @code_flag + end + if not @code_flag \ + and i !~/^%+\s/ + i=i.gsub(/^([123]|:?[ABC])~\? /,'% [conditional heading:] \1~ ') #off conditional heading (consider syntax) + if i =~/^@\S+?:/ + i=i.gsub(/\n/m,"\n% "). + gsub(/\n%\s+$/m,''). + gsub(/^@\S+?:/m,"\n% [imported header:] ") #off imported headers + end + end + file[:prepared] << i + if i !~/^%+\s/ \ + and i =~rgx_image + file[:images] << i.scan(rgx_image).uniq + end + end + file[:prepared] << "\n% end import" << "\n\n" + if file[:images].length > 0 + file[:images]=file[:images].flatten.uniq + file[:images].delete_if {|x| x =~/https?:\/\// } + end + file + end + def insertions? + data=@fns_array + tuned_file,imagedir=[],[] + SiSU_Screen::Ansi.new(@opt.cmd,'Composite Document',"[#{@opt.f_pth[:lng_is]}] #{@opt.fno}").grey_title_hi unless @opt.cmd =~/q/ + para=data.each do |para| + if para =~/^<<\s+(\S+?\.ss[it])$/ \ + or para =~/^<<\{(\S+?\.ss[it])\}$/ \ + or para =~/^(?:<<\s*)\|(\S+?)\|@\|.+?\|(?:req(?:quire)?\b|\s*\})?/ \ + or para =~/^r\{(.+?)\}/ #depreciated + loadfile=$1.strip + SiSU_Screen::Ansi.new(@opt.cmd,'loading:',loadfile).txt_grey if @opt.cmd =~/[MVv]/ + tuned_file << if loadfile =~ /(?:https?|file):\/\/\S+?\.ss[ti]$/ # and NetTest + imagedir = /((?:https?|file):\/\/\S+?)\/[^\/]+?\.ss[ti]$/.match(loadfile).captures.join + '/_sisu/image' #watch + require 'uri' + image_uri=URI.parse(imagedir) + require 'open-uri' + require 'pp' + insert=open(loadfile) + insert_array=insert.dup + insert.close + file=insertion(loadfile,insert_array) + @@imager[image_uri] ||=[] + @@imager[image_uri] << file[:images] + file[:prepared] + elsif loadfile =~ /\.ss[ti]$/ \ + and FileTest.file?(loadfile) + insert_array=IO.readlines(loadfile,'') + file=insertion(loadfile,insert_array) + file[:prepared] + else + cX=SiSU_Screen::Ansi.new(@opt.cmd).cX + STDERR.puts "\t #{cX.fuschia}ERROR#{cX.off} #{cX.brown}#{@opt.fns}#{cX.off} #{cX.fuschia}requires invalid or non-existent file:#{cX.off} #{cX.brown}#{loadfile}#{cX.off}" + para + end + else tuned_file << para + end + tuned_file=tuned_file.flatten.compact + end + if @@imager.length >0 + @@imager.each do |d,i| + i=i.flatten.uniq + image_info=d + i + download_images(image_info.flatten) + end + end + tuned_file + end + end + class CompositeFileList + @@imager={} + def initialize(opt) + @opt=opt + @env=SiSU_Env::InfoEnv.new + end + def read + begin + @opt.fns=@opt.fns.gsub(/\.ssm\.sst$/,'.ssm') #FIX earlier, hub + @fns_array=IO.readlines(@opt.fns,'') + files=insertions? + rescue + SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do + __LINE__.to_s + ':' + __FILE__ + end + ensure + end + end + def insertions? + data=@fns_array + tuned_file,imagedir=[],[] + SiSU_Screen::Ansi.new(@opt.cmd,'Composite Document',@opt.fno).grey_title_hi unless @opt.cmd =~/q/ + @ssm=[@opt.fns] + para=data.each do |para| + if para =~/^<<\s+(\S+?\.ss[it])$/ + loadfile=$1.strip + SiSU_Screen::Ansi.new(@opt.cmd,'loading:',loadfile).txt_grey if @opt.cmd =~/[MVv]/ + tuned_file << if loadfile =~ /(?:https?|file):\/\/\S+?\.ss[ti]$/ + @ssm << loadfile + elsif loadfile =~ /\.ss[ti]$/ \ + and FileTest.file?(loadfile) + @ssm << loadfile + else + cX=SiSU_Screen::Ansi.new(@opt.cmd).cX + STDERR.puts "\t #{cX.fuschia}ERROR#{cX.off} #{cX.brown}#{@opt.fns}#{cX.off} #{cX.fuschia}requires invalid or non-existent file:#{cX.off} #{cX.brown}#{loadfile}#{cX.off}" + para + end + end + end + @ssm + end + end +end +__END__ diff --git a/lib/sisu/v5/concordance.rb b/lib/sisu/v5/concordance.rb new file mode 100644 index 00000000..ac148f49 --- /dev/null +++ b/lib/sisu/v5/concordance.rb @@ -0,0 +1,376 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: concordance file (html concordance, wordmap, linked index of + words in document) + +=end +module SiSU_Concordance + require_relative 'particulars' # particulars.rb + include SiSU_Particulars + require_relative 'sysenv' # sysenv.rb + include SiSU_Env + require_relative 'defaults' # defaults.rb + include SiSU_Viz + require_relative 'html_format' # html_format.rb + include SiSU_HTML_Format + require_relative 'html_minitoc' # html_minitoc.rb + class Source + def initialize(opt) + @opt=opt + @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt) + end + def read + begin + @env,@md=@particulars.env,@particulars.md + loc=@env.url.output_tell + unless @md.opt.cmd =~/q/ + tool=(@md.opt.cmd =~/[MVv]/) \ + ? "#{@env.program.web_browser} #{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}" + : "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}" + @md.opt.cmd=~/[MVvz]/ \ + ? SiSU_Screen::Ansi.new(@opt.cmd,"Concordance",tool).green_hi_blue + : SiSU_Screen::Ansi.new(@opt.cmd,'Concordance',tool).green_title_hi + end + wordmax=@env.concord_max + unless @md.wc_words.nil? + if @md.wc_words < wordmax + SiSU_Concordance::Source::Words.new(@particulars).songsheet + else + SiSU_Screen::Ansi.new(@md.opt.cmd,"concordance skipped, large document has over #{wordmax} words (#{@md.wc_words})").warn unless @md.opt.cmd =~/q/ + end + else + SiSU_Screen::Ansi.new(@md.opt.cmd,"wc (word count) is off, concordance will be processed for all files including those over the max set size of: #{wordmax} words").warn unless @md.opt.cmd =~/q/ + SiSU_Concordance::Source::Words.new(@particulars).songsheet + end + rescue + SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do + __LINE__.to_s + ':' + __FILE__ + end + ensure + SiSU_Env::CreateSite.new(@opt.cmd).cp_css + end + end + private + class DocTitle + include SiSU_Viz + #revisit, both requires (html & shared_xml) needed for stand alone operation (sisu -w [filename]) + require_relative 'shared_xml' # shared_xml.rb + require_relative 'html' # html.rb + def initialize(particulars) + @particulars,@md=particulars,particulars.md + @data=SiSU_HTML::Source::HTML_Environment.new(particulars).tuned_file_instructions + @file=SiSU_Env::FileOp.new(@md) + txt_path=%{#{@md.dir_out}} + @fnb=@md.fnb + @lex_button=%{SiSU home -->} + @doc_details =< 

#{@md.title.full}

#{@md.author}

+WOK + @make=SiSU_Env::ProcessingSettings.new(@md) + end + def create + head_banner=SiSU_HTML_Format::HeadToc.new(@md) + minitoc=SiSU_HTML_MiniToc::TocMini.new(@md,@data).songsheet.join("\n") + stylesheet=SiSU_Style::CSS_HeadInfo.new(@md).stylesheet + if @make.build.manifest_minitoc? + toc='
' + minitoc + '
' + div_class='content' + else + toc='' + div_class='content0' + end + top_band=if @make.build.html_top_band? + head_banner.concordance_navigation_band + else '' + end + < + + + + SiSU created WordIndex for: #{@md.title.full} + + + + + + + + #{stylesheet.css_head_seg} + + + #{top_band} + #{toc} +
+ #{@doc_details} +

Word index links are to html versions of the text the segmented version followed by the scroll (single document) version.
[For segmented text references [T1], [T2] or [T3] appearing without a link, indicates that the word appears in a title (or subtitle) of the text (that is identifiable by the appended object citation number).]

+

(The word listing/index is Case sensitive: Capitalized words appear before lower case)

+

+ word (number of occurences)
linked references to word within document
+ [if number of occurences exceed number of references - word occurs more than once in at least one reference. Footnote/endnotes are either assigned to the paragraph from which they are referenced or ignored, so it is relevant to check the footnotes referenced from within a paragraph as well.] +

+

+ (After the page is fully loaded) you can jump directly to a word by appending a hash (#) and the word to the url for this text, (do not forget that words are case sensitive, and may be listed twice (starting with and without an upper case letter)), #your_word # [ http://[web host]/#{@fnb}/concordance.html#your_word ] +

+WOK + end + end + class Word + @@word_previous='' + def initialize(word,freq) + @word,@freq=word,freq + end + def html + w=if @word.capitalize==@@word_previous + %{\n

#{@word}

(#{@freq})

\n\t

} + else n=@word.strip.gsub(/\s+/,'_') #also need to convert extended character set to html + %{\n

#{@word}

(#{@freq})

\n\t

} + end + @@word_previous=@word.capitalize + w + end + end + class Words + require_relative 'i18n' # i18n.rb + include SiSU_i18n + require_relative 'defaults' # defaults.rb + include SiSU_Viz + require_relative 'html_format' # html_format.rb + include SiSU_HTML_Format + require_relative 'sysenv' # sysenv.rb + include SiSU_Screen + @@dp=nil + def initialize(particulars) + @particulars=particulars + begin + @env,@md,@dal_array=particulars.env,particulars.md,particulars.dal_array + @file=SiSU_Env::FileOp.new(@md) + @freq=Hash.new(0) + @dp=@@dp ||=SiSU_Env::InfoEnv.new.digest.pattern + @rxp_lv1=/^#{Mx[:lv_o]}1:/ #fix @rxp_lv # Mx[:lv_o] + @rxp_lv2=/^#{Mx[:lv_o]}2:/ #fix @rxp_lv # Mx[:lv_o] + @rxp_lv3=/^#{Mx[:lv_o]}3:/ #fix @rxp_lv # Mx[:lv_o] + @rxp_title=Regexp.new("^#{Mx[:meta_o]}title#{Mx[:meta_c]}\s*(.+?)\s*$") + @rxp_t1=Regexp.new('^T1') + @rxp_t2=Regexp.new('^T2') + @rxp_t3=Regexp.new('^T3') + @rxp_excluded1=/#{Mx[:url_o]}\S+?#{Mx[:url_c]}|#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}/ + @rxp_excluded0=/^(?:#{Mx[:fa_bold_o]}|#{Mx[:fa_italics_o]})?(?:to\d+|\d+| |#{Mx[:br_endnotes]}|EOF|#{Mx[:br_eof]}|thumb_\S+|snap_\S+|_+|-+|[(]?(?:ii+|iv|vi+|ix|xi+|xiv|xv|xvi+|xix|xx)[).]?|\S+?_\S+|[\d_]+\w\S+|[\w\d]{1,2}|\d{1,3}\w?|#{@dp}|[0-9a-f]{16,64}|\d{2,3}x\d{2,3}|\S{0,2}sha\d|\S{0,3}\d{4}w\d\d|\b\w\d+|\d_all\b|e\.?g\.?)(?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})?$/mi #this regex causes and cures a stack dump in ruby 1.9 !!! + @rgx_splitlist=%r{[—.,;:#{Mx[:nbsp]}-]}mi + @alph=SiSU_i18n::Alphabet.new(@md.opt.lng).hash_arrays + @alphlst=SiSU_i18n::Alphabet.new(@md.opt.lng).hash_strings + @rgx_scanlist=%r{#{Mx[:fa_italics_o]}[#{@alphlst[:l]}#{@alphlst[:u]}0-9"\s]{2,12}#{Mx[:fa_italics_c]}|#{Mx[:fa_bold_o]}[#{@alphlst[:l]}#{@alphlst[:u]}0-9"\s]{2,12}#{Mx[:fa_bold_c]}|(?:https?|file)://\S+|<\S+?>|[#{@alphlst[:l]}#{@alphlst[:u]}]+|\w+}mi + rescue + SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do + __LINE__.to_s + ':' + __FILE__ + end + end + end + def songsheet + begin + FileUtils::mkdir_p(@file.output_path.html_concordance.dir) unless FileTest.directory?(@file.output_path.html_concordance.dir) + @file_concordance=File.open(@file.place_file.html_concordance.dir,'w') + map_para + rescue + SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do + __LINE__.to_s + ':' + __FILE__ + end + ensure + @file_concordance.close + end + end + protected + def location_scroll(wordlocation,show) + %{#{wordlocation}; } + end + def location_seg(wordlocation,show) ##fix + @word_location_seg=wordlocation.gsub(/(.+?)\#(\d+)/,"\\1#{@md.lang_code_insert}#{Sfx[:html]}#\\2") unless wordlocation.nil? + case wordlocation + when @rxp_t1 + %{[H]#{show}, } + when @rxp_t2 + %{[H]#{show}, } + when @rxp_t3 + %{[H]#{show}, } + else %{#{show}, } + end + end + def map_para + @seg,ocn=nil,nil + @word_map={} + @dal_array.each do |line| + if defined? line.ocn \ + and line.ocn.to_s =~/\d/ + if (line.is ==:heading \ + || line.is ==:heading_insert) \ + && line.ln==4 + @seg=line.name + end + ocn=line.ocn.to_s + if ocn =~/\d+/ \ + and ocn !~/^0$/ + line.obj=line.obj.gsub(/#{@rxp_excluded1}/,' ') + line.obj=line.obj.split(@rgx_splitlist).join(' ') #%take in word or other match + for word in line.obj.scan(@rgx_scanlist) #%take in word or other match + if word =~ /^([#{@alphlst[:l]}])/ + firstletter=$1 + flu=firstletter.tr(@alphlst[:l],@alphlst[:u]) + word=word.gsub(/^#{firstletter}/,flu ) + end + word=word.gsub(/#{Mx[:lnk_o]}|#{Mx[:lnk_c]}|#{Mx[:url_o]}|#{Mx[:url_c]}/,''). + gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,''). + gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,''). + gsub(/#{Mx[:gl_o]}#[a-z]+#{Mx[:gl_c]}/,''). + gsub(/#{Mx[:gl_o]}#[0-9]+#{Mx[:gl_c]}/,'') + word=word.gsub(/[0-9a-f]{10,}/,' ') if word =~/[0-9]/ + word=word.gsub(/#{Mx[:br_line]}/,' '). + gsub(/^ +/,''). + gsub(/^\S$/,'') + word=nil if word.empty? + word=nil if word =~@rxp_excluded0 #watch + word=nil if word =~/^\S$/ + if word + word=word.gsub(/#{Mx[:br_nl]}|#{Mx[:br_line]}/,' '). + gsub(/#{Mx[:fa_o]}[a-z]{1,7}#{Mx[:fa_o_c]}|#{Mx[:fa_c_o]}[a-z]{1,7}#{Mx[:fa_c]}/,''). + gsub(/#{Mx[:en_a_o]}(?:\d|[*+])*|#{Mx[:en_b_o]}(?:\d|[*+])*|#{Mx[:en_a_c]}|#{Mx[:en_b_c]}/mi,''). + gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,''). + gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,''). + gsub(/<\/?\S+?>/,''). + gsub(/^\@+/,''). + strip. + gsub(/#{Mx[:tc_p]}.+/,''). + gsub(/[\.,;:"]$/,''). + gsub(/["]/,''). + gsub(/^\s*[\(]/,''). + gsub(/[\(]\s*$/,''). + gsub(/^(?:See|e\.?g\.?).+/,''). + gsub(/^\s*[.,;:]\s*/,''). + strip. + gsub(/^\(?[a-zA-Z]\)$/,''). + gsub(/^\d+(st|nd|rd|th)$/,''). + gsub(/^(\d+\.?)+$/, ''). + gsub(/#{Mx[:mk_o]}|#{Mx[:mk_c]}/,''). + gsub(/:name#\S+/,''). + gsub(/^\S$/,'') + word=nil if word =~/^\S$/ + word=nil if word =~/^\s*$/ #watch + if word + unless word =~/[A-Z][A-Z]/ \ + or word =~/\w+\s\w+/ + word=word.capitalize + end + @freq[word] +=1 + @word_map[word] ||= [] + if line !~@rxp_lv1 \ + and line !~@rxp_lv2 \ + and line !~@rxp_lv3 #fix @rxp_lv # Mx[:lv_o] + @word_map[word] << location_seg("#{@seg}\##{ocn}",ocn) + else + @word_map[word] << case line + when @rxp_lv1; location_seg('T1',ocn) #fix @rxp_lv # Mx[:lv_o] + when @rxp_lv2; location_seg('T2',ocn) #fix @rxp_lv # Mx[:lv_o] + when @rxp_lv3; location_seg('T3',ocn) #fix @rxp_lv # Mx[:lv_o] + end + end + end + end + end + end + end + end + scr='Full Text scroll: doc#  ' + seg='' + head=SiSU_Concordance::Source::DocTitle.new(@particulars).create + head=head.gsub(/#{Xx[:html_relative2]}/m,@file.path_rel_links.html_seg_2). + gsub(/#{Xx[:html_relative1]}/m,@file.path_rel_links.html_seg_1) + @file_concordance << head + @file_concordance << '

' + alph=@alph[:u] + alph.each {|x| @file_concordance << %{#{x}, }} + @file_concordance << '

' + letter=alph.shift + @file_concordance << %{\n

A

} + for word in @freq.keys.sort! {|a,b| a.downcase<=>b.downcase} + f=/^(\S)/.match(word)[1] + if letter < f.upcase + while letter < f.upcase + if alph.length > 0 + letter=alph.shift + @file_concordance << %{\n

#{letter}

} + else break + end + end + end + keyword=SiSU_Concordance::Source::Word.new(word,@freq[word]).html + if keyword !~ @rxp_excluded0 + if @word_map[word][0] =~ /\d+/ + wm=[] + @file_concordance << %{#{keyword}#{seg}#{@word_map[word].uniq.compact.join}} + end + @file_concordance << '

' + end + # special cases endnotes and header levels 1 - 3 + end + @file_concordance << %{
\n} # footer + SiSU_Screen::Ansi.new(@md.opt.cmd,@md.fns,"#{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}").flow if @md.opt.cmd =~/[MV]/ + end + end + end +end +__END__ diff --git a/lib/sisu/v5/conf.rb b/lib/sisu/v5/conf.rb new file mode 100644 index 00000000..4d5b58d2 --- /dev/null +++ b/lib/sisu/v5/conf.rb @@ -0,0 +1,258 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: configuration + +=end +module SiSU_Initialize + require_relative 'sysenv' # sysenv.rb + include SiSU_Env + include SiSU_Screen + require_relative 'relaxng' # relaxng.rb + include SiSU_Relaxng + require_relative 'css' # css.rb + include SiSU_Style + class Source + def initialize(opt) + @opt=opt + end + def read + ConfigSite.new(@opt).make_homepage + ConfigSite.new(@opt).css + ConfigSite.new(@opt).dtd + ConfigSite.new(@opt).cp_local_images + ConfigSite.new(@opt).cp_external_images + ConfigSite.new(@opt).cp_webserver_images #if @opt.mod.inspect =~/--init(?:ialize)?=site/ + end + end + class ConfigSite #config files such as css are not updated if they already exist unless forced using the --init=site modifier + require_relative 'sysenv' # sysenv.rb + def initialize(opt) + @opt=opt + @env=SiSU_Env::InfoEnv.new(@opt.fns) + @suffix,@path={},{} + @suffix[:rnc]='rnc' + @suffix[:rng]='rng' + @suffix[:xsd]='xsd' + @path[:xml]= @env.path.output + '/_sisu/xml' + @path[:xsd]= @env.path.output + '/_sisu/xml/xsd' + @path[:rnc]= @env.path.output + '/_sisu/xml/rnc' + @path[:rng]= @env.path.output + '/_sisu/xml/rng' + @pwd,@home=Dir.pwd,@env.path.home + end + def make_homepage + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Make homepage','').colorize unless @opt.cmd =~/q/ + SiSU_Env::CreateSite.new(@opt.cmd).homepage + end + def cp_local_images + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Copy images','').colorize unless @opt.cmd =~/q/ + SiSU_Env::CreateSite.new(@opt.cmd).cp_local_images + SiSU_Env::CreateSite.new(@opt.cmd).cp_webserver_images_local #this should not have been necessary + SiSU_Env::CreateSite.new(@opt.cmd).cp_base_images #base images (nav etc.) used by all html + end + def cp_external_images + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Copy external images','').colorize if @opt.cmd =~/V/ + SiSU_Env::CreateSite.new(@opt.cmd).cp_external_images + end + def cp_webserver_images + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Copy webserver/output file images','').colorize unless @opt.cmd =~/q/ + SiSU_Env::CreateSite.new(@opt.cmd).cp_webserver_images + end + def css + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Configuring CSSs','').colorize unless @opt.cmd =~/q/ + SiSU_Env::CreateSite.new(@opt.cmd).cp_css + end + def dtd + SiSU_Screen::Ansi.new(@opt.cmd,'invert','Configuring DTDs','').colorize unless @opt.cmd =~/q/ + @rxng=SiSU_Relaxng::RelaxNG.new + @path.each { |d| FileUtils::mkdir_p(d[1]) unless FileTest.directory?(d[1]) } + #ugly code, sort later + if @rxng.methods.join =~/[^_]dtd_sax\b/ + if @rxng.dtd_sax.length > 200 + dtd=File.new("#{@env.path.output}/#{@env.path.style}/#{@rxng.rng_name.output_sax}",'w') + dtd << @rxng.dtd_sax + dtd.close + else trang_rnc_model_output_sax + end + else trang_rnc_model_output_sax + end + if @rxng.methods.join =~/[^_]dtd_dom\b/ + if @rxng.dtd_dom.length > 200 + dtd=File.new("#{@env.path.output}/#{@env.path.style}/#{@rxng.rng_name.output_dom}",'w') + dtd << @rxng.dtd_dom + dtd.close + else trang_rnc_model_output_dom + end + else trang_rnc_model_output_dom + end + if @rxng.methods.join =~/[^_]dtd_node\b/ + if @rxng.dtd_node.length > 200 + dtd=File.new("#{@env.path.output}/#{@env.path.style}/#{@rxng.rng_name.input_node}",'w') + dtd << @rxng.dtd_node + dtd.close + else trang_rnc_model_input_node + end + else trang_rnc_model_input_node + end + if @rxng.methods.join =~/[^_]dtd_xhtml\b/ + if @rxng.dtd_xhtml.length > 200 + dtd=File.new("#{@env.path.output}/#{@env.path.style}/#{@rxng.rng_name.output_xhtml}",'w') + dtd << @rxng.dtd_xhtml + dtd.close + else trang_rnc_model_output_xhtml + end + else trang_rnc_model_output_xhtml + end + end + def trang_rnc_model_output_sax + s=@suffix + rnc_src=@env.processing_path.dal + '/sax.' + s[:rnc] + rnc_file=@path[:rnc] + '/' + @rxng.rnc_name.output_sax + rng_file=@path[:rng] + '/' + @rxng.rng_name.output_sax + xsd_file=@path[:xsd] + '/' + @rxng.xsd_name.output_sax + rnc=File.new(rnc_src,'w') + rnc << @rxng.rnc_model_output_sax + rnc.close + #xsd + schema=SiSU_Env::SystemCall.new(rnc_src,xsd_file) + schema.relaxng(@opt.cmd) + #rng + schema=SiSU_Env::SystemCall.new(rnc_src,rng_file) + schema.relaxng(@opt.cmd) + #rnc + if FileTest.file?(rnc_src) + FileUtils::cp(rnc_src,rnc_file) + FileUtils::chmod(0644,rnc_file) + else STDERR.puts %{\t*WARN* did not find rnc - "#{rnc_src}" [#{__FILE__}:#{__LINE__}]} + end + end + def trang_rnc_model_output_dom + s=@suffix + rnc_src=@env.processing_path.dal + '/dom.' + s[:rnc] + rnc_file=@path[:rnc] + '/' + @rxng.rnc_name.output_dom + rng_file=@path[:rng] + '/' + @rxng.rng_name.output_dom + xsd_file=@path[:xsd] + '/' + @rxng.xsd_name.output_dom + rnc=File.new(rnc_src,'w') + rnc << @rxng.rnc_model_output_dom + rnc.close + #xsd + schema=SiSU_Env::SystemCall.new(rnc_src,xsd_file) + schema.relaxng(@opt.cmd) + #rng + schema=SiSU_Env::SystemCall.new(rnc_src,rng_file) + schema.relaxng(@opt.cmd) + #rnc + if FileTest.file?(rnc_src) + FileUtils::cp(rnc_src,rnc_file) + FileUtils::chmod(0644,rnc_file) + else STDERR.puts %{\t*WARN* did not find rnc - "#{rnc_src}" [#{__FILE__}:#{__LINE__}]} + end + end + def trang_rnc_model_output_xhtml + s=@suffix + rnc_src=@env.processing_path.dal + '/xhtml.' + s[:rnc] + rnc_file=@path[:rnc] + '/' + @rxng.rnc_name.output_xhtml + rng_file=@path[:rng] + '/' + @rxng.rng_name.output_xhtml + xsd_file=@path[:xsd] + '/' + @rxng.xsd_name.output_xhtml + rnc=File.new(rnc_src,'w') + rnc << @rxng.rnc_model_output_xhtml + rnc.close + #xsd + schema=SiSU_Env::SystemCall.new(rnc_src,xsd_file) + schema.relaxng(@opt.cmd) + #rng + schema=SiSU_Env::SystemCall.new(rnc_src,rng_file) + schema.relaxng(@opt.cmd) + #rnc + if FileTest.file?(rnc_src) + FileUtils::cp(rnc_src,rnc_file) + FileUtils::chmod(0644,rnc_file) + else STDERR.puts %{\t*WARN* did not find rnc - "#{rnc_src}" [#{__FILE__}:#{__LINE__}]} + end + end + def trang_rnc_model_input_sax + rnc_file=@env.processing_path.dal + '/sax.rnc' + dtd_file=@path[:xsd] + '/' + @rxng.rng_name.input_sax + rnc=File.new(rnc_file,'w') + rnc << @rxng.rnc_model_output_sax + rnc.close + schema=SiSU_Env::SystemCall.new(rnc_file,dtd_file) + schema.relaxng(@opt.cmd) + end + def trang_rnc_model_input_dom + rnc_file=@env.processing_path.dal + '/dom.rnc' + dtd_file=@path[:xsd] + '/' + @rxng.rng_name.input_dom + rnc=File.new(rnc_file,'w') + rnc << @rxng.rnc_model_output_dom + rnc.close + schema=SiSU_Env::SystemCall.new(rnc_file,dtd_file) + schema.relaxng(@opt.cmd) + end + def trang_rnc_model_input_node + rnc_file=@env.processing_path.dal + '/node.rnc' + rng_file=@env.processing_path.dal + '/node.rng' + dtd_file=@path[:xsd] + '/' + @rxng.rng_name.input_node + rnc=File.new(rnc_file,'w') + rnc << @rxng.rnc_model_input_node + rnc.close + schema=SiSU_Env::SystemCall.new(rnc_file,dtd_file) + schema.relaxng(@opt.cmd) + end + end +end +__END__ diff --git a/lib/sisu/v5/constants.rb b/lib/sisu/v5/constants.rb new file mode 100644 index 00000000..d72049e9 --- /dev/null +++ b/lib/sisu/v5/constants.rb @@ -0,0 +1,344 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + constants + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +#Ax,Xx,Mx,Rx,Hx,Dx,Px,Ep,Db,Gt,Tex=Array.new(11){{}} +YEAR='2013' +Sfx={ txt: '.txt', html: '.html', xhtml: '.xhtml', xml: '.xml', epub: '.epub', epub_xhtml: '.xhtml', odt: '.odt', pdf: '.pdf'} +Ax={ + tab: "\t", +} +Xx={ + protect: '☞', + segment: 'Ф', + relative_path: '☼', + html_relative2: '※※', + html_relative1: '※', +} +Mx={ + meta_o: '〔@', meta_c: '〕', + lv_o_1: 1, + lv_o_2: 2, + lv_o_3: 3, + lv_o_4: 4, + lv_o_5: 5, + lv_o_6: 6, + lv_o_7: 7, + lv_o_8: 8, + lv_o_9: 9, + lv_o: '〔', lv_c: '〕', + en_a_o: '【', en_a_c: '】', #endnote Mx[:en_a_o]='~{'; Mx[:en_a_c]='}~' + en_b_o: '〖', en_b_c: '〗', #endnote Mx[:en_b_o]='~['; Mx[:en_b_c]=']~' + bl_o: '〔', bl_c: '〕', #block text mark + gr_o: '〔', gr_c: '〕', #group text mark #REPLACE & RETIRE + id_o: '〔', id_c: '〕', #object id mark + tc_o: '『', tc_c: "』", #table row mark #Mx[:tc_c]="』\n" + tc_p: '┆', #table col/misc mark + pa_o: '〔', pa_c: '〕', #affects paragraph mark + mk_o: '〔', mk_c: '〕', #generic mark + gl_o: '〔', gl_c: '〕', #glyph + fa_o: '〔', fa_o_c: '¤', fa_c_o: '¤', fa_c: '〕', + idx_o: '▢ ', idx_c: '▢ ', + nbsp: '░', #'▭ ' + br_line: '╱', #lB ▌ 9612 ┘ ¶ + br_nl: '╲', #lB ▌ 』 ┘ + br_paragraph: '█', #FB █ 9608 # PP ∥ 8741 #▐ #'┘' #'¶' #FB █ 9608 lB ▌ 9612 RB ▐ 9616 + br_obj: 'break_obj', + br_page_line: '▭', + br_page: '┼', + br_page_new: '╋', + lnk_o: '⌠', lnk_c: '⌡', #'⌈' '⌋' '⌠' '⌡' #Mx[:lnk_o: '◁'; Mx[:lnk_c: '▷' #‹ › + url_o: '「', url_c: '」', + rel_o: '⌈', rel_c: '⌋', + tag_o: '⌊', tag_c: '⌉', + sm_set_o: '《', sm_set_c: '》', + sm_subset_o: '《 ', sm_subset_c: '》', + vline: '┆', # ¦ | +} +Mx[:fa_bold_o]= "#{Mx[:fa_o]}b#{Mx[:fa_o_c]}" +Mx[:fa_bold_c]= "#{Mx[:fa_c_o]}b#{Mx[:fa_c]}" +Mx[:fa_italics_o]= "#{Mx[:fa_o]}i#{Mx[:fa_o_c]}" +Mx[:fa_italics_c]= "#{Mx[:fa_c_o]}i#{Mx[:fa_c]}" +Mx[:fa_underscore_o]= "#{Mx[:fa_o]}u#{Mx[:fa_o_c]}" +Mx[:fa_underscore_c]= "#{Mx[:fa_c_o]}u#{Mx[:fa_c]}" +Mx[:fa_cite_o]= "#{Mx[:fa_o]}cite#{Mx[:fa_o_c]}" +Mx[:fa_cite_c]= "#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}" +Mx[:fa_insert_o]= "#{Mx[:fa_o]}ins#{Mx[:fa_o_c]}" +Mx[:fa_insert_c]= "#{Mx[:fa_c_o]}ins#{Mx[:fa_c]}" +Mx[:fa_strike_o]= "#{Mx[:fa_o]}del#{Mx[:fa_o_c]}" +Mx[:fa_strike_c]= "#{Mx[:fa_c_o]}del#{Mx[:fa_c]}" +Mx[:fa_superscript_o]= "#{Mx[:fa_o]}sup#{Mx[:fa_o_c]}" +Mx[:fa_superscript_c]= "#{Mx[:fa_c_o]}sup#{Mx[:fa_c]}" +Mx[:fa_subscript_o]= "#{Mx[:fa_o]}sub#{Mx[:fa_o_c]}" +Mx[:fa_subscript_c]= "#{Mx[:fa_c_o]}sub#{Mx[:fa_c]}" +Mx[:fa_hilite_o]= "#{Mx[:fa_o]}hi#{Mx[:fa_o_c]}" +Mx[:fa_hilite_c]= "#{Mx[:fa_c_o]}hi#{Mx[:fa_c]}" +Mx[:fa_monospace_o]= "#{Mx[:fa_o]}mono#{Mx[:fa_o_c]}" +Mx[:fa_monospace_c]= "#{Mx[:fa_c_o]}mono#{Mx[:fa_c]}" +Mx[:gl_bullet]= "#{Mx[:gl_o]}●#{Mx[:gl_c]}" +Mx[:br_endnotes]= "#{Mx[:mk_o]}ENDNOTES#{Mx[:mk_c]}" +Mx[:br_eof]= "#{Mx[:mk_o]}EOF#{Mx[:mk_c]}" +Mx[:pa_non_object_dummy_heading]="#{Mx[:pa_o]}-##{Mx[:pa_c]}" #unnumbered paragraph, delete when not required [used in dummy headings, eg. for segmented html] (place marker at end of paragraph) +Mx[:pa_non_object_no_heading]="#{Mx[:pa_o]}~##{Mx[:pa_c]}" #unnumbered paragraph (place marker at end of paragraph) +Hx={ + br_obj: { obj: Mx[:br_obj] }, # line sep + br_page_line: { obj: Mx[:br_page_line] }, # line across page + br_page: { obj: Mx[:br_page] }, # newpage + br_page_new: { obj: Mx[:br_page_new] }, # clearpage +} +#Mx[:sm_set_o]='∈ '; Mx[:sm_set_c]='∋ ' +#Mx[:sm_subset_o]='∈ '; Mx[:sm_subset_c]='∋ ' +Rx={ + mx_fa_clean: /#{Mx[:fa_o]}.+?#{Mx[:fa_c]}|#{Mx[:pa_o]}.+?#{Mx[:pa_c]}|#{Mx[:mk_o]}.+?#{Mx[:mk_c]}/, + lv: /〔([1-9]):(\S*?)〕/, + lv_1: /#{Mx[:lv_o_1]}(\S*?)#{Mx[:lv_c]}/, + lv_2: /#{Mx[:lv_o_2]}(\S*?)#{Mx[:lv_c]}/, + lv_3: /#{Mx[:lv_o_3]}(\S*?)#{Mx[:lv_c]}/, + lv_4: /#{Mx[:lv_o_4]}(\S*?)#{Mx[:lv_c]}/, + lv_5: /#{Mx[:lv_o_5]}(\S*?)#{Mx[:lv_c]}/, + lv_6: /#{Mx[:lv_o_6]}(\S*?)#{Mx[:lv_c]}/, + lv_7: /#{Mx[:lv_o_7]}(\S*?)#{Mx[:lv_c]}/, + lv_8: /#{Mx[:lv_o_8]}(\S*?)#{Mx[:lv_c]}/, + lv_9: /#{Mx[:lv_o_9]}(\S*?)#{Mx[:lv_c]}/, + meta: /#{Mx[:meta_o]}(\S+?)#{Mx[:meta_c]}/, +} +Dx={ + url_o: '‹', url_c: '›', + url_o_xml: '<', url_c_xml: '>', + rel_o: '‹', rel_c: '›', + lt_xml: '<', gt_xml: '>', +} +Tex={ + backslash: "\\\\", + backslash: "\\\\", + tilde: '\\\\\\~', +} +Px={ + bold_o: '*', bold_c: '*', + italics_o: '/', italics_c: '/', + underscore_o: '_', underscore_c: '_', + #emphasis_o: '*', emphasis_c: '*', + #bold_o: '!', bold_c: '!', + cite_o: '"', cite_c: '"', + insert_o: '+', insert_c: '+', + strike_o: '-', strike_c: '-', + superscript_o: '^', superscript_c: '^', + subscript_o: '[', subscript_c: ']', + hilite_o: '*', hilite_c: '*', + monospace_o: '', monospace_c: '', + po_bold_o: '!{', po_bold_c: '}!', + po_italics_o: '/{', po_italics_c: '}/', + po_underscore_o: '_{', po_underscore_c: '}_', + po_cite_o: '"{', po_cite_c: '}"', + po_insert_o: '+{', po_insert_c: '}+', + po_strike_o: '-{', po_strike_c: '}-', + po_superscript_o: '^{', po_superscript_c: '}^', + po_subscript_o: ',{', po_subscript_c: '},', + po_hilite_o: '*{', po_hilite_c: '}*', + po_monospace_o: '#{', po_monospace_c: '}#', + lng_lst: %w[am bg bn br ca cs cy da de el en eo es et eu fi fr ga gl he hi hr hy ia is it la lo lt lv ml mr nl nn no oc pl pt pt_BR ro ru sa se sk sl sq sr sv ta te th tk tr uk ur us vi], + lv1: '*', + lv2: '=', + lv3: '=', + lv4: '-', + lv5: '.', + lv6: '.', +} +Px[:lng_lst_rgx]=Px[:lng_lst].join('|') +Ep={ + alt: :on, + d_oebps: 'OEBPS', + d_image: 'OEBPS/image', + d_css: 'OEBPS/css', + f_ncx: 'toc.ncx', + f_opf: 'content.opf', +} +$ep=if Ep[:alt]==:on + { + o: 'opf:', + hsp: ' ', + } +else + { + o: '', + hsp: ' ', + } +end +Db={ + name_prefix: "SiSU#{SiSU_version_dir}a_", + name_prefix_db: "sisu_#{SiSU_version_dir}a_", + col_title: 800, + col_title_part: 400, + col_title_edition: 10, + col_name: 600, + col_creator_misc_short: 100, + col_language: 100, + col_language_char: 6, + col_date_text: 10, + col_txt_long: 600, + col_txt_short: 200, + col_identify_hash: 256, + col_library: 30, + col_small: 16, + col_filename: 256, + col_digest: 64, + col_filesize: 10, + col_info_note: 2500, +} +Gt={ + grotto: 'sisu_src', + git: 'sisu:', + src: 'src', + pods: 'pods', + sisupod: 'sisupod', + pod: 'pod', + files: 'files', + doc: 'doc', + po: 'po4a/po', + pot: 'po4a/pot', + image: 'image', + audio: 'audio', + video: 'video', + conf: 'doc/_sisu', +} +S_CONF={ + header_make: 'sisu_document_make', + rc_yml: 'sisurc.yml', +} +ANSI_C={ + red: "\033[#{31}m", + green: "\033[#{32}m", + yellow: "\033[#{33}m", + blue: "\033[#{34}m", + fuchsia: "\033[#{35}m", + cyan: "\033[#{36}m", + inv_red: "\033[#{41}m", + inv_green: "\033[#{42}m", + inv_yellow: "\033[#{43}m", + inv_blue: "\033[#{44}m", + inv_fuchsia: "\033[#{45}m", + inv_cyan: "\033[#{46}m", + b_red: "\033[#{91}m", + b_green: "\033[#{92}m", + b_yellow: "\033[#{93}m", + b_blue: "\033[#{94}m", + b_fuchsia: "\033[#{95}m", + b_cyan: "\033[#{96}m", + off: "\033[m" +} +DISABLE={ + epub: { + internal_navigation: true, + per_section_title: true, + ncx_navpoint_unique_id: true, + }, +} +DEVELOPER={ + maintenance: :false, +} +__END__ +puts SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:red).set +puts SiSU_Utils::CodeMarker.new(__LINE__,__FILE__).set(:green) +puts SiSU_Utils::CodeMarker.new(__LINE__,__FILE__).set(:fuchsia) +puts ANSI_C[:red] + __LINE__.to_s + ANSI_C[:off] + ' ' + __FILE__ +puts "#{ANSI_C[:red]} #{__LINE__.to_s} #{ANSI_C[:off]} #{__FILE__}" +puts ANSI_C[:fuchsia] + __LINE__.to_s + ANSI_C[:off] + ' ' + __FILE__.gsub(/([^\/]+$)/,"#{ANSI_C[:fuchsia]}\\1#{ANSI_C[:off]}") +puts ANSI_C[:red] + __LINE__.to_s + ANSI_C[:off] + ' ' + __FILE__.gsub(/([^\/]+$)/,"#{ANSI_C[:red]}\\1#{ANSI_C[:off]}") +consider: + 〔comment〕 + 〔links?????〕 + import document? +check: + bold line + +┆┆⋮┇┊┋ +『』 +「」 +〔〕 +【】 + +· +¤ + #˝ " λ Ω β α π Ѫ Ж Я Ѳ ѳ Ф ㈣ + Ѳ ѳ Ф + ♩ ♭  ✠  ▭ ▬ ▪ +【】〖〗《》「」 + ‹ › ∗  +'〔lv1〕','〔lv2〕','〔lv3〕','〔lv4〕','〔lv5〕','〔lv6〕','〔lv7〕','〔lv8〕','〔lv9〕' +'〔 Ѳ1〕','〔 Ѳ2〕','〔 Ѳ3〕','〔 Ѳ4〕','〔 Ѳ5〕','〔Ѳ6〕','〔Ѳ7〕','〔Ѳ8〕','〔Ѳ9〕' +◁▷ +◀this is text or an image▶ http:// +p __FILE__ +':'+ __LINE__.to_s +p __FILE__ + ' ' + __LINE__.to_s + ' ' + html +puts "#{__FILE__} #{__LINE__} #{o.inspect}" +puts __FILE__ + ' ' + __LINE__.to_s + '--> ' + o.inspect +puts %{-\t#{__FILE__}::#{__LINE__}::#{caller}:\n"#{name}"} +p "\t" + txt.obj + " << #{__FILE__} #{__LINE__} >>" +p (__FILE__ + ' ' + __LINE__.to_s + '--> ' + dob.inspect) if dob.is==:heading +data.each {|o| p (__FILE__ + ' ' + __LINE__.to_s + '--> ' + o.inspect) if o.is==:heading} +puts "#{__FILE__} #{__LINE__} #{para}" if @opt.cmd =~/M/ +puts "#{__FILE__} #{__LINE__} #{t_o}" if @opt.cmd =~/M/ + dr ┌ 9484 dR ┍ 9485 Dr ┎ 9486 DR ┏ 9487 dl ┐ 9488 dL ┑ 9489 Dl ┒ 9490 LD ┓ 9491 ur └ 9492 uR ┕ 9493 Ur ┖ 9494 UR ┗ 9495 ul ┘ 9496 uL ┙ 9497 Ul ┚ 9498 UL ┛ 9499 vr ├ + dr ┌ 9484 dR ┍ 9485 Dr ┎ 9486 DR ┏ 9487 dl ┐ 9488 dL ┑ 9489 Dl ┒ 9490 LD ┓ 9491 ur └ 9492 uR ┕ 9493 Ur ┖ 9494 UR ┗ 9495 ul ┘ 9496 uL ┙ 9497 Ul ┚ 9498 UL ┛ 9499 vr ├ + └ ┘ +Iu ⌠ 8992 Il ⌡ <7 ⌈ 8968 >7 ⌉ 8969 7< ⌊ 8970 7> ⌋ 8971 +<" 『 12302 >" 』 12303 +<' 「 12300 >' 」 12301 diff --git a/lib/sisu/v5/css.rb b/lib/sisu/v5/css.rb new file mode 100644 index 00000000..4b682e5d --- /dev/null +++ b/lib/sisu/v5/css.rb @@ -0,0 +1,3444 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: css stylesheets + +=end +module SiSU_Style + require_relative 'sysenv' # sysenv.rb + require_relative 'defaults' # defaults.rb + class CSS_HeadInfo + def initialize(md,ft='html') + @md,@ft=md,ft + @env=SiSU_Env::InfoEnv.new('',md) + @fn_css ||=SiSU_Env::CSS_Default.new + @o_str ||=SiSU_Env::ProcessingSettings.new(md).output_dir_structure + css_copy + end + def stylesheet + def css_path + SiSU_Env::CSS_Stylesheet.new(@md) + end + def css_embed_content + @css_embed_content ||=SiSU_Style::CSS.new + end + def css_embed? + if @ft=='html' \ + && @o_str.dump_or_redirect? + true + else + false + end + end + def css_embed(css) + <<-WOK + + WOK + end + def css_action + style_css=SiSU_Style::CSS.new + css=case @ft + when 'html' + css=css_embed_content.html + css_embed(css) + when 'xhtml' + css_path.xhtml + when 'xml_sax' + css_path.xml_sax + when 'xml_dom' + css_path.xml_dom + else + css_embed_content.html + end + end + def css_head + (css_embed?) \ + ? css_action + : "#{css_path.html}#{css_path.html_seg}" + end + def css_head_seg + (css_embed?) \ + ? css_action + : css_path.html_seg + end + def css_head_xml + css_action + end + self + end + def css_copy + if @o_str.dump_or_redirect? + css=SiSU_Style::CSS.new + if @o_str.dump? + css_pth="#{@md.opt.opt_act[:dump][:inst]}/#{@env.path.style}" + elsif @o_str.redirect? + css_pth="#{@md.opt.opt_act[:redirect][:inst]}/#{@md.fnb}/#{@env.path.style}" + end + FileUtils::mkdir_p(css_pth) unless FileTest.directory?(css_pth) + case @ft + when 'html' + style=File.new("#{css_pth}/#{@fn_css.html}",'w') + style << css.html + style.close + when 'xhtml' + style=File.new("#{css_pth}/#{@fn_css.xhtml}",'w') + style << css.xhtml + style.close + when 'xml_sax' + style=File.new("#{css_pth}/#{@fn_css.xml_sax}",'w') + style << css.xml_sax + style.close + when 'xml_dom' + style=File.new("#{css_pth}/#{@fn_css.xml_dom}",'w') + style << css.xml_dom + style.close + css_path.xml_dom + end + end + end + end + class CSS + def initialize + @vz=SiSU_Viz::Defaults.new + end + def fonts + @vz.font_fonts + end + def html_tables #stylesheet for css table_pages +<. + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: preprocessing, (document abstraction), data abstraction used + in subsequent processing + +=end +module SiSU_DAL + require_relative 'defaults' # defaults.rb + include SiSU_Viz + require_relative 'sysenv' # sysenv.rb + include SiSU_Env + require_relative 'param' # param.rb + include SiSU_Param + require_relative 'dal_doc_objects' # dal_doc_objects.rb + require_relative 'dal_syntax' # dal_syntax.rb + include SiSU_DAL_Syntax + require_relative 'dal_doc_str' # dal_doc_str.rb + require_relative 'dal_idx' # dal_idx.rb + require_relative 'dal_numbering' # dal_numbering.rb + require_relative 'dal_hash_digest' # dal_hash_digest.rb + require_relative 'dal_endnotes' # dal_endnotes.rb + require_relative 'dal_images' # dal_images.rb + require_relative 'dal_metadata' # dal_metadata.rb + require_relative 'dal_character_check' # dal_character_check.rb + require_relative 'dal_substitutions_and_insertions' # dal_substitutions_and_insertions.rb + require_relative 'dal_expand_insertions' # dal_expand_insertions.rb + require_relative 'prog_text_translation' # prog_text_translation.rb + require_relative 'shared_sem' # shared_sem.rb + class Instantiate < SiSU_Param::Parameters::Instructions + def initialize + @@flag_vocab=0 + @@line_mode='' + end + end + class Source #{cf.meta}").txt_grey if @opt.cmd =~/M/ + end + dal.each {|s| dal_array << s} + dal_array + end + def read_fnm + dal=[] + dal=(FileTest.file?(@fnm)) \ + ? (File.open(@fnm,'r:utf-8'){ |f| dal=Marshal.load(f)}) + : SiSU_DAL::Source.new(@opt).create_dal + end + def read_fnc + dal=[] + dal=(FileTest.file?(@fnc)) \ + ? (File.open(@fnc,'r:utf-8'){ |f| dal=Marshal.load(f)}) + : SiSU_DAL::Source.new(@opt).create_dal + end + def read_idx_sst + m=[] + m=(FileTest.file?(@idx_sst)) \ + ? (File.open(@idx_sst,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + def read_idx_raw + m=[] + m=(FileTest.file?(@idx_raw)) \ + ? (File.open(@idx_raw,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + def read_idx_html + m=[] + m=(FileTest.file?(@idx_html)) \ + ? (File.open(@idx_html,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + def read_idx_xhtml + m=[] + m=(FileTest.file?(@idx_xhtml)) \ + ? (File.open(@idx_xhtml,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + def read_map_nametags + m=[] + m=(FileTest.file?(@map_nametags)) \ + ? (File.open(@map_nametags,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + def read_map_ocn_htmlseg + m=[] + m=(FileTest.file?(@map_ocn_htmlseg)) \ + ? (File.open(@map_ocn_htmlseg,'r:utf-8'){ |f| m=Marshal.load(f)}) + : nil + end + end + class Output + def initialize(fn,md,data) + @fn,@md,@data=fn,md,data + @cf=SiSU_Env::CreateFile.new(@fn) + @make=SiSU_Env::InfoFile.new(@fn) + @dir=SiSU_Env::InfoEnv.new(@fn) + end + def screen_dump(o) + if defined? o.of + print %{OF: #{o.of}; } + end + if defined? o.is + print %{IS: #{o.is.to_s}; } + end + if defined? o.ocn + print %{OCN: #{o.ocn}; } + end + if defined? o.node + print %{NODE: #{o.node}; } + end + if defined? o.parent + print %{Parent: #{o.parent}; } + end + if defined? o.obj and not o.obj.empty? + puts %{\n#{o.obj}; } + else "\n" + end + end + def screen_print(t_o) + if defined? t_o + print ' ' + t_o.to_s + end + end + def screen_output(data) + data.each do |o| + print o.class + screen_print(o.ocn) + screen_print(o.obj) + puts "\n" + end + end + def hard_output + if @md.opt.cmd =~/M/ + filename_meta=@cf.metaverse.file_meta + @data.each {|o| filename_meta.puts o.inspect.sub(/:0x[0-9a-f]{8}\s/,': ')} #to make diffing easier + filename_txt=@cf.metaverse.file_txt + @data.each do |o| + if defined? o.ocn + filename_txt.puts case o.is + when :heading + "[#{o.is.to_s} #{o.lv}~#{o.name} [#{o.ocn}]] #{o.obj}" + else "[#{o.is.to_s} [#{o.ocn}]] #{o.obj}" + end + else + filename_txt.puts case o.is + when :meta + "[m~#{o.tag}] #{o.obj}" + else "[#{o.is.to_s}] #{o.obj}" + end + end + end + filename_debug=@cf.file_debug + @data.each do |o| + if defined? o.ocn + case o.is + when :heading + filename_debug.puts "#{o.is.to_s} #{o.lv}~#{o.name} odv=#{o.odv} osp=#{o.osp} [#{o.ocn}] -->\n\t#{o.obj}" + end + end + end + else + hard="#{@dir.processing_path.dal}/#{@md.fns}.meta" + File.unlink(hard) if FileTest.file?(hard) + hard="#{@dir.processing_path.dal}/#{@md.fns}.txt" + File.unlink(hard) if FileTest.file?(hard) + hard="#{@dir.processing_path.dal}/#{@md.fns}.debug.txt" + File.unlink(hard) if FileTest.file?(hard) + end + end + def make_marshal_content + marshal_dal=@make.marshal.dal_content + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def make_marshal_metadata + marshal_dal=@make.marshal.dal_metadata + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def idx_html_hard_output + if @md.book_idx \ + and @md.opt.cmd =~/M/ + filename_meta=@cf.file_meta_idx_html + if @data.is_a?(Array) + @data.each {|s| p s.inspect + "\n" unless s.is_a?(String)} + @data.each {|s| filename_meta.puts s.strip + "\n" unless s.strip.empty?} + end + else + hard_idx_html="#{@dir.processing_path.dal}/#{@md.fns}.idx.html" + File.unlink(hard_idx_html) if FileTest.file?(hard_idx_html) + end + end + def make_marshal_idx_sst_html_seg + marshal_dal=@make.marshal.dal_idx_sst_rel_html_seg + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def make_marshal_idx_sst_rel + marshal_dal=@make.marshal.dal_idx_sst_rel + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def make_marshal_idx_html + marshal_dal=@make.marshal.dal_idx_html + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def make_marshal_idx_xhtml + marshal_dal=@make.marshal.dal_idx_xhtml + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Array) + end + def make_marshal_map_nametags + marshal_dal=@make.marshal.dal_map_nametags + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Hash) + end + def make_marshal_map_name_ocn_htmlseg + marshal_dal=@make.marshal.dal_map_ocn_htmlseg + File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.is_a?(Hash) + end + end + class Make + def initialize(fn,md,data) + @fn,@md,@data=fn,md,data + @env=SiSU_Env::InfoEnv.new(@md.fns) + end + def reset + @@flag_vocab=0 + @@line_mode='' + end + def song + reset + data=@data + data=SiSU_DAL_Insertions::Insertions.new(@md,data).expand_insertions? # dal_expand_insertions.rb + data=SiSU_DAL_SubstituteAndInsert::SI.new(@md,data).substitutions_and_insertions? # dal_substitutions_and_insertions.rb + data,metadata=SiSU_DAL_DocumentStructureExtract::Build.new(@md,data).identify_parts # dal_doc_str.rb + data=SiSU_DAL_Syntax::Markup.new(@md,data).songsheet # dal_syntax.rb + data,endnote_array=SiSU_DAL_CharacterCheck::Check.new(data).character_check_and_oldstyle_endnote_array # dal_character_check.rb + data=SiSU_DAL_Images::Images.new(@md,data).images # dal_images.rb + data,tags_map,ocn_html_seg_map=SiSU_DAL_Numbering::Numbering.new(@md,data).numbering_song # dal_numbering.rb + data,book_index_rel,book_index_rel_html_seg,html_idx,xhtml_idx=SiSU_DAL_BookIndex::BookIndex.new(@md,data,@env).indexing_song if @md.book_idx # dal_idx.rb + data=SiSU_DAL_Endnotes::Endnotes.new(@md,data,endnote_array).endnotes # dal_endnotes.rb + outputdata=data + if @md.opt.cmd =~/[mM]/ + SiSU_DAL::Output.new(@fn,@md,outputdata).hard_output + SiSU_DAL::Output.new(@fn,@md,outputdata).make_marshal_content + SiSU_DAL::Output.new(@fn,@md,metadata).make_marshal_metadata + SiSU_DAL::Output.new(@fn,@md,html_idx).idx_html_hard_output + SiSU_DAL::Output.new(@fn,@md,book_index_rel_html_seg).make_marshal_idx_sst_html_seg + SiSU_DAL::Output.new(@fn,@md,book_index_rel).make_marshal_idx_sst_rel + SiSU_DAL::Output.new(@fn,@md,html_idx).make_marshal_idx_html + SiSU_DAL::Output.new(@fn,@md,xhtml_idx).make_marshal_idx_xhtml + SiSU_DAL::Output.new(@fn,@md,tags_map).make_marshal_map_nametags + SiSU_DAL::Output.new(@fn,@md,ocn_html_seg_map).make_marshal_map_name_ocn_htmlseg + end + reset + outputdata + end + protected + end +end +__END__ diff --git a/lib/sisu/v5/dal_character_check.rb b/lib/sisu/v5/dal_character_check.rb new file mode 100644 index 00000000..62aa60c1 --- /dev/null +++ b/lib/sisu/v5/dal_character_check.rb @@ -0,0 +1,107 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +module SiSU_DAL_CharacterCheck + class Check + def initialize(data) + @data=data + @comment='%' + @endnote_array=[] + end + def character_check_and_oldstyle_endnote_array + data=@data + @tuned_file,@endnote_array=[],[] + endnote_no=1 + data.each do |dob| + unless dob.is ==:table + dob.obj=dob.obj.strip. + gsub(/^[{~}]\s*$/,''). + gsub(/~#\s*/,"#{Mx[:pa_non_object_no_heading]}"). + gsub(/-#\s*/,"#{Mx[:pa_non_object_dummy_heading]}"). + gsub(/(#{Mx[:en_a_o]})\s*\s+/,'\1 '). + gsub(/(~\{\s*)\s+/,'\1 '). + gsub(/ \/\//,"#{Mx[:br_line]}"). + gsub(/
/,"#{Mx[:br_line]}"). #needed by xml, xhtml etc. + gsub(/\t/,' '). + gsub(/\342\200\231/u,"'"). #if dob =~/’/ #Avoid #‘ ’ #“ ” + gsub(/\\copy(?:right)?\b/,'©'). + gsub(/\\trademark\b|\\tm\b/,'®') + dob.obj=dob.obj + "\n" + unless dob.is ==:code + case dob.obj + when /\^~/ #% Note must do this first (earlier loop) and then enter gathered data into ~^\d+ + sub_dob=dob.obj.dup + @endnote_array << sub_dob.gsub(/\n/,''). + gsub(/\^~\s+(.+)\s*/,%{#{Mx[:en_a_o]}#{endnote_no} \\1 #{Mx[:en_a_c]}}). + strip + endnote_no+=1 + dob=nil if dob.obj =~/\^~ .+/ #watch, removes 'binary' endnote now in endnote array for later insertion + end + end + end + @tuned_file << dob if dob.is_a?(Object) + end + @tuned_file=@tuned_file.flatten.compact + [@tuned_file,@endnote_array] + end + end +end +__END__ diff --git a/lib/sisu/v5/dal_doc_objects.rb b/lib/sisu/v5/dal_doc_objects.rb new file mode 100644 index 00000000..de9d568f --- /dev/null +++ b/lib/sisu/v5/dal_doc_objects.rb @@ -0,0 +1,452 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: document abstraction + +=end +module SiSU_DAL_DocumentStructure + class Extract + def extract(h,o) + obj=h ? h : o + end + end + class ObjectMetadata + attr_accessor :is,:of,:tags,:obj,:digest + def initialize + @tags={} + @is=@tmp=@digest=nil + @of=:meta + end + def metadata(tags) + of= @of #String, classification - group + is= :meta #String, classification - specific type + tags= tags || ((defined? o.tags) ? o.tags : {}) #String, metadata type/tag + obj= nil + @of,@is,@tags,@obj=of,is,tags,obj + self + end + end + class ObjectMeta + attr_accessor :obj,:is,:of,:tag,:digest,:tmp + def initialize + @is=@obj=@tag=@digest=@digest=@tmp=nil + @of=:meta + end + def metadata(h,o=nil) + of= @of #String, classification - group + is= :meta #String, classification - specific type + tag= h[:tag] || ((defined? o.tag) ? o.tag : nil) #String, metadata type/tag + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + @of,@is,@tag,@obj,@digest,@tmp=of,is,tag,obj,digest,tmp + self + end + end + class ObjectHeading + attr_accessor :obj,:is,:tags,:of,:lv,:ln,:toc_,:name,:idx,:ocn,:odv,:osp,:node,:parent,:ocn_,:note_,:autonum_,:digest,:tmp + def initialize + @of=:para + @is=@obj=@lv=@ln=@toc_=@name=@idx=@size=@ocn=@odv=@osp=@node=@parent=@ocn_=@note_=@autonum_=@digest=@tmp=nil + @tags=[] + end + def heading_ln(lv) + ln=case lv + when /A/; 1 + when /B/; 2 + when /C/; 3 + when /1/; 4 + when /2/; 5 + when /3/; 6 + when /4/; 7 + when /5/; 8 + when /6/; 9 + end + end + def heading_lv(ln) + lv=case ln.to_s + when /1/; 'A' + when /2/; 'B' + when /3/; 'C' + when /4/; '1' + when /5/; '2' + when /6/; '3' + when /7/; '4' + when /8/; '5' + when /9/; '6' + end + end + def heading(h,o=nil) + if not h[:ln] \ + and (h[:lv] and h[:lv]=~/[1-6A-C]/) + h[:ln]=heading_ln(h[:lv]) + elsif not h[:lv] \ + and (h[:ln] and h[:ln].to_s=~/[1-9]/) + h[:lv]=heading_lv(h[:ln]) + end + of= @of #String, classification - group + is= :heading #String, classification - specific type + name= h[:name] || ((defined? o.name) ? o.name : nil) #String, named object? + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + node= h[:node] || ((defined? o.node) ? o.node : nil) #[Node relationship doc structure info] + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + lv= h[:lv] || ((defined? o.lv) ? o.lv : nil) #Alpha-numeric, document structure as used in markup, A-C then 1-6 + ln= h[:ln] || ((defined? o.ln) ? o.ln : nil) #Integer, document structure level, for convenience in processing 1-9 + toc_= h[:toc_] || ((defined? o.toc_) ? o.toc_ : false) #Bool, do not include in toc, (relevant to headings) + ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + autonum_= if h[:autonum_].nil? then ((defined? o.autonum_) ? o.autonum_ : true) #Bool? auto-numbering if requested default on, false suppresses + else h[:autonum_] + end + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@lv,@ln,@name,@tags,@obj,@idx,@ocn,@odv,@osp,@node,@parent,@toc_,@ocn_,@note_,@autonum_,@digest,@tmp=of,is,lv,ln,name,tags,obj,idx,ocn,odv,osp,node,parent,toc_,ocn_,note_,autonum_,digest,tmp + self + end + def heading_insert(h,o=nil) + heading(h,o=nil) + @is= :heading_insert #String, classification - specific type + self + end + end + class ObjectPara + attr_accessor :obj,:is,:tags,:of,:name,:idx,:bullet_,:indent,:hang,:ocn,:odv,:osp,:parent,:note_,:image_,:ocn_,:digest,:tmp + def initialize + @of=:para + @is=@obj=@name=@idx=@bullet_=@indent=@hang=@size=@ocn=@odv=@osp=@parent=@note_=@image_=@ocn_=@digest=@tmp=nil + @tags=[] + end + def paragraph(h,o=nil) + of= @of #String, classification - group + is= :para #String, classification - specific type + name= h[:name] || ((defined? o.name) ? o.name : nil) #String, named object? + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + indent= h[:indent].to_s || ((defined? o.indent) ? o.indent.to_s : nil) #Integer, indent level + hang= h[:hang].to_s || ((defined? o.hang) ? o.hang.to_s : nil) #Integer, hanging indent level + bullet_=h[:bullet_] || ((defined? o.bullet_) ? o.bullet_ : false) #Bool, bulleted? + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + image_= h[:image_] || ((defined? o.image_) ? o.image_ : false) #Bool, images? (processing optimization) + ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@name,@tags,@obj,@indent,@hang,@bullet_,@idx,@ocn,@odv,@osp,@parent,@image_,@note_,@ocn_,@digest,@tmp=of,is,name,tags,obj,indent,hang,bullet_,idx,ocn,odv,osp,parent,image_,note_,ocn_,digest,tmp + self + end + def docinfo(h,o=nil) + of= @of #String, classification - group + is= :docinfo #String, classification - specific type + name= h[:name] || ((defined? o.name) ? o.name : nil) #String, named object? + tags= h[:tags] || ((defined? o.tags) ? o.tags : nil) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= nil #String, book index provided? + ocn= nil #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + indent= nil #Integer, indent level + hang= nil #Integer, indent level + bullet_=false #Bool, bulleted? + note_= false #Bool, endnotes/footnotes? (processing optimization) + image_= h[:image_] || ((defined? o.image_) ? o.image_ : false) #Bool, images? (processing optimization) + ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@name,@tags,@obj,@indent,@hang,@bullet_,@idx,@ocn,@odv,@osp,@parent,@image_,@note_,@ocn_,@digest,@tmp=of,is,name,tags,obj,indent,hang,bullet_,idx,ocn,odv,osp,parent,image_,note_,ocn_,digest,tmp + self + end + end + class ObjectBlockTxt + attr_accessor :obj,:is,:of,:tags,:idx,:ocn,:odv,:osp,:parent,:note_,:number_,:ocn_,:digest,:tmp + def initialize + @of=:block + @is=@obj=@idx=@ocn=@odv=@osp=@parent=@note_=@number_=@ocn_=@digest=@tmp=nil + @tags=[] + end + def code(h,o=nil) + of= @of #String, classification - group #alt 'code' + is= :code #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + number_= h[:number_] || ((defined? o.number_) ? o.number_ : false) #Bool, numbered or not? + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_= if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@number_,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,number_,note_,ocn_,digest,tmp + self + end + def block(h,o=nil) + of= @of #String, classification - group + is= :block #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_= if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp + self + end + def group(h,o=nil) + of= @of #String, classification - group + is= :group #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_= if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp + self + end + def alt(h,o=nil) #see block + of= @of #String, classification - group + is= :alt #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_= if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp + self + end + def verse(h,o=nil) #part of poem decide how you deal with this + of= @of #String, classification - group + is= :verse #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + ocn_= if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp + @h=nil + self + end + end + class ObjectTable + attr_accessor :obj,:is,:of,:lv,:tags,:name,:idx,:indent,:hang,:size,:ocn,:number,:head_,:cols,:widths,:odv,:osp,:parent,:note_,:ocn_,:digest,:tmp + def initialize + @of=:block + @is=@obj=@lv=@name=@idx=@indent=@hang=@size=@ocn,@number,@head_,@cols,@widths=@odv=@osp=@parent=@note_=@ocn_=@digest=@tmp=nil + @tags=[] + end + def table(h,o=nil) + of= @of #String, classification - group + is= :table #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + cols= h[:cols] || ((defined? o.cols) ? o.cols : nil) + widths= h[:widths] || ((defined? o.widths) ? o.widths : nil) + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + head_= h[:head_] || ((defined? o.head_) ? o.head_ : false) + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@cols,@widths,@obj,@idx,@ocn,@odv,@osp,@parent,@head_,@note_,@ocn_,@digest,@tmp=of,is,tags,cols,widths,obj,idx,ocn,odv,osp,parent,head_,note_,ocn_,digest,tmp + self + end + end + class ObjectImage + attr_accessor :obj,:is,:of,:lv,:idx,:size,:ocn,:parent,:note_,:ocn_,:digest,:tmp + def initialize + @of=:image + @is=@obj=@lv=@idx=@size=@ocn=@parent=@note_=@ocn_=@tmp=@digest=nil + @tags=[] + end + def image(h,o=nil) #not yet used, and what of a paragraph containing several images, consider + of= @of #String, classification - group + is= :image #String, classification - specific type + tags= h[:tags] || ((defined? o.tags) ? o.tags : []) #Array, associated object tags, names if any + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + size= h[:size] || ((defined? o.size) ? o.size : nil) + idx= h[:idx] || ((defined? o.idx) ? o.idx : nil) #String, book index provided? + ocn= h[:ocn] || ((defined? o.ocn) ? o.ocn : nil) #Integer, sequential on substantive-content objects + odv= h[:odv] || ((defined? o.odv) ? o.odv : nil) + osp= h[:osp] || ((defined? o.osp) ? o.osp : nil) + parent= h[:parent] || ((defined? o.parent) ? o.parent : nil) #[Node parent] + note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) + ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider + else h[:ocn_] + end + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@tags,@obj,@size,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,size,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp + self + end + end + class ObjectStructure + attr_accessor :obj,:tag,:node,:lv,:ln,:status,:is,:of,:tmp + def initialize + @of=:structure + @is=@obj=@node=@lv=@ln=@status=@tmp=nil + end + def xml_dom(h,o=nil) + of= @of #String, classification - group + is= :xml_dom #String, classification - specific type + obj= h[:obj] || ((defined? o.obj) ? o.obj : '') #String, text content + lv= h[:lv] || ((defined? o.lv) ? o.lv : nil) #Alpha-numeric, document structure as used in markup, A-C then 1-6 + ln= h[:ln] || ((defined? o.ln) ? o.ln : nil) #Integer, document structure level, for convenience in processing 1-9 + node= h[:node] || ((defined? o.node) ? o.node : nil) #[Node relationship doc structure info] + status= h[:status] || ((defined? o.status) ? o.status : nil) #tag status open or close + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@obj,@status,@node,@lv,@ln,@tmp=of,is,obj,status,node,lv,ln,tmp + self + end + end + class ObjectComment + attr_accessor :obj,:is,:of,:tmp + def initialize + @of=:comment + @is=@obj=@tmp=nil + end + def comment(h,o=nil) + of= @of #String, classification - group + is= :comment #String, classification - specific type + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@obj,@tmp=of,is,obj,tmp + self + end + end + class ObjectLayout + attr_accessor :obj,:is,:of,:from,:tmp + def initialize + @of=:layout + @is=@obj=@from=@tmp=nil + end + def break(h,f=nil) #decide how to deal with + of= @of #String, classification - group + is= :break #String, classification - specific type + obj= h[:obj] #String, text content + from= f + tmp= h[:tmp] #available for processing, empty after use + @of,@is,@obj,@from,@tmp=of,is,obj,from,tmp + self + end + def insert(h,o=nil) #decide how to deal with, could mimic paragraph? + of= @of #String, classification - group + is= :insert #String, classification - specific type + obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content + tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use + @of,@is,@obj,@tmp=of,is,obj,tmp + self + end + end +end +__END__ +# ~# |-# no paragraph number # -# not included in toc diff --git a/lib/sisu/v5/dal_doc_str.rb b/lib/sisu/v5/dal_doc_str.rb new file mode 100644 index 00000000..da417bd6 --- /dev/null +++ b/lib/sisu/v5/dal_doc_str.rb @@ -0,0 +1,1051 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: document abstraction + +=end +module SiSU_DAL_DocumentStructureExtract + class Instantiate < SiSU_Param::Parameters::Instructions + @@flag={} #Beware!! + def initialize + @@flag['table_to']=false + @@counter=@@column=@@columns=0 + @@line_mode='' + end + end + class Build + @@flag={} #Beware!! + def initialize(md,data) + @md,@data=md,data + SiSU_DAL_DocumentStructureExtract::Instantiate.new + @pb=SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page]) + @pbn=SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_new]) + @pbl=SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_line]) + end + def ln_get(lv) + ln=case lv + when /A/; 1 + when /B/; 2 + when /C/; 3 + when /1/; 4 + when /2/; 5 + when /3/; 6 + when /4/; 7 + when /5/; 8 + when /6/; 9 + end + end + def image_test(str) + boolean=(str=~/\{\s*\S+?\.png.+?\}https?:\/\/\S+/ ? true : false) + end + def bullet_test(str) + bool=((str=~/\*/) ? true : false) + end + def hang_and_indent_test(str) + hang_indent=if str=~/^_([1-9])[^_]/ + [$1,$1] + elsif str=~/^__([1-9])/ + [0,$1] + elsif str=~/^_([0-9])_([0-9])/ + [$1,$2] + else + [0,0] + end + hang,indent=hang_indent[0],hang_indent[1] + [hang,indent] + end + def hang_and_indent_def_test(str1,str2) + hang_indent=if str1=~/^_([1-9])[^_]/ + [$1,$1] + elsif str1=~/^__([1-9])/ + [0,$1] + elsif str1=~/^_([0-9])_([0-9])/ + [$1,$2] + else + [0,0] + end + obj=if str2 =~/^(.+?)\s+\\\\(?:\s+|\n)/ + str2.gsub(/^(.+?)(\s+\\\\(?:\s+|\n))/,"#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2") + else + str2.gsub(/^(.+?)\n/,"#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\n") + end + hang,indent=hang_indent[0],hang_indent[1] + [hang,indent,obj] + end + def endnote_test?(str) + bool=((str=~/~\{.+?\}~|~\[.+?\]~/) ? true : false) + end + def extract_tags(str,nametag=nil) + tags=[] + if str.nil? + else + if str =~/(?:^|[ ])\*~([a-z0-9._-]+)(?=[ #{Mx[:br_nl]}]|$)/ + str=str.gsub(/(^|[ ])\*~([a-z0-9._-]+)(?=[ #{Mx[:br_nl]}]|$)/i, + "\\1#{Mx[:tag_o]}\\2#{Mx[:tag_c]}"). + gsub(/ [ ]+/i,' ') + tags=str.scan(/#{Mx[:tag_o]}(\S+?)#{Mx[:tag_c]}/).flatten + str=str.gsub(/[ ]?#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}[ ]?/,' ') #may be issues with spaces would leave one, but "code" blocks? + end + tags=nametag ? (tags << nametag) : tags + end + [str,tags] + end + def identify_parts + data=@data + tuned_file=[] + @tuned_block,@tuned_code=[],[] + @@counter,@verse_count=0,0 + @metadata={} + @data.each do |t_o| + t_o=t_o.gsub(/(?:\n\s*\n)+/m,"\n") unless @@flag['code'] + if t_o !~/^(?:code|poem|alt|group|block)\{|^\}(?:code|poem|alt|group|block)|^(?:table\{|\{table)[ ~]/ \ + and not @@flag['code'] \ + and not @@flag['poem'] \ + and not @@flag['group'] \ + and not @@flag['block'] \ + and not @@flag['alt'] \ + and not @@flag['table'] + unless t_o =~/^(?:@\S+?:|%+)\s/ # extract book index for paragraph if any + idx=if t_o=~/^=\{(.+)\}\s*$\Z/m; m=$1 + t_o=t_o.gsub(/\n=\{.+\}\s*$\Z/m,'') + m + else nil + end + end + t_o=case t_o + when /^#{Mx[:meta_o]}\S+?#{Mx[:meta_c]}/ #metadata, header + if t_o=~/^#{Mx[:meta_o]}(\S+?)#{Mx[:meta_c]}\s*(.+)/m + tag,obj=$1,$2 + @metadata[tag]=obj + end + t_o=nil + when /^%+\s/ #comment + t_o=if t_o=~/^%+\s+(.+)/ + h={obj: $1} + SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) + else nil + end + when /^:?([A-C1-6])\~/ #heading / lv + lv=$1 + ln=ln_get(lv) + t_o=if t_o=~/^:?[A-C1-6]\~\s+(.+)/m + obj=$1 + note=endnote_test?(obj) + obj,tags=extract_tags(obj) + h={ lv: lv, ln: ln, obj: obj, idx: idx, tags: tags } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h) + elsif t_o=~/^:?[A-C1-6]\~(\S+?)-\s+(.+)/m + name,obj=$1,$2 + note=endnote_test?(obj) + obj,tags=extract_tags(obj) + h={ lv: lv, name: name, obj: obj, idx: idx, autonum_: false, tags: tags} + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h) + elsif t_o=~/^:?[A-C1-6]\~(\S+)\s+(.+)/m + name,obj=$1,$2 + note=endnote_test?(obj) + obj,tags=extract_tags(obj,name) + h={ lv: lv, name: name, obj: obj, idx: idx, tags: tags } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h) + else nil + end + when /^_(?:[1-9]!?|[1-9]?\*)\s+/ #indented and/or bullet paragraph + t_o=if t_o=~/^(_(?:[1-9]?\*|[1-9]!?)\s+)(.+)/m + tst,obj=$1,$2 + if t_o=~/^_[1-9]!\s+.+/m + hang,indent,obj=hang_and_indent_def_test(tst,obj) + else + hang,indent=hang_and_indent_test(tst) + end + bullet=bullet_test(tst) + image=image_test(obj) + note=endnote_test?(obj) + obj,tags=extract_tags(obj) + unless obj=~/\A\s*\Z/m + h={ bullet_: bullet, hang: hang, indent: indent, obj: obj, idx: idx, note_: note, image_: image, tags: tags } + SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h) + end + else nil + end + when /^_[0-9]?_[0-9]!?\s+/ #hanging indent paragraph + t_o=if t_o=~/^(_[0-9]?_[0-9]!?\s+)(.+)/m + tst,obj=$1,$2 + if t_o=~/^_[0-9]?_[0-9]!\s+.+/m + hang,indent,obj=hang_and_indent_def_test(tst,obj) + else + hang,indent=hang_and_indent_test(tst) + end + image=image_test(obj) + note=endnote_test?(obj) + obj,tags=extract_tags(obj) + unless obj=~/\A\s*\Z/m + h={ hang: hang, indent: indent, obj: obj, idx: idx, note_: note, image_: image, tags: tags } + SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h) + end + else nil + end + when /^<(?:br)?:(?:pa?r|o(?:bj|---)?)>\s*$/ #[br:par] #[br:obj] + SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_obj]) + when /^(?:-\\\\-|<:pb>)\s*$/ #[br:pg] + SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page],:markup) + when /^(?:=\\\\=|<:pn>)\s*$/ #[br:pgn] + SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_new],:markup) + when /^-\.\.-\s*$/ #[br:pgl] + SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_line],:markup) + else #paragraph + image=image_test(t_o) + note=endnote_test?(t_o) + obj,tags=extract_tags(t_o) + unless obj=~/\A\s*\Z/m + h={ bullet_: false, indent: 0, hang: 0, obj: obj, idx: idx, note_: note, image_: image, tags: tags } + SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h) + end + end + elsif not @@flag['code'] + if t_o =~/^code\{/ + @@flag['code']=true + @@counter=1 + @codeblock_numbered=(t_o =~/^code\{#/) ? true : false + h={ obj: 'code block start' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + elsif t_o =~/^poem\{/ + @@flag['poem']=true + h={ obj: 'poem start' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + tuned_file << t_o + elsif t_o =~/^group\{/ + @@flag['group']=true + h={ obj: 'group text start' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + tuned_file << t_o + elsif t_o =~/^block\{/ + @@flag['block']=true + h={ obj: 'block text start' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + tuned_file << t_o + elsif t_o =~/^alt\{/ + @@flag['alt']=true + h={ obj: 'alt text start' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + tuned_file << t_o + elsif t_o =~/^(?:table\{|\{table)[ ~]/ + h={ obj: 'table start' } #introduce a counter + ins=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #ins=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + tuned_file << ins + if t_o=~/^table\{(?:~h)?\s+/ + @@flag['table']=true + @rows='' + case t_o + when /table\{~h\s+c(\d+);\s+(.+)/ + cols=$1 + col=$2.scan(/\d+/) + heading=true + when /table\{\s+c(\d+);\s+(.+)/ + cols=$1 + col=$2.scan(/\d+/) + heading=false + end + @h={ head_: heading, cols: cols, widths: col, idx: idx } + elsif t_o=~/^\{table(?:~h)?(?:\s+\d+;?)?\}\n.+\Z/m + m1,m2,hd=nil,nil,nil + tbl=/^\{table(?:~h)?(?:\s+\d+;?)?\}\n(.+)\Z/m.match(t_o)[1] #two table representations should be consolidated as one + hd=((t_o =~/^\{table~h/) ? true : false) + tbl,tags=extract_tags(tbl) + rws=tbl.split(/\n/) + rows='' + cols=nil + rws.each do |r| + cols=(cols ? cols : (r.scan('|').length) +1) + r=r.gsub(/\s*\|\s*/m,"#{Mx[:tc_p]}") #r.gsub!(/\|/m,"#{Mx[:tc_p]}") + rows += r + Mx[:tc_c] + end + col=[] + if t_o =~/^\{table(?:~h)?\s+(\d+);?\}/ #width of col 1 given as %, usually when wider than rest that are even + c1=$1.to_i + width=(100 - c1)/(cols - 1) + col=[ c1 ] + (cols - 1).times { col << width } + else #all columns of equal width + width=100.00/cols + cols.times { col << width } + end + h={ head_: hd, cols: cols, widths: col, obj: rows, idx: idx, tags: tags } + t_o=SiSU_DAL_DocumentStructure::ObjectTable.new.table(h) unless h.nil? + tuned_file << t_o + h={ obj: 'table end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) + t_o + elsif t_o=~/^\{table(?:~h)?\s+/ + m1,m2,hd=nil,nil,nil + h=case t_o + when /\{table~h\s+(.+?)\}\n(.+)\Z/m #two table representations should be consolidated as one + m1,tbl,hd=$1,$2,true + when /\{table\s+(.+?)\}\n(.+)\Z/m #two table representations should be consolidated as one + m1,tbl,hd=$1,$2,false + else nil + end + tbl,tags=extract_tags(tbl) + col=m1.scan(/\d+/) + rws=tbl.split(/\n/) + rows='' + rws.each do |r| + r=r.gsub(/\s*\|\s*/m,"#{Mx[:tc_p]}") #r.gsub!(/\|/m,"#{Mx[:tc_p]}") + rows += r + Mx[:tc_c] + end + h={ head_: hd, cols: col.length, widths: col, obj: rows, idx: idx, tags: tags } + t_o=SiSU_DAL_DocumentStructure::ObjectTable.new.table(h) unless h.nil? + tuned_file << t_o + h={ obj: 'table end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) + t_o + end + end + t_o + end + if @@flag['table'] + if @@flag['table'] \ + and t_o =~/^\}table/ #two table representations should be consolidated as one + @@flag['table']=false + headings,columns,widths,idx=@h[:head_],@h[:cols],@h[:widths],@h[:idx] + @h={ head_: headings, cols: columns, widths: widths, idx: idx, obj: @rows } + t_o=SiSU_DAL_DocumentStructure::ObjectTable.new.table(@h) + tuned_file << t_o + @h,@rows=nil,'' + t_o + h={ obj: 'table end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + t_o + else + if t_o.is_a?(String) \ + and t_o !~/^table\{/ + t_o=t_o.gsub(/^\n+/m,''). #check added for ruby 1.9.2 not needed in 1.8 series (tested in v2) + gsub(/\n+/m,"#{Mx[:tc_p]}") + @rows += t_o + Mx[:tc_c] + end + t_o=nil + end + end + if @@flag['code'] + if t_o =~/^\}code/ + @@flag['code']=false + obj=@tuned_code.join("\n") + tags=[] + h={ obj: obj, tags: tags, number_: @codeblock_numbered } + t_o=SiSU_DAL_DocumentStructure::ObjectBlockTxt.new.code(h) + @tuned_code=[] + tuned_file << t_o + h={ obj: 'code block end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + end + if @@flag['code'] \ + and t_o.is_a?(String) + sub_array=t_o.dup + "#{Mx[:br_nl]}" + @line_mode=sub_array.scan(/.+/) + @line_mode=[] + sub_array.scan(/.+/) {|w| @line_mode << w if w =~/[\S]+/} + t_o=SiSU_DAL_DocumentStructureExtract::Build.new(@md,@line_mode).build_lines('code').join + @tuned_code << t_o + t_o=nil + end + elsif @@flag['poem'] \ + or @@flag['group'] \ + or @@flag['block'] \ + or @@flag['alt'] + if @@flag['poem'] \ + and t_o =~/^\}poem/ + @@flag['poem']=false + h={ obj: 'poem end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + elsif ( @@flag['group'] \ + and t_o =~/^\}group/ ) + @@flag['group']=false + obj,tags=extract_tags(@tuned_block.join("\n")) + h={ obj: obj, tags: tags } + @tuned_block=[] + t_o=SiSU_DAL_DocumentStructure::ObjectBlockTxt.new.group(h) + tuned_file << t_o + h={ obj: 'group text end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + elsif ( @@flag['block'] \ + and t_o =~/^\}block/ ) + @@flag['block']=false + obj,tags=extract_tags(@tuned_block.join("\n")) + h={ obj: obj, tags: tags } + @tuned_block=[] + t_o=SiSU_DAL_DocumentStructure::ObjectBlockTxt.new.block(h) + tuned_file << t_o + h={ obj: 'block text end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + elsif ( @@flag['alt'] \ + and t_o =~/^\}alt/ ) + @@flag['alt']=false + obj,tags=extract_tags(@tuned_block.join("\n")) + h={ obj: obj, tags: tags } + t_o=SiSU_DAL_DocumentStructure::ObjectBlockTxt.new.alt(h) + @tuned_block=[] + tuned_file << t_o + h={ obj: 'alt text end' } #introduce a counter + t_o=SiSU_DAL_DocumentStructure::ObjectComment.new.comment(h) #t_o=SiSU_DAL_DocumentStructure::ObjectLayout.new.insert(h) + end + if @@flag['poem'] \ + or @@flag['group'] \ + or @@flag['alt'] \ + and t_o =~/\S/ \ + and t_o !~/^(?:\}(?:verse|code|alt|group|block)|(?:verse|code|alt|group|block)\{)/ # fix logic + sub_array=t_o.dup + @line_mode=sub_array.scan(/.+/) + type=if @@flag['poem']; 'poem' + t_o=SiSU_DAL_DocumentStructureExtract::Build.new(@md,@line_mode).build_lines(type).join + poem=t_o.split(/\n\n/) + poem.each do |v| + v=v.gsub(/\n/m,"#{Mx[:br_nl]}\n") + obj,tags=extract_tags(v) + h={ obj: obj, tags: tags } + t_o=SiSU_DAL_DocumentStructure::ObjectBlockTxt.new.verse(h) + tuned_file << t_o + end + else 'group' + end + @verse_count+=1 if @@flag['poem'] + end + end + if not @@flag['code'] + if @@flag['poem'] \ + or @@flag['group'] \ + or @@flag['alt'] + if t_o.is_a?(String) + t_o=t_o.gsub(/\n/m,"#{Mx[:br_nl]}"). + gsub(/[ ][ ]/m,"#{Mx[:nbsp]*2}"). + gsub(/#{Mx[:nbsp]}\s/,"#{Mx[:nbsp]*2}") + t_o=t_o + Mx[:br_nl] if t_o =~/\S+/ + elsif t_o.is==:group \ + || t_o.is==:block \ + || t_o.is==:alt \ + || t_o.is==:verse + t_o.obj=t_o.obj.gsub(/\n/m,"#{Mx[:br_nl]}"). + gsub(/[ ][ ]/m,"#{Mx[:nbsp]*2}"). + gsub(/#{Mx[:nbsp]}\s/,"#{Mx[:nbsp]*2}") + end + @tuned_block << t_o if t_o =~/\S+/ + else tuned_file << t_o + end + else tuned_file << t_o + end + end + if @md.flag_endnotes + tuned_file << @pb + h={ ln: 2, obj: 'Endnotes', autonum_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + h={ ln: 4, obj: 'Endnotes', name: 'endnotes', autonum_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + h={ obj: 'Endnotes' } + end + if @md.book_idx + tuned_file << @pb + h={ ln: 2, obj: 'Index', autonum_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + h={ ln: 4, obj: 'Index', name: 'book_index', autonum_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + h={ obj: 'Index' } + end + tuned_file << @pb + h={ ln: 2, obj: 'Metadata', autonum_: false, ocn_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + h={ ln: 4, obj: 'SiSU Metadata, document information', name: 'metadata', autonum_: false, ocn_: false } + tuned_file << SiSU_DAL_DocumentStructure::ObjectHeading.new.heading_insert(h) + tuned_file + h={ obj: 'eof' } + meta=SiSU_DAL_DocumentStructure::ObjectMetadata.new.metadata(@metadata) + [tuned_file,meta] + end + def table_rows_and_columns_array(table_str) + table=[] + table_str.split(/#{Mx[:tc_c]}/).each do |table_row| + table_row_with_columns=table_row.split(/#{Mx[:tc_p]}/) + table << table_row_with_columns + end + table + end + def meta_heading(h) + h={ lv: h[:lv], ln: h[:ln], name: h[:name], obj: h[:obj], ocn: '0' } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h) + end + def meta_para(str) + h={ obj: str, ocn_: false } + SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h) + end + def build_lines(type='') + lines,lines_new=@data,[] + lines.each do |line| + line=if line =~/\S/ \ + and line !~/^code\{|^\}code/ \ + and not line.is_a?(Hash) + @@counter+=1 if @@flag['code'] + line=line.gsub(/\s\s/,"#{Mx[:nbsp]*2}"). + gsub(/#{Mx[:nbsp]}\s/,"#{Mx[:nbsp]*2}") + line=line.gsub(/^/,"#{Mx[:gr_o]}codeline#{Mx[:gr_c]}") if type=='code' # REMOVE try sort for texpdf special case + line=if line =~/(?:https?|file|ftp):\/\/\S+$/ + line.gsub(/\s*$/," #{Mx[:br_nl]}") + else line.gsub(/\s*$/,"#{Mx[:br_nl]}") #unless type=='code' + end + elsif line =~/^\s*$/ + line.gsub(/\s*$/,"#{Mx[:br_nl]}") + else line + end + lines_new << line + end + lines_new + end + end + class Structure # this must happen early + def initialize(md,dob) + @md,@dob=md,dob + end + def structure + structure_markup_normalize + structure_markup + @dob + end + def structure_markup #build structure where structure provided only in meta header + @dob=if @dob.is ==:para \ + && (((@dob.hang !~/[1-9]/) && (@dob.indent !~/[1-9]/)) \ + || (@dob.hang != @dob.indent)) \ + and not @dob.bullet_ + @dob=case @dob.obj + when /^#{@md.lv1}/ + h={ lv: 'A', ln: 1 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + when /^#{@md.lv2}/ + h={ lv: 'B', ln: 2 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + when /^#{@md.lv3}/ + h={ lv: 'C', ln: 3 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + when /^#{@md.lv4}/ + h={ lv: '1', ln: 4 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + when /^#{@md.lv5}/ + h={ lv: '2', ln: 5 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + when /^#{@md.lv6}/ + h={ lv: '3', ln: 6 } + SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,@dob) + else @dob + end + else @dob + end + @dob + end + def structure_markup_normalize #needs a bit of thinking + dob=if @md.markup_version.determined < 0.38 #%convert internal representation, consider making 0.38 structure default ([A-C1-6] instead of [1-9]), requires downstream changes + @dob=@dob.gsub(/^[456]~/,'!_'). + gsub(/^3~(\S+)/,"#{Mx[:lv_o]}6:\\1#{Mx[:lv_c]}"). + gsub(/^3~\s+/,"#{Mx[:lv_o]}6:#{Mx[:lv_c]}"). + gsub(/^2~(\S+)/,"#{Mx[:lv_o]}5:\\1#{Mx[:lv_c]}"). + gsub(/^2~\s+/,"#{Mx[:lv_o]}5:#{Mx[:lv_c]}"). + gsub(/^1~(\S+)/,"#{Mx[:lv_o]}4:\\1#{Mx[:lv_c]}"). + gsub(/^1~\s+/,"#{Mx[:lv_o]}4:#{Mx[:lv_c]}"). + gsub(/^:?C~(\S+)/,"#{Mx[:lv_o]}3:\\1#{Mx[:lv_c]}"). + gsub(/^:?C~\s+/,"#{Mx[:lv_o]}3:#{Mx[:lv_c]}"). + gsub(/^:?B~(\S+)/,"#{Mx[:lv_o]}2:\\1#{Mx[:lv_c]}"). + gsub(/^:?B~\s+/,"#{Mx[:lv_o]}2:#{Mx[:lv_c]}"). + gsub(/^:?A~(\S+)/,"#{Mx[:lv_o]}1:\\1#{Mx[:lv_c]}"). + gsub(/^:?A~\s+/,"#{Mx[:lv_o]}1:#{Mx[:lv_c]}") + @dob=if @dob =~/^@(?:level|markup):\s/ + @dob=@dob.gsub(/3/,'6'). + gsub(/2/,'5'). + gsub(/1/,'4'). + gsub(/:?C/,'3'). + gsub(/:?B/,'2'). + gsub(/:?A/,'1') + @dob + else @dob + end + else @dob + end + end + def structure_marks + t_o=if @md.markup_version.determined < 0.38 + @t_o=@t_o.gsub(/^1~(\S+)/,"#{Mx[:lv_o]}1:\\1#{Mx[:lv_c]}"). + gsub(/^1~\s+/,"#{Mx[:lv_o]}1:#{Mx[:lv_c]}"). + gsub(/^2~(\S+)/,"#{Mx[:lv_o]}2:\\1#{Mx[:lv_c]}"). + gsub(/^2~\s+/,"#{Mx[:lv_o]}2:#{Mx[:lv_c]}"). + gsub(/^3~(\S+)/,"#{Mx[:lv_o]}3:\\1#{Mx[:lv_c]}"). + gsub(/^3~\s+/,"#{Mx[:lv_o]}3:#{Mx[:lv_c]}"). + gsub(/^4~(\S+)/,"#{Mx[:lv_o]}4:\\1#{Mx[:lv_c]}"). + gsub(/^4~\s+/,"#{Mx[:lv_o]}4:#{Mx[:lv_c]}"). + gsub(/^5~(\S+)/,"#{Mx[:lv_o]}5:\\1#{Mx[:lv_c]}"). + gsub(/^5~\s+/,"#{Mx[:lv_o]}5:#{Mx[:lv_c]}"). + gsub(/^6~(\S+)/,"#{Mx[:lv_o]}6:\\1#{Mx[:lv_c]}"). + gsub(/^6~\s+/,"#{Mx[:lv_o]}6:#{Mx[:lv_c]}"). + gsub(/^[789]~/,'!_') + @t_o + else @t_o + end + end + end + class OCN + def initialize(md,data) + @md,@data=md,data + end + def ocn #and auto segment numbering increment + data=@data + @o_array=[] + node=ocn=ocn_dv=ocn_sp=ocnh=ocnh1=ocnh2=ocnh3=ocnh4=ocnh5=ocnh6=ocno=ocnp=ocnt=ocnc=ocng=ocni=ocnm=ocnu=ocnk=nm=0 # h heading, o other, t table, g group, i image + node_count_flag=false + regex_exclude_ocn_and_node = /#{Rx[:meta]}|^@\S+?:\s|^4~endnotes|^#{Mx[:lv_o]}4:endnotes#{Mx[:lv_c]}|^\^~ |<:e[:_]\d+?>|^<:\#|<:- |<[:!]!4|
+ if dob.is==:heading + ln=case dob.lv + when 'A'; 1 + when 'B'; 2 + when 'C'; 3 + when '1'; 4 + when '2'; 5 + when '3'; 6 + when '4'; 7 + when '5'; 8 + when '6'; 9 + end + end + if not dob.obj =~/<:#>|~#|-#/ \ + or not dob.toc_ # fix this no longer in dob.obj + ocn+=1 + if dob.is==:heading \ + and (ln.to_s =~/^[1-9]/ \ + or ln.to_s =~@md.lv1 \ + or ln.to_s =~@md.lv2 \ + or ln.to_s =~@md.lv3 \ + or ln.to_s =~@md.lv4 \ + or ln.to_s =~@md.lv5 \ + or ln.to_s =~@md.lv6) + ocnh+=1 + if ln==1 \ + or ln=~@md.lv1; ocnh1+=1 #heading + node1="1:#{ocnh1};#{ocn}" + node,ocn_sp,parent=node1,"h#{ocnh}",0 #FIX + elsif ln==2 \ + or ln=~@md.lv2; ocnh2+=1 + node2="2:#{ocnh2};#{ocn}" + node,ocn_sp,parent=node2,"h#{ocnh}",node1 + elsif ln==3 \ + or ln=~@md.lv3; ocnh3+=1 + node3="3:#{ocnh3};#{ocn}" + node,ocn_sp,parent=node3,"h#{ocnh}",node2 + elsif ln==4 \ + or ln=~@md.lv4; ocnh4+=1 + node4="4:#{ocnh4};#{ocn}" + node,ocn_sp,parent=node4,"h#{ocnh}",node3 + elsif ln==5 \ + or ln=~@md.lv5; ocnh5+=1 + node5="5:#{ocnh5};#{ocn}" + node,ocn_sp,parent=node5,"h#{ocnh}",node4 + elsif ln==6 \ + or ln=~@md.lv6; ocnh6+=1 + node6="6:#{ocnh6};#{ocn}" + node,ocn_sp,parent=node6,"h#{ocnh}",node5 + end + else + ocno+=1 + if dob.is==:table + ocnt+=1 + ocn_sp,parent="t#{ocnt}",node + elsif dob.is==:code + ocnc+=1 + ocn_sp,parent="c#{ocnc}",node + elsif dob.is==:group \ + || dob.is==:block \ + || dob.is==:alt \ + || dob.is==:verse + ocng+=1 #group, poem + ocn_sp,parent="g#{ocng}",node + elsif dob.is==:image #check + ocni+=1 + ocn_sp,parent="i#{ocni}",node + else ocnp+=1 #paragraph + ocn_sp,parent="p#{ocnp}",node + end + end + if dob.is==:heading + dob.ln,dob.node,dob.ocn,dob.odv,dob.osp,dob.parent=ln,node,ocn,ocn_dv,ocn_sp,parent + else + if dob.of !=:meta \ + && dob.of !=:comment \ + && dob.of !=:layout + dob.ocn,dob.odv,dob.osp,dob.parent=ocn,ocn_dv,ocn_sp,parent + end + end + else ocnu+=1 + dob.obj=dob.obj.gsub(/#{Mx[:fa_o]}~##{Mx[:fa_c]}/,'') if dob.obj + ocn_dv,ocn_sp="u#{ocnu}","u#{ocnu}" + dob.ocn,dob.odv,dob.osp=ocn,ocn_dv,ocn_sp + end + h + elsif dob.obj=~/#{Mx[:pa_non_object_no_heading]}/ + dob.obj=dob.obj.gsub(/#{Mx[:pa_non_object_no_heading]}/,'') + if dob.is==:para + h={ obj: dob.obj, ocn_: false, ocn: nil, hang: dob.hang, indent: dob.indent, bullet_: dob.bullet_, tags: dob.tags, parent: dob.parent } + dob=SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h,dob) + elsif dob.is==:heading + h={ obj: dob.obj, ocn_: false, ocn: nil, toc_: true, parent: dob.parent } + dob=SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,dob) + end + elsif dob.obj=~/#{Mx[:pa_non_object_dummy_heading]}/ + dob.obj=dob.obj.gsub(/#{Mx[:pa_non_object_dummy_heading]}/,'') + if dob.is==:para + h={ obj: dob.obj, ocn_: false, ocn: nil, hang: dob.hang, indent: dob.indent, bullet_: dob.bullet_, tags: dob.tags, parent: dob.parent } + dob=SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h,dob) + elsif dob.is==:heading + h={ obj: dob.obj, ocn_: false, ocn: nil, toc_: false, parent: dob.parent } + dob=SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h,dob) + end + else dob + end + if dob.is==:code \ + || dob.is==:verse \ + || dob.is==:alt \ + || dob.is==:group \ + || dob.is==:block + dob.obj=dob.obj.gsub(/\n\n/,"\n") #newlines taken out + end + @o_array << dob + end + @o_array + end + end + class XML + def initialize(md,data) + @data,@md=data,md + end + def dom + @s=['0', + 'A', + 'B', + 'C', + '1', + '2', + '3' + ] + @sp=' ' + tuned_file=structure_build + tuned_file + end + def structure_build + data=@data + tuned_file=[] + hs=[0,false,false,false] + t={ lv: @s[0], status: 'open' } + tuned_file << tags(t) + if @md.opt.cmd =~/V/ + puts "\nXML sisu structure outline --->\n" + puts "<#{@s[0]}>" + end + data.each_with_index do |o,i| + if o.is ==:heading \ + || o.is ==:heading_insert + case o.ln + when 1 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs=[1,true,false,false] + when 2 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs=[2,true,true,false] + when 3 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs=[3,true,true,true] + when 4 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs[0]=4 + when 5 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs[0]=5 + when 6 + tuned_file << tag_close(o.ln,hs) + tuned_file << tag_open(o,@s) + if @md.opt.cmd =~/V/ + puts_tag_close(o.ln,hs) + puts_tag_open(o,@s) + end + hs[0]=6 + end + end + tuned_file << o + end + puts_tag_close(0,hs) if @md.opt.cmd =~/V/ + tuned_file << tag_close(0,hs) + tuned_file=tuned_file.flatten + end + def tags(o) + tag=(o[:status]=='open') \ + ? %{<#{o[:lv]} id="#{o[:node]}">} + : "" + ln=case o[:lv] + when 'A'; 1 + when 'B'; 2 + when 'C'; 3 + when '1'; 4 + when '2'; 5 + when '3'; 6 + when '4'; 7 + when '5'; 8 + when '6'; 9 + end + h={ tag: tag, node: o[:node], lv: o[:lv], ln: ln, status: o[:status] } + SiSU_DAL_DocumentStructure::ObjectStructure.new.xml_dom(h) #downstream code utilise else ignore like comments + end + def tag_open(o,tag) + t={ lv: tag[o.ln], node: o.node, status: 'open' } + t_o=tags(t) + t_o + end + def tag_close(lev,hs) + ary=[] + case hs[0] + when 1 + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + when 2 + if (lev <= 2) and hs[2] + t={ lv: @s[2], status: 'close' } + ary << tags(t) + end + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + when 3 + if (lev <= 3) and hs[3] + t={ lv: @s[3], status: 'close' } + ary << tags(t) + end + if (lev <= 2) and hs[2] + t={ lv: @s[2], status: 'close' } + ary << tags(t) + end + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + when 4 + if (lev <= 4) + t={ lv: @s[4], status: 'close' } + ary << tags(t) + end + if (lev <= 3) and hs[3] + t={ lv: @s[3], status: 'close' } + ary << tags(t) + end + if (lev <= 2) and hs[2] + t={ lv: @s[2], status: 'close' } + ary << tags(t) + end + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + when 5 + if (lev <= 5) + t={ lv: @s[5], status: 'close' } + ary << tags(t) + end + if (lev <= 4) + t={ lv: @s[4], status: 'close' } + ary << tags(t) + end + if (lev <= 3) and hs[3] + t={ lv: @s[3], status: 'close' } + ary << tags(t) + end + if (lev <= 2) and hs[2] + t={ lv: @s[2], status: 'close' } + ary << tags(t) + end + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + when 6 + if (lev <= 6) + t={ lv: @s[6], status: 'close' } + ary << tags(t) + end + if (lev <= 5) + t={ lv: @s[5], status: 'close' } + ary << tags(t) + end + if (lev <= 4) + t={ lv: @s[4], status: 'close' } + ary << tags(t) + end + if (lev <= 3) and hs[3] + t={ lv: @s[3], status: 'close' } + ary << tags(t) + end + if (lev <= 2) and hs[2] + t={ lv: @s[2], status: 'close' } + ary << tags(t) + end + if (lev <= 1) and hs[1] + t={ lv: @s[1], status: 'close' } + ary << tags(t) + end + if (lev==0) + t={ lv: @s[0], status: 'close' } + ary << tags(t) + end + end + ary + end + def puts_tag_open(o,tag) + puts %{#{@sp*o.ln}<#{tag[o.ln]} id="#{o.node}">} + end + def puts_tag_close(lev,hs) + case hs[0] + when 1 + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + when 2 + puts "#{@sp*2}" if (lev <= 2) and hs[2] + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + when 3 + puts "#{@sp*3}" if (lev <= 3) and hs[3] + puts "#{@sp*2}" if (lev <= 2) and hs[2] + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + when 4 + puts "#{@sp*4}" if (lev <= 4) + puts "#{@sp*3}" if (lev <= 3) and hs[3] + puts "#{@sp*2}" if (lev <= 2) and hs[2] + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + when 5 + puts "#{@sp*5}" if (lev <= 5) + puts "#{@sp*4}" if (lev <= 4) + puts "#{@sp*3}" if (lev <= 3) and hs[3] + puts "#{@sp*2}" if (lev <= 2) and hs[2] + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + when 6 + puts "#{@sp*6}" if (lev <= 6) + puts "#{@sp*5}" if (lev <= 5) + puts "#{@sp*4}" if (lev <= 4) + puts "#{@sp*3}" if (lev <= 3) and hs[3] + puts "#{@sp*2}" if (lev <= 2) and hs[2] + puts "#{@sp*1}" if (lev <= 1) and hs[1] + puts "" if (lev==0) + end + end + end +end +__END__ diff --git a/lib/sisu/v5/dal_endnotes.rb b/lib/sisu/v5/dal_endnotes.rb new file mode 100644 index 00000000..8412b45e --- /dev/null +++ b/lib/sisu/v5/dal_endnotes.rb @@ -0,0 +1,128 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +module SiSU_DAL_Endnotes + class Endnotes + def initialize(md,data,endnote_array=nil) + @md,@data,@endnote_array=md,data,endnote_array + @endnote_counter,@endnote_counter_asterisk,@endnote_counter_dag=1,1,1 + end + def endnotes + data=@data + @tuned_file=[] + endnote_no,endnote_ref=1,1 + data.each do |dob| + # manually numbered endnotes --> + if @md.opt.mod.inspect =~/--no-asterisk|--no-annotate/ + dob.obj=dob.obj.gsub(/#{Mx[:en_b_o]}\s.+?#{Mx[:en_b_c]}/,'') + end + if @md.opt.mod.inspect =~/--no-dagger|--no-annotate/ + dob.obj=dob.obj.gsub(/#{Mx[:en_b_o]}[+]\s.+?#{Mx[:en_b_c]}/,'') + end + if (defined? dob.obj) \ + && (defined? dob.is) \ + && dob.is !=:code + case dob.obj # auto-numbered endnotes --> + when /#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}[*+]\s+.+?#{Mx[:en_b_c]}/ + dob.obj=dob.obj.gsub(/\s*(#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/,' \1') # required 2003w31 + word_mode=dob.obj.scan(/\S+/m) + word_mode=endnote_call_number(word_mode) + dob.obj=word_mode.join(' ') + endnote_ref+=1 + when /~\^(?:\s|$)|<:e>/ #%note inserts endnotes previously gathered from /^(|[-~]\{{3})/ (in earlier loop) + word_mode=dob.obj.scan(/\S+/m) + word_mode=endnote_call_number(word_mode) + dob.obj=word_mode.join(' ') + endnote_ref+=1 + end + end + @tuned_file << dob + end + @endnote_counter,@endnote_counter_asterisk,@endnote_counter_dag=1,1,1 + @tuned_file=@tuned_file.flatten + end + def endnote_call_number(words) + words.each do |word| + case word + when /#{Mx[:en_a_o]}/ + unless word =~/#{Mx[:en_a_o]}[*+]+/ + word.gsub!(/#{Mx[:en_a_o]}/,"#{Mx[:en_a_o]}#{@endnote_counter} ") + @endnote_counter+=1 + end + when /#{Mx[:en_b_o]}/ + if word =~/#{Mx[:en_b_o]}[+]/ + word.gsub!(/#{Mx[:en_b_o]}[+]/,"#{Mx[:en_b_o]}\+#{@endnote_counter_dag} ") + @endnote_counter_dag+=1 + else + word.gsub!(/#{Mx[:en_b_o]}[*]?/,"#{Mx[:en_b_o]}\*#{@endnote_counter_asterisk} ") + @endnote_counter_asterisk+=1 + end + when /~\^|<:e>/ + if @endnote_array + word.gsub!(/~\^|<:e>/,"#{@endnote_array[@endnote_counter-1]}") + @endnote_counter+=1 + end + end + end + end + end +end +__END__ diff --git a/lib/sisu/v5/dal_expand_insertions.rb b/lib/sisu/v5/dal_expand_insertions.rb new file mode 100644 index 00000000..46efaa63 --- /dev/null +++ b/lib/sisu/v5/dal_expand_insertions.rb @@ -0,0 +1,382 @@ +# encoding: utf-8 +=begin + + * Name: SiSU + + * Description: a framework for document structuring, publishing and search + + * Author: Ralph Amissah + + * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved. + + * License: GPL 3 or later: + + SiSU, a framework for document structuring, publishing and search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see . + + If you have Internet connection, the latest version of the GPL should be + available at these locations: + + + + + + * SiSU uses: + * Standard SiSU markup syntax, + * Standard SiSU meta-markup syntax, and the + * Standard SiSU object citation numbering and system + + * Hompages: + + + + * Download: + + + * Git + + + + * Ralph Amissah + + + + ** Description: system environment, resource control and configuration details + +=end +module SiSU_DAL_Insertions + class Insertions + def initialize(md,data) + @md,@data=md,data + end + def output_filetypes_in_cmd(cmd_shortcut,lnk=nil) #make list of file types in shortcut command (as configured), e.g. when sisu -3 is used + cf_defaults=SiSU_Env::InfoProcessingFlag.new + cmd_list=case cmd_shortcut.inspect + when /0/; cf_defaults.cf_0 + when /1/; cf_defaults.cf_1 + when /2/; cf_defaults.cf_2 + when /3/; cf_defaults.cf_3 + when /4/; cf_defaults.cf_4 + when /5/; cf_defaults.cf_5 + end + file_type_names={} + file_type_names[:gen],file_type_names[:src]=[],[] + file_type_names[:gen] <<= if cmd_list =~ /y/ then "~^ { document manifest }#{lnk[:manifest]}" + end + file_type_names[:gen] <<= if cmd_list =~ /h/ then [" { html, segmented text }#{lnk[:html_toc]}"," { html, scroll, document in one }#{lnk[:html_doc]}"] + end + file_type_names[:gen] <<= if cmd_list =~ /e/ then [" { epub }#{lnk[:epub]}"] + end + file_type_names[:gen] <<= if cmd_list =~ /p/ then [" { pdf, landscape }#{lnk[:pdf_landscape]}"," { pdf, portrait }#{lnk[:pdf_portrait]}"] + end + file_type_names[:gen] <<= if cmd_list =~ /o/ then " { odf:odt, open document text }#{lnk[:odt]}" + end + file_type_names[:gen] <<= if cmd_list =~ /b/ then " { xhtml scroll }#{lnk[:xhtml]}" + end + file_type_names[:gen] <<= if cmd_list =~ /x/ then " { xml, sax }#{lnk[:xml_sax]}" + end + file_type_names[:gen] <<= if cmd_list =~ /X/ then " { xml, dom }#{lnk[:xml_dom]}" + end + file_type_names[:gen] <<= if cmd_list =~ /a/ then " { plain text utf-8 }#{lnk[:txt]}" + end + file_type_names[:gen] <<= if cmd_list =~ /g/ then 'wiki.txt' + end + file_type_names[:gen] <<= if cmd_list =~ /w/ then " { concordance }#{lnk[:html_concordance]}" + end + file_type_names[:gen] <<= if cmd_list =~ /N/ then " { dcc, document content certificate (digests) }#{lnk[:digest]}" + end + file_type_names[:src] <<= if source and cmd_shortcut =~ /s/ then " { markup source text }#{lnk[:source]}" + end + file_type_names[:src] <<= if cmd_shortcut =~ /S/ then " { markup source (zipped) pod }#{lnk[:sisupod]}" + end + file_type_names[:gen]=file_type_names[:gen].flatten + file_type_names[:src]=file_type_names[:src].flatten + file_type_names + end + def by_language(linked_doc,lng,src=nil) + @linked_doc,@lng,@src=linked_doc,lng,src + @base_path="#{@md.file.output_path.base.url}/#{@lng}" + def manifest + "#{@base_path}/manifest/#{@linked_doc}.html" + end + def html_toc + "#{@base_path}/html/#{@linked_doc}.html" + end + def html_doc + "#{@base_path}/html/#{@linked_doc}.html" + end + def html_concordance + "#{@base_path}/html/#{@linked_doc}.html" + end + def epub + "#{@base_path}/epub/#{@linked_doc}.epub" + end + def pdf_landscape + "#{@base_path}/pdf/#{@linked_doc}.pdf" + end + def pdf_portrait + "#{@base_path}/pdf/#{@linked_doc}.pdf" + end + def odt + "#{@base_path}/odt/#{@linked_doc}.odt" + end + def xhtml + "#{@base_path}/xhtml/#{@linked_doc}.xhtml" + end + def xml_sax + "#{@base_path}/xml/#{@linked_doc}.xml" + end + def xml_dom + "#{@base_path}/xml/#{@linked_doc}.xml" + end + def txt + "#{@base_path}/txt/#{@linked_doc}.txt" + end + def digest + "#{@base_path}/digest/#{@linked_doc}.txt" + end + def source + "#{@base_path}/src/#{@src}" + end + def sisupod + "#{@base_path}/pod/#{@src}.zip" + end + self + end + def by_filetype(linked_doc,lng,src=nil) + @linked_doc,@lng,@src=linked_doc,lng,src + @base_path="#{@md.file.output_path.base.url}" + def manifest + "#{@base_path}/manifest/#{@linked_doc}.#{@lng}.html" + end + def html_toc + "#{@base_path}/html/#{@linked_doc}.#{@lng}.html" + end + def html_doc + "#{@base_path}/html/#{@linked_doc}.#{@lng}.html" + end + def html_concordance + "#{@base_path}/html/#{@linked_doc}.#{@lng}.html" + end + def epub + "#{@base_path}/epub/#{@linked_doc}.#{@lng}.epub" + end + def pdf_landscape + "#{@base_path}/pdf/#{@linked_doc}.#{@lng}.pdf" + end + def pdf_portrait + "#{@base_path}/pdf/#{@linked_doc}.#{@lng}.pdf" + end + def odt + "#{@base_path}/odt/#{@linked_doc}.#{@lng}.odt" + end + def xhtml + "#{@base_path}/xhtml/#{@linked_doc}.#{@lng}.xhtml" + end + def xml_sax + "#{@base_path}/xml_sax/#{@linked_doc}.#{@lng}.xml" + end + def xml_dom + "#{@base_path}/xml_dom/#{@linked_doc}.#{@lng}.xml" + end + def txt + "#{@base_path}/txt/#{@linked_doc}.#{@lng}.txt" + end + def digest + "#{@base_path}/digest/#{@linked_doc}.#{@lng}.txt" + end + def source + "#{@base_path}/src/#{@src}" + end + def sisupod + "#{@base_path}/pod/#{@src}.zip" + end + self + end + def by_filename(linked_doc,lng,src=nil) + @linked_doc,@lng,@src=linked_doc,lng,src + @base_path="#{@md.file.output_path.base.url}/#{@linked_doc}" + def manifest + "#{@base_path}/sisu_manifest.#{@lng}.html" + end + def html_toc + "#{@base_path}/toc.#{@lng}.html" + end + def html_doc + "#{@base_path}/scroll.#{@lng}.html" + end + def html_concordance + "#{@base_path}/concordance.#{@lng}.html" + end + def epub + "#{@base_path}/epub/#{@linked_doc}.#{@lng}.epub" + end + def pdf_landscape + "#{@base_path}/landscape.#{@lng}.a4.pdf" + end + def pdf_portrait + "#{@base_path}/portrait.#{@lng}.a4.pdf" + end + def odt + "#{@base_path}/opendocument.#{@lng}.odt" + end + def xhtml + "#{@base_path}/scroll.#{@lng}.xhtml" + end + def xml_sax + "#{@base_path}/scroll.#{@lng}.sax.xml" + end + def xml_dom + "#{@base_path}/scroll.#{@lng}.dom.xml" + end + def txt + "#{@base_path}/plain.#{@lng}.txt" + end + def digest + "#{@base_path}/digest.#{@lng}.txt" + end + def source + "#{@base_path}/#{@src}" + end + def sisupod + "#{@base_path}/#{@src}.zip" + end + self + end + def expand_insertions? + data=@data + tuned_file,tuned_file_tmp=[],[] + codeblock_=false + data.each do |para| + codeblock_=if para =~/^code\{/ + true + elsif para =~/^\}code/ + false + else codeblock_ + end + if para !~/^%+\s/ \ + and not codeblock_ \ + and para =~/\{(?:~\^\s+)?(.+?)\s\[(?:\d(?:[sS]*))\]\}(?:\.\.\/\S+?\/|\S+?\.ss[tm]\b)/ + txt,cmd,source,linked_doc,note,manifest=nil,nil,nil,nil,nil,nil + @u=SiSU_Env::InfoEnv.new.url + pre=txt=cmd=source=linked_doc=note='' + if defined? @u.remote + if /(?
.+?)\{(?.+?)\s\[(?\d[sS]*)\]\}(?(?\S+?)\.ss[tm]\b)(?.*)/m =~ para
+              pre=pre.strip
+            elsif /\{(?.+?)\s\[(?\d[sS]*)\]\}(?(?\S+?)\.ss[tm]\b)(?.*)/m =~ para
+            end
+            if linked_doc =~ /(\S+?)\/(\S+)/
+              linked_doc,linked_doc_lang=$1,$2
+            else
+              linked_doc,linked_doc_lang=linked_doc,@md.opt.lng_base
+            end
+          else
+            puts "error, does currently support relative paths (reltive paths were removed, as had problems for citation, and was not suited to all output types should possibly reconsider) #{__FILE__} #{__LINE__}"
+            if /\{(?:~\^\s+)?(?.+?)\s\[(?\d[sS]*)\]\}\.\.\/(?\S+?)\/(?\s+#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]})?/ =~ para
+            end
+          end
+          lnk={}
+          case @md.opt.dir_structure_by
+          when :language
+            lnk={
+              manifest:         by_language(linked_doc,linked_doc_lang).manifest,
+              html_toc:         by_language(linked_doc,linked_doc_lang).html_toc,
+              html_doc:         by_language(linked_doc,linked_doc_lang).html_doc,
+              epub:             by_language(linked_doc,linked_doc_lang).epub,
+              pdf_landscape:    by_language(linked_doc,linked_doc_lang).pdf_landscape,
+              pdf_portrait:     by_language(linked_doc,linked_doc_lang).pdf_landscape,
+              odt:              by_language(linked_doc,linked_doc_lang).odt,
+              xhtml:            by_language(linked_doc,linked_doc_lang).xhtml,
+              xml_sax:          by_language(linked_doc,linked_doc_lang).xml_sax,
+              xml_dom:          by_language(linked_doc,linked_doc_lang).xml_dom,
+              txt:              by_language(linked_doc,linked_doc_lang).txt,
+              html_concordance: by_language(linked_doc,linked_doc_lang).html_concordance,
+              digest:           by_language(linked_doc,linked_doc_lang).digest,
+              sisupod:          by_language(linked_doc,linked_doc_lang,source).sisupod,
+              source:           by_language(linked_doc,linked_doc_lang,source).source,
+            }
+          when :filetype
+            lnk={
+              manifest:         by_filetype(linked_doc,linked_doc_lang).manifest,
+              html_toc:         by_filetype(linked_doc,linked_doc_lang).html_toc,
+              html_doc:         by_filetype(linked_doc,linked_doc_lang).html_doc,
+              epub:             by_filetype(linked_doc,linked_doc_lang).epub,
+              pdf_landscape:    by_filetype(linked_doc,linked_doc_lang).pdf_landscape,
+              pdf_portrait:     by_filetype(linked_doc,linked_doc_lang).pdf_landscape,
+              odt:              by_filetype(linked_doc,linked_doc_lang).odt,
+              xhtml:            by_filetype(linked_doc,linked_doc_lang).xhtml,
+              xml_sax:          by_filetype(linked_doc,linked_doc_lang).xml_sax,
+              xml_dom:          by_filetype(linked_doc,linked_doc_lang).xml_dom,
+              txt:              by_filetype(linked_doc,linked_doc_lang).txt,
+              html_concordance: by_filetype(linked_doc,linked_doc_lang).html_concordance,
+              digest:           by_filetype(linked_doc,linked_doc_lang).digest,
+              sisupod:          by_filetype(linked_doc,linked_doc_lang,source).sisupod,
+              source:           by_filetype(linked_doc,linked_doc_lang,source).source,
+            }
+          else
+            lnk={
+              manifest:         by_filename(linked_doc,linked_doc_lang).manifest,
+              html_toc:         by_filename(linked_doc,linked_doc_lang).html_toc,
+              html_doc:         by_filename(linked_doc,linked_doc_lang).html_doc,
+              epub:             by_filename(linked_doc,linked_doc_lang).epub,
+              pdf_landscape:    by_filename(linked_doc,linked_doc_lang).pdf_landscape,
+              pdf_portrait:     by_filename(linked_doc,linked_doc_lang).pdf_landscape,
+              odt:              by_filename(linked_doc,linked_doc_lang).odt,
+              xhtml:            by_filename(linked_doc,linked_doc_lang).xhtml,
+              xml_sax:          by_filename(linked_doc,linked_doc_lang).xml_sax,
+              xml_dom:          by_filename(linked_doc,linked_doc_lang).xml_dom,
+              txt:              by_filename(linked_doc,linked_doc_lang).txt,
+              html_concordance: by_filename(linked_doc,linked_doc_lang).html_concordance,
+              digest:           by_filename(linked_doc,linked_doc_lang).digest,
+              sisupod:          by_filename(linked_doc,linked_doc_lang,source).sisupod,
+              source:           by_filename(linked_doc,linked_doc_lang,source).source,
+            }
+          end
+          linked_title="#{pre}{#{txt} }#{lnk[:manifest]}#{note}\n\n"
+          tuned_file_tmp << linked_title
+          output_filetypes=output_filetypes_in_cmd(cmd,lnk)
+          output_filetypes[:gen].each do |o_f|
+            describe = o_f
+            if describe
+              tuned_file_tmp << if @u.remote #to double space <:br> at beginning of entry
+                "#{Mx[:nbsp]*4} #{describe} "
+              else # remove ...
+                "[provide document placement host location]"
+              end
+            end
+          end
+          output_filetypes[:src].each do |o_f|
+            describe = o_f
+            if describe
+              tuned_file_tmp << if @u.remote
+                "#{Mx[:nbsp]*4} #{describe} "
+              else
+                "[provide document placement host location]"
+              end
+            end
+          end
+          tuned_file << 'group{' << tuned_file_tmp.join("\n") << '}group'
+          tuned_file_tmp=[]
+        else tuned_file << para
+        end
+      end
+      tuned_file
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_hash_digest.rb b/lib/sisu/v5/dal_hash_digest.rb
new file mode 100644
index 00000000..29d8a1ed
--- /dev/null
+++ b/lib/sisu/v5/dal_hash_digest.rb
@@ -0,0 +1,159 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_Hash
+  require_relative 'shared_markup_alt.rb'               #shared_markup_alt.rb
+  class ObjectDigest
+    def initialize(md,data,env=nil)
+      @md,@data,@env=md,data,env
+      @env ||=SiSU_Env::InfoEnv.new(@md.fns)
+    end
+    def object_digest
+    # 1. clean/stripped text without any markup, paragraph, headings etc. without endnotes
+    # 2. endnotes clean/stripped text digest only (there may be several endnotes within a paragraph)
+    # 3. whole object, text with markup and any endnotes, (question: with or without the endnote digests??? presumption better without, [however may be easier to check with?])
+    # [digests should not include other digests]
+      data=@data.compact
+      @tuned_file=[]
+      sha_ =(@env.digest.type=='sha256' ? true : false)
+      sha_ ? (require 'digest/sha2') : (require 'digest/md5')
+      data.each do |t_o|
+        unless t_o.obj.is_a?(Array)
+          t_o.obj=t_o.obj.strip
+        end
+        if (t_o.of !=:structure \
+        && t_o.of !=:comment \
+        && t_o.of !=:layout) \
+        && t_o.ocn.is_a?(Fixnum)
+          if sha_
+            for hash_class in [ Digest::SHA256 ]
+              @tuned_file << stamped(t_o,hash_class)
+            end
+          else
+            for hash_class in [ Digest::MD5 ]
+              @tuned_file << stamped(t_o,hash_class)
+            end
+          end
+        else @tuned_file << t_o unless t_o.nil?
+        end
+      end
+      @tuned_file=@tuned_file.flatten
+      #use md5 or to create hash of each dal object including ocn, & add into to each dal object
+    end
+    def endnote_digest(data)
+      t_o_bit=[]
+      data.each do |en_plus|
+        t_o_bit <<= case en_plus
+        when /#{Mx[:en_a_o]}|#{Mx[:en_b_o]}/
+          if en_plus =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/
+            t_o_txt,en_open,en_txt,en_close=/(.*?)(#{Mx[:en_a_o]}|#{Mx[:en_b_o]})(.+?)(#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/m.match(en_plus)[1..4]
+            stripped_en=SiSU_TextRepresentation::Alter.new(en_txt).strip_clean_of_markup
+            digest_en_strip=if @env.digest.type =~/sha256/
+              Digest::SHA256.hexdigest(stripped_en)
+            else
+              Digest::MD5.hexdigest(stripped_en)
+            end
+            t_o_txt + en_open + en_txt + Mx[:id_o] + digest_en_strip + Mx[:id_c] + en_close
+          else STDERR.puts "Error Exception - problem encountered with:\n#{en_plus}" #arbitrary exception, tidy up
+          end
+        else en_plus
+        end
+      end
+      t_o_bit.join
+    end
+    def stamped(t_o,hash_class) #decide what hash information is most useful, is compromise necessary?
+      t_o.obj=SiSU_TextRepresentation::Alter.new(t_o).strip_clean_of_extra_spaces
+      t_obj=t_o.inspect.sub(/:0x[0-9a-f]{8}\s/,': ')
+      stripped=SiSU_TextRepresentation::Alter.new(t_o).strip_clean_of_markup
+      markup=SiSU_TextRepresentation::Alter.new(t_o).semi_revert_markup
+      digests=SiSU_TextRepresentation::ModifiedTextPlusHashDigest.new(@md,t_o).composite.dgst
+      unless t_o.is==:code
+        case t_o.obj
+        when /#{Mx[:en_a_o]}[\d*+]+\s+.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}[*+]\d+\s+.+?#{Mx[:en_b_c]}/m
+          en_and_t_o,en_and_t_o_digest=[],[]
+          t_o.obj=t_o.obj.gsub(/\s*(#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/m,' \1') #watch
+          t_o_plus_en=t_o.obj.scan(/.*?#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|.*?#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/m)
+          t_o_tail=if t_o.obj =~/(?:.*?#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|.*?#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})+([\s\S]+)/m
+            /(?:.*?#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|.*?#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})+.*/m.match(t_o.obj)[1]
+          else ''
+          end
+          t_o_plus_en << t_o_tail
+          en_and_t_o_digest << endnote_digest(t_o_plus_en)
+          t_o_new=en_and_t_o_digest.join(' ')
+          #@tuned << t_o_new + Mx[:id_o] + digest_strip + ':' + digest_all + Mx[:id_c] unless t_o.nil?
+        else #@tuned << t_o + Mx[:id_o] + digest_strip + ':' + digest_all + Mx[:id_c] unless t_o.nil?
+        end
+      else #@tuned << t_o + Mx[:id_o] + digest_strip + ':' + digest_all + Mx[:id_c] unless t_o.nil?
+      end
+      t_o #KEEP intact
+    end
+    def strip_clean_extra_spaces(s)                                            # dal output tuned
+      s=s.dup
+      s=s.gsub(/[ ]+([,.;:?](?:$|\s))/,'\1') unless s =~/#{Mx[:en_a_o]}|#{Mx[:en_b_o]}/
+      s=s.gsub(/ [ ]+/,' ').
+        gsub(/^ [ ]+/,'').
+        gsub(/ [ ]+$/,'').
+        gsub(/((?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})')[ ]+(s )/,'\1\2').
+        gsub(/((?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})')[ ]+(s )/,'\1\2')
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_idx.rb b/lib/sisu/v5/dal_idx.rb
new file mode 100644
index 00000000..67f2805d
--- /dev/null
+++ b/lib/sisu/v5/dal_idx.rb
@@ -0,0 +1,366 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_BookIndex
+  class BookIndex
+    def initialize(md,data,env=nil)
+      @md,@data,@env=md,data,env
+      @rgx_idx=/#{Mx[:idx_o]}(?:.+?)#{Mx[:idx_c]}\s*/
+      @rgx_idx_ocn_seg=/(.+?)~(\d+)~(\S+)/
+      @rgx_idx_ocn=/(.+?)~(\d+)/
+      @env ||=SiSU_Env::InfoEnv.new(@md.fns)
+    end
+    def indexing_song
+      data=@data
+      data,sisu_markup_idx_rel,sisu_markup_idx_rel_html_seg,html_idx,xhtml_idx=extract_book_index(data)
+      data=clean_and_insert_index(data,sisu_markup_idx_rel_html_seg)
+      [data,sisu_markup_idx_rel,sisu_markup_idx_rel_html_seg,html_idx,xhtml_idx]
+    end
+    def extract_book_index(data)
+      tuned_file=[]
+      idx_array=[]
+      data.each do |dob|
+        if (dob.is ==:heading \
+        || dob.is ==:heading_insert) \
+        && dob.ln==4
+          @seg=dob.name
+        end
+        if defined? dob.idx \
+        and dob.idx.is_a?(String) \
+        and not dob.idx.empty?
+          idx_array << "#{dob.idx}~#{dob.ocn}~#{@seg}"
+        end
+        tuned_file << dob if dob
+      end
+      idx_array=construct_idx_array(idx_array) if idx_array.length > 0
+      if idx_array.length > 0
+        the_idx=construct_book_index(idx_array)
+        sisu_markup_idx_rel,sisu_markup_idx_rel_html_seg,html_idx,xhtml_idx=nil,nil,nil,nil
+        if @md.book_idx
+          idx=index(the_idx)
+          sisu_markup_idx_rel,sisu_markup_idx_rel_html_seg,html_idx,xhtml_idx=idx[:sst_rel],idx[:sst_rel_html_seg],idx[:html],idx[:xhtml]
+        end
+      end
+      [tuned_file,sisu_markup_idx_rel,sisu_markup_idx_rel_html_seg,html_idx,xhtml_idx]
+    end
+    def construct_idx_array(idx_array)
+      idx_lst=[]
+      idx_array.each do |idx|
+        idx_list,ocn,seg=@rgx_idx_ocn_seg.match(idx)[1..3]
+        idx_lst <<=if idx_list =~/;/
+          g=idx_list.scan(/[^;]+/)
+          idxl=[]
+          g.each do |i|
+            i=i.strip
+            idxl << { rough_idx: i, ocn: ocn, seg: seg }
+          end
+          idxl
+        else { rough_idx: idx_list, ocn: ocn, seg: seg }
+        end
+      end
+      idx_lst=idx_lst.flatten
+    end
+    def construct_book_index(idx_array)
+      the_idx={}
+      idx_array.each do |idx|
+        if idx[:rough_idx] =~/[|]/ \
+        && idx[:rough_idx] !~/[:]/
+          if @md.opt.cmd =~/[MVv]/
+            p 'book index error? --> ' + idx[:rough_idx]
+          end
+        else
+          idx_lst=idx[:rough_idx].scan(/[^|:]+/)
+          idx_lst[0]=idx_lst[0].strip
+          if idx_lst[0] =~/.+?\+\d+/
+            use,plus=/(.+?)\+(\d+)/.match(idx_lst[0])[1,2]
+          else use=idx_lst[0]
+          end
+          use=use[0].chr.capitalize + use[1,use.length]
+          the_idx[use]={} unless the_idx[use] and defined? the_idx[use]
+          idx_lst.each do |i|
+            i=i.strip
+            i,r=/(.+?)\+(\d+)/.match(i)[1,2] if i =~/.+?\+\d+/
+            x=if idx_lst.length==1 or idx_lst[0].gsub(/\+\d+/,'')==i
+              the_idx[use]['term_node_lev1']=[] unless the_idx[use]['term_node_lev1'] and defined? the_idx[use]['term_node_lev1']
+              x=if r
+                the_idx[use]['term_node_lev1'] << { ocn: idx[:ocn], range: "#{idx[:ocn]}-#{idx[:ocn].to_i+r.to_i}", seg: idx[:seg] }
+                "#{i} #{idx[:ocn]}-#{idx[:ocn].to_i+r.to_i}"
+              else
+                the_idx[use]['term_node_lev1'] << { ocn: idx[:ocn], seg: idx[:seg] }
+                "#{i} #{idx[:ocn]}"
+              end
+            else
+              the_idx[use]['term_node_lev2']={} unless the_idx[use]['term_node_lev2'] and defined? the_idx[use]['term_node_lev2']
+              the_idx[use]['term_node_lev2'][i]=[] unless the_idx[use]['term_node_lev2'][i] and defined? the_idx[use]['term_node_lev2'][i]
+              x=if r
+                the_idx[use]['term_node_lev2'][i] << { ocn: idx[:ocn], range: "#{idx[:ocn]}-#{idx[:ocn].to_i+r.to_i}", seg: idx[:seg] }
+                "#{idx_lst[0]}:#{i} #{idx[:ocn]}-#{idx[:ocn].to_i+r.to_i}"
+              else
+                the_idx[use]['term_node_lev2'][i] << { ocn: idx[:ocn], seg: idx[:seg] }
+                "#{idx_lst[0]}:#{i} #{idx[:ocn]}"
+              end
+            end
+          end
+        end
+      end
+      the_idx=the_idx.sort
+      the_idx
+    end
+    def clean_xml(str)
+      str=str.gsub(/&/,'&')
+      str
+    end
+    def index(the_idx)
+      @x=1
+      idx={}
+      idx[:sst_rel_html_seg],idx[:sst_rel],idx[:html],idx[:xhtml]=[],[],[],[]
+      h={obj: Mx[:br_page]}
+      o=SiSU_DAL_DocumentStructure::ObjectLayout.new.break(h)
+      idx[:sst_rel_html_seg] << o
+      idx[:sst_rel] << o
+      h={lv: '2', name: 'index', obj: "Index"}
+      o=SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h)
+      idx[:sst_rel_html_seg] << o
+      idx[:sst_rel] << o
+      h={lv: '4', name: 'idx', obj: " [Index] #{Mx[:pa_non_object_dummy_heading]}"}
+      o=SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h)
+      idx[:sst_rel_html_seg] << o
+      idx[:sst_rel] << o
+      alph=%W[9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+      idx[:html] << '
'
+      idx[:xhtml] << '
'
+      alph.each do |x|
+        if x =~/[0-9]/
+          idx[:html] << ''
+          idx[:xhtml] << ''
+        else
+          idx[:html] << %{#{x},#{$ep[:hsp]}}
+          idx[:xhtml] << %{#{x},#{$ep[:hsp]}}
+        end
+      end
+      idx[:html] << '
'
+      idx[:xhtml] << '
'
+      letter=alph.shift
+      idx[:html] << %{\n
}
+      idx[:xhtml] << %{\n
0 - 9
}
+      the_idx.each do |i|
+        i.each do |x|
+          if x.is_a?(String)
+            f=/^(\S)/.match(x)[1]
+            if letter < f
+              while letter < f
+                if alph.length > 0
+                  letter=alph.shift
+                  idx[:html] << %{\n
#{letter}
 
}
+                  idx[:xhtml] << %{\n
#{letter}
}
+                else break
+                end
+              end
+            end
+            idx[:sst_rel_html_seg] << %{\n\n#{Mx[:fa_bold_o]}#{x},#{Mx[:fa_bold_c]} }
+            idx[:sst_rel] << %{\n\n#{Mx[:fa_bold_o]}#{x},#{Mx[:fa_bold_c]} }
+            aname=x.gsub(/\s+/,'_')
+            idx[:html] << %{\n
#{x}, }
+            c=clean_xml(x.dup)
+            idx[:xhtml] << %{\n
#{c}, }
+            @o=idx[:sst_rel_html_seg].index(idx[:sst_rel_html_seg].last)
+            @t=idx[:sst_rel].index(idx[:sst_rel].last)
+            @q=idx[:html].index(idx[:html].last)
+            @r=idx[:xhtml].index(idx[:xhtml].last)
+            print "\n" + x + ', ' if @md.opt.cmd =~/V/
+          elsif x.is_a?(Array)
+            p 'array error? -->'
+            print x
+          elsif x.is_a?(Hash)
+            if x['term_node_lev1'].is_a?(Array)
+              x['term_node_lev1'].each do |a|
+                if a[:range]
+                  idx[:sst_rel_html_seg][@o]=idx[:sst_rel_html_seg][@o] + %{#{Mx[:lnk_o]}#{a[:range]}#{Mx[:lnk_c]}#{Mx[:rel_o]}/#{a[:seg]}.html##{a[:ocn]}#{Mx[:rel_c]}, }
+                  idx[:sst_rel][@t]=idx[:sst_rel][@t] + %{#{Mx[:lnk_o]}#{a[:range]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{a[:ocn]}#{Mx[:rel_c]}, }
+                  idx[:html][@q]=idx[:html][@q] + %{#{a[:range]}, }
+                  idx[:xhtml][@q]=idx[:xhtml][@q] + %{#{a[:range]}, }
+                  print a[:range] + ', ' if @md.opt.cmd =~/V/
+                elsif a[:ocn]
+                  idx[:sst_rel_html_seg][@o]=idx[:sst_rel_html_seg][@o] + %{#{Mx[:lnk_o]}#{a[:ocn]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{a[:seg]}.html##{a[:ocn]}#{Mx[:rel_c]}, }
+                  idx[:sst_rel][@t]=idx[:sst_rel][@t] + %{#{Mx[:lnk_o]}#{a[:ocn]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{a[:ocn]}#{Mx[:rel_c]}, }
+                  idx[:html][@q]=idx[:html][@q] + %{#{a[:ocn]}, }
+                  idx[:xhtml][@q]=idx[:xhtml][@q] + %{#{a[:ocn]}, }
+                  print a[:ocn] + ', ' if @md.opt.cmd =~/V/
+                else p 'error'
+                end
+              end
+              idx[:html][@q]=idx[:html][@q] + '
'
+              idx[:xhtml][@r]=idx[:xhtml][@r] + '
'
+            end
+            if x['term_node_lev2']
+              m=x['term_node_lev2']
+              m=m.sort
+              m.each do |k,y|
+                if k !~/term_node_lev1/
+                  idx[:sst_rel_html_seg][@o]=idx[:sst_rel_html_seg][@o] + %{#{k}, }
+                  idx[:sst_rel][@t]=idx[:sst_rel][@t] + %{#{k}, }
+                  idx[:html][@q]=idx[:html][@q] + %{\n
#{k}, }
+                  c=clean_xml(k.dup)
+                  idx[:xhtml][@r]=idx[:xhtml][@r] + %{\n
#{c}, }
+                  print "\n\t" + k + ', ' if @md.opt.cmd =~/V/
+                  y.each do |z|
+                    if z[:range]
+                      idx[:sst_rel_html_seg][@o]=idx[:sst_rel_html_seg][@o] + %{#{Mx[:lnk_o]}#{z[:range]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{z[:seg]}.html##{z[:ocn]}#{Mx[:rel_c]}, }
+                      idx[:sst_rel][@t]=idx[:sst_rel][@t] + %{#{Mx[:lnk_o]}#{z[:range]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{z[:ocn]}#{Mx[:rel_c]}, }
+                      idx[:html][@q]=idx[:html][@q] + %{#{z[:range]}, }
+                      idx[:xhtml][@q]=idx[:xhtml][@q] + %{#{z[:range]}, }
+                      print z[:range] + ', ' if @md.opt.cmd =~/V/
+                    elsif z[:ocn]
+                      idx[:sst_rel_html_seg][@o]=idx[:sst_rel_html_seg][@o] + %{#{Mx[:lnk_o]}#{z[:ocn]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{z[:seg]}.html##{z[:ocn]}#{Mx[:rel_c]}, }
+                      idx[:sst_rel][@t]=idx[:sst_rel][@t] + %{#{Mx[:lnk_o]}#{z[:ocn]}#{Mx[:lnk_c]}#{Mx[:rel_o]}#{z[:ocn]}#{Mx[:rel_c]}, }
+                      idx[:html][@q]=idx[:html][@q] + %{#{z[:ocn]}, }
+                      idx[:xhtml][@q]=idx[:xhtml][@q] + %{#{z[:ocn]}, }
+                      print z[:ocn] + ', ' if @md.opt.cmd =~/V/
+                    else p 'error'
+                    end
+                  end
+                  idx[:html][@q]=idx[:html][@q] + '
'
+                  idx[:xhtml][@r]=idx[:xhtml][@r] + '
'
+                end
+              end
+            end
+            idx
+            @x +=1
+          end
+        end
+      end
+      print "\n" if @md.opt.cmd =~/V/
+      idx
+    end
+    def screen_print(the_idx)
+      the_idx.each do |i|
+        i.each do |x|
+          if x.is_a?(String)
+            print "\n" + x + ', '
+          elsif x.is_a?(Array)
+            p 'array error? -->'
+            print x
+          elsif x.is_a?(Hash)
+            if x['term_node_lev1'].is_a?(Array)
+              x['term_node_lev1'].each do |a|
+                if a[:range]
+                  print a[:range] + ', '
+                elsif a[:ocn]
+                  print a[:ocn] + ', '
+                else p 'error'
+                end
+              end
+            end
+            if x['term_node_lev2']
+              m=x['term_node_lev2']
+              m=m.sort
+              m.each do |k,y|
+                if k !~/term_node_lev1/
+                  print "\n\t" + k + ', '
+                  y.each do |z|
+                    if z[:range]
+                      print z[:range] + ', '
+                    elsif z[:ocn]
+                      print z[:ocn] + ', '
+                    else p 'error'
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+    def output_idx(idx)
+      if @md.book_idx
+        path="#{@env.path.output}/#{@md.fnb}"
+        Dir.mkdir(path) unless FileTest.directory?(path)
+        puts "#{path}/#{@md.fn[:book_idx_html]} #{__FILE__}::#{__LINE__}"
+        html_index_file=File.new("#{path}/#{@md.fn[:book_idx_html]}",'w')
+        idx[:html].each {|x| html_index_file << x }
+        html_index_file.close
+      end
+    end
+    def clean_and_insert_index(data,sisu_markup_idx)
+      tuned_file=[]
+      data.each do |dob|
+        tuned_file << dob
+        if dob.obj =~/#{Mx[:br_endnotes]}/ \
+        and sisu_markup_idx
+          sisu_markup_idx.each do |idx|
+            tuned_file << idx
+          end
+        end
+      end
+      tuned_file
+    end
+    def clean_index(data)                                  #check on use of dob
+      tuned_file=[]
+      data.each do |para|
+        para=para.gsub(/\n*#{@rgx_idx}/m,'')
+        tuned_file << para
+      end
+      tuned_file
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_images.rb b/lib/sisu/v5/dal_images.rb
new file mode 100644
index 00000000..1141447d
--- /dev/null
+++ b/lib/sisu/v5/dal_images.rb
@@ -0,0 +1,161 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_Images
+  class Images
+    #require 'RMagick'
+    #include Magick
+    def initialize(md,data)
+      @md,@data=md,data
+    end
+    def images
+      data=@data
+      tuned_file=[]
+      @rmgk=false
+      imagemagick_=true      #imagemagick_=SiSU_Env::InfoSettings.new.program?('rmagick')
+      if imagemagick_
+        begin
+          @rmgk=SiSU_Env::Load.new('RMagick').prog
+        rescue
+          @rmgk=false
+        end
+      else
+        if @md.opt.cmd =~/[vVM]/
+          SiSU_Screen::Ansi.new(@md.opt.cmd,'*WARN* use of RMagick is not enabled in sisurc.yml').warn
+        end
+      end
+      data.each do |dob|
+        unless dob.is ==:table
+          dob.obj=dob.obj.strip
+          if dob.obj =~/#{Mx[:lnk_o]}\s*\S+\.(?:png|jpg|gif)(?:\s*|\s+.+)?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/
+            if dob.obj !~/#{Mx[:lnk_o]}\s*\S+\.(?:png|jpg|gif)\s+\d+x\d+/
+              m=/#{Mx[:lnk_o]}\s*(\S+\.(?:png|jpg|gif))/
+              if imagemagick_
+                imgs=dob.obj.scan(m).flatten
+                img_col=img_row=nil
+                images=imgs.each do |image|
+                  dir=SiSU_Env::InfoEnv.new(@md.fns)
+                  path_image=[dir.path.image_source_include_local,dir.path.image_source_include_remote,dir.path.image_source_include]
+                  image_path=nil
+                  path_image.each do |img_pth|
+                    image_path=img_pth
+                    break if FileTest.exist?("#{img_pth}/#{image}")
+                  end
+                  if FileTest.exist?("#{image_path}/#{image}")
+                    if @rmgk
+                      img=Magick::ImageList.new("#{image_path}/#{image}")
+                      img_col,img_row=img.columns,img.rows
+                    else
+                      if @md.opt.cmd =~/[vVM]/
+                        SiSU_Screen::Ansi.new(@md.opt.cmd,'*WARN* RMagick not present, will attempt to use imagemagick (identify) directly').warn
+                      end
+                      imgk=SiSU_Env::SystemCall.new.imagemagick
+                      gmgk=SiSU_Env::SystemCall.new.graphicksmagick
+                      if imgk or gmgk
+                        if imgk
+                          imgsys=`identify #{image_path}/#{image}`.strip                           #system call
+                        elsif gmgk
+                          imgsys=`gm identify #{image_path}/#{image}`.strip                        #system call
+                        end
+                        img_col,img_row=/(\d+)x(\d+)/m.match(imgsys)[1,2]
+                        img_col,img_row=img_col.to_i,img_row.to_i
+                      end
+                    end
+                    row=((img && defined? img.rows) ? img.rows : img_row)
+                    col=((img && defined? img.columns) ? img.columns : img_col)
+                    if img_col > img_row                                                           #landscape
+                      if img_col> 640
+                        img_col=640
+                        img_row=((1.00*img_col/col)*row).round
+                      end
+                    else                                                                           #portrait
+                      if img_col> 640
+                        img_col=640
+                        img_row=((1.00*img_col/col)*row).round
+                      end
+                      if img_row > 640
+                        img_row=640
+                        img_col=((1.00*img_row/row)*col).round
+                      end
+                    end
+                    dob.obj=dob.obj.gsub(/(#{image})/,"#{image} #{img_col}x#{img_row}")
+                  else dob.obj=dob.obj.gsub(/#{Mx[:lnk_o]}\s*(\S+)\.(png|jpg|gif).+?#{Mx[:lnk_c]}(#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/,'[ \1 (\2 missing) ]')
+                  end
+                end
+              else
+                images=dob.obj.scan(m) do |image|
+                  SiSU_Screen::Ansi.new(@md.opt.cmd,'*WARN* where image dimensions have not been provided RMagick or imagemagick is required',image).warn #unless @opt.cmd =~/q/
+                end
+              end
+            end
+          end
+          if dob.obj =~/#{Mx[:lnk_o]}\s*\S+\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/
+            dob.obj=dob.obj.gsub(/(#{Mx[:lnk_o]})\s*(\S+\.(?:png|jpg|gif))\s+/i,'\1\2 ')
+          end
+        end
+        tuned_file << dob unless dob.nil?
+      end
+      tuned_file
+    end
+  end
+end
+__END__
+imgsys=`identify #{image_path}/#{image}`.strip
diff --git a/lib/sisu/v5/dal_metadata.rb b/lib/sisu/v5/dal_metadata.rb
new file mode 100644
index 00000000..1fa5619c
--- /dev/null
+++ b/lib/sisu/v5/dal_metadata.rb
@@ -0,0 +1,82 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_Metadata
+  class Metadata
+    def initialize(md,metad)
+      @md,@metadata=md,metad
+      l=SiSU_Env::StandardiseLanguage.new(@md.opt.lng).language
+      language=l[:n]
+      @tr=SiSU_Translate::Source.new(md,language)
+    end
+    def make_para(obj,ocn)
+      h={ obj: obj, ocn: 0 }
+      SiSU_DAL_DocumentStructure::ObjectPara.new.paragraph(h)
+    end
+    def make_heading(obj,ocn,name,lv,ln)
+      h={ lv: lv, ln: ln, name: name, obj: obj, ocn: 0 }
+      SiSU_DAL_DocumentStructure::ObjectHeading.new.heading(h)
+    end
+    def metadata
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_numbering.rb b/lib/sisu/v5/dal_numbering.rb
new file mode 100644
index 00000000..39524cc9
--- /dev/null
+++ b/lib/sisu/v5/dal_numbering.rb
@@ -0,0 +1,475 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_Numbering
+  class Numbering
+    attr_accessor :obj,:osp,:ocn,:lv,:name,:index,:comment
+    def initialize(md,data)
+      @md,@data=md,data
+      @obj=@type=@ocn=@lv=@name=@index=@comment=nil
+    end
+    def numbering_song
+      data=@data
+      data=number_plaintext_para(data)
+      data=auto_number_heading_ie_title(data.compact) #tr issue
+      data=ocn(data.compact) #watch
+      data=xml(data.compact)
+      data=minor_numbering(data.compact)
+      data,tags_map,ocn_html_seg_map=name_para_seg_filename(data)
+      data=set_heading_top(data) unless @md.set_heading_top
+      [data,tags_map,ocn_html_seg_map]
+    end
+    def number_plaintext_para(data)
+      @tuned_file=[]
+      data.each do |dob|
+        if (dob.of !=:block \
+        && dob.of !=:comment \
+        && dob.of !=:layout) \
+        && dob.ocn_ #and dob.obj !~ /#{Mx[:gr_o]}Th|#{Mx[:tc_o]}#{Mx[:tc_p]}#{Mx[:tc_p]}/ #FIX
+          dob.obj=dob.obj.gsub(/(.+)\n/,'\1 ') #messy, but idea is that tables should retain breaks
+        end
+        unless dob.obj.is_a?(Array)
+          dob.obj=dob.obj.gsub(/^\s+/,'').
+            gsub(/\s$/,"\n")
+        end
+        @tuned_file << dob
+      end
+      @tuned_file=@tuned_file.flatten
+    end
+    def number_sub_heading(dob,num,title_no)
+      unless dob.obj =~/\d+\.|(?:chapter|article|section|clause)\s+\d+/i #name selection arbitrary, fix
+        dob.obj=case dob.name
+        when /-/;  dob.obj.gsub(/^/,"#{title_no} ")
+        when /^#/; dob.obj.gsub(/^/,"#{title_no} ")
+        when /^[a-z_\.]+/; dob.obj.gsub(/^/,"#{title_no} ")
+        else
+          dob.name=title_no if dob.name=~/^$/ #where title contains title number
+          dob.obj.gsub(/^/,"#{title_no} ") if title_no =~/\d+/ #main, where title number is to be provided #watch changed placement
+        end
+        if @md.toc_lev_limit \
+        and @md.toc_lev_limit < num
+          dob.obj=dob.obj.gsub(/^/,'!_ ') #bold line, watch
+        end
+      end
+      dob
+    end
+    def heading_tag_clean(heading_tag)
+      heading_tag=heading_tag.gsub(/[ ]+/,'_').
+        gsub(/["']/,'').
+        gsub(/[\/]/,'-').
+        gsub(/#{Mx[:fa_bold_o]}|#{Mx[:fa_bold_c]}/,'').
+        gsub(/#{Mx[:fa_italics_o]}|#{Mx[:fa_italics_c]}/,'').
+        gsub(/#{Mx[:fa_underscore_o]}|#{Mx[:fa_underscore_c]}/,'').
+        gsub(/#{Mx[:fa_cite_o]}|#{Mx[:fa_cite_c]}/,'').
+        gsub(/#{Mx[:fa_insert_o]}|#{Mx[:fa_insert_c]}/,'').
+        gsub(/#{Mx[:fa_strike_o]}|#{Mx[:fa_strike_c]}/,'').
+        gsub(/#{Mx[:fa_superscript_o]}|#{Mx[:fa_superscript_c]}/,'').
+        gsub(/#{Mx[:fa_subscript_o]}|#{Mx[:fa_subscript_c]}/,'').
+        gsub(/#{Mx[:fa_hilite_o]}|#{Mx[:fa_hilite_c]}/,'').
+        gsub(/#{Mx[:gl_bullet]}/,'')
+    end
+    def auto_number_heading_ie_title(data)                                             #also does some segment naming
+      @tuned_file=[]
+      if defined? @md.make.num_top \
+      and @md.make.num_top \
+      and @md.make.num_top !~/^$/
+        input||=@md.make.num_top
+      end
+      num_top=(input ? input.to_i : nil)
+      t_no1=t_no2=t_no3=t_no4=0
+      if num_top
+        no1=num_top; no2=(num_top + 1); no3=(num_top + 2);  no4=(num_top + 3)
+      end
+      t_not=0
+      chapter_number_counter=0
+      data=data.compact
+      data.each do |dob| #@md.seg_names << [additions to segment names]
+        title_no=nil
+        dob=SiSU_DAL_DocumentStructureExtract::Structure.new(@md,dob).structure_markup #must happen earlier, node info etc. require
+        if dob.is ==:heading \
+        && dob.autonum_ \
+        and defined? @md.make.num_top \
+        and @md.make.num_top !~/^$/
+          if  dob.lv=='1' \
+          and dob.obj =~/^#\s|\s#(?:\s|$)/
+            chapter_number_counter +=1
+            dob.obj=dob.obj.gsub(/^#\s/,"#{chapter_number_counter} ").
+              gsub(/#([:,]?\s|[.]?$)/,"#{chapter_number_counter}\\1")
+          end
+          if dob.ln==no1
+            @subnumber=1
+            @subnumber=0 if dob.ln==no1
+          end
+          if dob.ln.to_s =~/^[1-6]/ \
+          and not dob.toc_ \
+          and dob.obj !~/#{Mx[:fa_o]}(?:~#|-#)#{Mx[:fa_c]}/ # <-- fix
+            if dob.ln==no1
+              t_no1+=1; t_no2=0; t_no3=0
+              title_no="#{t_no1}"
+              if @md.seg_names.is_a?(Array) \
+              and not @md.seg_names.include?(title_no)
+                if dob.ln==no1
+                  dob.name="#{title_no}" if not dob.name
+                  dob.tags=[title_no,dob.tags].flatten if title_no !~/^\d+$/  #check whether will work across file types with stop signs
+                  tag=dob.obj.gsub(/(Article|Clause|Section|Chapter)\s+/,"\\1_#{title_no}").downcase
+                  tag=heading_tag_clean(tag)
+                  dob.tags=[tag,dob.tags].flatten if tag !~/^\d+$/ #check whether will work across file types with stop signs
+                  dob.obj=(dob.obj =~/(Article|Clause|Section)\s+/) \
+                  ? (dob.obj.gsub(/(Article|Clause|Section)\s+/,"\\1 #{title_no} "))
+                  : (dob.obj.gsub(/^/,"#{title_no}. ")) #fix stop later
+                end
+                if dob.ln !=no1 \
+                and dob.obj =~/^[\d.]+\s/ #fix -> if the title starts with a numbering scheme, do not auto-number, review
+                  dob.name ="#{title_no}" if not dob.name
+                  dob.tags=[title_no,dob.tags].flatten if title_no !~/^\d+$/ #check whether will work across file types with stop signs
+                  dob.obj=dob.obj.gsub(/^/,"#{title_no}. ")
+                end
+                @md.seg_names << title_no
+              end
+              if dob.ln!=no1 \
+              and dob.name!~/^[a-z_\.]+$/ \
+              and dob.obj !~/[A-Z]\.?\s/ #bug -> tmp fix, excludes A. B. C. lettering, but not roman numerals, is arbitrary, review required # not fixed, work on
+                dob.tags=[title_no,dob.tags].flatten if title_no !~/^\d+$/ #check whether will work across file types with stop signs
+                dob.obj=dob.obj.gsub(/^/i,"#{title_no}. ")
+              end
+            end
+            if dob.ln==no1         #watch because here you change dob.name
+              dob.tags=["h#{title_no}",dob.tags].flatten #check whether will work across file types with stop signs
+            end
+            if dob.ln==no2         #watch because here you change dob.name
+              t_no2+=1; t_no3=0
+              title_no="#{t_no1}.#{t_no2}"
+              dob.tags=["h#{title_no}",dob.tags].flatten #check whether will work across file types with stop signs
+              dob=number_sub_heading(dob,no2,title_no)
+            end
+            if dob.ln==no3         #watch because here you change dob.name
+              t_no3+=1
+              title_no="#{t_no1}.#{t_no2}.#{t_no3}"
+              dob.tags=["h#{title_no}",dob.tags].flatten #check whether will work across file types with stop signs
+              dob=number_sub_heading(dob,no3,title_no)
+            end
+          elsif dob.ln.to_s =~/^[1-6]/ \
+          and dob.name =~ /^[\w-]+-/ # endnotes, watch2005# endnotes, watch2005
+            dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/  #check whether will work across file types with stop signs
+            dob.name.gsub(/^([a-z_\.]+)-$/,'\1')
+          end
+        elsif dob.is ==:heading \
+        and dob.autonum_ \
+        and @md.markup =~/num_extract/ #AS DANGEROUS force enable with document, note already does this type of numbering for cisg, locate and coordinate logic, is currently misplaced in code, chengwei inspired 2004w23/4
+          #here lies a bug, as is nil when run from -Dv --update, FIX
+          if (dob.name.nil? or dob.name.empty?) \
+          and dob.ln.to_s =~/^[1-9]/ \
+          and dob.obj =~ /^([\d\.]+)/ #risky (must be unique) consider output to 4~~\d instead of 4~\d
+            dob.name=$1
+            dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/ #check whether will work across file types with stop signs
+          end
+          if @md.toc_lev_limit
+          end
+        elsif defined? dob.name \
+        and  dob.name
+          dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/ #check whether will work across file types with stop signs
+        end
+        dob.tags=dob.tags.uniq if defined? dob.tags
+        @tuned_file << dob
+      end
+      @tuned_file=@tuned_file.flatten
+    end
+    def ocn(data)                                                                      #and auto segment numbering increment
+      @tuned_file=SiSU_DAL_DocumentStructureExtract::OCN.new(@md,data).ocn
+      @tuned_file
+    end
+    def xml(data)
+      @tuned_file=SiSU_DAL_DocumentStructureExtract::XML.new(@md,data).dom
+      @tuned_file
+    end
+    def minor_numbering(data)                                                          #and auto segment numbering increment
+      @tuned_file=[]
+      number_small,letter_small=0,0
+      letter=%w( a b c d e f g h i j k l m n o p q r s t u v w x y z )
+      data.each do |dob|
+        if dob.of ==:heading \
+        || dob.of ==:heading_insert \
+        || dob.of ==:para \
+        || dob.of ==:block
+          if dob.is ==:heading \
+          and dob.ln.to_s=~/^[1-9]/                                                    #% sub-number system, (baby numbering) reset with any change of major number (more obviously should be placed in number titles, but that is conditionally executed, check and move later)
+            number_small,letter_small=0,0
+          elsif dob.is ==:para
+            if dob.obj =~/^#[ 1]/ \
+            and dob.obj !~/^#\s+(?:~#)?$/
+              letter_small=0
+              number_small=0 if dob.obj =~ /^#1/
+              number_small+=1
+              dob.obj=dob.obj.gsub(/^#[ 1]/,"#{number_small}. ")
+            end
+            if dob.obj =~/^_# /
+              dob.obj=dob.obj.gsub(/^_# /,"#{letter[letter_small]}. ")
+              dob.indent='1'
+              letter_small+=1
+            end
+          end
+        end
+        @tuned_file << dob
+      end
+      @tuned_file=@tuned_file.flatten
+    end
+    def name_para_seg_filename(data)                                                   #segment naming, remaining
+      # paragraph name/numbering rules
+      # manual naming overrides, manual naming may be
+      #   alpha-numeric characters mixed,
+      #   numeric only (a number), if
+      #     all segments have been named,
+      #     the numbers used are over 1000 or
+      #     it is  not minded that auto-numbering uses a funny scheme for naming segments (not yet implemented)
+      #       [for now a warning is printed for such documents on use of maintenance or very-verbose flag]
+      # auto-naming takes the form of giving numbers to segments
+      # the rules for which are as follows
+      #   if the title/heading text starts with a numeric, then that is used (1 3.1 3rd etc.)
+      #   otherwise the level 4 segment number from the embedded document structure info is used
+      #   if there is none a sequential number is designated, preceded by an underscore
+      @tuned_file,@unique_auto_name=[],[]
+      tags={}
+      art_filename_auto=1
+      @counter=1
+      if not @md.seg_autoname_safe and @md.opt.cmd =~/[MV]/
+        puts 'manual segment names, numbers used as names, risk warning (segmented html)'
+      end
+      ocn_html_seg=[]
+      data.each do |dob|
+        if dob.is==:heading \
+        && dob.ln \
+        and dob.ln.to_s =~/^[456]/
+          if dob.ln==4 \
+          and not dob.name \
+          and not @md.set_heading_seg
+            @md.set_heading_seg=true
+          end
+          if dob.name !~/^\S+/ \
+          and dob.obj =~/^\s*(?:\S+\s+)?([\d.,:-]+)/m      #heading starts with a recognised numeric or word followed by a recognised numerical construct, use that as name
+            possible_seg_name=$1
+            possible_seg_name=possible_seg_name.gsub(/(?:[:,-]|\W)/,'.').
+              gsub(/\.$/,'')
+            if @md.seg_names.is_a?(Array) \
+            and not @md.seg_names.include?(possible_seg_name)
+              dob.name=possible_seg_name
+              dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/
+              @md.seg_names << possible_seg_name
+            else puts 'warn, there may be a conflicting numbering scheme' if @md.opt.cmd =~/[VM]/
+            end
+          end
+          if dob.ln==4 \
+          and dob.name                                     #extract segment name from embedded document structure info
+            if @md.seg_names.is_a?(Array) \
+            and not @md.seg_names.include?(dob.name)
+              dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/
+              @md.seg_names << dob.name
+            end
+          end
+          if dob.ln==4 \
+          and not dob.name                                 #if still no segment name, provide a numerical one
+            pf='_'                                         #pg='' #may use e.g. '' or '~' or '_'
+            segn_auto="#{pf}#{art_filename_auto.to_s}"
+            if @md.seg_names.is_a?(Array) \
+            and not @md.seg_names.include?(segn_auto)
+             dob.name=segn_auto
+             dob.tags=[dob.name,dob.tags].flatten if dob.name !~/^\d+$/ #check whether will work across file types with stop signs
+              @md.seg_names << segn_auto
+            else puts 'segment name (numbering) error'
+            end
+            art_filename_auto+=1
+          end
+          if dob.ln==4 \
+          and not dob.name #should not occur
+            puts "e r r o r -\t#{__FILE__}::#{__LINE__}\n#{dob.inspect}"
+          end
+        end
+        if (dob.is ==:heading \
+        || dob.is ==:heading_insert) \
+        && dob.ln==4
+          @seg=dob.name
+        end
+        @tuned_file << if dob.is==:heading \
+        && (@md.pagenew || @md.pagebreak || @md.pageline)
+          m=dob.ln.to_s
+          dob_tmp=[]
+          if @md.pagenew.inspect =~/#{m}/
+            dob_tmp << SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_new]) << dob
+          elsif @md.pagebreak.inspect =~/#{m}/
+            dob_tmp << SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page]) << dob
+          elsif @md.pageline.inspect =~/#{m}/
+            dob_tmp << SiSU_DAL_DocumentStructure::ObjectLayout.new.break(Hx[:br_page_line]) << dob
+          end
+          para_result=unless dob_tmp.length > 0; dob
+          else                                   dob_tmp
+          end
+        else dob
+        end
+        if defined? dob.ocn \
+        and dob.ocn
+          @segname=((dob.is==:heading || dob.is==:heading_insert) && dob.ln==4 && (defined? dob.name)) \
+          ? (dob.name)
+          : @segname
+          tags["#{dob.ocn}"]={ segname: @segname }
+          ocn_html_seg[dob.ocn]=if (dob.is==:heading || dob.is==:heading_insert)
+            x=if dob.ln =~/[1-3]/
+              { seg: nil, level: dob.ln }
+            else #elsif dob.ln =~/[4-6]/
+              { seg: @seg, level: dob.ln }
+            end
+          else
+            { seg: @seg, level: nil }
+          end
+        end
+        dob.tags=dob.tags.uniq if defined? dob.tags
+        if defined? dob.tags \
+        and dob.tags.length > 0
+          #@segname=((dob.is=='heading'|| dob.is=='heading_insert') && dob.ln==4 && (defined? dob.name)) \
+          #? (dob.name) \
+          #: @segname
+          dob.tags.each do |x|
+            tags[x]={ ocn: dob.ocn.to_s, segname: @segname }
+          end
+        end
+        dob
+      end
+      ocn_html_seg.each_with_index do |ocn,i|
+        if ocn \
+        and ocn[:level].to_s=~/[1-3]/
+          ocn_seg=nil
+          (1..4).each do |x|
+            if ocn_html_seg[i+x] and ocn_html_seg[i+x][:level]==4
+              ocn[:seg]=ocn_html_seg[i+x][:seg]
+            end
+          end
+        end
+      end
+      if @md.seg_names.length > 0
+        @md.set_heading_seg=true
+      end
+      tuned_file=@tuned_file.flatten
+      [tuned_file,tags,ocn_html_seg]
+    end
+    def set_heading_top(data)                                                          #% make sure no false positives
+      unless @md.set_heading_top
+        puts "\tdocument contains no top level heading, (will have to manufacture one)" if @md.opt.cmd =~/[MV]/
+        @tuned_file=[]
+        data.each do |t_o|
+          unless @md.set_heading_top
+            if t_o !~/^(?:#{Rx[:meta]}|@\S+:)\s/m \
+            and t_o !~/\A\s*\Z/m
+              @md.set_heading_top=true
+              if defined? @md.title \
+              and @md.title \
+              and defined? @md.title.full \
+              and defined? @md.creator \
+              and @md.creator
+                head=@md.title.main ? ([@lv='1',@obj=@md.title.main]) : ([@lv='1',@obj='[no title provided]'])
+                @tuned_file << head
+              end
+            end
+          end
+          @tuned_file << t_o
+        end
+        @tuned_file=@tuned_file.flatten
+      end
+    end
+    def set_heading_seg(data)                                                          #% make sure no false positives
+      unless @md.set_heading_seg
+        puts "\tdocument contains no segment level, (will have to manufacture one)" if @md.opt.cmd =~/[MV]/
+        @tuned_file=[]
+        data.each do |dob|
+          unless @md.set_heading_seg
+            if defined? dob.ln and dob.ln.to_s !~/^[123]/m \
+            and dob.obj !~/\A\s*\Z/m \
+            and dob.is !=:layout
+              @md.set_heading_seg=true
+              head=@md.title.main \
+              ? (dob.ln,dob.name,dob.obj=4,'seg',@md.title.main)
+              : (dob.ln,dob.name,dob.obj=4,'seg','[segment]')
+              @tuned_file << head
+            end
+          end
+          @tuned_file << dob
+        end
+        @tuned_file=@tuned_file.flatten
+      end
+    end
+    def set_header_title(data)                                                         #% make sure no false positives
+      unless @md.set_header_title
+        puts "\t no document title provided, (will have to manufacture one)" if @md.opt.cmd =~/[MV]/
+        @tuned_file=[]
+        data.each do |t_o|
+          unless @md.set_header_title
+            if t_o !~/^%{1,2}\s/m \
+            and t_o !~/\A\s*\Z/m
+              @tuned_file << "#{Mx[:meta_o]}title#{Mx[:meta_c]} #{@md.heading_seg_first}"
+              @md.title.main=@md.heading_seg_first
+              @md.set_header_title=true
+            end
+          end
+          @tuned_file << t_o
+        end
+        @tuned_file=@tuned_file.flatten
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_substitutions_and_insertions.rb b/lib/sisu/v5/dal_substitutions_and_insertions.rb
new file mode 100644
index 00000000..051d2bfd
--- /dev/null
+++ b/lib/sisu/v5/dal_substitutions_and_insertions.rb
@@ -0,0 +1,150 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DAL_SubstituteAndInsert
+  class SI
+    def initialize(md,data)
+      @md,@data=md,data
+    end
+    def substitutions_and_insertions?
+      data=@data
+      data_expand=[]
+      if data[0] =~ /^#!\s*(?:\/usr\/bin\/env sisu|\/usr\/bin\/sisu)/ # remove bang from top #! (however file is stripped, so will be removed provided no content precedes it)
+        data[0]=data[0].gsub(/^#!\s*\/usr\/bin\/sisu/,'').
+          gsub(/^#!\s*\/usr\/bin\/env sisu/,'')
+      end
+      if data[0] =~ /^(SiSU\s+[\d.]*|sisu-[\d.]+)$/ # SiSU identifier
+        data[0]=data[0].gsub(/^(SiSU\s*[\d.]*)$/,'% \1').
+          gsub(/^(sisu-[\d.]+)$/,'% \1')
+      end
+      data.each do |para|
+        if para =~/<:(insert\d+)!?>/ \
+        and para !~/^%\s+/
+          ins=SiSU_Viz::Inserts.new
+          case para
+          when /^\s*<:(insert1)>\s*$/
+            i=$1
+            if defined? ins.insert1
+              para=[]
+              ins.insert1.split(/\n\n/).each {|x| para << x }
+            end
+          when /^\s*<:(insert2)>\s*$/
+            i=$1
+            if defined? ins.insert2
+              para=[]
+              ins.insert2.split(/\n\n/).each {|x| para << x }
+            end
+          when /^\s*<:(insert3)>\s*$/
+            i=$1
+            if defined? ins.insert3
+              para=[]
+              ins.insert3.split(/\n\n/).each {|x| para << x << "\n"}
+            end
+          when /^\s*<:(insert4)>\s*$/
+            i=$1
+            if defined? ins.insert4
+              para=[]
+              ins.insert4.split(/\n\n/).each {|x| para << x << "\n"}
+            end
+          when /^\s*<:(insert5)>\s*$/
+            i=$1
+            if defined? ins.insert5
+              para=[]
+              ins.insert5.split(/\n\n/).each {|x| para << x << "\n"}
+            end
+          when /^\s*<:(insert6)>\s*$/
+            i=$1
+            if defined? ins.insert6
+              para=[]
+              ins.insert6.split(/\n\n/).each {|x| para << x << "\n"}
+            end
+          when /^\s*<:(insert7)>\s*$/
+            i=$1
+            if defined? ins.insert7
+              para=[]
+              ins.insert7.split(/\n\n/).each {|x| para << x << "\n"}
+            end
+          end
+          para.each {|x| data_expand << x }
+        else data_expand << para
+        end
+        data_expand=data_expand.flatten.compact
+      end
+      data_new=[]
+      data_expand.each do |para|
+        para=if @md.markup_version.determined >= 0.38
+          SiSU_DAL_DocumentStructureExtract::Structure.new(@md,para).structure_markup_normalize
+        else
+          SiSU_DAL_DocumentStructureExtract::Structure.new(@md,para).structure_marks
+        end
+        para=para.gsub(/^(:?A~)\s*$/,'\1~ @title @author').                                    #conditional header
+          gsub(/^((?:[1-9]|:?[A-C])~\S*)\s*$/,'\1~ [Note: heading marker::required title missing]~#')    #conditional header for incorporated document 2004w12
+        if para =~/^@\S+?:/
+          para=para.gsub(/^@(\S+?):(\s+|$)/,"#{Mx[:meta_o]}\\1#{Mx[:meta_c]}\\2").
+            gsub(/^@(\S+?):([+-])(\s+|$)/,"#{Mx[:meta_o]}\\1\\2#{Mx[:meta_c]}\\3")
+        end
+        data_new << para
+      end
+      data_new
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dal_syntax.rb b/lib/sisu/v5/dal_syntax.rb
new file mode 100644
index 00000000..03d397b3
--- /dev/null
+++ b/lib/sisu/v5/dal_syntax.rb
@@ -0,0 +1,606 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: Syntax for markup, input markup syntaxes, determined here
+
+=end
+module SiSU_DAL_Syntax
+  class Words
+    def initialize(line,md,mkp)
+      @line,@md,@mkp=line,md,mkp
+    end
+  end
+  class Markup
+    def initialize(md='',data='')
+      @md,@data=md,data
+      @vz=SiSU_Viz::Defaults.new
+      @data_new=[]
+      url_and_stub=SiSU_Env::InfoEnv.new.url
+      @output_url="#{url_and_stub.remote}"
+      @env=SiSU_Env::InfoEnv.new
+      emph_set=if defined? @md.emphasis_set_to \
+      and not @md.emphasis_set_to.nil?
+        @md.emphasis_set_to
+      else @env.markup_emphasis
+      end
+      @emph=case emph_set
+      when /bold/
+        emph_italics=false
+        { o: Mx[:fa_bold_o], c: Mx[:fa_bold_c] }
+      when /italics/
+        emph_italics=true
+        { o: Mx[:fa_italics_o], c: Mx[:fa_italics_c] }
+      when /underscore/
+        emph_italics=false
+        { o: Mx[:fa_underscore_o], c: Mx[:fa_underscore_c] }
+      else p __LINE__.to_s + '::' + __FILE__
+      end
+      @http_m=%r{\{.+?\}https?://\S+|https?:\S+|:\S+|\.\.\/\S+|#\S+|\S+?\.png\b|[*]~\S+|^#{Mx[:meta_o]}.+|#{Mx[:gr_o]}(?:code|block|group|alt|verse)(?:-end)?#{Mx[:gr_c]}|#{Mx[:fa_o]}:br#{Mx[:fa_c]}}
+      @manmkp_ital=emph_italics \
+      ? '[i/*]\\{.+?\\}[i/*]'
+      : '[i/]\\{.+?\\}[i/]'
+      tail_m_ital=%q{(?:\s|[.,;:?!'")]|~\^|~\\\{\s|$)}
+      tail_m_bold=%{(?:(?:#{Mx[:fa_italics_c]})?(?:\s|[.,;:?!'")]|~\^|~\\\{\s|$))?}
+      bold_line=%{^!_\s.+?(?:#{Mx[:br_line]}|\n|$)}
+      ital_line=%{^/_\s.+?(?:#{Mx[:br_line]}|\n|$)}
+      @line_scan_ital=if defined? @md.italics_match_list[:str]
+        /#{@http_m}|#{bold_line}|#{@manmkp_ital}#{tail_m_ital}|#{@md.italics_match_list[:str]}#{tail_m_ital}|\S+|\n/i
+      elsif defined? @vz.markup_make_italic[:str]
+        /#{@http_m}|#{bold_line}|#{@manmkp_ital}#{tail_m_ital}|#{@vz.markup_make_italic[:str]}#{tail_m_ital}|\S+|\n/i
+      end
+      @manmkp_bold=emph_italics \
+      ? '^!_\s.+?(?:\n|$)|[!b]\\{.+?\\}[*!b]|[*!][a-zA-Z0-9\-_]+[!]'
+      : '^!_\s.+?(?:\n|$)|[*!b]\\{.+?\\}[*!b]|[*!][a-zA-Z0-9\-_]+[*!]'
+      @line_scan_bold=if (defined? @md.bold_match_list[:str] \
+      and @md.bold_match_list[:str]) \
+      and (defined? @vz.markup_make_bold[:str] \
+      and @vz.markup_make_bold[:str])
+        /#{@http_m}|#{bold_line}|(?:#{@manmkp_bold}|#{@md.bold_match_list[:str]}|#{@vz.markup_make_bold[:str]})#{tail_m_bold}|\S+|\n/i
+      elsif defined? @md.bold_match_list[:str] \
+      and @md.bold_match_list[:str]
+        /#{@http_m}|#{bold_line}|(?:#{@manmkp_bold}|#{@md.bold_match_list[:str]})#{tail_m_bold}|\S+|\n/i
+      elsif defined? @vz.markup_make_bold[:str] \
+      and @vz.markup_make_bold[:str]
+        /#{@http_m}|#{bold_line}|(?:#{@manmkp_bold}|#{@vz.markup_make_bold[:str]})#{tail_m_bold}|\S+|\n/i
+      end
+    end
+    def songsheet
+      @data=@data.compact
+      @data.each do |dob|
+        dob=breaks(dob)
+        dob=if @md.sem_tag then sem(dob) else dob end #revisit
+        dob=line_actions(dob)
+        dob=paragraph_set(dob)
+        dob=substitutions(dob)
+        dob=wordlist_italics(dob)
+        dob=wordlist_bold(dob)
+        dob=bodymarkup(dob)
+        @data_new << dob unless dob.nil?
+      end
+      @data_new
+    end
+    def sem(dob) #revisit
+      dob=SiSU_Sem::Tags.new(dob,@md).rm.all
+    end
+    def breaks(dob)
+      if dob.is !=:meta \
+      && dob.is !=:comment \
+      && dob.is !=:code \
+      && dob.is !=:table
+        dob.obj=dob.obj.gsub(/^-\\\\-\s*$/,"#{Mx[:br_page]}").
+          gsub(/^=\\\\=\s*$/,"#{Mx[:br_page_new]}").
+          gsub(/ \\\\(?: |$)/,"#{Mx[:br_line]}").
+          gsub(/(?:<:?pb>)/,"#{Mx[:br_page]}").                         # depreciated
+          gsub(/(?:<:?pn>)/,"#{Mx[:br_page_new]}").                     # depreciated
+          gsub(/(?:<:?br>|
)/,"#{Mx[:br_line]}").                 # depreciated
+          gsub(/(?:^-\.\.-\s*$)/,"#{Mx[:br_page_line]}")
+      end
+      dob
+    end
+    def wordlist_italics(dob)
+      dob=dob.dup
+      if (defined? @md.italics_match_list[:str] \
+      and @md.italics_match_list[:str]) \
+      or (defined? @vz.markup_make_italic[:str] \
+      and @vz.markup_make_italic[:str])
+        dob.obj=if dob.is !=:meta \
+        && dob.is !=:heading \
+        && dob.is !=:heading_insert \
+        && dob.is !=:code \
+        && dob.is !=:comment
+          word=dob.obj.scan(@line_scan_ital)
+          word=word.flatten.compact
+          line_array=[]
+          word.each do |w|
+            unless /#{@manmkp_ital}|#{@http_m}/.match(w)
+              if defined? @md.italics_match_list[:regx] \
+              and @md.italics_match_list[:regx]
+                w=w.gsub(@md.italics_match_list[:regx],
+                  "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}")
+              elsif defined? @vz.markup_make_italic \
+              and @vz.markup_make_italic
+                w=w.gsub(@vz.markup_make_italic,
+                  "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}")
+              else w
+              end
+            end
+            line_array << w
+          end
+          line_array.join(' ')
+        else dob.obj
+        end
+      end
+      dob
+    end
+    def embolden(given)
+      given=given.gsub(/^!_\s+((?:\{|#{Mx[:lnk_o]})(?:~^ )?.+?(?:\}|#{Mx[:lnk_o]})https?:\/\/\S+.*?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
+          "#{Mx[:fa_bold_o]} \\1 #{Mx[:fa_bold_c]}\\2").
+        gsub(/^!_\s+((?:\{|#{Mx[:lnk_o]})(?:~^ )?.+?(?:\}|#{Mx[:lnk_o]})https?:\/\/\S+.*)/,
+          "#{Mx[:fa_bold_o]} \\1 #{Mx[:fa_bold_c]}").
+        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
+          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
+        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)\s+((?:[*]~\S+\s*)+)/,
+          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
+        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)\s*([~-]#)$/,
+          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
+        gsub(/(?:^!_\s+|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]}\s*)(.*)?\s*$/,
+          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}")
+    end
+    def italicise(given)
+      given=given.gsub(/^\/_\s*(.+?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
+          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
+        gsub(/^\/_\s*(.+?)\s+((?:[*]~\S+\s*)+)/,
+          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
+        gsub(/^\/_\s*(.+?)\s*([~-]#)$/,
+          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
+        gsub(/^\/_\s+(.*)?\s*$/,
+          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}")
+    end
+    def line_actions(dob)
+      dob.obj=if (dob.is !=:heading \
+      && dob.is !=:heading_insert \
+      && dob.is !=:comment \
+      && dob.is !=:meta) \
+      and dob.obj =~ /^!_\s+/
+        embolden(dob.obj)
+      elsif dob.obj =~ /^\/_\s+/
+        italicise(dob.obj)
+      else dob.obj
+      end
+      dob
+    end
+    def paragraph_set(dob)
+      dob.obj=if dob.is !=:meta \
+      && dob.is !=:heading \
+      && dob.is !=:heading_insert \
+      && dob.is !=:code \
+      && dob.is !=:comment \
+      && dob.is !=:table
+        dob.obj.gsub(/\n/m,' ').
+          gsub(/ \s+/m,' ')
+      else dob.obj
+      end
+      dob
+    end
+    def substitutions(dob)
+      dob=dob.dup
+      dob=if defined? @md.substitution_match_list[:match_and_replace] \
+      and @md.substitution_match_list[:match_and_replace].is_a?(Array)
+        dob=if dob.is !=:meta \
+        && dob.is !=:heading_insert \
+        && dob.is !=:code \
+        && dob.is !=:comment \
+        && dob.is !=:table
+          if dob.obj =~/#{@md.substitution_match_list[:matches]}/
+            @md.substitution_match_list[:match_and_replace].each do |x|
+              dob.obj=if x[:case_s]==:i
+                dob.obj.gsub(/#{x[:match]}/mi,x[:replace])
+              else
+                dob.obj.gsub(/#{x[:match]}/m,x[:replace])
+              end
+            end
+          end
+          dob
+        else dob
+        end
+        dob
+      else dob
+      end
+    end
+    def wordlist_bold(dob)
+      dob=dob.dup
+      if (defined? @md.bold_match_list[:str] \
+      and @md.bold_match_list[:str]) \
+      or (defined? @vz.markup_make_bold[:str] \
+      and @vz.markup_make_bold[:str])
+        dob.obj=if dob.is !=:meta \
+        && dob.is !=:heading \
+        && dob.is !=:heading_insert \
+        && dob.is !=:code \
+        && dob.is !=:comment \
+        && dob.is !=:table
+          line_array=[]
+          word=dob.obj.scan(@line_scan_bold)
+          word=word.flatten.compact
+          word.each do |w|
+            unless /#{@manmkp_bold}|#{@http_m}/.match(w)
+              if defined? @md.bold_match_list[:regx] \
+              and @md.bold_match_list[:regx]                                                   #document header: @bold: [bold word list]
+                w=w.gsub(@md.bold_match_list[:regx],"#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}")
+              elsif defined? @vz.markup_make_bold \
+              and @vz.markup_make_bold                                                   #defaults adjusted bold word list
+                w=w.gsub(@vz.markup_make_bold,"#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}")
+              end
+            else
+              w=if w =~ /(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s+/
+                embolden(w)      #bold paragraph/emphasize #may wish to remove think about 7{ 8{ conversion not satisfactory, as information is lost!
+              elsif w =~/^\/_\s+/
+                italicise(w)
+              else w
+              end
+            end
+            line_array << w
+          end
+          line_array.join(' ')
+        else dob.obj
+        end
+      else
+        dob.obj=if dob.is==:heading \
+        and dob.ln.to_s =~/[7-9]/
+          embolden(dob.obj)
+        else dob.obj
+        end
+      end
+      dob
+    end
+    def fontface(dob)
+      dob.obj=dob.obj.gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|#{Mx[:lnk_o]}|#{Mx[:br_line]}|#{Mx[:br_paragraph]}|[\(\[\{]|\>)\*\{(.+?)\}\*/m,
+          "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                                                                                             #emphasis
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|#{Mx[:lnk_o]}|#{Mx[:br_line]}|#{Mx[:br_paragraph]}|[\(\[\{]|\>)!\{(.+?)\}!/m,
+          "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                                                                                                   #bold
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|#{Mx[:lnk_o]}|#{Mx[:br_line]}|#{Mx[:br_paragraph]}|[\(\[\{]|\(|\>)\/\{(.+?)\}\//m,
+          "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                                                                                             #italics
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|#{Mx[:lnk_o]}|#{Mx[:br_line]}|#{Mx[:br_paragraph]}|[\(\[\{]|\>)_\{(.+?)\}_/m,
+          "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                                                                                       #underscore
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|#{Mx[:lnk_o]}|#{Mx[:br_line]}|#{Mx[:br_paragraph]}|[\(\[\{]|\>)#\{(.+?)\}#/m,
+          "\\1#{Mx[:fa_monospace_o]}\\2#{Mx[:fa_monospace_c]}").                                                                                                         #monospace
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\"\{(.+?)\}\"/m,
+          "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").                                                                                                      #cite /blockquote?
+        gsub(/(^|[^\\])\^\{(.+?)\}\^/m,
+          "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").                                                                                                     #superscript
+        gsub(/(^|[^\\]),\{(.+?)\},/m,
+          "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").                                                                                                         #subscript
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\+\{(.+?)\}\+/m,
+          "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").                                                                                                               #inserted text
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)-\{(.+?)\}-/m,
+          "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").                                                                                                               #strikethrough - deleted text
+        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>|\d+)\^(\S+?)\^/,
+          "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}")                                                                                                      #superscript single word, watch digit added
+      dob
+    end
+    def bodymarkup(dob)
+      # << http://www.jus.uio.no/sisu/sisu_markup_table/markup >>
+      # See: data/sisu/sample/document_samples_sisu_markup/
+      ## fontface
+      # *{emphasis}*        e{emphasis}e       emphasis
+      # !{bold text}!       b{bold}b           bold text
+      # _{underline}_       u{underline}u      underline
+      # /{italics}/         i{italics}i        italics
+      # "{citation}"        c{citation}c       citation #blockquote?
+      # ^{superscript}^                        superscript
+      # ,{subscript},                          subscript
+      # +{inserted text}+                      inserted text
+      # -{deleted text}-                       deleted text
+      # #{monospace text}#
+      #
+      # {url address}:url
+      # {image.png}imageurl
+      # {image.png}png
+      # ~{endnote}~         
+      # !_                                    #bold/emphasise paragraph
+      # _"                                    #blockquote paragraph
+      # _1                  <:i1>            #indent paragraph 1 step
+      # _2                  <:i2>            #indent paragraph 2 steps
+      # _3                  <:i3>            #indent paragraph 3 steps
+      # _4                  <:i4>            #indent paragraph 4 steps
+      # _*                                    #bullet (list) ●
+      # _1*                                   #bullet (list) indented
+      # _1*                                   #bullet (list) indented
+      # #                                     #numbered (list) level 1
+      # _#                                    #numbered (list) level 2
+      dob=dob.dup
+      if dob.is !=:meta \
+      && dob.is !=:comment \
+      && dob.is !=:code \
+      && dob.is !=:table
+        line_array=[]
+        word=dob.obj.scan(/\S+|\n/) #unless line =~/^(?:#{Mx[:meta_o]}|%+\s)/ #visit
+        if word
+          word.each do |w| # _ - / # | : ! ^ ~
+            unless w =~/~\{|\}~|~\[|\]~|^\^~|~\^|\*~\S+|~#|\{t?~|\{table|https?:\/\/\S+/             # do something earlier about table!!
+              w=w.gsub(/\\?~/,"#{Mx[:gl_o]}#126#{Mx[:gl_c]}")                                       #escaped special character
+            end
+            w=w.gsub(/^\<$/,"#{Mx[:gl_o]}#lt#{Mx[:gl_c]}").gsub(/^\>$/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}")                                          #escaped special character
+            line_array << w
+          end
+          dob.obj=line_array.join(' ')
+          dob.obj=dob.obj.strip
+        end
+        dob.obj=dob.obj.gsub(/^([*#-.]{1,12})$/,'\1 ~#').                                                #ocn off for these paragraph separators
+          gsub(/~\{(.+?)\}~/m,Mx[:en_a_o] + '\1' + Mx[:en_a_c]).
+          gsub(/~\[([^*+].+?)\]~/m,Mx[:en_b_o] + '* \1' + Mx[:en_b_c]).                    #default if markup does not specify
+          gsub(/~\[(.+?)\]~/m,Mx[:en_b_o] + '\1' + Mx[:en_b_c])
+        if dob.is ==:heading \
+        and dob.ln ==1
+          dob.obj=dob.obj.gsub(/\s*@title\b/," #{@md.title.full}")
+          dob.obj=if defined? @md.creator.author \
+          and @md.creator.author
+            dob.obj.gsub(/\s+(?:@creator|@author)/,",#{Mx[:br_line]}#{@md.creator.author}")
+          else dob.obj.gsub(/\s+(?:@creator|@author)/,'')
+          end
+        end
+        if defined? @md.title \
+        and @md.title \
+        and defined? @md.title.full \
+        and defined? @md.creator \
+        and @md.creator
+          if dob.is ==:heading
+            dob.obj=dob.obj.gsub(/^\s*@title\s*$/,@md.title.full) if dob.lv =~/1/
+            dob.obj=if dob.lv =~/[23]/ \
+            and defined? @md.creator.author \
+            and @md.creator.author
+              dob.obj.gsub(/^\s*(?:(by\s+)?(?:@creator|@author))\s*$/,"\\1#{@md.creator.author}")
+            else dob.obj.gsub(/^\s*(?:(by\s+)?(?:@creator|@author))\s*$/,'\1')
+            end
+          end
+        end
+        dob.obj=dob.obj.gsub(/<(https?:\/\/\S+?)>/,'< \1 >').                                   #catch problem markup
+          gsub(/<:=(\S+?)>/,'{ c_\1.png 14x14 }image').
+          gsub(//,'<:\1>').                                              #escaped special character
+          gsub(/ /,"#{Mx[:nbsp]}").                                          #escaped special character
+          gsub(/\\~/,"#{Mx[:gl_o]}#126#{Mx[:gl_c]}").                             #escaped special character
+          gsub(/\\\{/,"#{Mx[:gl_o]}#123#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\}/,"#{Mx[:gl_o]}#125#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\<>/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}#{Mx[:gl_o]}#gt#{Mx[:gl_c]}"). #escaped special character
+          gsub(/\\\/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}").                             #escaped special character
+          gsub(/\\\_/,"#{Mx[:gl_o]}#095#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\-/,"#{Mx[:gl_o]}#045#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\+/,"#{Mx[:gl_o]}#043#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\//,"#{Mx[:gl_o]}#047#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\#/,"#{Mx[:gl_o]}#035#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\&/,"#{Mx[:gl_o]}#038#{Mx[:gl_c]}").                            #& #escaped special character
+          gsub(/\\\|/,"#{Mx[:gl_o]}#124#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
+          gsub(/\\\:/,"#{Mx[:gl_o]}#058#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
+          gsub(/\\\!/,"#{Mx[:gl_o]}#033#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
+          gsub(/\\\^/,"#{Mx[:gl_o]}#094#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
+          gsub(/\\\,/,"#{Mx[:gl_o]}#044#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
+          gsub(/\\\\/,"#{Mx[:gl_o]}#092#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\*/,"#{Mx[:gl_o]}#042#{Mx[:gl_c]}").                            #escaped special character
+          gsub(/\\\!/,"#{Mx[:gl_o]}#033#{Mx[:gl_c]}")                             #escaped special character
+        if dob.obj=~/(?:https?:|ftp:|\{([^{}]+?)\}(?:#|:|[.]{1,2}\/))\S+/m
+          if dob.obj=~/(?:^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (?:.+?)\s*\}(?:(?:https?:|ftp:|:|[.]{1,2}\/)\S+?)\s*#{Mx[:en_a_o]}(.+?)#{Mx[:en_a_c]}/m
+            dob.obj=dob.obj.gsub(/(^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ ([^}]+?)\s*\}((?:https?:|ftp:|:|[.]{1,2}\/)\S+?)\s*#{Mx[:en_a_o]}(.+?)#{Mx[:en_a_c]}/m,
+              "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3 #{Mx[:en_a_o]}\\3 \\4#{Mx[:en_a_c]}") # watch
+          end
+          if dob.obj=~/(?:^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (?:.+?)\s*\}(?:(?:https?:|ftp:|:|[.]{1,2}\/)\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m
+            dob.obj=dob.obj.gsub(/(^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (.+?)\s*\}((?:https?:|ftp:|:|[.]{1,2}\/)\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+              "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3\\4 #{Mx[:en_a_o]}\\3#{Mx[:en_a_c]} ")
+          end
+          dob.obj=dob.obj.gsub(/(^|[^#])\{\s*([^{}]+?)\s*\}((?:https?:|:|[.]{2}\/|#)\S+?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
+              "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3"). #linked (text or image, however text cannot include modified face, e.g. bold, ital, underline)
+            gsub(/(^|[#{Mx[:gl_c]}#{Mx[:lnk_c]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}(\s])((?:https?|ftp):\/\/\S+?\.[^>< ]+?)([,.;'"]?)(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+              %{\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3}).
+            gsub(/#{Mx[:lnk_c]}#(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}\\1#{Mx[:rel_c]}\\2}).
+            gsub(/#{Mx[:lnk_c]}:(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}:\\1#{Mx[:rel_c]}\\2}).
+            gsub(/#{Mx[:lnk_c]}[.]{2}\/(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}:\\1#{Mx[:rel_c]}\\2})
+        end
+        if dob.obj=~/_(?:https?|ftp):\S+/m           # _http://url #CHECK
+          dob.obj=dob.obj.gsub(/(^|[#{Mx[:gl_c]}#{Mx[:lnk_c]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}(\s])(_(?:https?|ftp):\/\/\S+?\.[^>< ]+?)([,.;'"]?)(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
+            %{\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3})
+        end
+        dob=fontface(dob)
+        dob.obj=dob.obj.gsub(/<[:e]\s+(.+?)!?>/,
+            "#{Mx[:en_a_o]}\\1#{Mx[:en_a_c]}").                                                                                                                             #not tested
+          gsub(/(^|#{Mx[:br_nl]})\s*_\*\s*/,
+            "\\1#{Mx[:gl_bullet]}").                                                                                                                                           #bullets, shortcut
+          gsub(/=\{(.+?)\}/,
+            "#{Mx[:idx_o]}\\1#{Mx[:idx_c]}").
+          gsub(/^\s*_([1-9])\*\s*/,
+            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}#{Mx[:gl_bullet]}").                                                                                                              #bullets, shortcut
+          gsub(/^\s*_([1-9])\s+/,
+            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}").                                                                                                                               #indent
+          gsub(/^\s*_([1-9])!\s+(.+?)\s*$/,
+            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]} ").                                                                                                                               #indent bold
+          gsub(/^\s*__([1-9])\s+/,
+            "#{Mx[:pa_o]}:i0:\\1#{Mx[:pa_c]}").                                                                                                                               #hang
+          gsub(/^\s*__([1-9])!\s+(.+?)\s*$/,
+            "#{Mx[:pa_o]}:i0:\\1#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]} ").                                                                                                                          #hangdef
+          gsub(/^\s*_([0-9])_([0-9])\s+/,
+            "#{Mx[:pa_o]}:i\\1:\\2#{Mx[:pa_c]}").                                                                                                                               #hang
+          gsub(/^\s*_([0-9])_([0-9])!\s+(.+?)\s*$/,
+            "#{Mx[:pa_o]}:i\\1:\\2#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\3#{Mx[:fa_bold_c]} ").                                                                                                                          #hangdef
+          gsub(/<:hi>/,"#{Mx[:fa_hilite_o]}"). #'')   # bright yellow rgb(255,255,0) pale yellow rgb(255,255,200)
+          gsub(/<:\/hi>/,"#{Mx[:fa_hilite_c]}"). #'')
+          gsub(/(#{Mx[:gr_o]}verse#{Mx[:gr_c]}.+)/m,"\\1\n").
+          gsub(/[ ]+($)/,'\1').
+          gsub(/\{\s*(.+?)\s*\}(https?:\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
+            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3"). #any remaining linked text or image
+          gsub(/\{\s*(.+?)\s*\}(#{Mx[:url_o]}\S+?#{Mx[:url_c]})/,
+            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2"). #any remaining linked text or image
+          gsub(/(^|\s)([a-zA-Z0-9._-]+\@\S+?\.[a-zA-Z0-9._-]+)/,"\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}").
+          gsub(/(^|[ ])\{\s*(.+?)\s*\}(\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
+            "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3\\4"). #any remaining linked text or image
+          gsub(/\{\s*(.+?)\s*\}#([a-zA-Z0-9][a-zA-Z0-9_-]*)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
+            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}#{Mx[:rel_o]}\\2#{Mx[:rel_c]}\\3"). #any remaining linked text or image, check need
+          gsub(/\{\s*(.+?)\s*\}(#{Mx[:rel_o]}\S+?#{Mx[:rel_c]})/,
+            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2"). #any remaining linked text or image, check need
+          gsub(/\{\s*(.+?)\s*\}(image)/,
+            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2")      #linked image
+      elsif dob.is==:table
+        dob=fontface(dob)
+      elsif dob.is ==:code
+        dob.obj=dob.obj.gsub(/#{Mx[:meta_o]}(\S+?)#{Mx[:meta_c]}\s*/,'@\1: ').
+          gsub(/(^|#{Mx[:gl_c]}|\s)<(br(?: \/)?)>([\s,.]|$)/,'\1<\2>\3') #convert 
 
 back, clumsy
+        if dob.number_
+          codeline=[]
+          ln=1
+          dob.obj.split(/#{Mx[:gr_o]}codeline#{Mx[:gr_c]}||\n/).each_with_index do |cl,i|
+            unless i == 0
+              cl=cl.gsub(Mx[:br_nl],'')
+              w=3-ln.to_s.length
+              cl = "#{ln}#{Mx[:nbsp]*w}#{Mx[:vline]}#{cl}#{Mx[:br_nl]}"
+              ln +=1
+            end
+            codeline << cl
+          end
+          codeline= codeline.join("")
+          dob.obj=codeline
+        else
+          dob.obj=dob.obj.gsub(/#{Mx[:gr_o]}codeline#{Mx[:gr_c]}/,"\n")
+        end
+        dob
+      else # @\S+?:
+      end
+      dob
+    end
+    def tech                                                                       #script markup planned to be more strict for technical documents
+      # *{emphasis}*        e{emphasis}e       emphasis
+      # !{bold text}!       b{bold}b           bold text
+      # _{underline}_       u{underline}u      underline
+      # /{italics}/         i{italics}i        italics
+      # "{citation}"        c{citation}c       citation
+      # ^{superscript}^                        superscript
+      # ,{subscript},                          subscript
+      # +{inserted text}+                      inserted text
+      # -{deleted text}-                       deleted text
+      # #{monospace text}#
+      # {url address}:url
+      # {image.png}imageurl
+      # {image.png}png
+      # ~{endnote}~         
+      # +1                  
+      # +2                  
+      puts 'tech'
+      @data.each do |line|
+        line=line.gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)e\{(.+?)\}e/,
+            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasis
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)b\{(.+?)\}b/,
+            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)u\{(.+?)\}u/,
+            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)c\{(.+?)\}c/,
+            "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").                                 #cite
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)i\{(.+?)\}i/,
+            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                        #italics
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)!\{(.+?)\}!/,
+            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)\*\{(.+?)\}\*/,
+            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasis
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)_\{(.+?)\}_/,
+            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\(|\>)\/\{(.+?)\}\//,
+            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                        #italics
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\"\{(.+?)\}\"/,
+            "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\^\{(.+?)\}\^/,
+            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)9\{(.+?)\}9/,
+            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>),\{(.+?)\},/,
+            "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)6\{(.+?)\}6/,
+            "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\+\{(.+?)\}\+/,
+            "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)v\{(.+?)\}v/,
+            "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)-\{(.+?)\}-/,
+            "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)x\{(.+?)\}x/,
+            "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\*(\S+?)\*/,
+            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasise single word, watch
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\!(\S+?)\!/,
+            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold single word, watch
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\/([\(\)a-zA-Z0-9']+?)\/([ ,.;:'"~$]|[^a-zA-Z0-9])/,
+            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}\\3").                                     #italics single word, watch
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)_(\S+?)_/,
+            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore single word, watch
+          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\^(\S+?)\^/,
+            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").                                #check  #superscript single word, watch digit added
+          gsub(/^\s*_\([1-9]\)\(\*\+\)\s*/,
+            "#{Mx[:pa_o]}:i\\1#{Mx[:pa_c]}#{Mx[:fa_o]}\\2#{Mx[:fa_c_o]}").                            #bullets, shortcut
+          gsub(/^\s*_\([1-9]\)\s+/,
+            "#{Mx[:pa_o]}:i\\1#{Mx[:pa_c]}"). #watch
+          gsub(/^\s*__\([1-9]\)\s+/,
+            "#{Mx[:pa_o]}:h\\1#{Mx[:pa_c]}"). #watch
+          #line.gsub(/^\s*__\([1-9]\)!\s+/,
+          #  "#{Mx[:pa_o]}:hd\\1#{Mx[:pa_c]}"). #watch
+          gsub(/#{Mx[:br_line]}\s*_[12]\s+/,
+            "#{Mx[:br_line]} ")                                                                      #indent used in endnotes, not implemented, replace when ready with: line.gsub(/(?:
|
)\s*_([12])\s+/,'
<:i\1> ')
+      end
+      @data
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_columns.rb b/lib/sisu/v5/db_columns.rb
new file mode 100644
index 00000000..14d3bb46
--- /dev/null
+++ b/lib/sisu/v5/db_columns.rb
@@ -0,0 +1,2001 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbColumns
+  require_relative 'sysenv'                            # sysenv.rb
+  require_relative 'db_sqltxt'                         # db_sqltxt.rb
+  class Columns < SiSU_DbText::Prepare
+    def initialize(md=nil)
+      @md=md
+      @db=SiSU_Env::InfoDb.new #watch
+      @lang ||=SiSU_i18n::Languages.new
+      if defined? md.opt.mod \
+      and md.opt.mod.inspect=~/import|update/ \
+      and FileTest.exist?(md.fns)
+        txt_arr=IO.readlines(md.fns,'')
+        src=txt_arr.join("\n")
+        if @db.share_source?
+          @sisutxt=special_character_escape(src)
+        else @sisutxt=''
+        end
+        @fulltext=clean_searchable_text(txt_arr)
+       else @sisutxt,@fulltext='',''
+      end
+    end
+#% structures
+    #def column_define
+    #  def varchar(name,size)
+    #    "#{name}                VARCHAR(#{size}) NULL,"
+    #  end
+    #end
+=begin
+#% title
+@title:
+ :subtitle:
+ :short:
+ :edition:
+ :language:
+ :note:
+=end
+    def column
+      def title                          # DublinCore 1 - title
+        def name
+          'title'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_title]}) NOT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata full document title [DC1]';}
+        end
+        def tuple
+          t=if defined? @md.title.full \
+          and @md.title.full=~/\S+/
+            txt=@md.title.full
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_main
+        def name
+          'title_main'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_title_part]}) NOT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata main document title';}
+        end
+        def tuple
+          t=if defined? @md.title.main \
+          and @md.title.main=~/\S+/
+            txt=@md.title.main
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_sub
+        def name
+          'title_sub'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_title_part]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document subtitle';}
+        end
+        def tuple
+          t=if defined? @md.title.sub \
+          and @md.title.sub=~/\S+/
+            txt=@md.title.sub
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_short
+        def name
+          'title_short'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_title_part]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document short title if any';}
+        end
+        def tuple
+          t=if defined? @md.title.short \
+          and @md.title.short=~/\S+/
+            txt=@md.title.short
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_edition
+        def name
+          'title_edition'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_title_edition]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document edition (version)';}
+        end
+        def tuple
+          t=if defined? @md.title.edition \
+          and @md.title.edition=~/\S+/
+            txt=@md.title.edition
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_note
+        def name
+          'title_note'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document notes associated with title';}
+        end
+        def tuple
+          t=if defined? @md.title.note \
+          and @md.title.note=~/\S+/
+            txt=@md.title.note
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_language
+        def name
+          'title_language'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document language [DC12]';}
+        end
+        def tuple
+          t=if @lang.list[@md.opt.lng][:n]
+            txt=@lang.list[@md.opt.lng][:n]
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def title_language_char            # consider
+        def name
+          'title_language_char'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language_char]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'metadata document language iso code';}
+        end
+        def tuple
+          t=if defined? @md.opt.lng \
+          and @md.opt.lng=~/\S+/
+            txt=@md.opt.lng
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% creator
+@creator:
+ :author:
+ :editor:
+ :contributor:
+ :illustrator:
+ :photographer:
+ :translator:
+ :prepared_by:
+ :digitized_by:
+ :audio:
+ :video:
+=end
+      def creator_author                 # DublinCore 2 - creator/author (author)
+        def name
+          'creator_author'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document author (creator) [DC2]';}
+        end
+        def tuple
+          t=if defined? @md.creator.author_detail \
+          and @md.creator.author_detail.is_a?(Array) \
+          and @md.creator.author_detail.length > 0
+            txt=''
+            @md.creator.author_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_author_honorific       # consider
+        def name
+          'creator_author_hon'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_creator_misc_short]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document author honorific (title e.g, Ms. Dr. Prof.)';}
+        end
+        def tuple
+          t=if defined? @md.creator.author_hon \
+          and @md.creator.author_hon=~/\S+/
+            txt=@md.creator.author_hon
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_author_nationality     # consider
+        def name
+          'creator_author_nationality'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_creator_misc_short]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata nationality of document author (creator)';}
+        end
+        def tuple
+          t=if defined? @md.creator.author_nationality_detail \
+          and @md.creator.author_nationality=~/\S+/
+            txt=@md.creator.author_nationality_detail
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_editor
+        def name
+          'creator_editor'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document editor name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.editor_detail \
+          and @md.creator.editor_detail.is_a?(Array) \
+          and @md.creator.editor_detail.length > 0
+            txt=@md.creator.editor_detail #dc
+            txt=''
+            @md.creator.editor_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_contributor            # DublinCore 6 - contributor
+        def name
+          'creator_contributor'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document contributor name(s) [DC6]';}
+        end
+        def tuple
+          t=if defined? @md.creator.contributor_detail \
+          and @md.creator.contributor_detail.is_a?(Array) \
+          and @md.creator.contributor_detail.length > 0
+            txt=@md.creator.contributor_detail #dc
+            txt=''
+            @md.creator.contributor_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_illustrator
+        def name
+          'creator_illustrator'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document illustrator name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.illustrator_detail \
+          and @md.creator.illustrator_detail.is_a?(Array) \
+          and @md.creator.illustrator_detail.length > 0
+            txt=@md.creator.illustrator_detail
+            txt=''
+            @md.creator.illustrator_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_photographer
+        def name
+          'creator_photographer'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document photographer name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.photographer_detail \
+          and @md.creator.photographer_detail.is_a?(Array) \
+          and @md.creator.photographer_detail.length > 0
+            txt=@md.creator.photographer_detail
+            txt=''
+            @md.creator.photographer_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_translator
+        def name
+          'creator_translator'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document translator name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.translator_detail \
+          and @md.creator.translator_detail.is_a?(Array) \
+          and @md.creator.translator_detail.length > 0
+            txt=''
+            @md.creator.translator_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_prepared_by
+        def name
+          'creator_prepared_by'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document prepared by name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.prepared_by_detail \
+          and @md.creator.prepared_by_detail.is_a?(Array) \
+          and @md.creator.prepared_by_detail.length > 0
+            txt=@md.creator.prepared_by_detail
+            txt=''
+            @md.creator.prepared_by_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_digitized_by
+        def name
+          'creator_digitized_by'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document digitized by name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.digitized_by_detail \
+          and @md.creator.digitized_by_detail.is_a?(Array) \
+          and @md.creator.digitized_by_detail.length > 0
+            txt=@md.creator.digitized_by_detail
+            txt=''
+            @md.creator.digitized_by_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_audio
+        def name
+          'creator_audio'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document audio by name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.audio_detail \
+          and @md.creator.audio_detail.is_a?(Array) \
+          and @md.creator.audio_detail.length > 0
+            txt=@md.creator.audio_detail
+            txt=''
+            @md.creator.audio_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def creator_video
+        def name
+          'creator_video'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document video by name(s)';}
+        end
+        def tuple
+          t=if defined? @md.creator.video_detail \
+          and @md.creator.video_detail.is_a?(Array) \
+          and @md.creator.video_detail.length > 0
+            txt=''
+            @md.creator.video_detail.each do |h|
+              txt=txt + %{#{h[:the]}, #{h[:others]}; }
+            end
+            txt=txt.gsub(/[;, ]+\s*$/,'')
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% language
+#taken from other fields
+@title:
+ :language:
+@original:
+ :language:
+#not available -->
+#@language:
+# :document:
+# :original:
+=end
+      def language_document
+        def name
+          'language_document'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document language';}
+        end
+        def tuple
+          t=if @lang.list[@md.opt.lng][:n]
+            txt=@lang.list[@md.opt.lng][:n]
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def language_document_char
+        def name
+          'language_document_char'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language_char]}) NOT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document language';}
+        end
+        def tuple
+          #modify check, is now required, SiSUv3d_
+          t=if defined? @md.opt.lng \
+          and @md.opt.lng=~/\S+/
+            txt=@md.opt.lng
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def language_original
+        def name
+          'language_original'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata original document/text language';}
+        end
+        def tuple
+          t=if defined? @md.language.original \
+          and @md.language.original=~/\S+/
+            txt=@md.language.original
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def language_original_char
+        def name
+          'language_original_char'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language_char]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document language';}
+        end
+        def tuple
+          t=if defined? @md.language.original_char \
+          and @md.language.original_char=~/\S+/
+            txt=@md.language.original_char
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% date
+@date:
+ :added_to_site:
+ :available:
+ :created:
+ :issued:
+ :modified:
+ :published:
+ :valid:
+ :translated:
+ :original_publication:
+=end
+      def date_added_to_site
+        def name
+          'date_added_to_site'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+          #"#{name}                DATE,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date added to site';}
+        end
+        def tuple
+          t=if defined? @md.date.added_to_site \
+          and @md.date.added_to_site=~/\S+/
+            txt=@md.date.added_to_site
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_available
+        def name
+          'date_available'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date added to site [DC]';}
+        end
+        def tuple
+          t=if defined? @md.date.available \
+          and @md.date.available=~/\S+/
+            txt=@md.date.available
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_created
+        def name
+          'date_created'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date created [DC]';}
+        end
+        def tuple
+          t=if defined? @md.date.created \
+          and @md.date.created=~/\S+/
+            txt=@md.date.created
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_issued
+        def name
+          'date_issued'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date issued [DC]';}
+        end
+        def tuple
+          t=if defined? @md.date.issued \
+          and @md.date.issued=~/\S+/
+            txt=@md.date.issued
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_modified
+        def name
+          'date_modified'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date modified [DC]';}
+        end
+        def tuple
+          t=if defined? @md.date.modified \
+          and @md.date.modified=~/\S+/
+            txt=@md.date.modified
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_published
+        def name
+          'date_published'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date published [DC7]';}
+        end
+        def tuple
+          t=if defined? @md.date.published \
+          and @md.date.published=~/\S+/
+            txt=@md.date.published
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_valid
+        def name
+          'date_valid'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date valid [DC]';}
+        end
+        def tuple
+          t=if defined? @md.date.valid \
+          and @md.date.valid=~/\S+/
+            txt=@md.date.valid
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_translated
+        def name
+          'date_translated'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date translated';}
+        end
+        def tuple
+          t=if defined? @md.date.translated \
+          and @md.date.translated=~/\S+/
+            txt=@md.date.translated
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_original_publication
+        def name
+          'date_original_publication'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_date_text]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date of original publication';}
+        end
+        def tuple
+          t=if defined? @md.date.original_publication \
+          and @md.date.original_publication=~/\S+/
+            txt=@md.date.original_publication
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def date_generated
+        def name
+          'date_generated'
+        end
+        def create_column              #choose other representation of time
+          "#{name}                VARCHAR(30) NULL,"
+          #"#{name}                VARCHAR(10) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata date of sisu generation of document, automatically populated';}
+        end
+        def tuple                      #choose other representation of time
+          t=if defined? @md.generated \
+          and @md.generated.to_s=~/\S+/
+            txt=@md.generated.to_s
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% publisher
+@publisher:
+=end
+      def publisher
+        def name
+          'publisher'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document publisher [DC5]';}
+        end
+        def tuple
+          t=if defined? @md.publisher \
+          and @md.publisher=~/\S+/
+            txt=@md.publisher
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+##% current
+#    def current_publisher
+#      def name
+#        'current_publisher'
+#      end
+#      def size
+#        10
+#      end
+#      def create_column
+#        "#{name}                VARCHAR(#{current_publisher.size}) NULL,"
+#      end
+#      def tuple
+#        t=if defined? @md.current.publisher \
+#        and @md.current.publisher=~/\S+/
+#          txt=@md.current.publisher
+#          txt=special_character_escape(txt)
+#          "'#{txt}', "
+#        end
+#      end
+#      self
+#    end
+=begin
+#% original
+@original:
+ :publisher:
+ #:date:                                #repeated under date
+ :language:
+ :institution:
+ :nationality:
+ :source:
+=end
+      def original_publisher
+        def name
+          'original_publisher'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original publisher [DC5]';}
+        end
+        def tuple
+          t=if defined? @md.original.publisher \
+          and @md.original.publisher=~/\S+/
+            txt=@md.original.publisher
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def original_language
+        def name
+          'original_language'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original language';}
+        end
+        def tuple
+          t=if defined? @md.original.language \
+          and @md.original.language=~/\S+/
+            txt=@md.original.language
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def original_language_char         # consider
+        def name
+          'original_language_char'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language_char]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original language iso character';}
+        end
+        def tuple
+          t=if defined? @md.original.language_char \
+          and @md.original.language_char=~/\S+/
+            txt=@md.original.language_char
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def original_source
+        def name
+          'original_source'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original source [DC11]';}
+        end
+        def tuple
+          t=if defined? @md.original.source \
+          and @md.original.source=~/\S+/
+            txt=@md.original.source
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def original_institution
+        def name
+          'original_institution'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_name]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original institution';}
+        end
+        def tuple
+          t=if defined? @md.original.institution \
+          and @md.original.institution=~/\S+/
+            txt=@md.original.institution
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def original_nationality
+        def name
+          'original_nationality'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_language]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document original nationality';}
+        end
+        def tuple
+          t=if defined? @md.original.nationality \
+          and @md.original.nationality=~/\S+/
+            txt=@md.original.nationality
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% rights
+@rights:
+ #:copyright:                          #mapped to :text: used where no other copyrights and included in :all:
+ :text:
+ :translation:
+ :illustrations:
+ :photographs:
+ :preparation:
+ :digitization:
+ :audio:
+ :video:
+ :license:
+ :all:
+=end
+      def rights_all
+        def name
+          'rights'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata rights associated with document (composite) [DC15]';}
+        end
+        def tuple
+          t=if defined? @md.rights.all \
+          and @md.rights.all=~/\S+/
+            txt=@md.rights.all
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_text
+        def name
+          'rights_copyright_text'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_text \
+          and @md.rights.copyright_text=~/\S+/
+            txt=@md.rights.copyright_text
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_translation
+        def name
+          'rights_copyright_translation'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text translation (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_translation \
+          and @md.rights.copyright_translation=~/\S+/
+            txt=@md.rights.copyright_translation
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_illustrations
+        def name
+          'rights_copyright_illustrations'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text illustrations (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_illustrations \
+          and @md.rights.copyright_illustrations=~/\S+/
+            txt=@md.rights.copyright_illustrations
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_photographs
+        def name
+          'rights_copyright_photographs'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text photographs (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_photographs \
+          and @md.rights.copyright_photographs=~/\S+/
+            txt=@md.rights.copyright_photographs
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_preparation
+        def name
+          'rights_copyright_preparation'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text preparation (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_preparation \
+          and @md.rights.copyright_preparation=~/\S+/
+            txt=@md.rights.copyright_preparation
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_digitization
+        def name
+          'rights_copyright_digitization'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text digitization (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_digitization \
+          and @md.rights.copyright_digitization=~/\S+/
+            txt=@md.rights.copyright_digitization
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_audio
+        def name
+          'rights_copyright_audio'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text audio (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_audio \
+          and @md.rights.copyright_audio=~/\S+/
+            txt=@md.rights.copyright_audio
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_copyright_video
+        def name
+          'rights_copyright_video'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata copyright associated for document text video (if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.copyright_video \
+          and @md.rights.copyright_video=~/\S+/
+            txt=@md.rights.copyright_video
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def rights_license
+        def name
+          'rights_license'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata license granted for use of document if any)';}
+        end
+        def tuple
+          t=if defined? @md.rights.license \
+          and @md.rights.license=~/\S+/
+            txt=@md.rights.license
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% identifier
+@identifier:
+ :oclc:
+ :isbn:
+=end
+      def identifier_oclc
+        def name
+          'identifier_oclc'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_library]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata identifier document Online Computer Library Center number';}
+        end
+        def tuple
+          t=if defined? @md.identifier.oclc \
+          and @md.identifier.oclc=~/\S+/
+            txt=@md.identifier.oclc
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def identifier_isbn
+        def name
+          'identifier_isbn'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_small]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata identifier document isbn (if any)';}
+        end
+        def tuple
+          t=if defined? @md.identifier.isbn \
+          and @md.identifier.isbn=~/\S+/
+            txt=@md.identifier.isbn
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% classify
+@classify:
+ :topic_register:
+ :subject:
+ :keywords:
+ :type:
+ :loc:
+ :dewey:
+=end
+      def classify_topic_register
+        def name
+          'classify_topic_register'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_info_note]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document topic register (semi-structured document subject information)';}
+        end
+        def tuple
+          t=if defined? @md.classify.topic_register \
+          and @md.classify.topic_register=~/\S+/
+            txt=@md.classify.topic_register
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def classify_subject
+        def name
+          'classify_subject'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_long]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document subject matter [DC3]';}
+        end
+        def tuple
+          t=if defined? @md.classify.subject \
+          and @md.classify.subject=~/\S+/
+            txt=@md.classify.subject
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def classify_loc
+        def name
+          'classify_loc'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_library]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document Library of Congress';}
+        end
+        def tuple
+          t=if defined? @md.classify.loc \
+          and @md.classify.loc=~/\S+/
+            txt=@md.classify.loc
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def classify_dewey
+        def name
+          'classify_dewey'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_library]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document Dewey';}
+        end
+        def tuple
+          t=if defined? @md.classify.dewey \
+          and @md.classify.dewey=~/\S+/
+            txt=@md.classify.dewey
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def classify_keywords
+        def name
+          'classify_keywords'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_long]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document keywords';}
+        end
+        def tuple
+          t=if defined? @md.classify.keywords \
+          and @md.classify.keywords=~/\S+/
+            txt=@md.classify.keywords
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% notes
+@notes:
+ :abstract:
+ :description:
+ :comment:
+ :coverage:
+ :relation:
+ :format:
+ :history:
+ :prefix:
+ :prefix_a:
+ :prefix_b:
+ :suffix:
+=end
+      def notes_abstract
+        def name
+          'notes_abstract'
+        end
+        def create_column
+          "#{name}                     TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes abstract';}
+        end
+        def tuple
+          t=if defined? @md.notes.abstract \
+          and @md.notes.abstract=~/\S+/
+            txt=@md.notes.abstract
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_description
+        def name
+          'notes_description'
+        end
+        def create_column
+          "#{name}                    TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes description [DC4]';}
+        end
+        def tuple
+          t=if defined? @md.notes.description \
+          and @md.notes.description=~/\S+/
+            txt=@md.notes.description
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_comment
+        def name
+          'notes_comment'
+        end
+        def create_column
+          "#{name}                       TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes comment';}
+        end
+        def tuple
+          t=if defined? @md.notes.comment \
+          and @md.notes.comment=~/\S+/
+            txt=@md.notes.comment
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_coverage
+        def name
+          'notes_coverage'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_short]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document coverage [DC14]';}
+        end
+        def tuple
+          t=if defined? @md.classify.coverage \
+          and @md.classify.coverage=~/\S+/
+            txt=@md.classify.coverage
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_relation
+        def name
+          'notes_relation'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_short]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document relation [DC13]';}
+        end
+        def tuple
+          t=if defined? @md.classify.relation \
+          and @md.classify.relation=~/\S+/
+            txt=@md.classify.relation
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_history   #check, consider removal
+        def name
+          'notes_history'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_long]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes history';}
+        end
+        def tuple
+          t=if defined? @md.notes.history \
+          and @md.notes.history=~/\S+/
+            txt=@md.notes.history
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_type #check
+        def name
+          'notes_type'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_long]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata notes document type [DC8]';}
+        end
+        def tuple
+          t=if defined? @md.notes.type \
+          and @md.notes.type=~/\S+/
+            txt=@md.notes.type
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_format
+        def name
+          'notes_format'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_txt_long]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata classify document format [DC9]';}
+        end
+        def tuple
+          t=if defined? @md.classify.format \
+          and @md.classify.format=~/\S+/
+            txt=@md.classify.format
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_prefix
+        def name
+          'notes_prefix'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes prefix';}
+        end
+        def tuple
+          t=if defined? @md.notes.prefix \
+          and @md.notes.prefix=~/\S+/
+            txt=@md.notes.prefix
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_prefix_a
+        def name
+          'notes_prefix_a'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes prefix_a';}
+        end
+        def tuple
+          t=if defined? @md.notes.prefix_a \
+          and @md.notes.prefix_a=~/\S+/
+            txt=@md.notes.prefix_a
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_prefix_b
+        def name
+          'notes_prefix_b'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes prefix_b';}
+        end
+        def tuple
+          t=if defined? @md.notes.prefix_b \
+          and @md.notes.prefix_b=~/\S+/
+            txt=@md.notes.prefix_b
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def notes_suffix
+        def name
+          'notes_suffix'
+        end
+        def create_column                # keep text
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document notes suffix';}
+        end
+        def tuple
+          t=if defined? @md.notes.suffix \
+          and @md.notes.suffix=~/\S+/
+            txt=@md.notes.suffix
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% src
+=end
+      def src_filename
+        def name
+          'src_filename'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_filename]}) NOT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'sisu markup source text filename';}
+        end
+        def tuple
+          t=if defined? @md.fns \
+          and @md.fns=~/\S+/
+            txt=@md.fns
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def src_fingerprint
+        def name
+          'src_fingerprint' #hash/digest, sha256 or md5
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_digest]}) NULL,"
+          #"#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'sisu markup source text fingerprint, hash digest sha256 (or md5)';}
+        end
+        def tuple
+          t=if defined? @md.dgst \
+          and @md.dgst.is_a?(Array) \
+          and @md.dgst[1]=~/\S+/
+            txt=@md.dgst[1]
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def src_filesize
+        def name
+          'src_filesize'
+        end
+        def create_column
+          "#{name}                VARCHAR(#{Db[:col_filesize]}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'sisu markup source text file size';}
+        end
+        def tuple
+         t=if defined? @md.filesize \
+         and @md.filesize=~/\S+/
+           txt=@md.filesize
+           txt=special_character_escape(txt)
+           ["#{name}, ","'#{txt}', "]
+         else ['','']
+         end
+        end
+        self
+      end
+      def src_word_count
+        def name
+          'src_word_count'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'sisu markup source text word count';}
+        end
+        def tuple
+          t=if defined? @md.wc_words \
+          and @md.wc_words=~/\S+/
+            txt=@md.wc_words
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def src_txt                      # consider naming sisusrc
+        def name
+          'src_text'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'sisu markup source text (if shared)';}
+        end
+        def tuple
+          t=if @md.opt.mod.inspect=~/import|update/ \
+          and FileTest.exist?(@md.fns)
+            ["#{name}, ","'#{@sisutxt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+=begin
+#% misc
+@links:
+=end
+      def fulltext
+        def name
+          'fulltext'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+            IS 'document full text clean, searchable';}
+        end
+        def tuple
+          t=if @md.opt.mod.inspect=~/import|update/ \
+          and  FileTest.exist?(@md.fns)
+            ["#{name}, ","'#{@fulltext}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      def links
+        def name
+          'links'
+        end
+        def create_column
+          "#{name}                TEXT NULL,"
+          #"#{name}                 VARCHAR(#{links.size}) NULL,"
+        end
+        def column_comment
+          %{COMMENT ON COLUMN metadata_and_text.#{name}
+           IS 'metadata document links';}
+        end
+        def tuple
+          t=if defined? @md.links \
+          and @md.links=~/\S+/
+            txt=@md.links
+            txt=special_character_escape(txt)
+            ["#{name}, ","'#{txt}', "]
+          else ['','']
+          end
+        end
+        self
+      end
+      self
+    end
+  end
+  class ColumnSize
+    def document_clean # restriction not necessary
+      60000
+    end
+    def document_body
+      16000
+    end
+    def document_seg
+      120
+    end
+    def document_seg_full
+      120
+    end
+    def endnote_clean # restriction not necessary
+      60000
+    end
+    def endnote_body
+      16000
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_create.rb b/lib/sisu/v5/db_create.rb
new file mode 100644
index 00000000..e2d1f916
--- /dev/null
+++ b/lib/sisu/v5/db_create.rb
@@ -0,0 +1,618 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbCreate
+  require_relative 'db_columns'                         # db_columns.rb
+  class Create < SiSU_DbColumns::Columns
+    require_relative 'sysenv'                           # sysenv.rb
+    @@dl=nil
+    def initialize(opt,conn,file,sql_type='pg')
+      @opt,@conn,@file,@sql_type=opt,conn,file,sql_type
+      @cX=SiSU_Screen::Ansi.new(@opt.cmd).cX
+      @comment=(@sql_type=='pg') \
+      ? (SiSU_DbCreate::Comment.new(@conn,@sql_type))
+      : nil
+      @@dl ||=SiSU_Env::InfoEnv.new.digest.length
+    end
+    def available
+      DBI.available_drivers.each do |driver|
+        puts "Driver: #{driver}"
+        DBI.data_sources(driver).each do |dsn|
+          puts "\tDatasource: #{dsn}"
+        end
+      end
+    end
+    def create_db
+      @env=SiSU_Env::InfoEnv.new(@opt.fns)
+      tell=(@sql_type=='sqlite') \
+      ? SiSU_Screen::Ansi.new(@opt.cmd,'Create SQLite db tables in:',%{"#{@file}"})
+      : SiSU_Screen::Ansi.new(@opt.cmd,'Create pgSQL db tables in:',%{"#{Db[:name_prefix]}#{@env.path.stub_pwd}"})
+      tell.dark_grey_title_hi if @opt.cmd =~/[vVM]/
+      SiSU_Env::SystemCall.new.create_pg_db(@env.path.stub_pwd) if @sql_type=='pg' #watch use of path.stub_pwd instead of stub
+    end
+    def output_dir?
+      dir=SiSU_Env::InfoEnv.new('')
+      if @opt.cmd =~/d/
+        dir.path.webserv_stub_ensure
+      end
+    end
+    def create_table
+      def metadata_and_text
+        print %{
+          currently using sisu dbi module
+          to be populated from document files
+          create tables metadata_and_text
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_metadata_and_text=%{
+          CREATE TABLE metadata_and_text (
+            tid                  BIGINT PRIMARY KEY,
+            /* title */
+            #{column.title.create_column}
+            #{column.title_main.create_column}
+            #{column.title_sub.create_column}
+            #{column.title_short.create_column}
+            #{column.title_edition.create_column}
+            #{column.title_note.create_column}
+            #{column.title_language.create_column}
+            #{column.title_language_char.create_column}
+            /* creator */
+            #{column.creator_author.create_column}
+            #{column.creator_author_honorific.create_column}
+            #{column.creator_author_nationality.create_column}
+            #{column.creator_editor.create_column}
+            #{column.creator_contributor.create_column}
+            #{column.creator_illustrator.create_column}
+            #{column.creator_photographer.create_column}
+            #{column.creator_translator.create_column}
+            #{column.creator_prepared_by.create_column}
+            #{column.creator_digitized_by.create_column}
+            #{column.creator_audio.create_column}
+            #{column.creator_video.create_column}
+            /* language */
+            #{column.language_document.create_column}
+            #{column.language_document_char.create_column}
+            #{column.language_original.create_column}
+            #{column.language_original_char.create_column}
+            /* date */
+            #{column.date_added_to_site.create_column}
+            #{column.date_available.create_column}
+            #{column.date_created.create_column}
+            #{column.date_issued.create_column}
+            #{column.date_modified.create_column}
+            #{column.date_published.create_column}
+            #{column.date_valid.create_column}
+            #{column.date_translated.create_column}
+            #{column.date_original_publication.create_column}
+            #{column.date_generated.create_column}
+            /* publisher */
+            #{column.publisher.create_column}
+            /* original */
+            #{column.original_publisher.create_column}
+            #{column.original_language.create_column}
+            #{column.original_language_char.create_column}
+            #{column.original_source.create_column}
+            #{column.original_institution.create_column}
+            #{column.original_nationality.create_column}
+            /* rights */
+            #{column.rights_all.create_column}
+            #{column.rights_copyright_text.create_column}
+            #{column.rights_copyright_translation.create_column}
+            #{column.rights_copyright_illustrations.create_column}
+            #{column.rights_copyright_photographs.create_column}
+            #{column.rights_copyright_preparation.create_column}
+            #{column.rights_copyright_digitization.create_column}
+            #{column.rights_copyright_audio.create_column}
+            #{column.rights_copyright_video.create_column}
+            #{column.rights_license.create_column}
+            /* classify */
+            #{column.classify_topic_register.create_column}
+            #{column.classify_subject.create_column}
+            #{column.classify_loc.create_column}
+            #{column.classify_dewey.create_column}
+            #{column.classify_keywords.create_column}
+            /* identifier */
+            #{column.identifier_oclc.create_column}
+            #{column.identifier_isbn.create_column}
+            /* notes */
+            #{column.notes_abstract.create_column}
+            #{column.notes_description.create_column}
+            #{column.notes_comment.create_column}
+            #{column.notes_history.create_column}
+            #{column.notes_coverage.create_column}
+            #{column.notes_relation.create_column}
+            /* column.notes_source.create_column */
+            #{column.notes_type.create_column}
+            #{column.notes_format.create_column}
+            #{column.notes_prefix.create_column}
+            #{column.notes_prefix_a.create_column}
+            #{column.notes_prefix_b.create_column}
+            #{column.notes_suffix.create_column}
+            /* src */
+            #{column.src_filename.create_column}
+            #{column.src_fingerprint.create_column}
+            #{column.src_filesize.create_column}
+            #{column.src_word_count.create_column}
+            #{column.src_txt.create_column}
+            /* misc */
+            #{column.fulltext.create_column}
+            #{column.links.create_column.gsub(/,$/,'')}
+/*          subj                 VARCHAR(64) NULL, */
+/*          contact              VARCHAR(100) NULL, */
+/*          information          VARCHAR(100) NULL, */
+/*          types                CHAR(1) NULL, */
+/*          writing_focus_nationality VARCHAR(100) NULL, */
+          );
+        }
+        @conn.execute(create_metadata_and_text)
+        @comment.psql.metadata_and_text if @comment
+      end
+      def doc_objects                                                 # create doc_objects base
+        print %{
+          to be populated from documents files
+          create tables doc_objects
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_doc_objects=%{
+          CREATE TABLE doc_objects (
+            lid             BIGINT PRIMARY KEY,
+            metadata_tid    BIGINT REFERENCES metadata_and_text,
+            ocn             SMALLINT,
+            ocnd            VARCHAR(6),
+            ocns            VARCHAR(6),
+            clean           TEXT NULL,
+            body            TEXT NULL,
+            book_idx        TEXT NULL,
+            seg             VARCHAR(256) NULL,
+            lev_an          VARCHAR(1),
+            lev             SMALLINT NULL,
+            lev1            SMALLINT,
+            lev2            SMALLINT,
+            lev3            SMALLINT,
+            lev4            SMALLINT,
+            lev5            SMALLINT,
+            lev6            SMALLINT,
+            en_a            SMALLINT NULL,
+            en_z            SMALLINT NULL,
+            en_a_asterisk   SMALLINT NULL,
+            en_z_asterisk   SMALLINT NULL,
+            en_a_plus       SMALLINT NULL,
+            en_z_plus       SMALLINT NULL,
+            t_of            VARCHAR(16),
+            t_is            VARCHAR(16),
+            node            VARCHAR(16) NULL,
+            parent          VARCHAR(16) NULL,
+            digest_clean    CHAR(#{@@dl}),
+            digest_all      CHAR(#{@@dl}),
+            types           CHAR(1) NULL
+          );
+        }
+        @conn.execute(create_doc_objects)
+        @comment.psql.doc_objects if @comment
+      end
+      def endnotes
+        print %{
+          to be populated from document files
+          create tables endnotes
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_endnotes=%{
+          CREATE TABLE endnotes (
+            nid             BIGINT PRIMARY KEY,
+            document_lid    BIGINT REFERENCES doc_objects,
+            nr              SMALLINT,
+            clean           TEXT NULL,
+            body            TEXT NULL,
+            ocn             SMALLINT,
+            ocnd            VARCHAR(6),
+            ocns            VARCHAR(6),
+            digest_clean    CHAR(#{@@dl}),
+            metadata_tid    BIGINT REFERENCES metadata_and_text
+          );
+        }
+        @conn.execute(create_endnotes)
+        @comment.psql.endnotes if @comment
+      end
+      def endnotes_asterisk
+        print %{
+          to be populated from document files
+          create tables endnotes_asterisk
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_endnotes_asterisk=%{
+          CREATE TABLE endnotes_asterisk (
+            nid             BIGINT PRIMARY KEY,
+            document_lid    BIGINT REFERENCES doc_objects,
+            nr              SMALLINT,
+            clean           TEXT NULL,
+            body            TEXT NULL,
+            ocn             SMALLINT,
+            ocnd            VARCHAR(6),
+            ocns            VARCHAR(6),
+            digest_clean    CHAR(#{@@dl}),
+            metadata_tid    BIGINT REFERENCES metadata_and_text
+          );
+        }
+        @conn.execute(create_endnotes_asterisk)
+        @comment.psql.endnotes_asterisk if @comment
+      end
+      def endnotes_plus
+        print %{
+          to be populated from document files
+          create tables endnotes_plus
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_endnotes_plus=%{
+          CREATE TABLE endnotes_plus (
+            nid             BIGINT PRIMARY KEY,
+            document_lid    BIGINT REFERENCES doc_objects,
+            nr              SMALLINT,
+            clean           TEXT NULL,
+            body            TEXT NULL,
+            ocn             SMALLINT,
+            ocnd            VARCHAR(6),
+            ocns            VARCHAR(6),
+            digest_clean    CHAR(#{@@dl}),
+            metadata_tid    BIGINT REFERENCES metadata_and_text
+          );
+        }
+        @conn.execute(create_endnotes_plus)
+        @comment.psql.endnotes_plus if @comment
+      end
+      def urls                                                       # create doc_objects file links mapping
+        print %{
+          currently using sisu dbi module
+          to be populated from doc_objects files
+          create tables urls
+          data import through ruby transfer
+        } if @opt.cmd =~/[VM]/
+        create_urls=%{
+          CREATE TABLE urls (
+            metadata_tid    BIGINT REFERENCES metadata_and_text,
+            plaintext       varchar(512),
+            html_toc        varchar(512),
+            html_doc        varchar(512),
+            xhtml           varchar(512),
+            xml_sax         varchar(512),
+            xml_dom         varchar(512),
+            odf             varchar(512),
+            pdf_p           varchar(512),
+            pdf_l           varchar(512),
+            concordance     varchar(512),
+            latex_p         varchar(512),
+            latex_l         varchar(512),
+            digest          varchar(512),
+            manifest        varchar(512),
+            markup          varchar(512),
+            sisupod         varchar(512)
+          );
+        }
+        @conn.execute(create_urls)
+        @comment.psql.urls if @comment
+      end
+      self
+    end
+  end
+  class Comment < SiSU_DbColumns::Columns
+    def initialize(conn,sql_type='pg')
+      @conn=conn
+      if sql_type =~ /pg/; psql
+      end
+    end
+    def psql
+      def conn_execute_array(sql_arr)
+        @conn.transaction do |conn|
+          sql_arr.each do |sql|
+            conn.execute(sql)
+          end
+        end
+      end
+      def metadata_and_text
+        sql_arr=[
+          %{COMMENT ON Table metadata_and_text
+            IS 'contains SiSU metadata and fulltext for search (including source .sst if shared)';},
+          %{COMMENT ON COLUMN metadata_and_text.tid
+            IS 'unique';},
+          %{#{column.title.column_comment}},
+          %{#{column.title_main.column_comment}},
+          %{#{column.title_sub.column_comment}},
+          %{#{column.title_short.column_comment}},
+          %{#{column.title_edition.column_comment}},
+          %{#{column.title_note.column_comment}},
+          %{#{column.title_language.column_comment}},
+          %{#{column.title_language_char.column_comment}},
+          %{#{column.creator_author.column_comment}},
+          %{#{column.creator_author_honorific.column_comment}},
+          %{#{column.creator_author_nationality.column_comment}},
+          %{#{column.creator_editor.column_comment}},
+          %{#{column.creator_contributor.column_comment}},
+          %{#{column.creator_illustrator.column_comment}},
+          %{#{column.creator_photographer.column_comment}},
+          %{#{column.creator_translator.column_comment}},
+          %{#{column.creator_prepared_by.column_comment}},
+          %{#{column.creator_digitized_by.column_comment}},
+          %{#{column.creator_audio.column_comment}},
+          %{#{column.creator_video.column_comment}},
+          %{#{column.language_document.column_comment}},
+          %{#{column.language_document_char.column_comment}},
+          %{#{column.language_original.column_comment}},
+          %{#{column.language_original_char.column_comment}},
+          %{#{column.date_added_to_site.column_comment}},
+          %{#{column.date_available.column_comment}},
+          %{#{column.date_created.column_comment}},
+          %{#{column.date_issued.column_comment}},
+          %{#{column.date_modified.column_comment}},
+          %{#{column.date_published.column_comment}},
+          %{#{column.date_valid.column_comment}},
+          %{#{column.date_translated.column_comment}},
+          %{#{column.date_original_publication.column_comment}},
+          %{#{column.date_generated.column_comment}},
+          %{#{column.publisher.column_comment}},
+          %{#{column.original_publisher.column_comment}},
+          %{#{column.original_language.column_comment}},
+          %{#{column.original_language_char.column_comment}},
+          %{#{column.original_source.column_comment}},
+          %{#{column.original_institution.column_comment}},
+          %{#{column.original_nationality.column_comment}},
+          %{#{column.rights_all.column_comment}},
+          %{#{column.rights_copyright_text.column_comment}},
+          %{#{column.rights_copyright_translation.column_comment}},
+          %{#{column.rights_copyright_illustrations.column_comment}},
+          %{#{column.rights_copyright_photographs.column_comment}},
+          %{#{column.rights_copyright_preparation.column_comment}},
+          %{#{column.rights_copyright_digitization.column_comment}},
+          %{#{column.rights_copyright_audio.column_comment}},
+          %{#{column.rights_copyright_video.column_comment}},
+          %{#{column.rights_license.column_comment}},
+          %{#{column.classify_topic_register.column_comment}},
+          %{#{column.classify_subject.column_comment}},
+          %{#{column.classify_loc.column_comment}},
+          %{#{column.classify_dewey.column_comment}},
+          %{#{column.classify_keywords.column_comment}},
+          %{#{column.identifier_oclc.column_comment}},
+          %{#{column.identifier_isbn.column_comment}},
+          %{#{column.notes_abstract.column_comment}},
+          %{#{column.notes_comment.column_comment}},
+          %{#{column.notes_description.column_comment}},
+          %{#{column.notes_history.column_comment}},
+          %{#{column.notes_coverage.column_comment}},
+          %{#{column.notes_relation.column_comment}},
+          %{#{column.notes_type.column_comment}},
+          %{#{column.notes_format.column_comment}},
+          %{#{column.notes_prefix.column_comment}},
+          %{#{column.notes_prefix_a.column_comment}},
+          %{#{column.notes_prefix_b.column_comment}},
+          %{#{column.notes_suffix.column_comment}},
+          %{#{column.src_filename.column_comment}},
+          %{#{column.src_fingerprint.column_comment}},
+          %{#{column.src_filesize.column_comment}},
+          %{#{column.src_word_count.column_comment}},
+          %{#{column.src_txt.column_comment}},
+          %{#{column.fulltext.column_comment}},
+          %{#{column.links.column_comment}},
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def doc_objects
+        sql_arr=[
+          %{COMMENT ON Table doc_objects
+            IS 'contains searchable text of SiSU document objects';},
+          %{COMMENT ON COLUMN doc_objects.lid
+            IS 'unique';},
+          %{COMMENT ON COLUMN doc_objects.metadata_tid
+            IS 'tie to title in metadata_and_text';},
+          %{COMMENT ON COLUMN doc_objects.lev_an
+            IS 'doc level A-C 1-6';},
+          %{COMMENT ON COLUMN doc_objects.lev
+            IS 'doc level 1-6 \d\~';},
+          %{COMMENT ON COLUMN doc_objects.seg
+            IS 'segment name from level number 4 (lv 1)';},
+          %{COMMENT ON COLUMN doc_objects.ocn
+            IS 'object citation number';},
+          %{COMMENT ON COLUMN doc_objects.en_a
+            IS 'first endnote number in text object (eg. NULL or 34) (used with en_z to create range)';},
+          %{COMMENT ON COLUMN doc_objects.en_z
+            IS 'last endnote number within text object (eg. NULL, 34 or say 47) (used with en_a to create range)';},
+          %{COMMENT ON COLUMN doc_objects.en_a_asterisk
+            IS 'first endnote number in text object (eg. NULL or 34) (used with en_z_asterisk to create range)';},
+          %{COMMENT ON COLUMN doc_objects.en_z_asterisk
+            IS 'last endnote number within text object (eg. NULL, 34 or say 47) (used with en_a_asterisk to create range)';},
+          %{COMMENT ON COLUMN doc_objects.en_a_plus
+            IS 'first endnote number in text object (eg. NULL or 34) (used with en_z_plus to create range)';},
+          %{COMMENT ON COLUMN doc_objects.en_z_plus
+            IS 'last endnote number within text object (eg. NULL, 34 or say 47) (used with en_a_plus to create range)';},
+          %{COMMENT ON COLUMN doc_objects.types
+            IS 'document types seg scroll';},
+          %{COMMENT ON COLUMN doc_objects.clean
+            IS 'text object - substantive text: clean, stripped of markup';},
+          %{COMMENT ON COLUMN doc_objects.body
+            IS 'text object - substantive text: light html markup';},
+          %{COMMENT ON COLUMN doc_objects.book_idx
+            IS 'book index creation information for paragraph, if provided';},
+          %{COMMENT ON COLUMN doc_objects.lev1
+            IS 'document structure, level number 1';},
+          %{COMMENT ON COLUMN doc_objects.lev2
+            IS 'document structure, level number 2';},
+          %{COMMENT ON COLUMN doc_objects.lev3
+            IS 'document structure, level number 3';},
+          %{COMMENT ON COLUMN doc_objects.lev4
+            IS 'document structure, level number 4';},
+          %{COMMENT ON COLUMN doc_objects.lev5
+            IS 'document structure, level number 5';},
+          %{COMMENT ON COLUMN doc_objects.lev6
+            IS 'document structure, level number 6';},
+          %{COMMENT ON COLUMN doc_objects.t_of
+            IS 'document structure, type of object (object is of)';},
+          %{COMMENT ON COLUMN doc_objects.t_is
+            IS 'document structure, object is';},
+          %{COMMENT ON COLUMN doc_objects.node
+            IS 'document structure, object node if heading';},
+          %{COMMENT ON COLUMN doc_objects.parent
+            IS 'document structure, object parent (is a heading)';}
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def endnotes
+        sql_arr=[
+          %{COMMENT ON Table endnotes
+            IS 'contains searchable text of SiSU documents endnotes';},
+          %{COMMENT ON COLUMN endnotes.nid
+            IS 'unique';},
+          %{COMMENT ON COLUMN endnotes.document_lid
+            IS 'ties to text block from which referenced';},
+          %{COMMENT ON COLUMN endnotes.nr
+            IS 'endnote number ';},
+          %{COMMENT ON COLUMN endnotes.clean
+            IS 'endnote substantive content, stripped of markup';},
+          %{COMMENT ON COLUMN endnotes.body
+            IS 'endnote substantive content';},
+          %{COMMENT ON COLUMN endnotes.ocn
+            IS 'object citation no# <\~(\d+)> from which endnote is referenced';},
+          %{COMMENT ON COLUMN doc_objects.metadata_tid
+            IS 'tie to title in metadata_and_text - unique for each document';}
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def endnotes_asterisk
+        sql_arr=[
+          %{COMMENT ON Table endnotes_asterisk
+            IS 'contains searchable text of SiSU documents endnotes marked with asterisk';},
+          %{COMMENT ON COLUMN endnotes_asterisk.nid
+            IS 'unique';},
+          %{COMMENT ON COLUMN endnotes_asterisk.document_lid
+            IS 'ties to text block from which referenced';},
+          %{COMMENT ON COLUMN endnotes_asterisk.nr
+            IS 'endnote number ';},
+          %{COMMENT ON COLUMN endnotes_asterisk.clean
+            IS 'endnote substantive content, stripped of markup';},
+          %{COMMENT ON COLUMN endnotes_asterisk.body
+            IS 'endnote substantive content';},
+          %{COMMENT ON COLUMN endnotes_asterisk.ocn
+            IS 'object citation no# <\~(\d+)> from which endnote is referenced';},
+          %{COMMENT ON COLUMN doc_objects.metadata_tid
+            IS 'tie to title in metadata_and_text - unique for each document';}
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def endnotes_plus
+        sql_arr=[
+          %{COMMENT ON Table endnotes_plus
+            IS 'contains searchable text of SiSU documents endnotes marked with plus';},
+          %{COMMENT ON COLUMN endnotes_plus.nid
+            IS 'unique';},
+          %{COMMENT ON COLUMN endnotes_plus.document_lid
+            IS 'ties to text block from which referenced';},
+          %{COMMENT ON COLUMN endnotes_plus.nr
+            IS 'endnote number ';},
+          %{COMMENT ON COLUMN endnotes_plus.clean
+            IS 'endnote substantive content, stripped of markup';},
+          %{COMMENT ON COLUMN endnotes_plus.body
+            IS 'endnote substantive content';},
+          %{COMMENT ON COLUMN endnotes_plus.ocn
+            IS 'object citation no# <\~(\d+)> from which endnote is referenced';},
+          %{COMMENT ON COLUMN doc_objects.metadata_tid
+            IS 'tie to title in metadata_and_text - unique for each document';},
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def urls
+        sql_arr=[
+          %{COMMENT ON Table urls
+            IS 'contains base url links to different SiSU output';},
+          %{COMMENT ON COLUMN doc_objects.metadata_tid
+            IS 'tie to title in metadata_and_text - unique for each document, the mapping of rows is one to one';},
+          %{COMMENT ON COLUMN urls.plaintext
+            IS 'plaintext utf-8';},
+          %{COMMENT ON COLUMN urls.html_toc
+            IS 'table of contents for segmented html document';},
+          %{COMMENT ON COLUMN urls.html_doc
+            IS 'html document (scroll)';},
+          %{COMMENT ON COLUMN urls.xhtml
+            IS 'xhtml document (scroll)';},
+          %{COMMENT ON COLUMN urls.xml_sax
+            IS 'xml sax oriented document (scroll)';},
+          %{COMMENT ON COLUMN urls.xml_dom
+            IS 'xml dom oriented document (scroll)';},
+          %{COMMENT ON COLUMN urls.odf
+            IS 'opendocument format text';},
+          %{COMMENT ON COLUMN urls.pdf_p
+            IS 'pdf portrait';},
+          %{COMMENT ON COLUMN urls.pdf_l
+            IS 'pdf landscape';},
+          %{COMMENT ON COLUMN urls.concordance
+            IS 'rudimentary document index linked to html';},
+          %{COMMENT ON COLUMN urls.latex_p
+            IS 'latex portrait';},
+          %{COMMENT ON COLUMN urls.latex_l
+            IS 'latex_landscape';},
+          %{COMMENT ON COLUMN urls.markup
+            IS 'markup';},
+          %{COMMENT ON COLUMN urls.sisupod
+            IS 'SiSU document format .tgz (all SiSU information on document)';},
+        ]
+        conn_execute_array(sql_arr)
+      end
+      self
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_dbi.rb b/lib/sisu/v5/db_dbi.rb
new file mode 100644
index 00000000..85640b3a
--- /dev/null
+++ b/lib/sisu/v5/db_dbi.rb
@@ -0,0 +1,96 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module  SiSU_DbDBI
+  require_relative 'db_columns'                                             # db_columns.rb
+  require_relative 'db_tests'                                               # db_tests.rb
+  require_relative 'db_create'                                              # db_create.rb
+  require_relative 'db_select'                                              # db_select.rb
+  require_relative 'db_indexes'                                             # db_indexes.rb
+  require_relative 'db_drop'                                                # db_drop.rb
+  require_relative 'db_remove'                                              # db_remove.rb
+  require_relative 'db_load_tuple'                                          # db_load_tuple.rb
+  require_relative 'db_import'                                              # db_import.rb
+  class ColumnSize < SiSU_DbColumns::ColumnSize                             # db_columns.rb
+  end
+  class Test < SiSU_DbTests::Test                                           # db_tests.rb
+  end
+  class Create .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbDrop
+  class Drop
+    require_relative 'response'                          # response.rb
+    def initialize(opt,conn,db_info,sql_type='')
+      @opt,@conn,@db_info,@sql_type=opt,conn,db_info,sql_type
+      @ans=SiSU_Response::Response.new
+      case @sql_type
+      when /sqlite/
+        cascade=''
+      else
+        cascade='CASCADE'
+      end
+      @drop_table=[
+        "DROP TABLE metadata_and_text #{cascade};",
+        "DROP TABLE doc_objects #{cascade};",
+        "DROP TABLE urls #{cascade};",
+        "DROP TABLE endnotes #{cascade};",
+        "DROP TABLE endnotes_asterisk #{cascade};",
+        "DROP TABLE endnotes_plus #{cascade};",
+      ]
+    end
+    def drop
+      def tables                                                              #% drop all tables
+        begin
+          msg_sqlite="as not all disk space is recovered after dropping the database << #{@db_info.sqlite.db} >>, you may be better off deleting the file, and recreating it as necessary"
+          case @sql_type
+          when /sqlite/
+            puts msg_sqlite
+            ans=@ans.response?('remove sql database?')
+            if ans \
+            and File.exist?(@db_info.sqlite.db)
+              @conn.close
+              File.unlink(@db_info.sqlite.db)
+              db=SiSU_Env::InfoDb.new
+              conn=db.sqlite.conn_sqlite3
+              sdb=SiSU_DbDBI::Create.new(@opt,conn,@db_info,@sql_type)
+              sdb_index=SiSU_DbDBI::Index.new(@opt,conn,@db_info,@sql_type)
+              sdb.output_dir?
+              begin
+                sdb.create_db
+                sdb.create_table.metadata_and_text
+                sdb.create_table.doc_objects
+                sdb.create_table.endnotes
+                sdb.create_table.endnotes_asterisk
+                sdb.create_table.endnotes_plus
+                sdb.create_table.urls
+                sdb_index.create_indexes
+              rescue
+                SiSU_Errors::InfoError.new($!,$@,'-D').error do
+                  __LINE__.to_s + ':' + __FILE__
+                end
+                sdb.output_dir?
+              end
+              exit
+            else
+              @conn.transaction
+              @drop_table.each do |d|
+                @conn.execute(d)
+              end
+              @conn.commit
+            end
+          else
+            @drop_table.each do |d|
+              @conn.execute(d)
+            end
+          end
+        rescue
+          case @sql_type
+          when /sqlite/
+            ans=@ans.response?('remove sql database?')
+            if ans and File.exist?(@db_info.sqlite.db); File.unlink(@db_info.sqlite.db)
+            end
+          else
+            @drop_table.each do |d|
+              @conn.execute(d)
+            end
+          end
+        ensure
+        end
+      end
+      def indexes
+        def conn_execute_array(sql_arr)
+          @conn.transaction do |conn|
+            sql_arr.each do |sql|
+              conn.execute(sql)
+            end
+          end
+        end
+        def base                                                             #% drop base indexes
+          print "\n          drop documents common indexes\n" unless @opt.cmd =~/q/
+          sql_arr=[
+            %{DROP INDEX idx_title;},
+            %{DROP INDEX idx_author;},
+            %{DROP INDEX idx_filename;},
+            %{DROP INDEX idx_topics;},
+            %{DROP INDEX idx_ocn;},
+            %{DROP INDEX idx_digest_clean;},
+            %{DROP INDEX idx_digest_all;},
+            %{DROP INDEX idx_lev1;},
+            %{DROP INDEX idx_lev2;},
+            %{DROP INDEX idx_lev3;},
+            %{DROP INDEX idx_lev4;},
+            %{DROP INDEX idx_lev5;},
+            %{DROP INDEX idx_lev6;},
+            %{DROP INDEX idx_endnote_nr;},
+            %{DROP INDEX idx_digest_en;},
+            %{DROP INDEX idx_endnote_nr_asterisk;},
+            %{DROP INDEX idx_endnote_asterisk;},
+            %{DROP INDEX idx_digest_en_asterisk;},
+            %{DROP INDEX idx_endnote_nr_plus;},
+            %{DROP INDEX idx_endnote_plus;},
+            %{DROP INDEX idx_digest_en_plus},
+          ]
+          conn_execute_array(sql_arr)
+        end
+        def text                                                             #% drop TEXT indexes, sqlite
+          print "\n          drop documents TEXT indexes\n" unless @opt.cmd =~/q/
+          sql_arr=[
+            %{DROP INDEX idx_clean;},
+            %{DROP INDEX idx_endnote},
+          ]
+          conn_execute_array(sql_arr)
+        end
+        self
+      end
+      indexes.base
+      @opt.cmd=~/D/ || ((@opt.mod=~/psql/) ? '' : indexes.text)
+      self
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_import.rb b/lib/sisu/v5/db_import.rb
new file mode 100644
index 00000000..a1d332bb
--- /dev/null
+++ b/lib/sisu/v5/db_import.rb
@@ -0,0 +1,717 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+   sqlite
+
+=end
+module SiSU_DbImport
+  require_relative 'db_columns'                         # db_columns.rb
+  require_relative 'db_load_tuple'                      # db_load_tuple.rb
+  require_relative 'db_sqltxt'                          # db_sqltxt.rb
+  require_relative 'shared_html_lite'                   # shared_html_lite.rb
+  require 'sqlite3'
+  class Import < SiSU_DbText::Prepare
+    include SiSU_Param
+    include SiSU_Screen
+    @@dl=nil
+    @@hname=nil
+    attr_accessor :tp
+    def initialize(opt,conn,file_maint,sql_type='pg')
+      @opt,@conn,@file_maint,@sql_type=opt,conn,file_maint,sql_type
+      @cX=SiSU_Screen::Ansi.new(@opt.cmd).cX
+      @env=SiSU_Env::InfoEnv.new(@opt.fns)
+      @dal="#{@env.processing_path.dal}"
+      @fnb=if @opt.fns.empty? \
+      or @opt.cmd.empty?
+        ''
+      else
+        @md=SiSU_Param::Parameters.new(@opt).get
+        @md.fnb
+      end
+      @fnc="#{@dal}/#{@opt.fns}.content.rbm"
+      @@seg,@@seg_full='',''                                  #create? consider placing field just before clean text as opposed to seg which contains seg(.html) name info seg_full would contain seg info for levels 5 & 6 where available eg seg_full may be 7.3 (level 5) and 7.3.1 (level 6) where seg  is 7
+      @col=Hash.new('')
+      @col[:ocn]=''
+      @counter={}
+      @db=SiSU_Env::InfoDb.new
+      if @sql_type=='sqlite'
+        @driver_sqlite3=(@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \
+        ? true
+        : false
+      end
+      sql='SELECT MAX(lid) FROM doc_objects'
+      begin
+        @col[:lid] ||=0
+        @col[:lid]=@driver_sqlite3 \
+        ? @conn.execute( sql ).join.to_i
+        : @conn.execute( sql ) { |x| x.fetch_all.flatten[0] }
+      rescue
+        puts "#{__FILE__}:#{__LINE__}" if @opt.cmd =~/M/
+      end
+      @col[:lid]=0 if @col[:lid].nil? or @col[:lid].to_s.empty?
+      sql='SELECT MAX(nid) FROM endnotes'
+      begin
+        @id_n=@driver_sqlite3 \
+        ? @conn.execute( sql ).join.to_i
+        : @id_n=@conn.execute( sql ) { |x| x.fetch_all.flatten[0] }
+        @id_n ||=0
+      rescue
+        puts "#{__FILE__}:#{__LINE__}" if @opt.cmd =~/M/
+      end
+      @id_n =0 if @col[:lid].nil? or @col[:lid].to_s.empty?
+      @col[:lv1]=@col[:lv2]=@col[:lv3]=@col[:lv4]=@col[:lv5]=@col[:lv6]=0
+      @db=SiSU_Env::InfoDb.new
+      @pdf_fn=SiSU_Env::FileOp.new(@md).base_filename
+      @@dl ||=SiSU_Env::InfoEnv.new.digest.length
+    end
+    def marshal_load
+      require_relative 'dal'                            # dal.rb
+      @dal_array=SiSU_DAL::Source.new(@opt).get            # dal file drawn here
+      SiSU_Screen::Ansi.new(@opt.cmd,"#{@db.psql.db}::#{@opt.fns}").puts_blue if @opt.cmd =~/vVM/
+      SiSU_Screen::Ansi.new(@opt.cmd,'Marshal Load',@fnc).puts_grey if @opt.cmd =~/v/
+#%
+      select_first_match=%{
+        SELECT metadata_and_text.tid
+        FROM metadata_and_text
+        WHERE metadata_and_text.src_filename = '#{@opt.fns}'
+        AND metadata_and_text.language_document_char = '#{@opt.lng}'
+      ;}
+      file_exist=@sql_type=~/sqlite/ \
+      ? @conn.get_first_value(select_first_match)
+      : @conn.select_one(select_first_match)
+      if not file_exist
+        t_d=[]                                                              # transaction_data
+        t_d << db_import_metadata
+        t_d << db_import_documents(@dal_array)
+        t_d << db_import_urls(@dal_array,@fnc)                              #import OID on/off
+        t_d=t_d.flatten
+        if @opt.cmd =~/[MV]/
+          puts @conn.class if defined? @conn.class
+          puts @conn.driver_name if defined? @conn.driver_name
+          puts @conn.driver if defined? @conn.driver
+        end
+        begin
+          sql=''
+          if @sql_type=~/sqlite/
+            @conn.transaction do |conn|
+              t_d.each do |sql|
+                conn.execute(sql)
+              end
+            end
+            #also 'execute' works for sqlite
+            #@conn.execute("BEGIN")
+            #  t_d.each do |sql|
+            #    @conn.execute(sql)
+            #  end
+            #@conn.execute("COMMIT")
+          else
+            #'do' works for postgresql
+            @conn.do("BEGIN")
+              t_d.each do |sql|
+                @conn.do(sql)
+              end
+            @conn.do("COMMIT")
+          end
+        rescue DBI::DatabaseError => e
+          STDERR.puts "Error code: #{e.err}"
+          STDERR.puts "Error message: #{e.errstr}"
+          STDERR.puts "Error SQLSTATE: #{e.state}"
+          SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+          sqlfn="#{@env.processing_path.sql}/#{@md.fnb}.sql"
+          sql=File.new(sqlfn,'w')
+          t_d.each {|i| sql.puts i}
+          p sqlfn
+          if @opt.cmd =~/M/
+            puts sql
+            p @conn.methods.sort
+            puts "#{__FILE__}:#{__LINE__}"
+          end
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+          sqlfn="#{@env.processing_path.sql}/#{@md.fnb}.sql"
+          sql=File.new(sqlfn,'w')
+          t_d.each {|i| sql.puts i}
+          p sqlfn
+          if @opt.cmd =~/M/
+            puts sql
+            p @conn.methods.sort
+            puts "#{__FILE__}:#{__LINE__}"
+          end
+        ensure
+        end
+      else
+        if file_exist
+          @db=SiSU_Env::InfoDb.new
+          puts "\n#{@cX.grey}file #{@cX.off} #{@cX.blue}#{@opt.fns}#{@cX.off} in language code #{cX.blue}#{@opt.lng}#{cX.off} #{@cX.grey}already exists in database#{@cX.off} #{@cX.blue}#{@db.psql.db}#{@cX.off} #{@cX.brown}update instead?#{@cX.off}"
+        end
+      end
+    end
+    def pf_db_import_transaction_open
+    end
+    def pf_db_import_transaction_close
+    end
+    def db_import_metadata                                                       #% import documents - populate database
+      print %{ #{@cX.grey}import documents dbi_unit #{@cX.off} } if @opt.cmd =~/vVM/
+      @tp={}
+      @md=SiSU_Param::Parameters.new(@opt).get
+#% sisutxt & fulltxt
+      if FileTest.exist?(@md.fns)
+        txt_arr=IO.readlines(@md.fns,'')
+        src=txt_arr.join("\n")
+        src=special_character_escape(src)
+        @tp[:sisutxt_f],@tp[:sisutxt_i]='sisutxt, ',"'#{src}', "
+        txt=clean_searchable_text(txt_arr)
+        #txt=special_character_escape(txt)
+        @tp[:fulltxt_f],@tp[:fulltxt_i]='fulltxt, ',"'#{txt}', "
+      end
+#% title
+      if defined? @md.title.full \
+      and @md.title.full=~/\S+/                                              # DublinCore 1 - title
+        #@tp[:title]=@md.title.full
+        #special_character_escape(@tp[:title])
+        #@tp[:title_f],@tp[:title_i]='title, ',"'#{@tp[:title]}', "
+        sql='SELECT MAX(tid) FROM metadata_and_text;'
+        begin
+          @@id_t ||=0
+          id_t=if @driver_sqlite3
+            @conn.execute( sql ).join.to_i # { |x| id_t=x.join.to_i }
+          else
+            @conn.execute( sql ) { |x| x.fetch_all.flatten[0] }
+          end
+          @@id_t=id_t if id_t
+        rescue
+          puts "#{__FILE__} #{__LINE__}" if @opt.cmd =~/M/
+        end
+        @@id_t+=1 #bug related, needs to be performed once at start of file, but consider moving, as, placed here it means program will fail if document header lacks @title:
+        puts %{\n#{@cX.grey}Processing file number#{@cX.off}: #{@cX.green}#{@@id_t}#{@@cX.off}} if @opt.cmd =~/vVM/
+      end
+      ################ CLEAR ##############
+      SiSU_DbDBI::Test.new(self,@opt).verify                          #% import title names, filenames (tuple)
+      t=SiSU_DbTuple::LoadMetadata.new(@conn,@@id_t,@md,@file_maint)
+      tuple=t.tuple
+      tuple
+    end
+    def db_import_documents(dal_array)                                     #% import documents - populate main database table, import into substantive database tables (tuple)
+      begin
+        @col[:tid]=@@id_t
+        @en,@en_ast,@en_pls,@tuple_array=[],[],[],[]
+        @col[:en_a],@col[:en_z]=nil,nil
+        dal_array.each do |data|
+          data.obj.gsub!(/#{Mx[:fa_bold_o]}(.+?)#{Mx[:fa_bold_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_italics_o]}(.+?)#{Mx[:fa_italics_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_underscore_o]}(.+?)#{Mx[:fa_underscore_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_superscript_o]}(.+?)#{Mx[:fa_superscript_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_subscript_o]}(.+?)#{Mx[:fa_subscript_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_insert_o]}(.+?)#{Mx[:fa_insert_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_cite_o]}(.+?)#{Mx[:fa_cite_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_strike_o]}(.+?)#{Mx[:fa_strike_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:fa_monospace_o]}(.+?)#{Mx[:fa_monospace_c]}/,'\1')
+          data.obj.gsub!(/#{Mx[:gl_o]}(●)#{Mx[:gl_c]}\s*/,'\1 ')
+          data.obj.gsub!(/#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}/,'') #check
+          @col[:seg]=@@seg
+          if data.of ==:para \
+          || :heading \
+          || :heading_insert \
+          || :block \
+          || :group      # regular text what of code-blocks grouped text etc.
+            notedata=data.obj.dup
+                                                                               #% :headings
+            if data.is==:heading \
+            && (data.ln.inspect=~/[123]/)
+              @col[:lev],txt,@col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=data.ln,data.obj,data.ocn,data.lv,data.odv,data.osp,data.of,data.is,data.node,data.parent,'',''
+              @col[:lid]+=1
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_minus
+              @col[:body]=special_character_escape(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text(plaintext)
+              book_idx=data.idx ? data.idx : ''
+              @col[:book_idx]=clean_searchable_text(book_idx)
+              if @en[0] then @en_a,@en_z=@en[0].first,@en[0].last
+              end
+              if @en_ast[0] then @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last
+              end
+              if @en_pls[0] then @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last
+              end
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              case @col[:lev]
+              when /1/; @col[:lv1]+=1
+              when /2/; @col[:lv2]+=1
+              when /3/; @col[:lv3]+=1
+              end
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==4
+              @@seg,txt,@col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=data.name,data.obj,data.ocn,data.lv,data.odv,data.osp,data.of,data.is,data.node,data.parent,'',''
+              @col[:seg]=@@seg
+              @col[:lv4]+=1
+              @col[:lid]+=1
+              @col[:lev]=4
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=special_character_escape(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text(plaintext)
+              book_idx=data.idx ? data.idx : ''
+              @col[:book_idx]=clean_searchable_text(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==5
+              txt,@col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=data.obj,data.ocn,data.lv,data.odv,data.osp,data.of,data.is,data.node,data.parent,'',''
+              @@seg_full=data.name if data.is==:heading \
+              && data.ln==5 \
+              && data.name #check data.name
+              @@seg ||='' #nil # watch
+              @col[:seg]=@@seg
+              @col[:lv5]+=1
+              @col[:lid]+=1
+              @col[:lev]=5
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=special_character_escape(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text(plaintext)
+              book_idx=data.idx ? data.idx : ''
+              @col[:book_idx]=clean_searchable_text(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==6
+              txt,@col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=data.obj,data.ocn,data.lv,data.odv,data.osp,data.of,data.is,data.node,data.parent,'',''
+              @@seg_full=data.name if data.is==:heading && data.ln==6 && data.name #check data.name
+              @@seg ||='' #nil # watch
+              @col[:seg]=@@seg
+              @col[:lv6]+=1
+              @col[:lid]+=1
+              @col[:lev]=6
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=special_character_escape(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text(plaintext)
+              book_idx=data.idx ? data.idx : ''
+              @col[:book_idx]=clean_searchable_text(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+                                                                               #% :structure :layout :comment
+            elsif data.of==:structure \
+            || data.of==:layout \
+            || data.of==:comment
+              #added watch
+                                                                               #% :
+            else                                                               #% regular text
+              @col[:lid]+=1
+              txt=''
+              txt,@col[:ocn],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=data.obj,data.ocn,data.odv,data.osp,data.of,data.is,'',data.parent,'',''
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              if @sql_type=~/pg/ \
+              and txt.size > (SiSU_DbColumns::ColumnSize.new.document_clean - 1)             # examine pg build & remove limitation
+                puts "\n\nTOO LARGE (TXT - see error log)\n\n"
+                open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                  error.puts("\n#{@opt.fns}\nTEXT BODY\n#{@col[:body].size} object #{@col[:ocn]} -> #{@col[:body].slice(0..500)}")
+                end
+                txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+              end
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              body=if data.is==:table
+                SiSU_FormatShared::CSS_Format.new(@md,data).html_table
+              elsif data.is==:code
+                SiSU_FormatShared::CSS_Format.new(@md,data).code
+              elsif defined? data.indent \
+              and defined? data.hang \
+              and data.indent =~/[1-9]/ \
+              and data.indent == data.hang
+                SiSU_FormatShared::CSS_Format.new(@md,data).indent(data.indent)
+              elsif defined? data.indent \
+              and defined? data.hang \
+              and data.hang =~/[0-9]/ \
+              and data.indent != data.hang
+                SiSU_FormatShared::CSS_Format.new(@md,data).hang_indent(data.hang,data.indent)
+              else
+                SiSU_FormatShared::CSS_Format.new(@md,data).norm
+              end
+              @col[:body]=special_character_escape(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text(plaintext)
+              book_idx=data.idx ? data.idx : ''
+              @col[:book_idx]=clean_searchable_text(book_idx)
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @en,@en_ast,@en_pls=[],[],[]
+              @col[:en_a]=@col[:en_z]=nil
+              @col[:lev]=@col[:plaintext]=@col[:body]=@col[:words]=''
+            end
+            if notedata =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/                                         #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_a_o]}\d+.+?#{Mx[:en_a_c]}/]
+                  if inf[/#{Mx[:en_a_o]}(\d+)(.+?)#{Mx[:en_a_c]}/]
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+            if notedata =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/                                      #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_b_o]}\*\d+.+?#{Mx[:en_b_c]}/]                    # dal new endnotes 2003w31/1
+                  if inf[/#{Mx[:en_b_o]}[*](\d+)(.+?)#{Mx[:en_b_c]}/]           # dal new endnotes 2003w31/1
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes_asterisk',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+            if notedata =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/                                           #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_b_o]}\+\d+.+?#{Mx[:en_b_c]}/]                        # dal new endnotes 2003w31/1
+                  if inf[/#{Mx[:en_b_o]}[+](\d+)(.+?)#{Mx[:en_b_c]}/]               # dal new endnotes 2003w31/1
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes_plus',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+          end
+        end
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+      @tuple_array
+    end
+    def endnotes(txt)
+      @txt=txt
+      def extract_any
+        if @txt =~/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[*+]?(\d+)\s+.+?(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/
+          endnotes(@txt).range
+          @en << endnotes(@txt).standard if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/
+          @en_ast << endnotes(@txt).asterisk if @txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/
+          @en_pls << endnotes(@txt).plus if @txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/
+          @txt=endnotes(@txt).clean_text
+        end
+        @txt
+      end
+      def standard
+        x=(@txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/) \
+        ? @txt.scan(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/)
+        : nil
+      end
+      def asterisk
+        x=(@txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/) \
+        ? @txt.scan(/#{Mx[:en_b_o]}[*](\d+).+?#{Mx[:en_b_c]}/)
+        : nil
+      end
+      def plus
+        x=(@txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/) \
+        ? @txt.scan(/#{Mx[:en_b_o]}[+](\d+).+?#{Mx[:en_b_c]}/)
+        : nil
+      end
+      def clean_text(base_url=nil)
+        @txt=if base_url
+          @txt.gsub(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/,%{\\1}).
+            gsub(/#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}/,%{\\1}).
+            gsub(/#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/,%{\\1})
+        else
+          @txt.gsub(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/,'\1').
+            gsub(/#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}/,'\1').
+            gsub(/#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/,'\1')
+        end
+        @txt
+      end
+      def range
+        @col[:en_a]=@col[:en_z]=nil
+        if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}|#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/
+          word_array=@txt.scan(/\S+/)
+          word_array.each do |w|
+            if w[/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[*+]?(\d+)\s+.+?(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/]                                                # not tested since change 2003w31
+              @col[:en_a]=$1 unless @col[:en_a]
+              @col[:en_z]=@col[:en_a].dup unless @col[:en_a]
+              @col[:en_z]=$1 if @col[:en_a]
+            end
+          end
+        end
+        @col
+      end
+      self
+    end
+    def db_import_urls(dbi_unit,content)                                           #% import documents OID - populate database
+      begin
+        @fnc=content
+        @env=SiSU_Env::InfoEnv.new(@opt.fns)
+        base=@env.url.root
+        out=@env.path.output
+        f,u={},{}
+        if @fnb.empty? \
+        or @fnb.nil?
+          p 'file output path error' #remove
+        end
+        if FileTest.file?("#{@md.file.output_path.txt.dir}/#{@md.file.base_filename.txt}")==true
+          f[:txt],u[:txt]='plaintext,', "'#{@md.file.output_path.txt.url}/#{@md.file.base_filename.txt}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_seg.dir}/#{@md.file.base_filename.html_seg}")==true
+          f[:html_toc],u[:html_toc]='html_toc,', "'#{@md.file.output_path.html_seg.url}/#{@md.file.base_filename.html_seg}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_scroll.dir}/#{@md.file.base_filename.html_scroll}")==true
+          f[:html_doc],u[:html_doc]='html_doc,', "'#{@md.file.output_path.html_scroll.url}/#{@md.file.base_filename.html_scroll}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xhtml.dir}/#{@md.file.base_filename.xhtml}")==true
+          f[:xhtml],u[:xhtml]='xhtml,', "'#{@md.file.output_path.xhtml.url}/#{@md.file.base_filename.xhtml}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xml_sax.dir}/#{@md.file.base_filename.xml_sax}")==true
+          f[:xml_sax],u[:xml_sax]='xml_sax,', "'#{@md.file.output_path.xml_sax.url}/#{@md.file.base_filename.xml_sax}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xml_dom.dir}/#{@md.file.base_filename.xml_dom}")==true
+          f[:xml_dom],u[:xml_dom]='xml_dom,', "'#{@md.file.output_path.xml_dom.url}/#{@md.file.base_filename.xml_dom}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}")==true
+          f[:epub],u[:epub]='epub,', "'#{@md.file.output_path.epub.url}/#{@md.file.base_filename.epub}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.odt.dir}/#{@md.file.base_filename.odt}")==true
+          f[:odf],u[:odf]='odf,', "'#{@md.file.output_path.odt.url}/#{@md.file.base_filename.odt}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_p_a4}")==true #\
+        #or FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_p_letter}")==true
+          f[:pdf_p],u[:pdf_p]='pdf_p,', "'#{@md.file.output_path.pdf.url}/#{@pdf_fn.pdf_p_a4}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_l_a4}")==true #\
+        #or FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_l_letter}")==true
+          f[:pdf_l],u[:pdf_l]='pdf_l,', "'#{@md.file.output_path.pdf.url}/#{@pdf_fn.pdf_l_a4}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}")==true
+          f[:concordance],u[:concordance]='concordance,', "'#{@md.file.output_path.html_concordance.url}/#{@md.file.base_filename.html_concordance}',"
+        end
+        #if FileTest.file?("#{@md.file.output_path.x.dir}/#{@md.file.base_filename.x}")==true
+        #  f[:latex_p],u[:latex_p]='latex_p,', "'#{@md.file.output_path.x.url}/#{@md.file.base_filename.x}',"
+        #end
+        ##if FileTest.file?("#{out}/#{@fnb}/#{@opt.fns}.tex")==true
+        ##  f[:latex_p],u[:latex_p]='latex_p,', "'#{base}/#{@fnb}/#{@opt.fns}.tex',"
+        ##end
+        #if FileTest.file?("#{@md.file.output_path.x.dir}/#{@md.file.base_filename.x}")==true
+        #  f[:latex_l],u[:latex_l]='latex_l,', "'#{@md.file.output_path.x.url}/#{@md.file.base_filename.x}',"
+        #end
+        ##if FileTest.file?("#{out}/#{@fnb}/#{@opt.fns}.landscape.tex")==true
+        ##  f[:latex_l],u[:latex_l]='latex_l,', "'#{base}/#{@fnb}/#{@opt}.fns}.landscape.tex',"
+        ##end
+        if FileTest.file?("#{@md.file.output_path.digest.dir}/#{@md.file.base_filename.digest}")==true
+          f[:digest],u[:digest]='digest,', "'#{@md.file.output_path.digest.url}/#{@md.file.base_filename.digest}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.manifest.dir}/#{@md.file.base_filename.manifest}")==true #revisit, was to be text, this is html
+          f[:manifest],u[:manifest]='manifest,', "'#{@md.file.output_path.manifest.url}/#{@md.file.base_filename.manifest}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.src.dir}/#{@md.file.base_filename.src}")==true
+          f[:markup],u[:markup]='markup,', "'#{@md.file.output_path.src.url}/#{@md.file.base_filename.src}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.sisupod.dir}/#{@md.file.base_filename.sisupod}")==true
+          f[:sisupod],u[:sisupod]='sisupod,', "'#{@md.file.output_path.sisupod.url}/#{@md.file.base_filename.sisupod}',"
+        end
+        t=SiSU_DbTuple::LoadUrls.new(@conn,f,u,@@id_t,@opt,@file_maint)
+        tuple=t.tuple
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+      tuple
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_indexes.rb b/lib/sisu/v5/db_indexes.rb
new file mode 100644
index 00000000..265d5169
--- /dev/null
+++ b/lib/sisu/v5/db_indexes.rb
@@ -0,0 +1,117 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+   sqlite
+
+=end
+module SiSU_DbIndex
+  class Index                                           # create documents Indexes def initialize(opt,conn='',sql_type='')
+    def initialize(opt,conn,file,sql_type='')
+      @opt,@conn,@file,@sql_type=opt,conn,file,sql_type
+    end
+    def create_indexes                                                           # check added from pg not tested
+      def conn_execute_array(sql_arr)
+        @conn.transaction do |conn|
+          sql_arr.each do |sql|
+            conn.execute(sql)
+          end
+        end
+      end
+      def base
+        print "\n          create documents common indexes\n" if @opt.cmd =~/[VM]/
+        sql_arr=[
+          %{CREATE INDEX idx_ocn ON doc_objects(ocn);},
+          %{CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);},
+          %{CREATE INDEX idx_digest_all ON doc_objects(digest_all);},
+          %{CREATE INDEX idx_lev1 ON doc_objects(lev1);},
+          %{CREATE INDEX idx_lev2 ON doc_objects(lev2);},
+          %{CREATE INDEX idx_lev3 ON doc_objects(lev3);},
+          %{CREATE INDEX idx_lev4 ON doc_objects(lev4);},
+          %{CREATE INDEX idx_lev5 ON doc_objects(lev5);},
+          %{CREATE INDEX idx_lev6 ON doc_objects(lev6);},
+          %{CREATE INDEX idx_endnote_nr ON endnotes(nr);},
+          %{CREATE INDEX idx_digest_en ON endnotes(digest_clean);},
+          %{CREATE INDEX idx_endnote_nr_asterisk ON endnotes_asterisk(nr);},
+          %{CREATE INDEX idx_endnote_asterisk ON endnotes_asterisk(clean);},
+          %{CREATE INDEX idx_digest_en_asterisk ON endnotes_asterisk(digest_clean);},
+          %{CREATE INDEX idx_endnote_nr_plus ON endnotes_plus(nr);},
+          %{CREATE INDEX idx_endnote_plus ON endnotes_plus(clean);},
+          %{CREATE INDEX idx_digest_en_plus ON endnotes_plus(digest_clean);},
+          %{CREATE INDEX idx_title ON metadata_and_text(title);},
+          %{CREATE INDEX idx_author ON metadata_and_text(creator_author);},
+          %{CREATE INDEX idx_filename ON metadata_and_text(src_filename);},
+          %{CREATE INDEX idx_language ON metadata_and_text(language_document_char);},
+          %{CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register)},
+        ]
+        conn_execute_array(sql_arr)
+      end
+      def text
+        print "\n          create documents TEXT indexes\n" if @opt.cmd =~/[VM]/
+        sql_arr=[
+          %{CREATE INDEX idx_clean ON doc_objects(clean);},
+          %{CREATE INDEX idx_endnote ON endnotes(clean);}
+        ]
+        conn_execute_array(sql_arr)
+      end
+      base
+      @opt.cmd=~/D/ || ((@opt.mod=~/psql/) ? '' : text)
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_load_tuple.rb b/lib/sisu/v5/db_load_tuple.rb
new file mode 100644
index 00000000..bdaf285a
--- /dev/null
+++ b/lib/sisu/v5/db_load_tuple.rb
@@ -0,0 +1,326 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbTuple
+  require_relative 'db_columns'                         # db_columns.rb
+  class LoadDocuments
+    require_relative 'param'                            # param.rb
+      include SiSU_Param
+    def initialize(conn,col,opt,file_maint)
+      @conn,@col,@opt,@file_maint=conn,col,opt,file_maint
+      @col[:lev]=@col[:lev].to_i
+      unless @col[:lev]=~/^[1-6]/ \
+      or @col[:lev]==1 \
+      or @col[:lev]==2 \
+      or @col[:lev]==3 \
+      or @col[:lev]==4 \
+      or @col[:lev]==5 \
+      or @col[:lev]==6 #changed from \d+ ??
+        @col[:lev]=0
+      end
+      @col[:ocn]=0 unless @col[:ocn].inspect=~/\d+/
+      @cX=SiSU_Screen::Ansi.new(@opt.cmd).cX
+    end
+    def tuple                                                                    #% import line
+      sql_entry=if @col[:en_a]
+        "INSERT INTO doc_objects (lid, metadata_tid, lev, lev_an, clean, body, book_idx, ocn, ocnd, ocns, seg, lev1, lev2, lev3, lev4, lev5, lev6, en_a, en_z, t_of, t_is, node, parent, digest_clean, digest_all) " +
+        "VALUES (#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:lev_an]}', '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:book_idx]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}', '#{@col[:en_a]}', '#{@col[:en_z]}', '#{@col[:t_of]}', '#{@col[:t_is]}', '#{@col[:node]}', '#{@col[:parent]}', '#{@col[:digest_clean]}', '#{@col[:digest_all]}');"
+      else
+        "INSERT INTO doc_objects (lid, metadata_tid, lev, lev_an, clean, body, book_idx, ocn, ocnd, ocns, seg, lev1, lev2, lev3, lev4, lev5, lev6, t_of, t_is, node, parent, digest_clean, digest_all) " +
+        "VALUES (#{@col[:lid]}, #{@col[:tid]}, #{@col[:lev]}, '#{@col[:lev_an]}', '#{@col[:plaintext]}', '#{@col[:body]}', '#{@col[:book_idx]}', '#{@col[:ocn]}', '#{@col[:ocnd]}', '#{@col[:ocns]}', '#{@col[:seg]}', '#{@col[:lv1]}', '#{@col[:lv2]}', '#{@col[:lv3]}', '#{@col[:lv4]}', '#{@col[:lv5]}', '#{@col[:lv6]}', '#{@col[:t_of]}', '#{@col[:t_is]}', '#{@col[:node]}', '#{@col[:parent]}', '#{@col[:digest_clean]}', '#{@col[:digest_all]}');"
+      end
+      if @opt.cmd =~/M/
+        if @opt.cmd =~/V/
+          puts @file_maint.inspect
+          puts sql_entry
+        end
+        @file_maint.puts sql_entry
+      else
+        if @opt.cmd =~/V/
+          puts sql_entry
+          @file_maint.puts sql_entry
+        end
+      end
+      if @opt.cmd =~/v/
+        if @col[:lev].inspect =~/[12356789]/
+          lev=case @col[:lev].inspect
+          when /1/; ':A'
+          when /2/; ':B'
+          when /3/; ':C'
+          when /5/; ' 2'
+          when /6/; ' 3'
+          end
+          puts %{#{lev}>\t#{@col[:lv1]}\t#{@col[:lv2]}\t#{@col[:lv3]}\t#{@col[:lv4]}\t#{@col[:lv5]}\t#{@col[:lv6]}\t#{@col[:ocn]}\t#{@col[:node]}\t#{@col[:ocns]}}
+        elsif @col[:lev].inspect =~/[4]/
+          puts %{ #{@cX.green}1>#{@cX.off}\t#{@col[:lv1]}\t#{@col[:lv2]}\t#{@col[:lv3]}\t#{@col[:lv4]}\t#{@col[:lv5]}\t#{@col[:lv6]}\t#{@col[:ocn]}\t#{@col[:node]}\t#{@col[:ocns]}\t#{@col[:seg]}}
+        end
+      end
+      sql_entry
+    end
+  end
+  class LoadMetadata #< SiSU_DbColumns::Columns
+    def initialize(conn,id,md,file_maint)
+      @conn,@id,@md,@file_maint=conn,id,md,file_maint
+      @tp=SiSU_DbColumns::Columns.new(md)
+    end
+    def tuple
+      sql_entry="INSERT INTO metadata_and_text (
+#{@tp.column.title.tuple[0]}
+#{@tp.column.title_main.tuple[0]}
+#{@tp.column.title_sub.tuple[0]}
+#{@tp.column.title_short.tuple[0]}
+#{@tp.column.title_edition.tuple[0]}
+#{@tp.column.title_note.tuple[0]}
+#{@tp.column.title_language.tuple[0]}
+#{@tp.column.title_language_char.tuple[0]}
+#{@tp.column.creator_author.tuple[0]}
+#{@tp.column.creator_author_honorific.tuple[0]}
+#{@tp.column.creator_author_nationality.tuple[0]}
+#{@tp.column.creator_editor.tuple[0]}
+#{@tp.column.creator_contributor.tuple[0]}
+#{@tp.column.creator_illustrator.tuple[0]}
+#{@tp.column.creator_photographer.tuple[0]}
+#{@tp.column.creator_translator.tuple[0]}
+#{@tp.column.creator_prepared_by.tuple[0]}
+#{@tp.column.creator_digitized_by.tuple[0]}
+#{@tp.column.creator_audio.tuple[0]}
+#{@tp.column.creator_video.tuple[0]}
+#{@tp.column.language_document.tuple[0]}
+#{@tp.column.language_document_char.tuple[0]}
+#{@tp.column.language_original.tuple[0]}
+#{@tp.column.language_original_char.tuple[0]}
+#{@tp.column.date_added_to_site.tuple[0]}
+#{@tp.column.date_available.tuple[0]}
+#{@tp.column.date_created.tuple[0]}
+#{@tp.column.date_issued.tuple[0]}
+#{@tp.column.date_modified.tuple[0]}
+#{@tp.column.date_published.tuple[0]}
+#{@tp.column.date_valid.tuple[0]}
+#{@tp.column.date_translated.tuple[0]}
+#{@tp.column.date_original_publication.tuple[0]}
+#{@tp.column.date_generated.tuple[0]}
+#{@tp.column.publisher.tuple[0]}
+#{@tp.column.original_publisher.tuple[0]}
+#{@tp.column.original_language.tuple[0]}
+#{@tp.column.original_language_char.tuple[0]}
+#{@tp.column.original_source.tuple[0]}
+#{@tp.column.original_institution.tuple[0]}
+#{@tp.column.original_nationality.tuple[0]}
+#{@tp.column.rights_all.tuple[0]}
+#{@tp.column.rights_copyright_text.tuple[0]}
+#{@tp.column.rights_copyright_translation.tuple[0]}
+#{@tp.column.rights_copyright_illustrations.tuple[0]}
+#{@tp.column.rights_copyright_photographs.tuple[0]}
+#{@tp.column.rights_copyright_preparation.tuple[0]}
+#{@tp.column.rights_copyright_digitization.tuple[0]}
+#{@tp.column.rights_copyright_audio.tuple[0]}
+#{@tp.column.rights_copyright_video.tuple[0]}
+#{@tp.column.rights_license.tuple[0]}
+#{@tp.column.classify_topic_register.tuple[0]}
+#{@tp.column.classify_subject.tuple[0]}
+#{@tp.column.classify_loc.tuple[0]}
+#{@tp.column.classify_dewey.tuple[0]}
+#{@tp.column.classify_keywords.tuple[0]}
+#{@tp.column.identifier_oclc.tuple[0]}
+#{@tp.column.identifier_isbn.tuple[0]}
+#{@tp.column.notes_abstract.tuple[0]}
+#{@tp.column.notes_description.tuple[0]}
+#{@tp.column.notes_comment.tuple[0]}
+#{@tp.column.notes_history.tuple[0]}
+#{@tp.column.notes_format.tuple[0]}
+#{@tp.column.notes_relation.tuple[0]}
+#{@tp.column.notes_coverage.tuple[0]}
+#{@tp.column.notes_type.tuple[0]}
+#{@tp.column.notes_prefix.tuple[0]}
+#{@tp.column.notes_prefix_a.tuple[0]}
+#{@tp.column.notes_prefix_b.tuple[0]}
+#{@tp.column.notes_suffix.tuple[0]}
+#{@tp.column.src_filename.tuple[0]}
+#{@tp.column.src_fingerprint.tuple[0]}
+#{@tp.column.src_filesize.tuple[0]}
+#{@tp.column.src_word_count.tuple[0]}
+#{@tp.column.src_txt.tuple[0]}
+#{@tp.column.fulltext.tuple[0]}
+#{@tp.column.links.tuple[0]}
+tid)
+" +
+       "VALUES (
+#{@tp.column.title.tuple[1]}
+#{@tp.column.title_main.tuple[1]}
+#{@tp.column.title_sub.tuple[1]}
+#{@tp.column.title_short.tuple[1]}
+#{@tp.column.title_edition.tuple[1]}
+#{@tp.column.title_note.tuple[1]}
+#{@tp.column.title_language.tuple[1]}
+#{@tp.column.title_language_char.tuple[1]}
+#{@tp.column.creator_author.tuple[1]}
+#{@tp.column.creator_author_honorific.tuple[1]}
+#{@tp.column.creator_author_nationality.tuple[1]}
+#{@tp.column.creator_editor.tuple[1]}
+#{@tp.column.creator_contributor.tuple[1]}
+#{@tp.column.creator_illustrator.tuple[1]}
+#{@tp.column.creator_photographer.tuple[1]}
+#{@tp.column.creator_translator.tuple[1]}
+#{@tp.column.creator_prepared_by.tuple[1]}
+#{@tp.column.creator_digitized_by.tuple[1]}
+#{@tp.column.creator_audio.tuple[1]}
+#{@tp.column.creator_video.tuple[1]}
+#{@tp.column.language_document.tuple[1]}
+#{@tp.column.language_document_char.tuple[1]}
+#{@tp.column.language_original.tuple[1]}
+#{@tp.column.language_original_char.tuple[1]}
+#{@tp.column.date_added_to_site.tuple[1]}
+#{@tp.column.date_available.tuple[1]}
+#{@tp.column.date_created.tuple[1]}
+#{@tp.column.date_issued.tuple[1]}
+#{@tp.column.date_modified.tuple[1]}
+#{@tp.column.date_published.tuple[1]}
+#{@tp.column.date_valid.tuple[1]}
+#{@tp.column.date_translated.tuple[1]}
+#{@tp.column.date_original_publication.tuple[1]}
+#{@tp.column.date_generated.tuple[1]}
+#{@tp.column.publisher.tuple[1]}
+#{@tp.column.original_publisher.tuple[1]}
+#{@tp.column.original_language.tuple[1]}
+#{@tp.column.original_language_char.tuple[1]}
+#{@tp.column.original_source.tuple[1]}
+#{@tp.column.original_institution.tuple[1]}
+#{@tp.column.original_nationality.tuple[1]}
+#{@tp.column.rights_all.tuple[1]}
+#{@tp.column.rights_copyright_text.tuple[1]}
+#{@tp.column.rights_copyright_translation.tuple[1]}
+#{@tp.column.rights_copyright_illustrations.tuple[1]}
+#{@tp.column.rights_copyright_photographs.tuple[1]}
+#{@tp.column.rights_copyright_preparation.tuple[1]}
+#{@tp.column.rights_copyright_digitization.tuple[1]}
+#{@tp.column.rights_copyright_audio.tuple[1]}
+#{@tp.column.rights_copyright_video.tuple[1]}
+#{@tp.column.rights_license.tuple[1]}
+#{@tp.column.classify_topic_register.tuple[1]}
+#{@tp.column.classify_subject.tuple[1]}
+#{@tp.column.classify_loc.tuple[1]}
+#{@tp.column.classify_dewey.tuple[1]}
+#{@tp.column.classify_keywords.tuple[1]}
+#{@tp.column.identifier_oclc.tuple[1]}
+#{@tp.column.identifier_isbn.tuple[1]}
+#{@tp.column.notes_abstract.tuple[1]}
+#{@tp.column.notes_comment.tuple[1]}
+#{@tp.column.notes_description.tuple[1]}
+#{@tp.column.notes_history.tuple[1]}
+#{@tp.column.notes_format.tuple[1]}
+#{@tp.column.notes_relation.tuple[1]}
+#{@tp.column.notes_coverage.tuple[1]}
+#{@tp.column.notes_type.tuple[1]}
+#{@tp.column.notes_prefix.tuple[1]}
+#{@tp.column.notes_prefix_a.tuple[1]}
+#{@tp.column.notes_prefix_b.tuple[1]}
+#{@tp.column.notes_suffix.tuple[1]}
+#{@tp.column.src_filename.tuple[1]}
+#{@tp.column.src_fingerprint.tuple[1]}
+#{@tp.column.src_filesize.tuple[1]}
+#{@tp.column.src_word_count.tuple[1]}
+#{@tp.column.src_txt.tuple[1]}
+#{@tp.column.fulltext.tuple[1]}
+#{@tp.column.links.tuple[1]}
+#{@id}
+);"
+      if @md.opt.cmd =~/M/
+        puts "maintenance mode on: creating sql transaction file (for last transaction set (document) only):\n\t#{@file_maint.inspect}"
+        @file_maint.puts sql_entry
+      else
+        @file_maint.puts sql_entry if @md.opt.cmd =~/V/
+      end
+      sql_entry
+    end
+  end
+  class LoadUrls
+    def initialize(conn,f,u,id,opt,file_maint)
+      @conn,@f,@u,@id,@opt,@file_maint=conn,f,u,id,opt,file_maint
+    end
+    def tuple
+      sql_entry="INSERT INTO urls (#{@f[:txt]} #{@f[:html_toc]} #{@f[:html_doc]} #{@f[:xhtml]} #{@f[:xml_sax]} #{@f[:xml_dom]} #{@f[:odf]} #{@f[:pdf_p]} #{@f[:pdf_l]} #{@f[:concordance]} #{@f[:latex_p]} #{@f[:latex_l]} #{@f[:manifest]} #{@f[:digest]} #{@f[:markup]} #{@f[:sisupod]} metadata_tid) " +
+      "VALUES (#{@u[:txt]} #{@u[:html_toc]} #{@u[:html_doc]} #{@u[:xhtml]} #{@u[:xml_sax]} #{@u[:xml_dom]} #{@u[:odf]} #{@u[:pdf_p]} #{@u[:pdf_l]} #{@u[:concordance]} #{@u[:latex_p]} #{@u[:latex_l]} #{@u[:manifest]} #{@u[:digest]} #{@u[:markup]} #{@u[:sisupod]} #{@id});"
+      if @opt.cmd =~/M/
+        @file_maint.puts sql_entry
+      else
+        @file_maint.puts sql_entry if @opt.cmd =~/V/
+      end
+      sql_entry
+    end
+  end
+  class LoadEndnotes
+    def initialize(conn,en,opt,file_maint)
+      @conn,@en,@opt,@file_maint=conn,en,opt,file_maint
+    end
+    def tuple
+      sql_entry="INSERT INTO #{@en[:type]} (nid, document_lid, nr, clean, body, ocn, ocnd, ocns, metadata_tid, digest_clean) " +
+      "VALUES ('#{@en[:id]}', '#{@en[:lid]}', '#{@en[:nr]}', '#{@en[:txt]}', '#{@en[:body]}', '#{@en[:ocn]}', '#{@en[:ocnd]}', '#{@en[:ocns]}', '#{@en[:id_t]}', '#{@en[:hash]}');"
+      if @opt.cmd =~/M/
+        @file_maint.puts sql_entry
+      else
+        @file_maint.puts sql_entry if @opt.cmd =~/V/
+      end
+      sql_entry
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_remove.rb b/lib/sisu/v5/db_remove.rb
new file mode 100644
index 00000000..1fb40e30
--- /dev/null
+++ b/lib/sisu/v5/db_remove.rb
@@ -0,0 +1,124 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbRemove
+  class Remove
+    def initialize(opt,conn,file,sql_type)
+      @opt,@conn,@file,@sql_type=opt,conn,file,sql_type
+      @md=SiSU_Param::Parameters.new(@opt).get
+      @fnb=@md.fnb
+      @db=SiSU_Env::InfoDb.new
+    end
+    def remove
+      driver_sqlite3=if @sql_type=='sqlite'
+        (@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \
+        ? true
+        : false
+      end
+      del_id=if driver_sqlite3
+        remove_selected=%{
+          SELECT tid
+          FROM metadata_and_text
+          WHERE src_filename = '#{@opt.fns}'
+          AND metadata_and_text.language_document_char = '#{@opt.lng}'
+        ;}
+        @conn.get_first_value(remove_selected).to_i
+      else
+        remove_selected=%{
+          SELECT metadata_and_text.tid
+          FROM metadata_and_text
+          WHERE metadata_and_text.src_filename = '#{@opt.fns}'
+          AND metadata_and_text.language_document_char = '#{@opt.lng}'
+        ;}
+        x=@conn.select_one(remove_selected)
+        x ? (x.join.to_i) : nil
+      end
+      if del_id
+        sql_entry=[
+          "DELETE FROM endnotes WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM endnotes_asterisk WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM endnotes_plus WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM doc_objects WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM urls WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM metadata_and_text WHERE metadata_and_text.tid = '#{del_id}';",
+        ]
+        if driver_sqlite3
+          @conn.transaction
+          sql_entry.each do |s|
+            @conn.execute(s)
+          end
+          @conn.commit if driver_sqlite3
+        else
+          sql_entry.each do |s|
+            @conn.execute(s)
+          end
+        end
+        if @opt.cmd =~/M/
+          @file.puts sql_entry if @opt.cmd =~/M/
+        end
+      else
+        SiSU_Screen::Ansi.new(@opt.cmd,"no such file in database #{@db.psql.db}::#{@opt.fns}").puts_grey if @opt.cmd =~/vVM/
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_select.rb b/lib/sisu/v5/db_select.rb
new file mode 100644
index 00000000..f6930fa4
--- /dev/null
+++ b/lib/sisu/v5/db_select.rb
@@ -0,0 +1,227 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbSelect
+  class Case
+    def initialize(opt,conn='',sql_type='pg')
+      @opt,@conn,@sql_type=opt,conn,sql_type
+      @db=SiSU_Env::InfoDb.new
+      @file_maint=sql_maintenance_file
+      @sdb=SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type)      # db_dbi.rb
+      @sdb_index=SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type) # db_dbi.rb
+      @sdb_no=SiSU_DbDBI::Drop.new(@opt,@conn,@db,@sql_type)       # db_dbi.rb
+      if @opt.mod.inspect =~/update|import/
+        @sdb_import=SiSU_DbDBI::Import.new(@opt,@conn,@file_maint,@sql_type)
+        @sdb_remove_doc=SiSU_DbDBI::Remove.new(@opt,@conn,@file_maint,@sql_type)
+      elsif @opt.mod.inspect =~/remove/
+        @sdb_remove_doc=SiSU_DbDBI::Remove.new(@opt,@conn,@file_maint,@sql_type)
+      end
+    end
+    def db_exist?
+      if @sql_type=='sqlite' \
+      and (not (FileTest.file?(@db.sqlite.db)) or FileTest.zero?(@db.sqlite.db))
+        puts %{no connection with #{@sql_type} database established, createdb "#{@db.sqlite.db}"?}
+        exit
+      end
+      if @conn.is_a?(NilClass)
+        db=@sql_type=='sqlite' \
+        ? @db.sqlite.db
+        : @db.psql.db
+        puts %{no connection with #{@sql_type} database established, createdb "#{db}"?}
+        exit
+      end
+    end
+    def sql_maintenance_file
+      file=if @opt.inspect =~/M/
+        x=if @opt.fns and not @opt.fns.empty?
+          @env=SiSU_Env::InfoEnv.new(@opt.fns) if @opt.fns
+          puts "\n#{@env.processing_path.sqlite}/#{@opt.fns}.sql" if @sql_type =~/sqlite/ and @opt.cmd =~/M/
+          @db=SiSU_Env::InfoDb.new
+          @job="sqlite3 #{@db.sqlite.db} < #{@env.processing_path.sqlite}/#{@opt.fns}.sql"
+          File.new("#{@env.processing_path.sqlite}/#{@opt.fns}.sql",'w+')
+        elsif @opt.fns \
+        and @opt.fns.inspect =~/create/
+          nil #sort variations later
+        else nil
+        end
+      else nil
+      end
+      file
+    end
+    def cases
+      @opt.mod.each do |mod|
+        case mod
+        when /^--createdb$/
+          @sdb.output_dir?
+          begin
+            @sdb.create_db
+          rescue; @sdb.output_dir?
+          end
+        when /^--(?:init(?:ialize)?|create(?:all)?)$/
+          @sdb.output_dir?
+          begin
+            @sdb.create_table.metadata_and_text
+            @sdb.create_table.doc_objects
+            @sdb.create_table.endnotes
+            @sdb.create_table.endnotes_asterisk
+            @sdb.create_table.endnotes_plus
+            @sdb.create_table.urls
+            @sdb_index.create_indexes
+          rescue
+            SiSU_Errors::InfoError.new($!,$@,'-D').error; @sdb.output_dir? do
+              __LINE__.to_s + ':' + __FILE__
+            end
+          end
+        when /^--createtables?$/
+          @sdb.output_dir?
+          begin
+            @sdb.create_table.metadata_and_text
+            @sdb.create_table.doc_objects
+            @sdb.create_table.endnotes
+            @sdb.create_table.endnotes_asterisk
+            @sdb.create_table.endnotes_plus
+            @sdb.create_table.urls
+            @sdb_index.create_indexes
+          rescue; @sdb.output_dir?
+          end
+        when /^--recreate$/
+          @sdb.output_dir?
+          begin
+            @sdb_no.drop.tables
+            @sdb.create_table.metadata_and_text
+            @sdb.create_table.doc_objects
+            @sdb.create_table.endnotes
+            @sdb.create_table.endnotes_asterisk
+            @sdb.create_table.endnotes_plus
+            @sdb.create_table.urls
+            @sdb_index.create_indexes
+          rescue; @sdb.output_dir?
+          end
+        when /^--cr(eate)?lex$/
+          @sdb.output_dir?
+          begin
+            @sdb.create_table.doc_objects
+          rescue; @sdb.output_dir?
+          end
+        when /^--cr(eate)?metadata$/
+          @sdb.output_dir?
+          begin
+            @sdb.create_table.metadata_and_text
+          rescue; @sdb.output_dir?
+          end
+        when /^--import$/
+          db_exist?
+          @sdb_import.marshal_load
+          tell=case @sql_type
+          when /sqlite/; SiSU_Screen::Ansi.new(@opt.cmd,"sqlite3 #{@db.sqlite.db} database?")
+          when /pg/;     SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.psql.db} database?")
+          else '???'
+          end
+          tell.puts_grey if @opt.cmd =~/v/
+        when /^--update$/
+          db_exist?
+          @sdb_remove_doc.remove
+          @sdb_import.marshal_load
+          tell=case @sql_type
+          when /sqlite/; SiSU_Screen::Ansi.new(@opt.cmd,"sqlite3 #{@db.sqlite.db} database?")
+          when /pg/;     SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.psql.db} database?")
+          else '???'
+          end
+          tell.puts_grey if @opt.cmd =~/v/
+        when /^--remove$/
+          db_exist?
+          @sdb_remove_doc.remove
+        when /^--index$/
+          db_exist?
+          @sdb_index.create_indexes
+        when /^--droptable(s)?$/
+          db_exist?
+          @sdb_no.drop.tables
+        when /^--dropindex(es)?$/
+          db_exist?
+          @sdb_no.drop.indexes
+        when /^--(?:dropall|drop)$/
+          db_exist?
+          @sdb_no.drop.tables
+        when /^--(?:db=)?(?:(?:sq)?lite|pg(?:sql)?|my(?:sql)?)$/
+        when /^--(?:v\d+|dev)$/
+        else
+          #help=SiSU_Help::Help.new
+          #help.summary
+          #help.commands
+        end
+        if @opt.cmd =~/M/ \
+        and @opt.cmd  =~/d/
+          puts @job
+        end
+      end
+      if @opt.act[:psql][:set]==:on \
+      and @opt.mod.inspect !~/--(?:createdb|init(?:ialize)?|create(?:all)?|createtables?|recreate|cr(eate)?lex|cr(eate)?metadata|import|update|remove|index|droptable(s)?|dropindex(es)?|dropall|drop)/
+        SiSU_Screen::Ansi.new(@opt.cmd,"--pg requires further instruction").warn unless @opt.cmd =~/[q]/
+      end
+      begin
+      rescue; @sdb.output_dir?
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/db_sqltxt.rb b/lib/sisu/v5/db_sqltxt.rb
new file mode 100644
index 00000000..eb52f300
--- /dev/null
+++ b/lib/sisu/v5/db_sqltxt.rb
@@ -0,0 +1,133 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_DbText
+  class Prepare
+    def special_character_escape(str)
+      str=str.gsub(/'/,"''"). #string.gsub!(/'/,"\047") #string.gsub!(/'/,"\\'")
+        gsub(/(\\)/m,'\1\1'). #ok but with warnings, double backslash on sqlite #str.gsub!(/[\\]/m,'\\x5C') #ok but with warnings, but not for sqlite #str.gsub!(/(\\)/m,'\1') #ok for sqlite not for pgsql
+        gsub(/#{Mx[:br_line]}|#{Mx[:br_nl]}/,"
\n").
+        gsub(/#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}/,''). #check
+        gsub(/#{Mx[:lnk_o]}\s*(\S+?\.(?:png|jpg))(?:\s+\d+x\d+)?(.+?)#{Mx[:lnk_c]}\S+/,'[image: \1] \2').
+        gsub(/#{Mx[:lnk_o]}\s*(.+?)\s*#{Mx[:lnk_c]}(?:file|ftp):\/\/\S+?([.,!?]?(?:\s|$))/,'\1\2').
+        gsub(/#{Mx[:lnk_o]}\s*(.+?)\s*#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/,'\1')
+    end
+    def clean_searchable_text(arr) #produce clean, searchable, plaintext from document source
+      txt_arr,en=[],[]
+      arr=(arr.is_a?(String)) ? arr.split(/\n+/m) : arr
+      arr.each do |s|
+        s=s.gsub(/([*\/_-])\{(.+?)\}\1/m,'\2').
+          gsub(/^(?:block|group|poem|code)\{/m,'').gsub(/^\}(?:block|group|poem|code)/m,'').
+          gsub(/\A(?:@\S+:\s+.+)\Z/m,'')
+        if s =~/^:A~/
+          if defined? @md.creator \
+          and defined? @md.creator.author \
+          and not @md.creator.author.empty?
+            s=s.gsub(/@author/,@md.creator.author)
+          else
+            SiSU_Screen::Ansi.new('v','WARNING Document Author information missing; provide @creator: :author:',@md.fnb).warn unless @md.opt.cmd.inspect =~/q/
+          end
+          if defined? @md.title \
+          and defined? @md.title.full \
+          and not @md.title.full.empty?
+            s=s.gsub(/@title/,@md.title.full)
+          else
+            SiSU_Screen::Ansi.new('v','WARNING Document Title missing; provide @title:',@md.fnb).warn unless @md.opt.cmd.inspect =~/q/
+          end
+        end
+        s=s.gsub(/^(?:_[1-9]\*?|_\*)\s+/m,'').
+          gsub(/^(?:[1-9]\~(\S+)?)\s+/m,'').
+          gsub(/^(?::?[A-C]\~(\S+)?)\s+/m,'').
+          gsub(/^%{1,3} .+/m,''). #removed even if contained in code block
+          gsub(/
/m,' ')
+        en << s.scan(/~\{\s*(.+?)\s*\}~/m)
+        s=s.gsub(/~\{.+?\}~/m,'').
+          gsub(/ \s+/m,' ')
+        #special_character_escape(s)
+        s
+      end
+      txt_arr << arr << en
+      #txt_arr=txt_arr.flatten
+      txt=txt_arr.flatten.join("\n")
+      txt=special_character_escape(txt)
+      txt
+    end
+    def strip_markup(str) #define rules, make same as in dal clean
+      str=str.gsub(/#{Mx[:fa_superscript_o]}(\d+)#{Mx[:fa_superscript_c]}/,'[\1]').
+        gsub(/(?: \\;|#{Mx[:nbsp]})+/,' ').
+        gsub(/#{Mx[:tc_o]}#{Mx[:tc_p]}#{Mx[:tc_p]}\d+(.+)#{Mx[:tc_c]}/u,'\1').         #tables
+        gsub(/#{Mx[:tc_p]}#{Mx[:tc_p]}\d+#{Mx[:tc_p]}/u,' ').                          #tables
+        gsub(/#{Mx[:tc_p]}/u,' ').                                                     #tables tidy later
+        gsub(/<.+?>/,'').
+        gsub(/#{Mx[:lnk_o]}.+?\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}(?:file|ftp)\/\/:\S+ /,' [image] '). # else image names found in search
+        gsub(/#{Mx[:lnk_o]}.+?\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/,' [image]'). # else image names found in search
+        gsub(/\s\s+/,' ').
+        strip
+    end
+    def unique_words(str)
+      a=str.scan(/[a-zA-Z0-9\\\/_-]{2,}/) #a=str.scan(/\S+{2,}/)
+      str=a.uniq.sort.join(' ')
+      str
+    end
+  end
+end
+__END__
+
diff --git a/lib/sisu/v5/db_tests.rb b/lib/sisu/v5/db_tests.rb
new file mode 100644
index 00000000..5cc7c982
--- /dev/null
+++ b/lib/sisu/v5/db_tests.rb
@@ -0,0 +1,120 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: modules shared by the different db types, dbi, postgresql,
+    sqlite
+
+=end
+module SiSU_DbTests
+  class Test
+    def initialize(info,opt)
+      @ck,@opt=info,opt
+      unless @opt.cmd =~/q/
+        puts @ck.tp[:fns]  if @ck.tp[:fns] and not @ck.tp[:fns].empty?
+        puts @ck.tp[:title] if @ck.tp[:title] and not @ck.tp[:title].empty?
+        puts @ck.tp[:creator] if @ck.tp[:creator] and not @ck.tp[:creator].empty?
+      end
+    end
+    def verify
+      unless @opt.cmd =~/q/
+        puts @ck.tp[:fns].length.to_s                + ' checklength ' + @ck.tp[:fns]                if @ck.tp[:fns]                and @ck.tp[:fns].length                >@ck.lt_filename
+        puts @ck.tp[:title].length.to_s              + ' checklength ' + @ck.tp[:title]              if @ck.tp[:title]              and @ck.tp[:title].length              >@ck.lt_title
+        puts @ck.tp[:subtitle].length.to_s           + ' checklength ' + @ck.tp[:subtitle]           if @ck.tp[:subtitle]           and @ck.tp[:subtitle].length           >@ck.lt_subtitle
+        puts @ck.tp[:creator].length.to_s            + ' checklength ' + @ck.tp[:creator]            if @ck.tp[:creator]            and @ck.tp[:creator].length            >@ck.lt_creator
+        puts @ck.tp[:author_title].length.to_s       + ' checklength ' + @ck.tp[:author_title]       if @ck.tp[:author_title]       and @ck.tp[:author_title].length       >@ck.lt_author_title
+        puts @ck.tp[:illustrator].length.to_s        + ' checklength ' + @ck.tp[:illustrator]        if @ck.tp[:illustrator]        and @ck.tp[:illustrator].length        >@ck.lt_illustrator
+        puts @ck.tp[:translator].length.to_s         + ' checklength ' + @ck.tp[:translator]         if @ck.tp[:translator]         and @ck.tp[:translator].length         >@ck.lt_translator
+        puts @ck.tp[:prepared_by].length.to_s        + ' checklength ' + @ck.tp[:prepared_by]        if @ck.tp[:prepared_by]        and @ck.tp[:prepared_by].length        >@ck.lt_prepared_by
+        puts @ck.tp[:digitized_by].length.to_s       + ' checklength ' + @ck.tp[:digitized_by]       if @ck.tp[:digitized_by]       and @ck.tp[:digitized_by].length       >@ck.lt_digitized_by
+        puts @ck.tp[:subject].length.to_s            + ' checklength ' + @ck.tp[:subject]            if @ck.tp[:subject]            and @ck.tp[:subject].length            >@ck.lt_subject
+        puts @ck.tp[:description].length.to_s        + ' checklength ' + @ck.tp[:description]        if @ck.tp[:description]        and @ck.tp[:description].length        >@ck.lt_description
+        puts @ck.tp[:publisher].length.to_s          + ' checklength ' + @ck.tp[:publisher]          if @ck.tp[:publisher]          and @ck.tp[:publisher].length          >@ck.lt_publisher
+        puts @ck.tp[:contributor].length.to_s        + ' checklength ' + @ck.tp[:contributor]        if @ck.tp[:contributor]        and @ck.tp[:contributor].length        >@ck.lt_contributor
+        puts @ck.tp[:date].length.to_s               + ' checklength ' + @ck.tp[:date]               if @ck.tp[:date]               and @ck.tp[:date].length               >@ck.lt_date
+        puts @ck.tp[:date_created].length.to_s       + ' checklength ' + @ck.tp[:date_created]       if @ck.tp[:date_created]       and @ck.tp[:date_created].length       >@ck.lt_date
+        puts @ck.tp[:date_issued].length.to_s        + ' checklength ' + @ck.tp[:date_issued]        if @ck.tp[:date_issued]        and @ck.tp[:date_issued].length        >@ck.lt_date
+        puts @ck.tp[:date_valid].length.to_s         + ' checklength ' + @ck.tp[:date_valid]         if @ck.tp[:date_valid]         and @ck.tp[:date_valid].length         >@ck.lt_date
+        puts @ck.tp[:date_available].length.to_s     + ' checklength ' + @ck.tp[:date_available]     if @ck.tp[:date_available]     and @ck.tp[:date_available].length     >@ck.lt_date
+        puts @ck.tp[:date_modified].length.to_s      + ' checklength ' + @ck.tp[:date_modified]      if @ck.tp[:date_modified]      and @ck.tp[:date_modified].length      >@ck.lt_date
+        puts @ck.tp[:date_translated].length.to_s    + ' checklength ' + @ck.tp[:date_translated]    if @ck.tp[:date_translated]    and @ck.tp[:date_translated].length    >@ck.lt_date
+        puts @ck.tp[:date_added_to_site].length.to_s + ' checklength ' + @ck.tp[:date_added_to_site] if @ck.tp[:date_added_to_site] and @ck.tp[:date_added_to_site].length >@ck.lt_date
+        puts @ck.tp[:type].length.to_s               + ' checklength ' + @ck.tp[:type]               if @ck.tp[:type]               and @ck.tp[:type].length               >@ck.lt_type
+        puts @ck.tp[:format].length.to_s             + ' checklength ' + @ck.tp[:format]             if @ck.tp[:format]             and @ck.tp[:format].length             >@ck.lt_format
+        puts @ck.tp[:identifier].length.to_s         + ' checklength ' + @ck.tp[:identifier]         if @ck.tp[:identifier]         and @ck.tp[:identifier].length         >@ck.lt_identifier
+        puts @ck.tp[:source].length.to_s             + ' checklength ' + @ck.tp[:source]             if @ck.tp[:source]             and @ck.tp[:source].length             >@ck.lt_source
+        puts @ck.tp[:language].length.to_s           + ' checklength ' + @ck.tp[:language]           if @ck.tp[:language]           and @ck.tp[:language].length           >@ck.lt_language
+        puts @ck.tp[:language_original].length.to_s  + ' checklength ' + @ck.tp[:language_original]  if @ck.tp[:language_original]  and @ck.tp[:language_original].length  >@ck.lt_language_original
+        puts @ck.tp[:relation].length.to_s           + ' checklength ' + @ck.tp[:relation]           if @ck.tp[:relation]           and @ck.tp[:relation].length           >@ck.lt_relation
+        puts @ck.tp[:coverage].length.to_s           + ' checklength ' + @ck.tp[:coverage]           if @ck.tp[:coverage]           and @ck.tp[:coverage].length           >@ck.lt_coverage
+        puts @ck.tp[:rights].length.to_s             + ' checklength ' + @ck.tp[:rights]             if @ck.tp[:rights]             and @ck.tp[:rights].length             >@ck.lt_rights
+        puts @ck.tp[:copyright].length.to_s          + ' checklength ' + @ck.tp[:copyright]          if @ck.tp[:copyright]          and @ck.tp[:copyright].length          >@ck.lt_copyright
+        puts @ck.tp[:owner].length.to_s              + ' checklength ' + @ck.tp[:owner]              if @ck.tp[:owner]              and @ck.tp[:owner].length              >@ck.lt_owner
+        puts @ck.tp[:keywords].length.to_s           + ' checklength ' + @ck.tp[:keywords]           if @ck.tp[:keywords]           and @ck.tp[:keywords].length           >@ck.lt_keywords
+        puts @ck.tp[:abstract].length.to_s           + ' checklength ' + @ck.tp[:abstract]           if @ck.tp[:abstract]           and @ck.tp[:abstract].length           >@ck.lt_abstract
+        puts @ck.tp[:comment].length.to_s            + ' checklength ' + @ck.tp[:comment]            if @ck.tp[:comment]            and @ck.tp[:comment].length            >@ck.lt_comment
+        puts @ck.tp[:loc].length.to_s                + ' checklength ' + @ck.tp[:loc]                if @ck.tp[:loc]                and @ck.tp[:loc].length                >@ck.lt_loc
+        puts @ck.tp[:dewey].length.to_s              + ' checklength ' + @ck.tp[:dewey]              if @ck.tp[:dewey]              and @ck.tp[:dewey].length              >@ck.lt_dewey
+        puts @ck.tp[:isbn].length.to_s               + ' checklength ' + @ck.tp[:isbn]               if @ck.tp[:isbn]               and @ck.tp[:isbn].length               >@ck.lt_isbn
+        puts @ck.tp[:pg].length.to_s                 + ' checklength ' + @ck.tp[:pg]                 if @ck.tp[:pg]                 and @ck.tp[:pg].length                 >@ck.lt_pg
+        puts @ck.tp[:topic_register].length.to_s     + ' checklength ' + @ck.tp[:topic_register]     if @ck.tp[:topic_register]     and @ck.tp[:topci_register].length     >@ck.lt_topic_register
+        puts @ck.tp[:date]                                                                           if @ck.tp[:date] and not @ck.tp[:date].empty? and @ck.tp[:date] !~/\d\d-\d\d-\d\d/
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dbi.rb b/lib/sisu/v5/dbi.rb
new file mode 100644
index 00000000..33592727
--- /dev/null
+++ b/lib/sisu/v5/dbi.rb
@@ -0,0 +1,146 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: postgresql module, dbi import frame
+
+=end
+module  SiSU_DBI                                                                 #% database building
+  require_relative 'help'                               # help.rb
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env; include SiSU_Screen
+  require_relative 'param'                              # param.rb
+    include SiSU_Param
+  require_relative 'db_dbi'                             # db_dbi.rb
+    include SiSU_DbDBI
+  require_relative 'shared_html_lite'                   # shared_html_lite.rb
+    include SiSU_FormatShared
+  class SQL
+    def initialize(opt)
+      SiSU_Env::Load.new('dbi',true).prog
+      @opt=opt
+      @db=SiSU_Env::InfoDb.new
+      if @opt.cmd =~/[Dd]/ \
+      or  @opt.mod.inspect =~/--(pg(?:sql)?|(?:sq)?lite)/
+        @sql_type=if @opt.cmd=~/D/ \
+        or @opt.mod.inspect =~/--pg(?:sql)?/
+          maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+          'pg'
+        elsif @opt.cmd =~/d/ \
+        and @opt.mod.inspect =~/--(?:db[=-])?pg(?:sql)?/
+          maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+          'pg'
+        elsif @opt.cmd=~/d/ \
+        or @opt.mod.inspect =~/--(?:sq)?lite/
+          maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+          'sqlite'
+        elsif @opt.cmd =~/d/ \
+        and @opt.mod.inspect =~/--(?:db[=-])?(?:sq)?lite/
+          maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+          'sqlite'
+        else
+          maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+          'sqlite'
+        end
+      end
+    end
+    def maintenance_check(opt,file,line)
+      p opt.mod
+      p opt.cmd
+      p "at #{file} #{line}"
+    end
+    def read_psql
+      begin
+        @conn=@db.psql.conn_dbi
+      rescue
+        if @opt.mod.inspect=~/--(createall|create)/
+          puts %{manually create the database: "#{@db.db}" if it does not yet exist}
+          #sudo su -p postgres;  createdb #{@db.db}; #[createuser?]
+        end
+      ensure
+      end
+    end
+    def read_sqlite
+      begin
+        sql_type='sqlite'
+        @conn=@db.sqlite.conn_sqlite3
+      rescue
+      ensure
+      end
+    end
+    def connect
+      case @sql_type
+      when /pg/;     read_psql
+      when /sqlite/; read_sqlite
+      end
+      SiSU_Screen::Ansi.new(@opt.cmd,"DBI (#{@sql_type}) #{@opt.mod}",@opt.fno).dbi_title unless @opt.cmd =~/q/
+      begin
+        SiSU_DbDBI::Case.new(@opt,@conn,@sql_type).cases
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@cf,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+      begin
+      rescue
+        connect
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/dbi_discrete.rb b/lib/sisu/v5/dbi_discrete.rb
new file mode 100644
index 00000000..89fd3898
--- /dev/null
+++ b/lib/sisu/v5/dbi_discrete.rb
@@ -0,0 +1,186 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: postgresql module, dbi import frame
+
+=end
+module  SiSU_DBI_Discrete                               #% database building
+  require_relative 'help'                               # help.rb
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env; include SiSU_Screen
+  require_relative 'param'                              # param.rb
+    include SiSU_Param
+  require_relative 'db_dbi'                             # db_dbi.rb
+    include SiSU_DbDBI
+  require_relative 'shared_html_lite'                   # shared_html_lite.rb
+    include SiSU_FormatShared
+  require 'fileutils'
+    include FileUtils::Verbose
+  class SQL
+    def initialize(opt)
+      SiSU_Env::Load.new('dbi',true).prog
+      @opt=opt
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+      @md=@particulars.md
+      if @opt.cmd =~/[d]/ \
+      or  @opt.mod.inspect =~/--((?:sq)?lite)/
+        @sql_type='sqlite'
+        maintenance_check(@opt,__FILE__,__LINE__) if @opt.cmd.inspect =~/M/
+      end
+      @output_path=@md.file.output_path.sqlite_discrete.dir
+      @filename=@md.file.base_filename.sqlite_discrete
+      @file_maint=sql_maintenance_file
+      @file="#{@output_path}/#{@filename}"
+    end
+    def build
+      prepare
+      create_and_populate
+    end
+    def maintenance_check(opt,file,line)
+      p opt.mod
+      p opt.cmd
+      p "at #{file} #{line}"
+    end
+    def prepare
+      if not FileTest.directory?(@output_path)
+        FileUtils::mkdir_p(@output_path)
+      elsif @file
+        FileUtils::rm_rf(@file)
+      end
+    end
+    def db_exist?(db,conn)
+      msg=%{no connection with sqlite database established, createdb "#{db.sqlite.db}"?}
+      if (not (FileTest.file?(db.sqlite.db)) \
+      or FileTest.zero?(db.sqlite.db))
+        puts msg
+        exit
+      end
+      if conn.is_a?(NilClass)
+        puts msg
+        exit
+      end
+    end
+    def create_and_populate
+      db=SiSU_Env::DbOp.new(@md)
+      conn=db.sqlite_discrete.conn_sqlite3
+      sdb=SiSU_DbDBI::Create.new(@opt,conn,@file,'sqlite')
+      sdb_index=SiSU_DbDBI::Index.new(@opt,conn,@file,'sqlite')
+      sdb.output_dir?
+      begin
+        SiSU_Screen::Ansi.new(@opt.cmd,'SQLite',"[#{@opt.f_pth[:lng_is]}] #{@opt.fno}").green_title_hi unless @opt.cmd =~/q/
+        sdb.create_db
+        sdb.create_table.metadata_and_text
+        sdb.create_table.doc_objects
+        sdb.create_table.endnotes
+        sdb.create_table.endnotes_asterisk
+        sdb.create_table.endnotes_plus
+        sdb.create_table.urls
+        sdb_index.create_indexes
+        db_exist?(db,conn)
+        sdb_import=SiSU_DbDBI::Import.new(@opt,conn,@file_maint,'sqlite')
+        sdb_import.marshal_load
+        tell=SiSU_Screen::Ansi.new(@opt.cmd,"sqlite3 #{db.sqlite.db} database?")
+        tell.puts_grey if @opt.cmd =~/v/
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,'-d').error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+        sdb.output_dir?
+      end
+    end
+    def read_sqlite
+      begin
+        @conn=@db.sqlite.conn_sqlite3
+      rescue
+      ensure
+      end
+    end
+    def connect
+      begin
+        sql_type='sqlite'
+        SiSU_Screen::Ansi.new(@opt.cmd,"DBI (#{@sql_type}) #{@opt.mod}",@opt.fns).dbi_title unless @opt.cmd =~/q/
+        @db.sqlite_discrete.conn_sqlite3
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@cf,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    def populate
+    end
+    def sql_maintenance_file
+      file=if @opt.inspect =~/M/
+        x=if @opt.fns and not @opt.fns.empty?
+          @env=SiSU_Env::InfoEnv.new(@opt.fns) if @opt.fns
+          puts "\n#{@env.processing_path.sqlite}/#{@opt.fns}.sql" if @sql_type =~/sqlite/ and @opt.cmd =~/M/
+          @db=SiSU_Env::InfoDb.new
+          @job="sqlite3 #{@db.sqlite.db} < #{@env.processing_path.sqlite}/#{@opt.fns}.sql"
+          File.new("#{@env.processing_path.sqlite}/#{@opt.fns}.sql",'w+')
+        elsif @opt.fns \
+        and @opt.fns.inspect =~/create/
+          nil #sort variations later
+        else nil
+        end
+      else nil
+      end
+      file
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/defaults.rb b/lib/sisu/v5/defaults.rb
new file mode 100644
index 00000000..6b3004ef
--- /dev/null
+++ b/lib/sisu/v5/defaults.rb
@@ -0,0 +1,1212 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: Default values
+
+=end
+$latex_run=nil
+module SiSU_Viz
+  require 'uri'
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env
+  require_relative 'css'                                # css.rb
+    include SiSU_Style
+  class Defaults
+    def initialize
+      @fonts='verdana, arial, georgia, tahoma, sans-serif, helvetica, times, roman' # 'verdana, arial, georgia, tahoma, sans-serif, helvetica, "times new roman", times, roman'
+      @dir=SiSU_Env::InfoEnv.new
+      @date=SiSU_Env::InfoDate.new #{@date.year}
+      @v=SiSU_Env::InfoVersion.instance.get_version
+    end
+    #% glyph
+    def glyph_bullet # •
+      '• ' # [• flagged]
+    end
+    #% html
+    def html_hardspace
+      ' '
+    end
+    def semantic_tags
+      def default
+        {
+          pub:     'publication',
+          conv:    'convention',
+          vol:     'volume',
+          pg:      'page',
+          cty:     'city',
+          org:     'organization',
+          uni:     'university',
+          dept:    'department',
+          fac:     'faculty',
+          inst:    'institute',
+          co:      'company',
+          com:     'company',
+          conv:    'convention',
+          dt:      'date',
+          y:       'year',
+          m:       'month',
+          d:       'day',
+          ti:      'title',
+          au:      'author',
+          ed:      'editor', #editor?
+          v:       'version', #edition
+          n:       'name',
+          fn:      'firstname',
+          mn:      'middlename',
+          ln:      'lastname',
+          in:      'initials',
+          qt:      'quote',
+          ct:      'cite',
+          ref:     'reference',
+          ab:      'abreviation',
+          def:     'define',
+          desc:    'description',
+          trans:   'translate',
+        }
+      end
+      self
+    end
+    #% decorate
+    def decorate_italics
+      'title|article|book|journal'
+    end
+    def decorate_bold
+    end
+    def decorate_uppercase
+      'surname'
+    end
+    #% semantic
+    def sem_title #dc 1
+      'title'
+    end
+    def sem_article
+      'article'
+    end
+    def sem_book
+      'book'
+    end
+    def sem_journal
+      'journal'
+    end
+    def sem_fullname # (contains: firstname, surname) #issues arise as contains surname etc.
+      'fullname'
+    end
+    def sem_first
+      'first'
+    end
+    def sem_surname
+      'surname'
+    end
+    def sem_middle
+      'middle'
+    end
+    def sem_creator #dc 2 #==fullname (contains: firstname, surname)
+      'creator'
+    end
+    def sem_author #==fullname (contains: firstname, surname)
+      'author'
+    end
+    def sem_editor #==fullname (contains: firstname, surname)
+      'editor'
+    end
+    def sem_illustrator #==fullname (contains: firstname, surname)
+      'illustrator'
+    end
+    def sem_translator #==fullname (contains: firstname, surname)
+      'translator'
+    end
+    def sem_isbn # 10 or 13
+      'isbn'
+    end
+    def sem_isbn_10
+      'isbn10'
+    end
+    def sem_isbn_13
+      'isbn13'
+    end
+    def sem_loc # library of congress
+      'loc'
+    end
+    def sem_dewey
+      'dewey'
+    end
+    def sem_pg # project gutenberg number
+      'pg'
+    end
+    def sem_subject #dc 3
+      'subject'
+    end
+    def sem_date #dc 7
+      'date'
+    end
+    def sem_date_created
+      'date_created'
+    end
+    def sem_date_issued
+      'date_issued'
+    end
+    def sem_date_available
+      'date_available'
+    end
+    def sem_date_valid
+      'date_valid'
+    end
+    def sem_date_modified
+      'date_modified'
+    end
+    def sem_type #dc 8
+      'type'
+    end
+    def sem_description #dc 4
+      'description'
+    end
+    def sem_publisher #dc 5
+      'publisher'
+    end
+    def sem_contributor #dc 6
+      'contributor'
+    end
+    def sem_format #dc 9
+      'format'
+    end
+    def sem_identifier #dc 10
+      'identifier'
+    end
+    def sem_source #dc 11
+      'source'
+    end
+    def sem_language #dc 12
+      'language'
+    end
+    def sem_relation #dc 13
+      'source'
+    end
+    def sem_coverage #dc 14
+      'coverage'
+    end
+    def sem_rights #dc 15
+      'rights'
+    end
+    def sem_copyright
+      'copyright'
+    end
+    def sem_license
+      'license'
+    end
+    def sem_prepared_by
+      'prepared_by'
+    end
+    def sem_digitized_by
+      'digitized_by'
+    end
+    def sem_keywords
+      'keywords'
+    end
+    def sem_comments
+      'comments'
+    end
+    def sem_abstract
+      'abstract'
+    end
+    #% path
+    def path_stylesheet_home
+      %{  }
+    end
+    #% text #changed from txt to avoid naming conflicts #FOLLOW
+    def txt_generator
+      %{  
+    }
+    end
+    def txt_generator_comment
+      %{  }
+    end
+    def txt_hp
+      ' SiSU'
+    end
+    def txt_hp_alias
+      'SiSU'
+    end
+    def txt_home
+      'SiSU'
+    end
+    def txt_signature # used in latex/pdf footer
+      'SiSU'
+    end
+    #% url
+    def url_urify(uri)
+      URI.parse(uri)
+    end
+    def url_sisu
+      'http://www.sisudoc.org/'
+    end
+    def url_sisudoc
+      'http://www.sisudoc.org'
+    end
+    def url_footer_signature
+      'http://www.sisudoc.org/'
+    end
+    def url_root
+      '/sisu' #watch
+    end
+    def url_root_http
+      'http://www.sisudoc.org/' #watch
+    end
+    def url_home
+      'http://www.sisudoc.org/' # used in pdf header
+    end
+    def url_site #used as stub... where there are subdirectories and is different from home
+      url_home
+      #'http://www.sisudoc.org/' # used in pdf header
+    end
+    def url_txt
+      'www.sisudoc.org/'
+    end
+    def url_path_image_base #used for html image display
+      "#{Xx[:html_relative2]}_sisu/image"
+    end
+    def url_path_image #used for html image display
+      "#{Xx[:html_relative2]}_sisu/image"
+    end
+    def url_path_image_sys #used for html image display
+      "#{Xx[:html_relative2]}_sisu/image_sys"
+    end
+    def url_decoration
+      def tex_open                     #'{\UseTextSymbol{OML}{<}}'
+        Dx[:url_o]
+      end
+      def tex_close                    #'{\UseTextSymbol{OML}{>}}'
+        Dx[:url_c]
+      end
+      def xml_open                     #'<'
+        Dx[:url_o]
+      end
+      def xml_close                    #'>'
+        Dx[:url_c]
+      end
+      def txt_open
+        '<'
+      end
+      def txt_close
+        '>'
+      end
+      self
+    end
+    def rel_decoration
+      def tex_open                     #'{\UseTextSymbol{OML}{<}}'
+        Dx[:rel_o]
+      end
+      def tex_close                    #'{\UseTextSymbol{OML}{>}}'
+        Dx[:rel_c]
+      end
+      def xml_open                     #'<'
+        Dx[:rel_o]
+      end
+      def xml_close                    #'>'
+        Dx[:rel_c]
+      end
+      def txt_open
+        '<'
+      end
+      def txt_close
+        '>'
+      end
+      self
+    end
+    #% color
+    def color_body
+      %{}
+    end
+    def color_white
+      '"#ffffff"'
+    end
+    def color_black
+      '#000000'
+    end
+    def color_blue_ink
+      '#003399'
+    end
+    def color_blue_tinge
+      '#e3ecef'
+    end
+    def color_blue_grey
+      '#8faebf'
+    end
+    def color_blue_murky
+      '#437389'
+    end
+    def color_beige
+      '#f1e8de'
+    end
+    def color_subtleglow
+      '#dddccc'
+    end
+    def color_glow
+      '#fff0c3'
+    end
+    def color_rose
+      '#ffdec9'
+    end
+    def color_turquoise
+      '#1c869b'
+    end
+    def color_grey_pale
+      '#eeeeee'
+    end
+    def color_grey_medium
+      '#cccccc'
+    end
+    def color_grey
+      '#999999'
+    end
+    def color_yellow_light
+      '#fff3b6'
+    end
+    def color_yellow
+      '#ffde14'
+    end
+    def color_yellow_dark
+      '#ffcc00'
+    end
+    def color_green_light
+      '#b7d398'  #  #e2efd5 #b7d398 #b1c999 # '#aed19e'
+    end
+    def color_green
+      '#0a8400'
+    end
+    def color_green_dark
+      '#086800'
+    end
+    def color_ruby
+      '#a00000'
+    end
+    def color_maroon
+      '#800000'
+    end
+    def color_paper
+      %{"#{color_white}"}
+    end
+    def color_band1
+      %{"#{color_white}"}
+    end
+    def color_band2
+      %{"#{color_white}"}
+    end
+    def color_body
+      %{}
+    end
+    def color_font_face #was font WATCH
+      "#{color_black}"
+    end
+    def color_surround
+      %{"#{color_white}"}
+    end
+    def color_band
+      %{"#{color_white}"}
+    end
+    def color_table1
+      'ffffcc'
+    end
+    def color_table2
+      'c0d0f0'
+    end
+    def color_band1
+      '"#ffffff"'
+    end
+    def color_band2
+      '"#ffffff"'
+    end
+    #% icon
+    def icon_ico
+      'rb7.ico'
+    end
+    def icon_sisu
+      'sisu.png'
+    end
+    def icon_manifest
+      'b_info.png'
+    end
+    def icon_doc
+      'b_doc.png'
+    end
+    def icon_toc
+      'b_toc.png'
+    end
+    def icon_wmp
+      'b_wmp.png'
+    end
+    def icon_odf
+      'b_odf.png'
+    end
+    def icon_epub
+      'b_epub.png'
+    end
+    def icon_pdf
+      'b_pdf.png'
+    end
+    def icon_pdf_portrait
+      'b_pdf.png'
+    end
+    def icon_pdf_landscape
+      'b_pdf.png'
+    end
+    def icon_choice
+      'b_choice.png'
+    end
+    def icon_new
+      'b_new.png'
+    end
+    def icon_dot_clear
+      'dot_clear.png'
+    end
+    def icon_dot_white
+      'dot_white.png'
+    end
+    def icon_dot
+      icon_dot_white
+    end
+    def icon_home_button
+      'sisu.png'
+    end
+    def icon_next
+      'arrow_next_red.png'
+    end
+    def icon_previous
+      'arrow_prev_red.png'
+    end
+    def icon_up
+      'arrow_up_red.png'
+    end
+    #% font
+    def font_fonts
+      @fonts
+    end
+    def font_face
+      %{face="#{font_fonts}"}
+    end
+    def font_color
+      'color="#000000"'
+    end
+    def font_size_endnote
+      'size="3"'
+    end
+    def font_small
+      'size="3"'
+    end
+    def font_tiny
+      'size="2"'
+    end
+    #% markup
+    def markup_italics_list #regular expression of words to be italised
+    end
+    def markup_bold_list #regular expression of words to be made bold
+      'SiSU'
+    end
+    def markup_make_italic
+      if defined? italics_list \
+      and italics_list
+        make={}
+        if italics_list
+          r=italics_list.dup
+          x=case r
+          when /\/i$/; 'i'
+          else         ''
+          end
+          r=r.gsub(/^\/(.+?)\/i?/,'\1').
+            gsub(/\(/,'(?:') # avoid need to escape use of brackets within regex provided
+          m='\b(' + r + ')\b'
+          make[:str]
+          make[:regx]=(x =~/i/) \
+          ? (/#{m}/i)
+          : (/#{m}/)
+        else nil
+        end
+      end
+    end
+    def markup_make_bold
+      if defined? bold_list \
+      and not bold_list.empty?
+        make={}
+        if bold_list
+          r=bold_list.dup
+          x=case r
+          when /\/i$/; 'i'
+          else         ''
+          end
+          r.gsub(/^\/(.+?)\/i?/,'\1').
+            gsub(/\(/,'(?:') # avoid need to escape use of brackets within regex provided
+          m='\b(' + r + ')\b'
+          make[:str]
+          make[:regx]=(x =~/i/) \
+          ? (/#{m}/i)
+          : (/#{m}/)
+        else nil
+        end
+        make
+      end
+    end
+    #% paragraph
+    def paragraph_txt
+      %{
}
+    end
+    def paragraph_endnote
+      %{
}
+    end
+    def paragraph_font_tiny
+      %{}
+    end
+    def paragraph_font_small
+      %{}
+    end
+    #% table
+    def table_close
+      '
+'
+    end
+    def table_width_1
+      '"100%"'
+    end
+    def table_width_2
+      '"99%"'
+    end
+    def table_width_txt
+      '"94%"'
+    end
+    def table_width_txt_r
+      '"96%"'
+    end
+    def table_cellpad_box
+      '"20"'
+    end
+    #% indent
+    def indent_level_0
+      '"1%"'
+    end
+    def indent_level_1
+      '"4%"'
+    end
+    def indent_level_2
+      '"6%"'
+    end
+    def indent_level_3
+      '"8%"'
+    end
+    def indent_level_4
+      '"10%"'
+    end
+    #% margin
+    def margin_num
+      '
 '
+    end
+    def margin_numless
+      ''
+    end
+    def margin_num_css
+      '
+  '
+    end
+    def margin_txt_w1
+      %{
+
+
 }
+    end
+    def margin_txt_w2
+      %{
+
+
 }
+    end
+    def margin_txt_0
+      %{
+

+}
+    end
+    def margin_txt_1
+      %{
+
}
+    end
+    def margin_txt_2
+      %{
+
+

+}
+    end
+    def margin_txt_3
+      %{
+
+

+}
+    end
+    def margin_css
+      '
+
 '
+    end
+    #% png
+    def png_ico
+      %{  }
+    end
+    def png_sisu #check url path
+      %{  
+    SiSU
+    }
+    end
+    def png_site
+      %{@}
+    end
+    def png_nav
+      %{Contents}
+    end
+    def png_manifest
+      %{Document Manifest}
+    end
+    def png_doc
+      %{Full Text}
+    end
+    def png_toc
+      %{TOC linked}
+    end
+    def png_odf
+      %{ODF/ODT}
+    end
+    def png_epub
+      %{EPUB}
+    end
+    def png_pdf
+      %{PDF}
+    end
+    def png_pdf_portrait
+      %{PDF portrait}
+    end
+    def png_pdf_landscape
+      %{PDF landscape}
+    end
+    def png_home
+      #dir=SiSU_Env::InfoEnv.new #(@fns)
+      %{#{txt_home} -->}
+      #%{#{txt_home} -->}
+    end
+    def png_home_button
+      rel=@dir.path_rel_links.html_scroll_2
+      %{#{txt_home} -->}
+    end
+    #% png_nav #not currently used
+    #% nav_txt
+    def nav_txt_homepage
+      %{  
+     home 
+   }
+    end
+    def nav_txt_toc_link
+      %{  
+      toc 
+   }
+    end
+    def nav_txt_doc_link
+      %{  
+     scroll 
+   }
+    end
+    def nav_txt_manifest
+      #{png_manifest} document manifest
+      %{  
+    [ document manifest ]
+   }
+    end
+    def nav_txt_concordance
+      %{  
+      A-Z 
+   }
+    end
+    def nav_txt_pdf_portrait
+      %{  
+     pdf 
+   }
+    end
+    def nav_txt_pdf_landscape
+      %{  
+     pdf 
+   }
+    end
+    #% banner
+    def banner_home_button_only
+      %{
+    #{png_home_button}
+  }
+    end
+    def banner_band #yellow_dark now white
+      %{
+
+

+  
+    #{png_home}
+  
+
+#{table_close}}
+    end
+    def banner_instrument_cover_band_scr
+      '
+
'
+    end
+    def banner_instrument_cover_band_seg
+      '
+
+WOK
+    end
+    def widget_sisu_verbose
+<
+
+WOK
+    end
+    def widget_way_better
+      <
+
+WOK
+    end
+    #% credits
+    def credits_sisu_manifest
+      widget_sisu_text
+    end
+    def credits_sisu
+      x=%{

+
'
+    end
+    #% widget
+    def widget_promo # Array used to build promo from list.yml and promo.yml
+    #  ['sisu_icon','sisu','sisu_search_libre','open_society','fsf','ruby']
+    end
+    def widgets_open
+<
+WOK
+    end
+    def widget_sisu_text
+<
+    Output generated by
+    
+      #{@v[:project]}
+    
+    #{@v[:version]} #{@v[:date]} (#{@v[:date_stamp]})
+  

+WOK
+    end
+    def widget_sisu
+<
+

+
+#{widget_sisu_text}
+

+  

+    
+      SiSU
+    
+  

+		
+
+  

+    Output generated by
+    
+      #{@v[:project]}
+    
+    #{@v[:version]} #{@v[:date]} (#{@v[:date_stamp]})
+    

+    
+      #{@v[:project]}
+    
+    Copyright © Ralph Amissah
+    1997, current #{@date.year_static}.
+    All Rights Reserved.
+    

+    
+      #{@v[:project]}
+    
+    is software for document structuring, publishing and search,
+    

+    
+      www.sisudoc.org/
+    
+    and
+    
+      www.sisudoc.org
+    
+  

+    w3 since October 3 1993
+    
+      ralph@amissah.com
+    
+  

+		
+  

+    #{@v[:project]} using:
+    
Standard SiSU markup syntax,
+    
Standard SiSU meta-markup syntax, and the
+    
Standard SiSU object citation numbering and system, (object/text positioning system)
+  

+    Copyright © Ralph Amissah 1997, current #{@date.year_static}.
+    All Rights Reserved.
+  

+

+  

+    
+      GPLv3
+    
+  

+		
+  

+    
+      #{@v[:project]}
+    
+    is released under
+    GPLv3
+    or later,
+    #{url_decoration.xml_open}http://www.gnu.org/licenses/gpl.html#{url_decoration.xml_close}
+  

+		
+  

+    #{@v[:project]}, developed using
+    
+      Ruby
+    
+    on
+    
+      Debian/Gnu/Linux
+    
+    software infrastructure,
+    with the usual GPL (or OSS) suspects.
+  

+    Better - "performance, reliability, scalability, security & total cost of ownership"
+    [not to mention flexibility & choice] use of and adherence to open standards (where practical and fair) and it is software libré.
+  

+    Get With the Future
+    
+      Way Better!
+    
+  

+

+

+  #{widget_sisu}
+  #{widget_way_better}
+
}
+      ''
+    end
+    def credits_splash
+    end
+    def credits_sisu_epub
+      x=%{

+
EPUB generated by #{@v[:project]} v#{@v[:version]}, GPL3

+
}
+      ''
+    end
+  end
+  class Home < Defaults
+    def initialize
+      @v=SiSU_Env::InfoVersion.instance.get_version
+      @dir=SiSU_Env::InfoEnv.new
+      @date=SiSU_Env::InfoDate.new #{@date.year}
+    end
+    def redirect
+      <
+SiSU
+
+
+
+SiSU informtion provided at www.sisudoc.org/sisu/SiSU

+If your browser supports redirection, you will be escorted there shortly.
+
+
+WOK
+    end
+    def homepage
+      <
+
+
+SiSU information Structuring Universe - Structured information, Serialized Units - software for electronic texts, documents, books, digital libraries in plaintext, HTML, EPUB, XHTML, XML, ODF (OpenDocument), LaTeX, PDF, SQL (PostgreSQL and SQLite), and for search
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
SiSU

+

+[Manual]
+

+

+[Source]
+[List Info (sisu@lists.sisudoc.org)]
+

+
+
SiSU Markup Samples

+

+[Source]
+[Output]
+

+
+

+
+

+  Structured information, Serialized Units
+    
+  
+    <www.sisudoc.org>
+  
+    or  
+  
+    <www.jus.uio.no/sisu/>
+  
+software for electronic texts, document collections, books, digital libraries & search, with "atomic search" & text locating system (shared object citation numbering: "ocn").
+Outputs include: plaintext, HTML, EPUB, ODT (OpenDocumentText), (XHTML, XML,) LaTeX, PDF, SQL (PostgreSQL and SQLite).
+

+

+
+<sisu@lists.sisudoc.org>
+
+
+<http://lists.sisudoc.org/listinfo/sisu>
+
+

+

+
+<ralph@amissah.com>
+
+
+<ralph.amissah@gmail.com>
+
+

+

+#{@v[:project]} #{@v[:version]} of #{@v[:date_stamp]} (#{@v[:date]}) (n*x and Ruby!), #{@date.year_static}.
+

+

+w3 since October 3 1993.
+

+
+
+WOK
+    end
+    def home_toc
+      ' '
+    end
+  end
+  class Inserts
+  end
+  class TeX < Defaults
+    def initialize(papersize='')
+      @papersize=papersize
+    end
+    def a4
+      def portrait
+        def w
+          160
+        end
+        def h
+          228
+        end
+        def img_px
+          450
+        end
+        self
+      end
+      def landscape
+        def w
+          238
+        end
+        def h
+          160
+        end
+        def img_px
+          300
+        end
+        self
+      end
+      self
+    end
+    def letter
+      def portrait
+        def w
+          166
+        end
+        def h
+          212
+        end
+        def img_px
+          468
+        end
+        self
+      end
+      def landscape
+        def w
+          226
+        end
+        def h
+          166
+        end
+        def img_px
+          290
+        end
+        self
+      end
+      self
+    end
+    def legal
+      def portrait
+        def w
+          168
+        end
+        def h
+          286
+        end
+        def img_px
+          474
+        end
+        self
+      end
+      def landscape
+        def w
+          296
+        end
+        def h
+          166
+        end
+        def img_px
+          420
+        end
+        self
+      end
+      self
+    end
+    def b5
+      def portrait
+        def w
+          140
+        end
+        def h
+          204
+        end
+        def img_px
+          356
+        end
+        self
+      end
+      def landscape
+        def w
+          200
+        end
+        def h
+          130
+        end
+        def img_px
+          260
+        end
+        self
+      end
+      self
+    end
+    def a5
+      def portrait
+        def w
+          112
+        end
+        def h
+          162
+        end
+        def img_px
+          280
+        end
+        self
+      end
+      def landscape
+        def w
+          152
+        end
+        def h
+          100
+        end
+        def img_px
+          190
+        end
+        self
+      end
+      self
+    end
+    def dimensions
+      d=case @papersize
+      when /a4/;     a4
+      when /letter/; letter
+      when /legal/;  legal
+      when /b5/;     b5
+      when /a5/;     a5
+      else           a4
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/digests.rb b/lib/sisu/v5/digests.rb
new file mode 100644
index 00000000..902a0e94
--- /dev/null
+++ b/lib/sisu/v5/digests.rb
@@ -0,0 +1,391 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: document digests (md5|sha256) and structure processing
+
+=end
+module SiSU_DigestView
+  require_relative 'particulars'                        # particulars.rb
+  require_relative 'prog_text_translation'              # prog_text_translation.rb
+  require_relative 'shared_markup_alt.rb'               # shared_markup_alt.rb
+  pwd=Dir.pwd
+  class Source
+    @@dg=nil
+    def initialize(opt)
+      @opt=opt
+      @fnb=@opt.fnb
+      @@endnotes_para=[]
+      @@dg=nil
+      @dg=@@dg ||=SiSU_Env::InfoEnv.new.digest.type
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+    end
+    def read
+      begin
+        @env,@md,@dal_array=@particulars.env,@particulars.md,@particulars.dal_array
+        unless @opt.cmd =~/q/
+          tool=(@opt.cmd =~/[MVv]/) \
+          ? "#{@env.program.text_editor} file://#{@md.file.output_path.hash_digest.dir}/#{@md.file.base_filename.hash_digest}"
+          : "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}"
+          @opt.cmd=~/[MVvz]/ \
+          ? SiSU_Screen::Ansi.new(@opt.cmd,"Document #{@dg} Digests",tool).green_hi_blue
+          : SiSU_Screen::Ansi.new(@opt.cmd,"Document #{@dg} Digests",tool).green_title_hi
+          SiSU_Screen::Ansi.new(@opt.cmd,@opt.fns,"file://#{@md.file.output_path.hash_digest.dir}/#{@md.file.base_filename.hash_digest}").flow if @opt.cmd =~/[MV]/
+        end
+        SiSU_DigestView::Source::Scroll.new(@particulars).songsheet
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    private
+    class Scroll \1').
+              gsub(/#{Mx[:fa_italics_o]}(.+?)#{Mx[:fa_italics_c]}/,'\1').
+              gsub(/#{Mx[:fa_underscore_o]}(.+?)#{Mx[:fa_underscore_c]}/,'\1')
+            if para=~/#{Mx[:en_a_o]}[\d*+]+.+?#{Mx[:id_o]}[0-9a-f]{#{@dl}}#{Mx[:id_c]}#{Mx[:en_a_c]}/
+              para_endnotes << para.scan(/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})([\d*+]+).+?#{Mx[:id_o]}([0-9a-f]{#{@dl}})#{Mx[:id_c]}(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/)
+            end
+            ima=[]
+            if para !~/^%+\s/ \
+            and para =~/(?:^|[^_\\])#{Mx[:lnk_o]}(\S+\.(png|jpg|gif))\s.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/
+              images=para.scan(/(?:^|[^_\\])#{Mx[:lnk_o]}(\S+\.(?:png|jpg|gif))\s.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/).flatten
+            else image=nil
+            end
+            x=case para
+            when /^#{Mx[:meta_o]}title#{Mx[:meta_c]}/
+              "\n" + '  '*0 +'@' + '  '*9
+            when /^#{Mx[:meta_o]}subtitle#{Mx[:meta_c]}/
+              "\n" + '  '*1 +'@' + '  '*8
+            when /^#{Mx[:lv_o]}1:/ #fix Mx[:lv_o]
+              "\n" + '  '*2 +':A ' + '  '*6 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            when /^#{Mx[:lv_o]}2:/ #fix Mx[:lv_o]
+              "\n" + '  '*3 +':B ' + '  '*5 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            when /^#{Mx[:lv_o]}3:/ #fix Mx[:lv_o]
+              "\n" + '  '*4 +':C ' + '  '*4 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            when /^#{Mx[:lv_o]}4:/ #fix Mx[:lv_o]
+              "\n" + '  '*5 +'1' + '  '*4 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            when /^#{Mx[:lv_o]}5:/ #fix Mx[:lv_o]
+              "\n" + '  '*6 +'2' + '  '*3 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            when /^#{Mx[:lv_o]}6:/ #fix Mx[:lv_o]
+              "\n" + '  '*7 +'3' + '  '*2 +'- ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+            else
+              if para =~/MD5\(\S+?\.sst\)=\s*([0-9a-f]{#{@dl}})<\/u>/                                        #watch
+                @n,@s=/MD5\((\S+?\.sst)\)=\s*([0-9a-f]{#{@dl}})<\/u>/.match(para)[1,2]
+              end
+              x=unless ocn =~ /^0$/
+                if images \
+                and images.length > 0 # then get path of image & produce digest
+                  @image_name,@image_dgst,@img=[],[],[]
+                  images.each do |i|
+                    image_source=if FileTest.file?("#{@env.path.image_source_include_local}/#{i}")
+                      @env.path.image_source_include_local
+                    elsif FileTest.file?("#{@env.path.image_source_include_remote}/#{i}")
+                      @env.path.image_source_include_remote
+                    elsif FileTest.file?("#{@env.path.image_source_include}/#{i}")
+                      @env.path.image_source_include
+                    else
+                      SiSU_Screen::Ansi.new(@md.opt.cmd,"ERROR - image:", %{"#{i}" missing}, "search locations: #{@env.path.image_source_include_local}, #{@env.path.image_source_include_remote} and #{@env.path.image_source_include}").error2 unless @md.opt.cmd =~/q/
+                      nil
+                    end
+                    @img << /\S+\.(png|jpg|gif)/.match(i)[1]
+                    not_found_msg='image not found'
+                    if image_source
+                      para_image = image_source + '/' + i
+                      @image_name << i
+                      @image_dgst << (@dg =~/^sha(?:2|256)$/) \
+                      ? sys.sha256(para_image)
+                      : sys.md5(para_image)
+                    else
+                      @image_name << '  '*16 + i + ' [image missing]'
+                      @image_dgst << ''
+                      @image_dgst[1]=not_found_msg + ' '*(32-not_found_msg.length)
+                    end
+                  end
+                  line= "\n" + '  '*9 + ' - ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all + "\n"
+                  line_image=[]
+                  c=0
+                  @image_name.each do |ok|
+                    line_image << %{                     #{@img[c]}       #{@image_dgst[c][1]}                                  #{@image_name[c]}}
+                    c +=1
+                  end
+                  line=line + line_image.join("\n")
+                else "\n" + '  '*9 + ' - ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all
+                end
+              else
+                prefix=''
+                metad=[@tr.full_title,@tr.author,@tr.translator,@tr.illustrator,@tr.prepared_by,@tr.digitized_by,@tr.description,@tr.subject,@tr.abstract,@tr.publisher,@tr.contributor,@tr.date_created,@tr.date_issued,@tr.date_available,@tr.date_modified,@tr.date_valid,@tr.date,@tr.type,@tr.format,@tr.rights,@tr.identifier,@tr.source,@tr.language,@tr.language_original,@tr.relation,@tr.coverage,@tr.keywords,@tr.comments,@tr.cls_loc,@tr.cls_dewey,@tr.cls_gutenberg,@tr.cls_isbn,@tr.prefix_a,@tr.prefix_b,@tr.sourcefile,@tr.sourcefile_digest,@tr.last_generated,@tr.sisu_version,@tr.ruby_version,@tr.sc_number,@tr.sc_date,'Generated by: ','Ruby version: ']
+                metad.each do |n|
+                  m=rgx_txt(n)
+                  if m=~/\S+/ \
+                  and para=~/^#{m}:/
+                    x,o=0,18
+                    while x < 2; o = o + 2
+                      x=o - n.length
+                    end
+                    space=' '*x
+                    prefix="#{n.downcase}#{space}"
+                    break
+                  else prefix='  '*9
+                  end
+                end
+                m_title=rgx_txt(@tr.full_title)
+                m_author=rgx_txt(@tr.author)
+                m_sourcefile_digest=rgx_txt(@tr.sourcefile_digest)
+                m_sisu_version=rgx_txt(@tr.sisu_version)
+                m_last_generated=rgx_txt(@tr.last_generated)
+                m_ruby_version=rgx_txt(@tr.ruby_version)
+                case para
+                when /#{m_title}: /
+                  @t=/#{m_title}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip
+                when /#{m_author}: /
+                  @c=/#{m_author}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip
+                when /#{m_sourcefile_digest}.+?/                                        #watch
+                  dgst_extra="\n" + ' '*21 +'source' +' '*4 + @md.dgst[1] + ' '*34 + @md.fns
+                when /#{m_sisu_version}: /
+                  @v=/#{m_sisu_version}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip
+                when /#{m_last_generated}: /
+                  @g=/#{m_last_generated}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip
+                when /#{m_ruby_version}: /
+                  @r=/#{m_ruby_version}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip
+                end
+                dgst_extra ||=''
+                "\n" + prefix +' - ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all + dgst_extra + "\n"
+              end
+            end
+            para_endnotes[0].each { |e| y << "\n" + ' '*(28-e[0].length) + "[#{e[0].to_s}] #{e[1].to_s}" } if para_endnotes[0]
+            if y; digests(x,y)
+            else  digests(x)
+            end
+          end
+        end
+        manifest="#{@env.url.root}/#{@md.fnb}/sisu_manifest.html"
+        a=%{ocn      digest clean (no markup/notes),#{@sp*33}digest all (includes markup & endnotes)\n}
+        description("#{@md.title.full}\n")
+        description("#{@md.author}\n")
+        description("#{@md.fns}\n")
+        description("----------------------------------------------\n")
+        description("SiSU Document Content Certificate (Digest/DCC)\n")
+        description("----------------------------------------------\n")
+        description("                               #{@dg} digests\n")
+        description("------------\n")
+        description("Sourcefile digest:             #{@md.dgst[1]}\n")
+        description("  source filename:             #{@md.fns}\n")
+        description("available outputs:             #{manifest}\n")
+       #description("  time generated:                #{@g}\n")
+       #description("  SiSU version used:             #{@v}\n")
+       #description("  Ruby version used:             #{@r}\n")
+        description("------------\n")
+        description("Document Digests\n")
+        description(a)
+      end
+      def dal_structure
+        data=@data
+        endnotes=nil
+        data.each do |t_o|
+          dgst=SiSU_TextRepresentation::ModifiedTextPlusHashDigest.new(@md,t_o).composite.dgst
+          if dgst
+            if t_o.is==:heading
+              digests("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}")
+            elsif t_o.is==:heading_insert
+              digests("#{@sp*0}[#{dgst[:ocn]}]#{@sp*(6-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}")
+            else
+              digests("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]}")
+              if dgst[:images]
+                dgst[:images].each do |img|
+                  digests("#{@sp*8}#{img[:img_dgst]}#{@sp*66}#{img[:img_type]} #{img[:img_name]}")
+                end
+              end
+            end
+            if dgst[:endnotes]
+              dgst[:endnotes].each do |en|
+                digests("#{@sp*8}#{en[:note_dgst]}#{@sp*66}note [#{en[:note_number]}]")
+                endnotes=en[:note_number]
+              end
+            end
+          end
+        end
+        l=Hash.new(0)
+        ocn=nil
+        dal_structure_tree("------------\n")
+        dal_structure_tree("document structure[*]\n")
+        data.each do |t_o|
+          if t_o.is==:heading
+            x=case t_o.ln
+            when 1; l[1] +=1 #fix Mx[:lv_o]
+              '  '*0 +':A'
+            when 2; l[2] +=1 #fix Mx[:lv_o]
+              '  '*1 +':B'
+            when 3; l[3] +=1 #fix Mx[:lv_o]
+              '  '*2 +':C'
+            when 4; l[4] +=1 #fix Mx[:lv_o]
+              '  '*3 +'1'
+            when 5; l[5] +=1 #fix Mx[:lv_o]
+              '  '*4 +'2'
+            when 6; l[6] +=1 #fix Mx[:lv_o]
+              '  '*5 +'3'
+            else nil
+            end
+          end
+          ocn=t_o.ocn if defined? t_o.ocn and t_o.is !=:heading_insert
+          dal_structure_tree("#{x}\n") if x and not x.empty?
+        end
+        dal_structure_tree("  [*] heading levels\n")
+        dal_structure_summary("------------\n")
+        dal_structure_summary("document structure[*]\n")
+        [0,1,2,3,4,5,6].each do |y|
+          v=case y
+          when 1; ':A'
+          when 2; ':B'
+          when 3; ':C'
+          when 4; '1 '
+          when 5; '2 '
+          when 6; '3 '
+          end
+          dal_structure_summary("#{v}            = #{l[y]}\n") if l[y] > 0
+        end
+        dal_structure_summary("objects (ocn) = #{ocn}\n")
+        dal_structure_summary("endnotes      = #{endnotes}\n")
+        dal_structure_summary("  [*] number of headers (@) and of each heading level (:A to :C and 1 to 3)\n")
+      end
+      def supplementary
+        if defined? @md.sc_number \
+        and @md.sc_number
+          rcinfo("------------\n")
+          rcinfo("source control information\n")
+          rcinfo("  (the following information while not important for document content certification\n   may help the publisher in locating the version referred to)\n")
+          rcinfo("  rcs version number:            #{@md.sc_number}\n")
+          if defined? @md.sc_date \
+          and @md.sc_date
+            rcinfo("  rcs date:                      #{@md.sc_date}\n")
+          end
+          if defined? @md.sc_time \
+          and @md.sc_time
+            rcinfo("  rcs time:                      #{@md.sc_time}\n")
+          end
+        end
+        rcinfo("------------\n")
+        rcinfo("Note: the time generated related fields (text and digests) will vary between otherwise identical document outputs\n")
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/embedded.rb b/lib/sisu/v5/embedded.rb
new file mode 100644
index 00000000..a75b1690
--- /dev/null
+++ b/lib/sisu/v5/embedded.rb
@@ -0,0 +1,148 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: sitemap created from parameters extracted from input file(s)
+
+=end
+module SiSU_Embedded
+  require_relative 'param'                              # param.rb
+    include SiSU_Param
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env
+  class Source
+    def initialize(opt)
+      @opt=opt
+      @md=SiSU_Param::Parameters.new(@opt).get
+      @env=SiSU_Env::InfoEnv.new(@md.fns)
+      @rhost=SiSU_Env::InfoRemote.new(@opt).remote_host_base
+      @base_src_dir=@opt.f_pth[:pth].sub(/\/#{@opt.f_pth[:lng]}$/,'')
+      @f=SiSU_Env::FileOp.new(@md)
+    end
+    def read
+      songsheet
+    end
+    def songsheet
+      images
+      audio
+      multimedia
+      begin
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    def images
+      src="#{@base_src_dir}/_sisu/image"
+      src_img='_sisu/image'
+      ldest=@env.path.output
+      img_dir="#{@env.path.output}/_sisu/image"
+      @rhost.each do |remote_conn|
+        rdest="#{remote_conn[:name]}/#{@env.path.stub_pwd}/_sisu/image"
+        remote_rel=remote_conn[:name] + '/' + @f.output_path.stub.rcp
+        if @md.opt.cmd.inspect =~/[vVMR]/ \
+        and FileTest.directory?(src)
+          FileUtils::mkdir_p(img_dir) unless FileTest.directory?(img_dir)
+          src_ec=@f.place_file.images.rel + '/' + @md.ec[:image].join(" #{@f.output_path.images.rel}/")
+          unless @opt.fns =~/\.-sst$/
+            SiSU_Env::SystemCall.new(src_ec,ldest,'q').rsync('--relative',@opt.base_path)
+            #if @md.opt.cmd.inspect =~/R/ #rsync to remote image directory
+            #  SiSU_Env::SystemCall.new(src_ec,remote_rel,'q').rsync('--relative')
+            #end
+          end
+        end
+      end
+    end
+    def audio
+      #p @md.ec[:audio]
+      src="#{@base_src_dir}/_sisu/mm/audio"
+      ldest="#{@env.path.webserv}/#{@env.path.stub_pwd}/_sisu/mm/audio"
+      @rhost.each do |remote_conn|
+        rdest="#{remote_conn[:name]}/#{@env.path.stub_pwd}/_sisu/mm/audio"
+        if @md.opt.cmd.inspect =~/[vVMR]/ \
+        and FileTest.directory?(src)
+          FileUtils::mkdir_p(ldest) unless FileTest.directory?(ldest)
+          src_ec="#{src}/" + @md.ec[:audio].join(" #{src}/")
+          SiSU_Env::SystemCall.new(src_ec,"#{ldest}/.",'q').rsync
+          if @md.opt.cmd.inspect =~/R/ #rsync to remote audio directory
+            SiSU_Env::SystemCall.new(src_ec,"#{rdest}/.",'q').rsync
+          end
+        end
+      end
+    end
+    def multimedia
+      #p @md.ec[:multimedia]
+      src="#{@base_src_dir}/_sisu/mm/video"
+      ldest="#{@env.path.webserv}/#{@env.path.stub_pwd}/_sisu/mm/video"
+      @rhost.each do |remote_conn|
+        rdest="#{remote_conn[:name]}/#{@env.path.stub_pwd}/_sisu/mm/video"
+        if @md.opt.cmd.inspect =~/[vVMR]/ \
+        and FileTest.directory?(src)
+          FileUtils::mkdir_p(ldest) unless FileTest.directory?(ldest)
+          src_ec="#{src}/" + @md.ec[:multimedia].join(" #{src}/")
+          SiSU_Env::SystemCall.new(src_ec,"#{ldest}/.",'q').rsync
+          if @md.opt.cmd.inspect =~/R/ #rsync to remote video directory
+            SiSU_Env::SystemCall.new(src_ec,"#{rdest}/.",'q').rsync
+          end
+        end
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/epub.rb b/lib/sisu/v5/epub.rb
new file mode 100644
index 00000000..480474e1
--- /dev/null
+++ b/lib/sisu/v5/epub.rb
@@ -0,0 +1,762 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: epub generation, processing
+
+=end
+module SiSU_EPUB
+  require 'pstore'
+  require_relative 'particulars'                        # particulars.rb
+    include SiSU_Particulars
+  require_relative 'defaults'                           # defaults.rb
+    include SiSU_Viz
+  require_relative 'xhtml_table'                        # xhtml_table.rb
+  require_relative 'epub_format'                        # epub_format.rb
+    include SiSU_EPUB_Format
+  require_relative 'epub_segments'                      # epub_segments.rb
+    include SiSU_EPUB_Seg
+  require_relative 'epub_tune'                          # epub_tune.rb
+    include SiSU_EPUB_Tune
+  require_relative 'epub_concordance'                   # epub_concordance.rb
+  class Source
+    def initialize(opt)
+      @opt=opt
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+    end
+    def read
+      songsheet
+    end
+    def songsheet
+      begin
+        @md=@particulars.md
+        @fnb=@md.fnb
+        @env=@particulars.env
+        loc=@env.path.url.output_tell
+        unless @opt.cmd =~/q/
+          tool=(@opt.cmd =~/[MVvz]/) \
+          ? "#{@env.program.epub_viewer} #{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}"
+          : "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}"
+          @opt.cmd=~/[MVvz]/ \
+          ? SiSU_Screen::Ansi.new(@opt.cmd,'EPUB',tool).green_hi_blue
+          : SiSU_Screen::Ansi.new(@opt.cmd,'EPUB',tool).green_title_hi
+          SiSU_Screen::Ansi.new(@opt.cmd,@opt.fns,"#{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}").flow if @opt.cmd =~/[MV]/
+        end
+        @env.processing_path.epub_bld #(@md)
+        @env.processing_path.epub_cp_images(@md)
+        dir_epub=@env.processing_path.epub
+        data=nil
+        SiSU_Env::FileOp.new(@md).mkdir.output.epub
+        @tuned_file_array=SiSU_EPUB::Source::XHTML_Environment.new(@particulars).tuned_file_instructions
+        data=@tuned_file_array
+        scr_endnotes=SiSU_EPUB::Source::Endnotes.new(@md,data).scroll
+        toc=SiSU_EPUB::Source::Toc.new(@md,data).songsheet
+        data=@tuned_file_array
+        scr_toc=SiSU_EPUB::Source::ScrollHeadAndSegToc.new(@md,toc).in_common #watch
+        SiSU_EPUB::Source::Seg.new(@md,data).songsheet
+        SiSU_EPUB::Source::Output.new(@md).songsheet
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+        unless @opt.cmd =~/[MV]/ #check maintenance flag
+          texfiles=Dir["#{@env.processing_path.tune}/#{@opt.fns}*"]
+          texfiles.each do |f|
+            if FileTest.file?(f)
+              File.unlink(f)
+            end
+          end
+        end
+        SiSU_Env::Clear.new(@opt.cmd,@opt.fns).param_instantiate
+        @@flag,@@scr,@@seg,@@seg_endnotes,@@seg_subtoc,@@seg_ad={},{},{},{},{},{}
+        @@seg_total,@@tracker,@@loop_count,@@tablehead,@@number_of_cols=0,0,0,0,0
+        @@seg_name,@@seg_name_html,@@seg_subtoc_array,@@seg_endnotes_array,@@segtocband,@@tablefoot=Array.new(7){[]}
+        @@filename_seg,@@seg_url,@@fn,@@to_lev4,@@get_hash_to,@@get_hash_fn='','','','','','',''
+        @@is4=@@is3=@@is2=@@is1=@@heading1=@@heading2=@@heading3=@@heading4=0
+      end
+    end
+    private
+    class XHTML_Environment
+      def initialize(particulars)
+        @particulars=particulars
+        @md,@env=particulars.md,particulars.env
+        @vz=SiSU_Viz::Defaults.new
+        @env,@css=particulars.env,SiSU_Style::CSS.new
+      end
+      def directories
+        title=File.basename(@md.fns,'.rb')
+        SiSU_Env::FileOp.new(@md).mkdir.output.epub
+      end
+      def tuned_file_instructions
+        @tell=SiSU_Screen::Ansi.new(@md.opt.cmd)
+        directories
+        dal_array=@particulars.dal_array # dal file drawn here
+        @tuned_file_array=SiSU_EPUB_Tune::Tune.new(dal_array,@md).songsheet
+        @tuned_file_array
+      end
+    end
+    class Endnotes
+      include SiSU_EPUB_Format
+      def initialize(md,data)
+        @md,@data=md,data
+      end
+      def scroll
+        @scr_endnotes=[]
+        format_head_scroll=SiSU_EPUB_Format::HeadScroll.new(@md)
+        @data.each do |dob|
+          pg=dob.dup
+          unless pg.is ==:code
+            if pg.obj =~/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[\d*+]+ /
+              endnote_array=[]
+              if pg.obj=~/#{Mx[:en_a_o]}[\d*+].+?#{Mx[:en_a_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_a_o]}[\d*+]+(.+?)#{Mx[:en_a_c]}/m)
+              end
+              if pg.obj=~/#{Mx[:en_b_o]}[\d*]+\s.+?#{Mx[:en_b_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_b_o]}[\d*]+(.+?)#{Mx[:en_b_c]}/m)
+              end
+              if pg.obj=~/#{Mx[:en_b_o]}[\d+]+\s.+?#{Mx[:en_b_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_b_o]}[\d+]+(.+?)#{Mx[:en_b_c]}/m)
+              end
+              endnote_array.flatten.each do |note|
+                txt_obj={ txt: note }
+                format_scroll=SiSU_EPUB_Format::FormatScroll.new(@md,txt_obj)
+                @scr_endnotes << format_scroll.endnote_body
+              end
+            end
+          end
+        end
+        @scr_endnotes
+      end
+    end
+    class Toc
+      @@toc={ seg: [], seg_mini: [], scr: [], ncx: [], opf: [] }
+      @@seg_url=''
+      @@firstseg=nil
+      def initialize(md=nil,data='')
+        @md,@data=md,data
+        @vz=SiSU_Viz::Defaults.new
+        @epub=SiSU_EPUB_Format::HeadInformation.new(@md)
+        @tell=SiSU_Screen::Ansi.new(@md.opt.cmd) if @md
+        @make=SiSU_Env::ProcessingSettings.new(@md)
+      end
+      def songsheet #extracts toc for scroll & seg
+        SiSU_Screen::Ansi.new(@md.opt.cmd,'Toc').txt_grey if @md.opt.cmd =~/[MVv]/
+        toc=nil
+        @@firstseg=nil
+        @@toc={ seg: [], seg_mini: [], scr: [], ncx: [], opf: [] }
+        md_opf_a_content,md_opf_a_spine,md_opf_a_guide=[],[],[]
+        @nav_no=0
+        @s_a_no,@s_b_no,@s_c_no,@lv5_no,@lv6_no=0,0,0,0,0
+        @@toc[:ncx] << @epub.toc_ncx.open #epub ncx navmap
+        @@toc[:ncx] << @epub.toc_ncx.head_open << @epub.toc_ncx.head << @epub.toc_ncx.head_close
+        @@toc[:ncx] << @epub.toc_ncx.doc_title << @epub.toc_ncx.doc_author
+        @@toc[:ncx] << @epub.toc_ncx.navmap_open
+        @@toc[:opf] << @epub.metadata_opf.package_open
+        @@toc[:opf] << @epub.metadata_opf.metadata
+        @@toc[:opf] << @epub.metadata_opf.manifest_open
+        @@toc[:seg] << %{
\n
}
+        @@toc[:scr] << %{
\n
}
+        if defined? @md.make.cover_image \
+        and @md.make.cover_image.is_a?(Hash) \
+        and @md.make.cover_image[:cover] =~/\S+/
+          md_opf_a_content << @epub.metadata_opf.manifest_cover_image_information(@md)
+          md_opf_a_spine << @epub.metadata_opf.spine_cover_image
+          md_opf_a_guide << @epub.metadata_opf.guide_cover_image
+        end
+        md_opf_a_content << @epub.metadata_opf.manifest_content_sisu_toc
+        if @make.build.toc?
+          md_opf_a_spine << @epub.metadata_opf.spine_sisu_toc
+          md_opf_a_guide << @epub.metadata_opf.guide_sisu_toc
+        end
+        @ncxo=[nil,false,false,false,false,false,false]
+        @dob_toc2,@dob_toc3=nil,nil
+        @ncx_cls=[]
+        @level_a_first_occurrence=true
+        @data.each do |dob|
+          if dob.is==:heading \
+          || dob.is==:heading_insert
+            dob_toc=dob.dup
+            toc=case dob_toc.ln
+            when 1
+              @s_a_no +=1
+              lv_name='section_a' + @s_a_no.to_s
+              @nav_no+=1
+              @nav_no2=@nav_no
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[4]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[3]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[2]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[1]
+              @ncxo[1],@ncxo[2],@ncxo[3],@ncxo[4],@ncxo[5],@ncxo[6]=true,false,false,false,false,false
+              @epub.sections(dob_toc,lv_name)
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name) if dob_toc
+              if @level_a_first_occurrence \
+              && @make.build.toc?
+                @nav_no+=1
+                @@toc[:ncx] << @epub.toc_ncx.navmap_sisu_toc(@nav_no) #epub ncx navmap, toc
+                @level_a_first_occurrence=false
+              end
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_1
+            when 2
+              @s_b_no +=1
+              lv_name='section_b' + @s_b_no.to_s
+              @nav_no+=1
+              @nav_no2=@nav_no
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[4]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[3]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[2]
+              @ncxo[2],@ncxo[3],@ncxo[4],@ncxo[5],@ncxo[6]=true,false,false,false,false
+              @epub.sections(dob_toc,lv_name)
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name) if dob_toc
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_2
+            when 3
+              @s_c_no +=1
+              lv_name='section_c' + @s_c_no.to_s
+              @nav_no+=1
+              @nav_no3=@nav_no
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[4]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[3]
+              @ncxo[3],@ncxo[4],@ncxo[5],@ncxo[6]=true,false,false,false
+              @epub.sections(dob_toc,lv_name)
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name) if dob_toc
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_3
+            when 4
+              @ncx_cls=[]
+              lv_name=dob_toc.name
+              @nav_no+=1
+              @dob_name=dob.name
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[4]
+              @ncxo[4],@ncxo[5],@ncxo[6]=true,false,false
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name) if dob_toc
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_4
+            when 5
+              @ncx_cls=[]
+              hashtag='#o' + dob_toc.ocn.to_s
+              lv_name=@dob_name
+              @nav_no+=1
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+              @ncxo[5],@ncxo[6]=true,false
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name,hashtag) if dob_toc
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name,hashtag)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name,hashtag)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name,hashtag)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_5
+            when 6
+              @ncx_cls=[]
+              hashtag='#o' + dob_toc.ocn.to_s
+              lv_name=@dob_name
+              @nav_no+=1
+              @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+              @ncxo[6]=true
+              @@toc[:ncx] << @epub.toc_ncx.navpoint(dob_toc,@nav_no,lv_name,hashtag) if dob_toc
+              md_opf_a_content << @epub.metadata_opf.manifest_content(dob_toc,lv_name,hashtag)
+              md_opf_a_spine << @epub.metadata_opf.spine(dob_toc,lv_name,hashtag)
+              md_opf_a_guide << @epub.metadata_opf.guide(dob_toc,lv_name,hashtag)
+              SiSU_EPUB::Source::Toc.new(@md,dob_toc).level_6
+            else nil
+            end
+            toc.each do |k,d|
+              d.gsub!(/(?:#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})\s*/m,' ')
+            end if toc
+            if @@firstseg.nil? \
+            and dob.ln==4 \
+            and dob.name =~/\S+/
+              @@firstseg=dob.name
+            end
+            if toc
+              begin
+                @@toc[:seg] << toc[:seg]
+                @@toc[:scr] << toc[:seg]
+              rescue
+                SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+                  __LINE__.to_s + ':' + __FILE__
+                end
+              end
+            end
+          end
+        end
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[6]
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[5]
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[4]
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[3]
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[2]
+        @@toc[:ncx] << @epub.toc_ncx.navpoint_close if @ncxo[1]
+        @ncxo[1],@ncxo[2],@ncxo[3],@ncxo[4],@ncxo[5],@ncxo[6]=false,false,false,false,false,false
+        md_opf_a_content << @epub.metadata_opf.manifest_images(@md.ec[:image])
+        @@toc[:seg] << "
\n
"
+        @@toc[:scr] << "
\n
"
+        @@toc[:ncx] << @epub.toc_ncx.navmap_close
+        @@toc[:ncx] << @epub.toc_ncx.close
+        @@toc[:opf] << md_opf_a_content << @epub.metadata_opf.manifest_close
+        @@toc[:opf] << @epub.metadata_opf.spine_open << md_opf_a_spine << @epub.metadata_opf.spine_close
+        @@toc[:opf] << @epub.metadata_opf.guide_open << md_opf_a_guide << @epub.metadata_opf.guide_close
+        @@toc[:opf] << @epub.metadata_opf.package_close
+        @@toc[:opf]=@@toc[:opf].flatten
+        SiSU_EPUB::Source::Output.new(@md,@@toc[:opf]).epub_metadata_opf
+        SiSU_EPUB::Source::Output.new(@md,@@toc[:ncx]).epub_toc_ncx
+        @md.firstseg=@@firstseg
+        @@toc
+      end
+    protected
+      def level_1
+        dob=@data
+        linkname,link=dob.obj.strip,dob.ocn
+        if link \
+        and link !~/#/ #% keep eye on link
+          p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,link)
+        end
+        title=if dob.obj !~/Document Information/
+          linkname
+        else
+          link='metadata'
+          %{#{linkname}}
+        end
+        toc={}
+        txt_obj={ txt: title }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc[:seg]=if dob.name =~/^meta/ \
+        and dob.obj =~/Document Information/ #check
+          format_toc.lev0
+        else format_toc.lev1
+        end
+        title=if dob.ocn ==0
+          if dob.name =~/^meta/ \
+          and dob.obj =~/Document Information/
+            %{#{linkname}}
+          else linkname
+          end
+        else
+          @@toc[:scr] <<  '
'
+          link=(dob.ln) \
+          ? dob.ln
+          : ''
+          %{#{linkname}}
+        end
+        txt_obj={ txt: title }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc[:scr]=if dob.name =~/^meta/ \
+        and dob.obj =~/Document Information/
+          format_toc.lev0
+        else format_toc.lev1
+        end
+        toc
+      end
+      def level_2
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        end
+        txt_obj={ txt: linkname }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev2
+        if p_num
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev2
+        end
+        toc
+      end
+      def level_3
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        end
+        txt_obj={ txt: linkname }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev3
+        if p_num
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev3
+        end
+        toc
+      end
+      def level_4
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn) if ocn
+        if dob.ln==4
+          seg_link=%{  
+    #{dob.obj}
+   }
+          @@seg_url=dob.name
+        elsif dob.obj =~/\d+.\d+.\d+.\d+|\d+.\d+.\d+|\d+.\d+|\d+/
+          seg_link=dob.obj.gsub(/^(\d+.\d+.\d+.\d+|\d+.\d+.\d+|\d+.\d+|\d+)(.*)/,
+            %{} +
+            %{\\1 \\2 })
+        end
+        p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn) if ocn
+        txt_obj={ txt: seg_link }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev4
+        title=%{#{p_num.goto}#{linkname}} if p_num
+        txt_obj={ txt: title }
+        format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+        toc[:scr]=format_toc.lev4
+        toc
+      end
+      def level_5
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        toc={}
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+          lnk_n_txt=%{  
+    #{linkname}
+  }
+          txt_obj={ txt: lnk_n_txt }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:seg]=format_toc.lev5
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev5
+        end
+        toc
+      end
+      def level_6
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        toc={}
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+          lnk_n_txt=%{  
+  #{linkname}
+}
+          txt_obj={ txt: lnk_n_txt }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:seg]=format_toc.lev6
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_EPUB_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev6
+        end
+        toc
+      end
+    end
+    class ScrollHeadAndSegToc < Toc
+      def initialize(md='',toc='',links_guide_toc='')
+        @md,@toc,@links_guide_toc=md,toc,links_guide_toc
+        @vz=SiSU_Viz::Defaults.new
+      end
+      def in_common
+        toc_shared=[]
+        segtoc=[]
+        SiSU_Screen::Ansi.new(@md.opt.cmd,'Scroll & Segtoc').txt_grey if @md.opt.cmd =~/[MVv]/
+        format_head_toc=SiSU_EPUB_Format::HeadToc.new(@md)
+        dochead=format_head_toc.head
+        dochead=dochead.gsub(/toc\.(html)/,'doc.\1') #kludge
+        toc_shared << dochead #<< ads.div.major
+        segtoc << format_head_toc.head #<< ads.div.major
+        if defined? @md.rights.all \
+        and @md.rights.all
+          rights=format_head_toc.rights.all
+          rights=SiSU_EPUB_Tune::CleanXHTML.new(rights).clean
+        end
+        if defined? @md.notes.prefix_b \
+        and @md.notes.prefix_b
+          prefix_b=format_head_toc.prefix_b
+          prefix_b=SiSU_EPUB_Tune::CleanXHTML.new(prefix_b).clean
+        end
+        tmp_head=nil
+        doc_title_endnote=@md.title.full.gsub(/(\*+)/,'\1')
+        tmp_head=doc_title_endnote + "\n"
+        txt_obj={ txt: tmp_head }
+        format_txt_obj=SiSU_EPUB_Format::FormatTextObject.new(@md,txt_obj)
+        toc_shared << format_txt_obj.center_bold
+        segtoc << format_txt_obj.center_bold
+        if defined? @md.creator.author \
+        and @md.creator.author
+          creator_endnote=@md.creator.author.gsub(/(\*+)/,%{#{$ep[:hsp]}\\1})
+          tmp_head=creator_endnote + "\n"
+          txt_obj={ txt: tmp_head }
+          format_txt_obj=SiSU_EPUB_Format::FormatTextObject.new(@md,txt_obj)
+          toc_shared << format_txt_obj.center_bold
+          segtoc << format_txt_obj.center_bold
+        end
+        tmp_head=nil
+        if defined? @md.prefix_a \
+        and @md.prefix_a
+          tmp_head ||= %{#{@md.prefix_a}\n}
+          toc_shared << tmp_head.dup
+          segtoc << tmp_head.dup
+        end
+        tmp_head=nil
+        toc_shared << @links_guide_toc
+        if defined? @md.rights.all \
+        and @md.rights.all
+          toc_shared << rights
+        end
+        if defined? @md.prefix_b \
+        and @md.prefix_b
+          toc_shared << prefix_b
+        end
+        #Table of Contents added/appended here
+        toc_shared << @toc[:scr]
+        segtoc << @links_guide_toc
+        segtoc << @toc[:seg]
+        if defined? @md.rights.all \
+        and @md.rights.all
+          segtoc << rights
+        end
+        if defined? @md.prefix_b \
+        and @md.prefix_b
+          segtoc << prefix_b
+        end
+        #Segtoc tail added here
+        segtoc << format_head_toc.xhtml_close
+        segtoc=segtoc.flatten.compact #watch
+        SiSU_EPUB::Source::Output.new(@md).make_cover_image
+        SiSU_EPUB::Source::Output.new(@md,segtoc).make_segtoc
+        segtoc=[]
+        @toc[:scr],@toc[:seg]=[],[]
+        toc_shared
+      end
+    end
+    class Table < SiSU_XHTML_Table::TableXHTML
+    end
+    class Seg < SiSU_EPUB_Seg::Seg
+    end
+    class Output
+      def initialize(md,output='')
+        @md,@output=md,output
+        @epub_doc="#{@md.fnb}.epub"
+        @epub_header=SiSU_EPUB_Format::HeadInformation.new(@md)
+        @make=SiSU_Env::ProcessingSettings.new(@md)
+        @make_file=SiSU_Env::CreateFile.new(@md.fns)
+      end
+      def songsheet
+        mimetype
+        metainf_container
+        css
+        images if @md.ec[:image]
+       #concordance #uncomment to enable inclusion of concordance file
+        output_zip
+      end
+      def mimetype
+        out=@make_file.epub.mimetype
+        out<<@epub_header.mimetype
+        out.close
+      end
+      def metainf_container #container.xml file in META-INF directory
+        out=@make_file.epub.metainf_cont
+        out<<@epub_header.metainf_container
+        out.close
+      end
+      def css
+        out=@make_file.epub.xhtml_css
+        out << SiSU_EPUB_Format::CSS.new.css_epub_xhtml
+        out.close
+      end
+      def epub_toc_ncx
+        begin
+          out=@make_file.epub.toc_ncx
+          @output.each do |para|
+            unless para =~/\A\s*\Z/
+              out.puts para
+            end
+          end
+          out.close
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        end
+      end
+      def epub_metadata_opf
+        begin
+          out=@make_file.epub.metadata
+          @output.each do |para|
+            unless para =~/\A\s*\Z/
+              out.puts para
+            end
+          end
+          out.close
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        end
+      end
+      def images
+        img_pth=@md.env.path.image_source_include
+        img_src_pth=unless @md.opt.f_pth[:pth] =~/\/\S+?\/sisupod\/\S+?\/sisupod\/doc/
+          @md.file.output_path.epub.rel_image
+        else
+          pt=/(\/\S+?\/sisupod\/\S+?\/sisupod)\/doc/.match(@md.opt.f_pth[:pth])[1]
+          pt + '/image'
+        end
+        @md.ec[:image].each do |x|
+          if FileTest.directory?("#{@md.env.processing_path.epub}/#{Ep[:d_oebps]}/image") \
+          && FileTest.file?("#{img_src_pth}/#{x}")
+            FileUtils::cp("#{img_src_pth}/#{x}","#{@md.env.processing_path.epub}/#{Ep[:d_oebps]}/image")
+          elsif FileTest.directory?("#{@md.env.processing_path.epub}/#{Ep[:d_oebps]}/image") \
+          && FileTest.file?("#{img_pth}/#{x}")
+            FileUtils::cp("#{img_pth}/#{x}","#{@md.env.processing_path.epub}/#{Ep[:d_oebps]}/image")
+          else STDERR.puts %{\t*WARN* did not find image - "#{x}" in #{img_src_pth} or #{img_pth} [#{__FILE__}:#{__LINE__}]}
+          end
+        end
+      end
+      def concordance
+        SiSU_EPUB_Concordance::Source.new(@md.opt).read
+      end
+      def output_zip
+        FileUtils::mkdir_p(@md.file.output_path.epub.dir) unless FileTest.directory?(@md.file.output_path.epub.dir)
+        if FileTest.directory?(@md.env.processing_path.epub)
+          pwd=Dir.pwd
+          Dir.chdir(@md.env.processing_path.epub)
+          system("
+            zip -qXr9D #{@epub_doc} *
+          ")
+          FileUtils::mv(@epub_doc, @md.file.place_file.epub.dir)
+          Dir.chdir(pwd)
+          unless @md.opt.cmd.inspect =~/M/
+            FileUtils::rm_r(@md.env.processing_path.epub)
+          end
+        end
+      end
+      def make_cover_image
+        begin
+          if @md.make.cover_image? \
+          and @md.make.cover_image.is_a?(Hash) \
+          and @md.make.cover_image[:cover] =~/\S+/
+            filename_xhtml=@make_file.epub.xhtml_cover_image
+            cover_image=<
+
+
+  
+    Cover
+    
+    
+    
+  
+  
+    

+      

+        
+        
+        
+      

+    

+  
+
+WOK
+            filename_xhtml.puts cover_image,"\n"
+            filename_xhtml.close
+          end
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        end
+      end
+      def make_segtoc
+        begin
+          if @make.build.toc?
+            filename_xhtml=@make_file.epub.xhtml_index
+            @output.each do |para|
+              para=para.strip
+              unless para =~/\A\s*\Z/
+                filename_xhtml.puts para,"\n"
+              end
+            end
+            filename_xhtml.close
+          end
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        end
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/epub_concordance.rb b/lib/sisu/v5/epub_concordance.rb
new file mode 100644
index 00000000..e528a788
--- /dev/null
+++ b/lib/sisu/v5/epub_concordance.rb
@@ -0,0 +1,327 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: epub concordance file (html concordance, wordmap, linked index
+    of words in document)
+
+=end
+module SiSU_EPUB_Concordance
+  require_relative 'particulars'                        # particulars.rb
+    include SiSU_Particulars
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env
+  require_relative 'defaults'                           # defaults.rb
+    include SiSU_Viz
+  require_relative 'epub_format'                        # epub_format.rb
+    include SiSU_EPUB_Format
+  class Source
+    def initialize(opt)
+      @opt=opt
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+    end
+    def read
+      begin
+        @env,@md=@particulars.env,@particulars.md
+        wordmax=@env.concord_max
+        unless @md.wc_words.nil?
+          if @md.wc_words < wordmax
+            SiSU_EPUB_Concordance::Source::Words.new(@particulars).songsheet
+          else
+            SiSU_Screen::Ansi.new(@md.opt.cmd,"*WARN* concordance skipped, large document has over #{wordmax} words (#{@md.wc_words})").warn unless @md.opt.cmd =~/q/
+          end
+        else
+          SiSU_Screen::Ansi.new(@md.opt.cmd,"*WARN* wc (word count) is off, concordance will be processed for all files including those over the max set size of: #{wordmax} words").warn unless @md.opt.cmd =~/q/
+          SiSU_EPUB_Concordance::Source::Words.new(@particulars).songsheet
+        end
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    private
+    class DocTitle
+      include SiSU_Viz
+      #revisit, both requires (html & shared_xml) needed for stand alone operation (sisu -w [filename])
+      require_relative 'epub'                           # epub.rb
+      def initialize(particulars)
+        @particulars,@md=particulars,particulars.md
+        @data=SiSU_EPUB::Source::XHTML_Environment.new(particulars).tuned_file_instructions
+        @vz=SiSU_Viz::Defaults.new
+        txt_path=%{#{@md.dir_out}}
+        @fnb=@md.fnb
+        @lex_button=%{SiSU home}
+        @doc_details =<
#{$ep[:hsp]}
#{@md.title.full}
#{@md.creator.author}

+WOK
+      end
+      def create
+        @css=SiSU_Env::CSS_Stylesheet.new(@particulars.md)
+        format_head_toc=SiSU_EPUB_Format::HeadToc.new(@md)
+        dochead=format_head_toc.head
+        <
+ #{@doc_details}
+
Word index links are to html versions of the text the segmented version followed by the scroll (single document) version.
[For segmented text references [T1], [T2] or [T3] appearing without a link, indicates that the word appears in a title (or subtitle) of the text (that is identifiable by the appended object citation number).]

+
(The word listing/index is Case sensitive: Capitalized words appear before lower case)

+  

+    word (number of occurences)
linked references to word within document 

+    [if number of occurences exceed number of references - word occurs more than once in at least one reference. Footnote/endnotes are either assigned to the paragraph from which they are referenced or ignored, so it is relevant to check the footnotes referenced from within a paragraph as well.]
+  

+  

+    (After the page is fully loaded) you can jump directly to a word by appending a hash (#) and the word to the url for this text, (do not forget that words are case sensitive, and may be listed twice (starting with and without an upper case letter)), #your_word # [#{$ep[:hsp]}http://[web host]/#{@fnb}/concordance.html#your_word#{$ep[:hsp]}]
+  

+WOK
+      end
+    end
+    class Word
+      @@word_previous=''
+      def initialize(word,freq)
+        @word,@freq=word,freq
+      end
+      def html
+        w=if @word.capitalize==@@word_previous
+          %{\n
#{@word}
(#{@freq})
\n\t
 }
+        else n=@word.strip.gsub(/\s+/,'_') #also need to convert extended character set to html
+          %{\n
#{@word}
(#{@freq})
\n\t
 }
+        end
+        @@word_previous=@word.capitalize
+        w
+      end
+    end
+    class Words
+      require_relative 'defaults'                       # defaults.rb
+        include SiSU_Viz
+      require_relative 'epub_format'                    # epub_format.rb
+        include SiSU_EPUB_Format
+      require_relative 'sysenv'                         # sysenv.rb
+        include SiSU_Screen
+      def initialize(particulars)
+        @particulars=particulars
+        begin
+          @vz=SiSU_Viz::Defaults.new
+          @env,@md,@dal_array=particulars.env,particulars.md,particulars.dal_array
+          @path="#{@env.processing_path.epub}"
+          @freq=Hash.new(0)
+          @rxp_lv1=/^#{Mx[:lv_o]}1:/ #fix Mx[:lv_o]
+          @rxp_lv2=/^#{Mx[:lv_o]}2:/ #fix Mx[:lv_o]
+          @rxp_lv3=/^#{Mx[:lv_o]}3:/ #fix Mx[:lv_o]
+          @rxp_seg=/^#{Mx[:lv_o]}4:(\S+?)#{Mx[:lv_c]}/
+          @rxp_title=Regexp.new("^#{Mx[:meta_o]}title#{Mx[:meta_c]}\s*(.+?)\s*$")
+          @rxp_t1=Regexp.new('^T1')
+          @rxp_t2=Regexp.new('^T2')
+          @rxp_t3=Regexp.new('^T3')
+          @rxp_excluded1=/(?:https?|file|ftp):\/\/\S+/
+          @rxp_excluded0=/^(?:#{Mx[:fa_bold_o]}|#{Mx[:fa_italics_o]})?(?:to\d+|\d+| |#{Mx[:br_endnotes]}|EOF|#{Mx[:br_eof]}|thumb_\S+|snap_\S+|_+|-+|[(]?(?:ii+|iv|vi+|ix|xi+|xiv|xv|xvi+|xix|xx)[).]?|\S+?_\S+|[\d_]+\w\S+|[\w\d]{1,2}|\d{1,3}\w?|[0-9a-f]{16,64}|\d{2,3}x\d{2,3}|\S{0,2}sha\d|\S{0,3}\d{4}w\d\d|\b\w\d+|\d_all\b|e\.?g\.?)(?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})?$/mi #this regex causes and cures a stack dump in ruby 1.9 !!!
+          @rgx_splitlist=%r{[—.,;:-]+|#{Mx[:nbsp]}+}mi
+          @rgx_scanlist=%r{#{Mx[:fa_italics_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_italics_c]}|#{Mx[:fa_bold_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_bold_c]}|#{Mx[:url_o]}https?://\S+?#{Mx[:url_c]}|file://\S+|<\S+?>|\w+|[a-zA-Z]+}mi
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        end
+      end
+      def songsheet
+        begin
+          #fix to use
+          p __LINE__.to_s + ':' + __FILE__
+          p "#{@path}/content/#{@md.fn[:epub_concord]}"
+          p "#{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}"
+          @file_concordance=File.open("#{@path}/content/#{@md.fn[:epub_concord]}",'w')
+          map_para
+        rescue
+          SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        ensure
+          @file_concordance.close
+        end
+      end
+    protected
+      def location_scroll(wordlocation,show)
+        @wordlocation=wordlocation
+        %{#{@wordlocation};  }
+      end
+      def location_seg(wordlocation,show)
+        @wordlocation,@show=wordlocation,show
+        @word_location_seg=wordlocation.gsub(/(.+?)\#(\d+)/,"\\1#{Sfx[:epub_xhtml]}#o\\2") unless wordlocation.nil?
+        case @wordlocation
+        when @rxp_t1
+          %{[H]#{@show},  }
+        when @rxp_t2
+          %{[H]#{@show},  }
+        when @rxp_t3
+          %{[H]#{@show},  }
+        else %{#{@show},  }
+        end
+      end
+      def map_para
+        @seg,toy=nil,nil
+        @word_map={}
+        @dal_array.each do |line|
+          if defined? line.ocn
+            if (line.is ==:heading \
+            || line.is ==:heading_insert) \
+            && line.ln==4
+              @seg=line.name
+            end
+            if line.ocn.to_s =~/\d+/ then toy=line.ocn.to_s
+            end
+            if toy =~/\d+/ \
+            and toy !~/^0$/
+              line.obj=line.obj.split(@rgx_splitlist).join(' ') #%take in word or other match
+              for word in line.obj.scan(@rgx_scanlist) #%take in word or other match
+                word=word.gsub(/#{Mx[:lnk_o]}|#{Mx[:lnk_c]}|#{Mx[:url_o]}|#{Mx[:url_c]}/,'').
+                  gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'').
+                  gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'').
+                  gsub(/#{Mx[:gl_o]}#[a-z]+#{Mx[:gl_c]}/,'').
+                  gsub(/#{Mx[:gl_o]}#[0-9]+#{Mx[:gl_c]}/,'').
+                  gsub(/^\S$/,'')
+                word=nil if word.empty?
+                word=nil if word =~@rxp_excluded0 #watch
+                word=nil if word =~@rxp_excluded1 #watch
+                word=nil if word =~/^\S$/
+                if word
+                  word=word.gsub(/#{Mx[:br_nl]}|#{Mx[:br_line]}/,' ').
+                    gsub(/#{Mx[:fa_o]}[a-z]{1,7}#{Mx[:fa_o_c]}|#{Mx[:fa_c_o]}[a-z]{1,7}#{Mx[:fa_c]}/,'').
+                    gsub(/#{Mx[:mk_o]}(?:[0-9a-f]{32}:[0-9a-f]{32}|[0-9a-f]{64}:[0-9a-f]{64})#{Mx[:mk_c]}/,'').
+                    gsub(/#{Mx[:mk_o]}(?:[0-9a-f]{32}|[0-9a-f]{64})#{Mx[:mk_c]}/,'').
+                    gsub(/#{Mx[:en_a_o]}(?:\d|[*+])*|#{Mx[:en_b_o]}(?:\d|[*+])*|#{Mx[:en_a_c]}|#{Mx[:en_b_c]}/mi,'').
+                    gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'').gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'').
+                    gsub(/<\/?\S+?>/,'').
+                    gsub(/^\@+/,'').
+                    strip.
+                    gsub(/#{Mx[:tc_p]}.+/,'').
+                    gsub(/[\.,;:"]$/,'').
+                    gsub(/["]/,'').
+                    gsub(/^\s*[\(]/,'').
+                    gsub(/[\(]\s*$/,'').
+                    gsub(/^(?:See|e\.?g\.?).+/,'').
+                    gsub(/^\s*[.,;:]\s*/,'').
+                    strip.
+                    gsub(/^\(?[a-zA-Z]\)$/,'').
+                    gsub(/^\d+(st|nd|rd|th)$/,'').
+                    gsub(/^(\d+\.?)+$/, '').
+                    gsub(/#{Mx[:mk_o]}|#{Mx[:mk_c]}/,'').
+                    gsub(/:name#\S+/,'').
+                    gsub(/^\S$/,'')
+                  word=nil if word =~/^\S$/
+                  word=nil if word =~/^\s*$/ #watch
+                  if word
+                    unless word =~/[A-Z][A-Z]/ \
+                    or word =~/\w+\s\w+/
+                      word=word.capitalize
+                    end
+                    @freq[word] +=1
+                    @word_map[word] ||= []
+                    if line !~@rxp_lv1 \
+                    and line !~@rxp_lv2 \
+                    and line !~@rxp_lv3
+                      @word_map[word] << location_seg("#{@seg}\##{toy}",toy)
+                    else
+                      @word_map[word] << case line
+                      when @rxp_lv1; location_seg('T1',toy)
+                      when @rxp_lv2; location_seg('T2',toy)
+                      when @rxp_lv3; location_seg('T3',toy)
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+        scr=%{Full Text#{$ep[:hsp]}scroll:#{$ep[:hsp]}doc##{$ep[:hsp]} }
+        seg=''
+        @file_concordance << SiSU_EPUB_Concordance::Source::DocTitle.new(@particulars).create
+        alph=%W[A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+        @file_concordance << '
'
+        alph.each {|x| @file_concordance << %{#{x},#{$ep[:hsp]}}}
+        @file_concordance << '
'
+        letter=alph.shift
+        @file_concordance << %{\n
A
}
+        for word in @freq.keys.sort! {|a,b| a.downcase<=>b.downcase}
+          f=/^(\S)/.match(word)[1]
+          if letter < f.upcase
+            while letter < f.upcase
+              if alph.length > 0
+                letter=alph.shift
+                @file_concordance << %{\n
#{letter}
}
+              else break
+              end
+            end
+          end
+          keyword=SiSU_EPUB_Concordance::Source::Word.new(word,@freq[word]).html
+          if keyword !~ @rxp_excluded0
+            if @word_map[word][0] =~ /\d+/
+              wm=[]
+              @file_concordance << %{#{keyword}#{seg}#{@word_map[word].uniq.compact.join}}
+            end
+            @file_concordance << '
'
+          end
+          # special cases endnotes and header levels 1 - 3
+        end
+        credits=@vz.credits_sisu_epub
+        @file_concordance << %{>#{credits}\n} # footer
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/epub_format.rb b/lib/sisu/v5/epub_format.rb
new file mode 100644
index 00000000..75967653
--- /dev/null
+++ b/lib/sisu/v5/epub_format.rb
@@ -0,0 +1,2236 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: epub formating, css template
+
+=end
+module SiSU_EPUB_Format
+  include SiSU_Viz
+  class ParagraphNumber
+    def initialize(md,ocn)
+      @md,@ocn=md,ocn.to_s
+      @ocn ||=''
+    end
+    def ocn_display
+      make=SiSU_Env::ProcessingSettings.new(@md)
+      if make.build.ocn?
+        ocn_class='ocn'
+        if @ocn==nil \
+        or @ocn.to_i==0 \
+        or @ocn.empty?
+          %{}
+        else
+          @ocn.gsub(/^(\d+|)$/,
+            %{})
+        end
+      else
+        %{}
+      end
+    end
+    def name
+      (@ocn==nil || @ocn.empty?) ? '' : %{}
+    end
+    def id #w3c? "tidy" complains about numbers as identifiers ! annoying
+      (@ocn==nil || @ocn.empty?) ? '' : %{id="o#{@ocn}"}
+    end
+    def goto
+      (@ocn==nil || @ocn.empty?) ? '' : %{}
+    end
+  end
+  class CSS
+    def css_epub_xhtml
+      <<-WOK
+/* SiSU epub css default stylesheet */
+  body {
+    color: black;
+    background: #ffffff;
+    background-color: #ffffff;
+  }
+/*
+    table {
+      margin-left: 5%;
+      display: block;
+    }
+    tr {
+      display: block;
+    }
+    th,td {
+      display: inline;
+      vertical-align: top;
+    }
+*/
+  a:link {
+    color: #003399;
+    text-decoration: none;
+  }
+  a:visited {
+    color: #003399;
+    text-decoration: none;
+  }
+  a:hover {
+    color: #000000;
+    background-color: #f9f9aa;
+  }
+/*
+  a:hover {
+    border-bottom: 2px solid #777777;
+    background-color: #fff3b6;
+  }
+*/
+  a:hover img {
+    background-color: #ffffff;
+  }
+  a:active {
+    color: #003399;
+    text-decoration: underline;
+  }
+  a.lnkocn:link {
+    color: #777777;
+    text-decoration: none;
+  }
+  div {
+    margin-left: 0;
+    margin-right: 0;
+  }
+  div.p {
+    margin-left: 5%;
+    margin-right: 1%;
+  }
+
+  .norm, .bold, .verse, .group, .block, .alt {
+    line-height: 133%;
+    margin-left: 0em;
+    margin-right: 2em;
+    margin-top: 12px;
+    margin-bottom: 0px;
+    padding-left: 0em;
+    text-indent: 0mm;
+  }
+  p, h0, h1, h2, h3, h4, h5, h6 {
+    display: block;
+    font-family: verdana, arial, georgia, tahoma, sans-serif, helvetica, times, roman;
+    font-size: 100%;
+    font-weight: normal;
+    line-height: 133%;
+    text-align: justify;
+    margin-left: 0em;
+    margin-right: 2em;
+    text-indent: 0mm;
+    margin-top: 4px;
+    margin-bottom: 4px;
+  }
+  p.norm { }
+  p.i1 {padding-left: 1em;}
+  p.i2 {padding-left: 2em;}
+  p.i3 {padding-left: 3em;}
+  p.i4 {padding-left: 4em;}
+  p.i5 {padding-left: 5em;}
+  p.i6 {padding-left: 6em;}
+  p.i7 {padding-left: 7em;}
+  p.i8 {padding-left: 8em;}
+  p.i9 {padding-left: 9em;}
+
+  p.h0i0 {
+    padding-left: 0em;
+    text-indent:  0em;
+  }
+  p.h0i1 {
+    padding-left: 1em;
+    text-indent: -1em;
+  }
+  p.h0i2 {
+    padding-left: 2em;
+    text-indent: -2em;
+  }
+  p.h0i3 {
+    padding-left: 3em;
+    text-indent: -3em;
+  }
+  p.h0i4 {
+    padding-left: 4em;
+    text-indent: -4em;
+  }
+  p.h0i5 {
+    padding-left: 5em;
+    text-indent: -5em;
+  }
+  p.h0i6 {
+    padding-left: 6em;
+    text-indent: -6em;
+  }
+  p.h0i7 {
+    padding-left: 7em;
+    text-indent: -7em;
+  }
+  p.h0i8 {
+    padding-left: 8em;
+    text-indent: -8em;
+  }
+  p.h0i9 {
+    padding-left: 9em;
+    text-indent: -9em;
+  }
+
+  p.h1i0 {
+    padding-left: 0em;
+    text-indent:  1em;
+  }
+  p.h1i1 {
+    padding-left: 1em;
+    text-indent:  0em;
+  }
+  p.h1i2 {
+    padding-left: 2em;
+    text-indent: -1em;
+  }
+  p.h1i3 {
+    padding-left: 3em;
+    text-indent: -2em;
+  }
+  p.h1i4 {
+    padding-left: 4em;
+    text-indent: -3em;
+  }
+  p.h1i5 {
+    padding-left: 5em;
+    text-indent: -4em;
+  }
+  p.h1i6 {
+    padding-left: 6em;
+    text-indent: -5em;
+  }
+  p.h1i7 {
+    padding-left: 7em;
+    text-indent: -6em;
+  }
+  p.h1i8 {
+    padding-left: 8em;
+    text-indent: -7em;
+  }
+  p.h1i9 {
+    padding-left: 9em;
+    text-indent: -8em;
+  }
+
+  p.h2i0 {
+    padding-left: 0em;
+    text-indent:  2em;
+  }
+  p.h2i1 {
+    padding-left: 1em;
+    text-indent:  1em;
+  }
+  p.h2i2 {
+    padding-left: 2em;
+    text-indent:  0em;
+  }
+  p.h2i3 {
+    padding-left: 3em;
+    text-indent: -1em;
+  }
+  p.h2i4 {
+    padding-left: 4em;
+    text-indent: -2em;
+  }
+  p.h2i5 {
+    padding-left: 5em;
+    text-indent: -3em;
+  }
+  p.h2i6 {
+    padding-left: 6em;
+    text-indent: -4em;
+  }
+  p.h2i7 {
+    padding-left: 7em;
+    text-indent: -5em;
+  }
+  p.h2i8 {
+    padding-left: 8em;
+    text-indent: -6em;
+  }
+  p.h2i9 {
+    padding-left: 9em;
+    text-indent: -7em;
+  }
+
+  p.h3i0 {
+    padding-left: 0em;
+    text-indent:  3em;
+  }
+  p.h3i1 {
+    padding-left: 1em;
+    text-indent:  2em;
+  }
+  p.h3i2 {
+    padding-left: 2em;
+    text-indent:  1em;
+  }
+  p.h3i3 {
+    padding-left: 3em;
+    text-indent:  0em;
+  }
+  p.h3i4 {
+    padding-left: 4em;
+    text-indent: -1em;
+  }
+  p.h3i5 {
+    padding-left: 5em;
+    text-indent: -2em;
+  }
+  p.h3i6 {
+    padding-left: 6em;
+    text-indent: -3em;
+  }
+  p.h3i7 {
+    padding-left: 7em;
+    text-indent: -4em;
+  }
+  p.h3i8 {
+    padding-left: 8em;
+    text-indent: -5em;
+  }
+  p.h3i9 {
+    padding-left: 9em;
+    text-indent: -6em;
+  }
+
+  p.h4i0 {
+    padding-left: 0em;
+    text-indent:  4em;
+  }
+  p.h4i1 {
+    padding-left: 1em;
+    text-indent:  3em;
+  }
+  p.h4i2 {
+    padding-left: 2em;
+    text-indent:  2em;
+  }
+  p.h4i3 {
+    padding-left: 3em;
+    text-indent:  1em;
+  }
+  p.h4i4 {
+    padding-left: 4em;
+    text-indent:  0em;
+  }
+  p.h4i5 {
+    padding-left: 5em;
+    text-indent: -1em;
+  }
+  p.h4i6 {
+    padding-left: 6em;
+    text-indent: -2em;
+  }
+  p.h4i7 {
+    padding-left: 7em;
+    text-indent: -3em;
+  }
+  p.h4i8 {
+    padding-left: 8em;
+    text-indent: -4em;
+  }
+  p.h4i9 {
+    padding-left: 9em;
+    text-indent: -5em;
+  }
+
+  p.h5i0 {
+    padding-left: 0em;
+    text-indent:  5em;
+  }
+  p.h5i1 {
+    padding-left: 1em;
+    text-indent:  4em;
+  }
+  p.h5i2 {
+    padding-left: 2em;
+    text-indent:  3em;
+  }
+  p.h5i3 {
+    padding-left: 3em;
+    text-indent:  2em;
+  }
+  p.h5i4 {
+    padding-left: 4em;
+    text-indent:  1em;
+  }
+  p.h5i5 {
+    padding-left: 5em;
+    text-indent:  0em;
+  }
+  p.h5i6 {
+    padding-left: 6em;
+    text-indent: -1em;
+  }
+  p.h5i7 {
+    padding-left: 7em;
+    text-indent: -2em;
+  }
+  p.h5i8 {
+    padding-left: 8em;
+    text-indent: -3em;
+  }
+  p.h5i9 {
+    padding-left: 9em;
+    text-indent: -4em;
+  }
+
+  p.h6i0 {
+    padding-left: 0em;
+    text-indent:  6em;
+  }
+  p.h6i1 {
+    padding-left: 1em;
+    text-indent:  5em;
+  }
+  p.h6i2 {
+    padding-left: 2em;
+    text-indent:  4em;
+  }
+  p.h6i3 {
+    padding-left: 3em;
+    text-indent:  3em;
+  }
+  p.h6i4 {
+    padding-left: 4em;
+    text-indent:  2em;
+  }
+  p.h6i5 {
+    padding-left: 5em;
+    text-indent:  1em;
+  }
+  p.h6i6 {
+    padding-left: 6em;
+    text-indent:  0em;
+  }
+  p.h6i7 {
+    padding-left: 7em;
+    text-indent: -1em;
+  }
+  p.h6i8 {
+    padding-left: 8em;
+    text-indent: -2em;
+  }
+  p.h6i9 {
+    padding-left: 9em;
+    text-indent: -3em;
+  }
+
+  p.h7i0 {
+    padding-left: 0em;
+    text-indent:  7em;
+  }
+  p.h7i1 {
+    padding-left: 1em;
+    text-indent:  6em;
+  }
+  p.h7i2 {
+    padding-left: 2em;
+    text-indent:  5em;
+  }
+  p.h7i3 {
+    padding-left: 3em;
+    text-indent:  4em;
+  }
+  p.h7i4 {
+    padding-left: 4em;
+    text-indent:  3em;
+  }
+  p.h7i5 {
+    padding-left: 5em;
+    text-indent:  2em;
+  }
+  p.h7i6 {
+    padding-left: 6em;
+    text-indent:  1em;
+  }
+  p.h7i7 {
+    padding-left: 7em;
+    text-indent:  0em;
+  }
+  p.h7i8 {
+    padding-left: 8em;
+    text-indent: -1em;
+  }
+  p.h7i9 {
+    padding-left: 9em;
+    text-indent: -2em;
+  }
+
+  p.h8i0 {
+    padding-left: 0em;
+    text-indent:  8em;
+  }
+  p.h8i1 {
+    padding-left: 1em;
+    text-indent:  7em;
+  }
+  p.h8i2 {
+    padding-left: 2em;
+    text-indent:  6em;
+  }
+  p.h8i3 {
+    padding-left: 3em;
+    text-indent:  5em;
+  }
+  p.h8i4 {
+    padding-left: 4em;
+    text-indent:  4em;
+  }
+  p.h8i5 {
+    padding-left: 5em;
+    text-indent:  3em;
+  }
+  p.h8i6 {
+    padding-left: 6em;
+    text-indent:  2em;
+  }
+  p.h8i7 {
+    padding-left: 7em;
+    text-indent:  1em;
+  }
+  p.h8i8 {
+    padding-left: 8em;
+    text-indent:  0em;
+  }
+  p.h8i9 {
+    padding-left: 9em;
+    text-indent: -1em;
+  }
+
+  p.h9i0 {
+    padding-left: 0em;
+    text-indent:  9em;
+  }
+  p.h9i1 {
+    padding-left: 1em;
+    text-indent:  8em;
+  }
+  p.h9i2 {
+    padding-left: 2em;
+    text-indent:  7em;
+  }
+  p.h9i3 {
+    padding-left: 3em;
+    text-indent:  6em;
+  }
+  p.h9i4 {
+    padding-left: 4em;
+    text-indent:  5em;
+  }
+  p.h9i5 {
+    padding-left: 5em;
+    text-indent:  4em;
+  }
+  p.h9i6 {
+    padding-left: 6em;
+    text-indent:  3em;
+  }
+  p.h9i7 {
+    padding-left: 7em;
+    text-indent:  2em;
+  }
+  p.h9i8 {
+    padding-left: 8em;
+    text-indent:  1em;
+  }
+  p.h9i9 {
+    padding-left: 9em;
+    text-indent:  0em;
+  }
+
+  p.it0 {
+    margin-left: 0em;
+    margin-top: 6px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it1 {
+    margin-left: 1em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it2 {
+    margin-left: 2em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it3 {
+    margin-left: 3em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it4 {
+    margin-left: 4em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it5 {
+    margin-left: 5em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it6 {
+    margin-left: 6em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it7 {
+    margin-left: 7em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it8 {
+    margin-left: 8em;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    line-height: 100%;
+  }
+  p.it9 {
+    margin-left: 9em;
+    margin-bottom: 0px;
+    margin-top: 0px;
+    line-height: 100%;
+  }
+
+  p.group { }
+
+  p.block { }
+
+  p.alt { }
+
+  p.verse {
+    margin-bottom: 6px;
+  }
+
+  p.code {
+    font-family: inconsolata, andale mono, courier new, courier, monospace;
+    font-size: 90%;
+    text-align: left;
+    background-color: #eeeeee;
+  }
+
+  p.caption {
+    text-align: left;
+    font-size: 80%;
+    display: inline;
+  }
+
+  p.endnote {
+    font-size: 96%;
+    line-height: 120%;
+    text-align: left;
+    margin-right: 2em;
+  }
+  p.endnote_indent {
+    font-size: 96%;
+    line-height: 120%;
+    text-align: left;
+    margin-left: 2em;
+    margin-right: 2em;
+  }
+
+  p.center {
+    text-align: center;
+  }
+  p.align_right {
+    text-align: right;
+  }
+  p.bold {
+    font-weight: bold;
+  }
+  p.bold_left {
+    font-weight: bold;
+    text-align: left;
+  }
+  p.centerbold {
+    text-align: center;
+    font-weight: bold;
+  }
+  p.em {
+    font-weight: bold;
+    font-style: normal;
+    background: #fff3b6;
+  }
+
+  p.small {
+    font-size: 80%;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    margin-right: 6px;
+    text-align: left;
+  }
+
+  .tiny, .tiny_left, .tiny_right, .tiny_center {
+    font-size: 10px;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    color: #777777;
+    margin-right: 6px;
+    text-align: left;
+  }
+  p.tiny { }
+  p.tiny_left {
+    margin-left: 0px;
+    margin-right: 0px;
+    text-align: left;
+  }
+  p.tiny_right {
+    margin-right: 1em;
+    text-align: right;
+  }
+  p.tiny_center {
+    margin-left: 0px;
+    margin-right: 0px;
+    text-align: center;
+  }
+
+  p.concordance_word {
+    line-height: 150%;
+    font-weight: bold;
+    display: inline;
+    margin-top: 4px;
+    margin-bottom: 1px;
+  }
+  p.concordance_count {
+    font-size: 80%;
+    color: #777777;
+    display: inline;
+    margin-left: 0em;
+  }
+  p.concordance_object {
+    font-size: 80%;
+    line-height: 120%;
+    text-align: left;
+    margin-left: 3em;
+    margin-top: 1px;
+    margin-bottom: 3px;
+  }
+  p.book_index_lev1 {
+    line-height: 100%;
+    margin-top: 4px;
+    margin-bottom: 1px;
+  }
+  p.book_index_lev2 {
+    line-height: 100%;
+    text-align: left;
+    margin-left: 3em;
+    margin-top: 1px;
+    margin-bottom: 3px;
+  }
+
+  p.quickref {
+    font-size: 10px;
+    font-style: italic;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    color: #777777;
+    margin-right: 5px;
+    text-align: left;
+  }
+  p.bigref {
+    font-size: 11px;
+    font-weight: bold;
+    margin-top: 0px;
+    margin-bottom: 0px;
+    color: #777777;
+    margin-right: 5px;
+    text-align: center;
+  }
+
+  p.letter {
+    font-weight: bold;
+    font-size: 80%;
+    margin-left: 0em;
+    margin-top: 2px;
+    margin-bottom: 2px;
+    margin-right: 6px;
+    text-align: left;
+    color: white;
+    background: #880000;
+  }
+
+  tt {
+    font-family: inconsolata, andale mono, courier new, courier, monospace;
+    background-color: #eeeeee;
+  }
+
+  label.ocn {
+    width: 2%;
+    float: right;
+    top: 0;
+    font-size: 10px;
+    margin-top: 0px;
+    margin-bottom: 5px;
+    color: #777777;
+    margin-right: 5px;
+    text-align: right;
+    background-color: #ffffff;
+  }
+
+  table { }
+  tr { }
+  th,td {
+    vertical-align: top;
+    text-align: left;
+  }
+  th {
+    font-weight: bold;
+  }
+
+  p.left, th.left, td.left {
+    text-align: left;
+  }
+  p.small_left, th.small_left, td.small_left {
+    text-align: left;
+    font-size: 80%;
+  }
+  p.right, th.right, td.right {
+    text-align: right;
+  }
+
+  #horizontal_links {
+    background: #eeeeee;
+    margin-left: 5%;
+    margin-right: 5%;
+  }
+  #horizontal {
+    margin: 0;
+    padding: 0 0 0 10px;
+    border-top: 1px solid #000077;
+    border-bottom: 1px solid #000077;
+  }
+  #horizontal li {
+    margin: 0 0 0 0;
+    padding: 0 16px 0 0;
+    display: inline;
+    list-style-type: none;
+    text-align: left;
+    background: none;
+  }
+  #horizontal a {
+    line-height: 12px;
+    margin: 0 0 0 0;
+    text-decoration: none;
+    color: #000077;
+  }
+  #horizontal a.active, #horizontal a:hover {
+    border-bottom: 2px solid #777777;
+    padding-bottom: 2px;
+    color: #000077;
+  }
+  #horizontal a:hover {
+    color: #000077;
+  }
+
+  #document_versions {
+    position: absolute;
+    top: 10mm;
+    right: 2%;
+    width: 12%;
+    float: right;
+  }
+
+  #vertical_links {
+    position: absolute;
+    top: 10mm;
+    right: 0px;
+    width: 20%;
+    background: #dddddd;
+    float: right;
+  }
+  #vertical {
+    padding: 0 12px 0px 0px;
+    margin-left: 2%;
+    margin-right: 2%;
+  }
+  #vertical li {
+    display: block;
+    list-style-type: none;
+  }
+  #vertical a {
+    line-height: 12px;
+    text-decoration: none;
+    color: #000077;
+  }
+  #vertical a.active, #vertical a:hover {
+    border-bottom: 2px solid #777777;
+    padding-bottom: 2px;
+    color: #000077;
+  }
+
+  ul, li {
+    list-style-type: none;
+    list-style: none;
+    padding-left: 20px;
+    display: block;
+    font-family: verdana, arial, georgia, tahoma, sans-serif, helvetica, times, roman;
+    font-weight: normal;
+    line-height: 150%;
+    text-align: justify;
+    text-indent: 0mm;
+    margin-left: 1em;
+    margin-right: 2em;
+    margin-top: 3px;
+    margin-bottom: 3px;
+  }
+
+  li {
+    background: url(../image/bullet_09.png) no-repeat 0px 6px;
+  }
+
+  ul {
+  }
+  li.bullet { margin-left: 0em; }
+  li.i1 { margin-left: 1em; }
+  li.i2 { margin-left: 2em; }
+  li.i3 { margin-left: 3em; }
+  li.i4 { margin-left: 4em; }
+  li.i5 { margin-left: 5em; }
+  li.i6 { margin-left: 6em; }
+  li.i7 { margin-left: 7em; }
+  li.i8 { margin-left: 8em; }
+  li.i9 { margin-left: 9em; }
+
+  li.doc, li.ref, li.refcenter {
+    margin-top: 0px;
+    margin-bottom: 0px;
+    margin-right: 0px;
+    font-size: 8px;
+    font-style: normal;
+    text-align: left;
+  }
+  li.doc {
+    background: url(../image/bullet_09.png) no-repeat 0px 6px;
+    padding-left: 16px;
+    margin-left: 10px;
+    margin-right: 0px;
+  }
+  li.ref {
+    background: none;
+    padding-left: 0;
+    margin-left: 0;
+    color: #777777;
+  }
+  li.refcenter {
+    background: url(../image/bullet_09.png) no-repeat 0px 6px;
+    padding-left: 20px;
+    margin-left: 10%;
+    font-size: 9px;
+    color: #777777;
+    text-align: center;
+  }
+  li.refbold {
+    list-style-type: none;
+    padding-left: 16px;
+    margin-left: 0;
+    margin-right: 10mm;
+    font-weight: bold;
+  }
+
+  h0, h1, h2, h3, h4, h5, h6 {
+    font-weight: bold;
+    line-height: 120%;
+    text-align: left;
+    margin-top: 20px;
+    margin-bottom: 10px;
+  }
+  h4.norm, h5.norm, h6.norm {
+    margin-top: 10px;
+    margin-bottom: 0px;
+  }
+  h1.center, h2.center, h3.center, h4.center, h5.center, h6.center {
+    text-align: center;
+  }
+  h1 { font-size: 120%; }
+  h2 { font-size: 115%; }
+  h3 { font-size: 110%; }
+  h4 { font-size: 105%; }
+  h5 { font-size: 100%; }
+  h6 { font-size: 100%; }
+  h0 { font-size: 80%; }
+
+  h1.i {margin-left: 2em;}
+  h2.i {margin-left: 3em;}
+  h3.i {margin-left: 4em;}
+  h4.i {margin-left: 5em;}
+  h5.i {margin-left: 6em;}
+  h6.i {margin-left: 7em;}
+  h7.i {margin-left: 8em;}
+  h8.i {margin-left: 9em;}
+  h9.i {margin-left: 10em;}
+
+  .toc {
+    font-weight: normal;
+    margin-top: 6px;
+    margin-bottom: 6px;
+  }
+  h1.toc {
+    margin-left: 1em;
+    font-size: 115%;
+    line-height: 150%;
+  }
+  h2.toc {
+    margin-left: 2em;
+    font-size: 110%;
+    line-height: 140%;
+  }
+  h3.toc {
+    margin-left: 3em;
+    font-size: 105%;
+    line-height: 120%;
+  }
+  h4.toc {
+    margin-left: 4em;
+    font-size: 100%;
+    line-height: 120%;
+  }
+  h5.toc {
+    margin-left: 5em;
+    font-size: 95%;
+    line-height: 110%;
+  }
+  h6.toc {
+    margin-left: 6em;
+    font-size: 90%;
+    line-height: 110%;
+  }
+
+  .microtoc {
+    margin-top: 2px;
+    margin-bottom: 2px;
+  }
+
+  h1.microtoc {
+    margin-left: 0mm;
+    font-size: 115%;
+  }
+  h2.microtoc {
+    margin-left: 5mm;
+    font-size: 110%;
+  }
+  h3.microtoc {
+    margin-left: 10mm;
+    font-size: 105%;
+  }
+  h4.microtoc {
+    margin-left: 15mm;
+    font-weight: normal;
+    font-size: 100%;
+  }
+  h5.microtoc {
+    margin-left: 20mm;
+    font-weight: normal;
+    font-size: 95%;
+  }
+  h6.microtoc {
+    margin-left: 25mm;
+    font-weight: normal;
+    font-size: 90%;
+  }
+
+  .subtoc {
+    margin-right: 34%;
+    font-weight: normal;
+  }
+  h5.subtoc {
+    margin-left: 2em;
+    font-size: 80%;
+    margin-top: 2px;
+    margin-bottom: 2px;
+  }
+  h6.subtoc {
+    margin-left: 3em;
+    font-size: 75%;
+    margin-top: 0px;
+    margin-bottom: 0px;
+  }
+
+  div.substance {
+    width: 100%;
+    background-color: #ffffff;
+  }
+  div.ocn {
+    width: 5%;
+    float: right;
+    top: 0;
+    background-color: #ffffff;
+  }
+  div.endnote {
+    width: 100%;
+    background-color: #fffffff;
+  }
+  div.toc {
+    position: absolute;
+    float: left;
+    margin: 0;
+    padding: 0;
+    padding-top: 0.5em;
+    border: 0;
+    width: 5%;
+    background-color: #eeeeee;
+    margin-right:1em;
+  }
+  div.summary {
+    margin: 0;
+    padding: 0;
+    border-left: 2em solid #eeeeee;
+    padding-left: 0em;
+    background-color: #eeeeee;
+  }
+  div.content, div.main_column {
+    margin: 0;
+    padding: 0;
+    border-left: 0% solid #ffffff;
+    padding-left: 5%;
+  }
+  div.content:after {
+    content:' ';
+    clear:both;
+    display:block;
+    height:0;
+    overflow:hidden
+  }
+  div.footer {
+    clear:left;
+    padding: 0.5em;
+    font-size: 80%;
+    margin: 0;
+  }
+  div.toc ul {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+  }
+  div.toc li ul a, li ul span.currentlink
+  {
+    font-weight: normal;
+    font-size: 90%;
+    padding-left: 2em;
+    background-color: #eeeeee;
+  }
+  div.toc a, span.currentlink{
+    display:block;
+    text-decoration: none;
+    padding-left: 0.5em;
+    color: #0000aa;
+  }
+  hr {
+    width: 90%;
+  }
+
+  span.currentlink {
+    text-decoration: none;
+    background-color: #aaaaf9;
+  }
+
+  div.toc a:visited {
+    color: #0000aa;
+  }
+  div.toc a:hover {
+    color: #000000;
+    background-color: #f9f9aa;
+  }
+
+  h1.c, h2.c, h3.c, h4.c, h5.c, h6.c, p.c {
+    text-align: center
+  }
+  h1.red, h2.red, h3.red, h4.red, h5.red, h6.red {
+    text-align: center;
+    color: #ff0000;
+    margin-left: 5mm;
+    text-indent: 5mm;
+    margin-top: 30px;
+    margin-bottom: 20px;
+    margin-right: 15mm;
+  }
+  h1.ruby, h2.ruby, h3.ruby, h4.ruby, h5.ruby, h6.ruby {
+    text-align: center;
+    color: #990000;
+    margin-left: 5mm;
+    text-indent: 5mm;
+    margin-top: 30px;
+    margin-bottom: 20px;
+    margin-right: 15mm;
+  }
+      WOK
+    end
+  end
+  module SanitizeXML
+    def self.xml(x)
+      if x.is_a?(String)
+        x=x.gsub(/ /,' ') if Ep[:alt]==:on
+        x.gsub(/&/,'&').
+          gsub(//,">").
+          gsub(/#{Dx[:url_o]}/,Dx[:url_o_xml]).gsub(/#{Dx[:url_c]}/,Dx[:url_o_xml]).
+          #gsub(//,'>').
+          gsub(/\\\\/,'
').
+          gsub(/<br(?: \/)?>/,'
')
+      else x
+      end
+    end
+  end
+  class HeadInformation
+    include SiSU_Viz
+    attr_reader :md,:rdf,:vz
+    def initialize(md)
+      @md=md
+      # DublinCore 1 - title
+      @vz=SiSU_Viz::Defaults.new
+      @css=SiSU_Env::CSS_Stylesheet.new(md)
+      @seg_name_xhtml=(SiSU_EPUB::Source::Seg.new.seg_name_xhtml || [])
+      @seg_name_xhtml_tracker=(SiSU_EPUB::Source::Seg.new.seg_name_xhtml_tracker || [])
+      @tocband_scroll,@tocband_segtoc=nil,nil
+      @index,@metalink='index','#metadata'
+    end
+    def doc_type_xhtml
+      <<-WOK
+
+
+      WOK
+    end
+=begin
+~/epub
+  |-- META-INF
+  |   `-- container.xml                # simple, make sure full-path of rootfile points to metadata.opf
+  |-- content
+  |   |-- 1.xhtml
+  |   |-- 2.xhtml
+  |   |-- 3.xhtml
+  |   |-- ... .xhtml
+  |   |-- concordance.xhtml
+  |   |-- css
+  |   |   `-- xhtml.css
+  |   |-- endnotes.xhtml
+  |   |-- image
+  |   |   |-- arrow_next_red.png
+  |   |   |-- arrow_prev_red.png
+  |   |   |-- arrow_up_red.png
+  |   |   `-- bullet_09.png
+  |   |-- index.xhtml
+  |   |-- meta.xhtml
+  |   |-- metadata.xhtml
+  |   `-- toc.xhtml
+  |-- metadata.opf                     #(i) metadata dc; (ii) manifest (contents); (iii) spine (mimetypes)
+  |-- mimetype                         # application/epub+zip
+  `-- toc.ncx                          #(i) head (ii) doc title (iii) navmap, list of navigation points (like chapters)
+=end
+    def doc_type
+      doc_type_xhtml
+    end
+    def mimetype
+      <<-WOK
+application/epub+zip
+      WOK
+    end
+    def metainf_container #container.xml file in META-INF directory
+      #simple, make sure full-path of rootfile points to metadata.opf
+      #epub_metadata.opf content.opf
+      <<-WOK
+
+
+  
+    
+  
+
+      WOK
+    end
+    def sections(dob,fn_base)
+      name=fn_base + Sfx[:epub_xhtml]
+      dir_epub_cont=@md.env.processing_path.epub + '/' + Ep[:d_oebps]
+      segfilename=dir_epub_cont + '/' + name
+      output_epub_cont_seg=File.new(segfilename,'w')
+      output_epub_cont_seg << %{#{doc_type}
+  
+    
+      #{dob.obj} -
+      #{@md.html_title}
+    
+    
+    #{@css.xhtml_epub}
+  
+  #{@vz.color_body}
+    

+      

+        #{dob.ocn}
+        

+          #{dob.obj}
+        

+      

+    

+    
+  }
+output_epub_cont_seg.close
+    end
+    def toc_ncx #list of navigation points (like chapters), table of contents, listing each navigation point (chapters and such) under the navigation map
+      def structure
+        open
+        head_open
+        head
+        head_close
+        doc_title
+        doc_author
+        navmap_open
+       #navmap ...
+        navmap_close
+        close
+      end
+      def open
+        <<-WOK
+
+
+        WOK
+      end
+      def close
+        <<-WOK
+
+        WOK
+      end
+      def head_open
+        <<-WOK
+  
+        WOK
+      end
+      def head
+        depth=@md.lvs[1] + @md.lvs[2] + @md.lvs[3] + @md.lvs[4]
+        title=SanitizeXML.xml(@md.title.full)
+        author=SanitizeXML.xml(@md.author)
+        <<-WOK
+    
+    #{title} by #{author}
+    
+    
+    
+    
+    
+    
+        WOK
+      end
+      def head_close
+        <<-WOK
+  
+        WOK
+      end
+      def doc_title
+        txt=SanitizeXML.xml(@md.title.full)
+        <<-WOK
+  
+    #{txt}
+  
+        WOK
+      end
+      def doc_author
+        txt=SanitizeXML.xml(@md.author)
+        <<-WOK
+  
+    #{txt}
+  
+        WOK
+      end
+      def navmap_open
+        <<-WOK
+  
+        WOK
+      end
+      def navmap_sisu_toc(no)
+        id_u=DISABLE[:epub][:ncx_navpoint_unique_id] \
+        ? ''
+        : "-#{no}"
+        <<-WOK
+      
+        
+          Table of Contents
+        
+        
+      
+        WOK
+      end
+      def navpoint(dob,no,fn_base,hashtag=nil)
+         fn=fn_base + Sfx[:epub_xhtml]
+         name=hashtag ? fn + hashtag : fn
+        id_u=DISABLE[:epub][:ncx_navpoint_unique_id] \
+        ? ''
+        : "-#{no}"
+        <<-WOK
+      
+        
+          #{dob.obj}
+        
+        
+        WOK
+      end
+      def navpoint_close
+        <<-WOK
+      
+        WOK
+      end
+      def navmap_close
+        <<-WOK
+  
+        WOK
+      end
+      self
+    end
+    def metadata_opf #(i) metadata dc; (ii) manifest (contents); (iii) spine (mimetypes)
+      def structure
+        package_open
+        metadata_open
+        metadata_close
+        manifest_open
+        manifest_close
+        spine_open
+        spine_close
+        guide_open
+        guide_close
+        package_close
+      end
+      def package_open
+        <<-WOK
+
+
+        WOK
+      end
+      def package_close
+        <<-WOK
+
+        WOK
+      end
+      def metadata #metadata dc
+        cover_image=if defined? @md.make.cover_image \
+        and @md.make.cover_image.is_a?(Hash) \
+        and @md.make.cover_image[:cover] =~/\S+/
+          %{\n    <#{$ep[:o]}meta name="cover" content="cover_image" />}
+        else ''
+        end
+        author=if defined? @md.creator.author \
+        and @md.creator.author =~/\S+/
+          m=''
+          @md.creator.author_detail.each do |x|
+            surname=x[:the] \
+            ? x[:the]
+            : ''
+            other_names=x[:others] \
+            ? ', ' + x[:others]
+            : ''
+            m=(m.empty?) \
+            ? (surname + other_names)
+            : (m + '; ' + surname + ', ' + other_names)
+            m=SanitizeXML.xml(m)
+          end
+          x=@md.creator.author.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        editor=if defined? @md.creator.editor \
+        and @md.creator.editor =~/\S+/
+          m=''
+          @md.creator.editor_detail.each do |x|
+            surname=x[:the] \
+            ? x[:the]
+            : ''
+            other_names=x[:others] \
+            ? ', ' + x[:others]
+            : ''
+            m=(m.empty?) \
+            ? (surname + other_names)
+            : (m + '; ' + surname + ', ' + other_names)
+            m=SanitizeXML.xml(m)
+          end
+          x=@md.creator.editor.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        translator=if defined? @md.creator.translator \
+        and @md.creator.translator =~/\S+/
+          m=''
+          @md.creator.translator_detail.each do |x|
+            surname=x[:the] \
+            ? x[:the]
+            : ''
+            other_names=x[:others] \
+            ? ', ' + x[:others]
+            : ''
+            m=(m.empty?) \
+            ? (surname + other_names)
+            : (m + '; ' + surname + ', ' + other_names)
+            m=SanitizeXML.xml(m)
+          end
+          x=@md.creator.translator.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        illustrator=if defined? @md.creator.illustrator \
+        and @md.creator.illustrator =~/\S+/
+          m=''
+          @md.creator.illustrator_detail.each do |x|
+            surname=x[:the] \
+            ? x[:the]
+            : ''
+            other_names=x[:others] \
+            ? ', ' + x[:others]
+            : ''
+            m=(m.empty?) \
+            ? (surname + other_names)
+            : (m + '; ' + surname + ', ' + other_names)
+            m=SanitizeXML.xml(m)
+          end
+          x=@md.creator.illustrator.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        date_published=if defined? @md.date.published \
+        and @md.date.published =~/\S+/
+          x=@md.date.published.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        subject=if defined? @md.classify.subject \
+        and @md.classify.subject =~/\S+/
+          x=@md.classify.subject.dup
+          x=SanitizeXML.xml(x)
+          %{\n    #{x}}
+        else ''
+        end
+        language=if defined? @md.opt.lng \
+        and @md.opt.lng =~/\S+/
+          language=@md.opt.lng.gsub(/
/,'
')
+          %{\n    #{language}}
+        else ''
+        end
+        rights=if defined? @md.rights.all \
+        and @md.rights.all =~/\S+/
+          rights=SanitizeXML.xml(@md.rights.all)
+          rights=rights.gsub(//,' ')
+          %{\n    #{rights}}
+        else ''
+        end
+        f=SiSU_Env::FileOp.new(@md)
+        <<-WOK
+  <#{$ep[:o]}metadata
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:opf="http://www.idpf.org/2007/opf"
+    xmlns:dcterms="http://purl.org/dc/terms/"
+    xmlns:dc="http://purl.org/dc/elements/1.1/"
+    unique-identifier="urn:uuid:#{@md.dgst[1]}" version="2.0">
+    #{@md.title.full}
+    #{cover_image}#{author}#{editor}#{translator}#{illustrator}#{language}#{date_published}#{subject}#{rights}
+    #{f.output_path.epub.url.gsub(/http:\/\//,'')}/#{f.base_filename.epub}
+    urn:uuid:#{@md.dgst[1]}
+    
+  
+        WOK
+      end
+      def manifest_open
+        <<-WOK
+  
+    
+    
+    
+    
+    
+        WOK
+      end
+      def manifest_content_sisu_toc
+        <<-WOK
+    
+        WOK
+      end
+      def manifest_cover_image_information(md)
+        cover_image=if defined? md.make.cover_image \
+        and @md.make.cover_image.is_a?(Hash) \
+        and md.make.cover_image[:cover] =~/\S+/
+          <<-WOK
+    
+          WOK
+        else ''
+        end
+      end
+      def manifest_content(dob,fn_base,hashtag=nil)
+         fn=fn_base + Sfx[:epub_xhtml]
+         name=hashtag ? fn + hashtag : fn
+        <<-WOK
+    
+        WOK
+      end
+      def manifest_images(imgs)
+        imgs=imgs + ['arrow_next_red.png','arrow_prev_red.png','arrow_up_red.png','bullet_09.png']
+        images=["    \n"]
+        imgs.each do |i|
+          image,type=/(\S+?)\.(png|jpg|gif)/.match(i)[1,2]
+          type=type.sub(/jpg/,'jpeg')
+          images<<<<-WOK
+    
+          WOK
+        end
+        images=images.join('')
+        images
+    end
+    def manifest_close
+      <<-WOK
+  
+        WOK
+      end
+      def spine_open
+        #spine: reading order of XHTML files from manifest, idref attribute refers back to id in manifest (exclude images, CSS etc.).
+        <<-WOK
+  
+        WOK
+      end
+      def spine_cover_image
+        <<-WOK
+    
+        WOK
+      end
+      def spine_sisu_toc
+        <<-WOK
+    
+        WOK
+      end
+      def spine(dob,fn_base,hashtag=nil)
+         fn=fn_base + Sfx[:epub_xhtml]
+         name=hashtag ? fn + hashtag : fn
+        <<-WOK
+    
+        WOK
+      end
+      def spine_close
+        <<-WOK
+  
+        WOK
+      end
+      def guide_open
+        #guide: presentation order of XHTML files by reader).
+        <<-WOK
+  
+        WOK
+      end
+      def guide_cover_image
+        <<-WOK
+    
+        WOK
+      end
+      def guide_sisu_toc
+        <<-WOK
+    
+        WOK
+      end
+      def guide(dob,fn_base,hashtag=nil)
+         fn=fn_base + Sfx[:epub_xhtml]
+         name=hashtag ? fn + hashtag : fn
+        name=name ? name : dob.name
+        guide_name=(name =~/#{Sfx[:epub_xhtml]}/) ? name : (name + Sfx[:epub_xhtml])
+        <<-WOK
+    
+        WOK
+      end
+      def guide_close
+        <<-WOK
+  
+        WOK
+      end
+      self
+    end
+    def table_close
+      %{  
+#{@vz.table_close}}
+    end
+    def xhtml_close
+    %{#{@vz.credits_sisu_epub}
+  
+}
+    end
+  end
+  class HeadToc < HeadInformation
+    def initialize(md)
+      super(md)
+      @md=md
+      @tocband_segtoc=make_seg
+    end
+    def manifest_link(text)
+  %{ #{text}}
+    end
+    def concordance_link(text)
+      if @md.concord_make
+  %{
+      #{text}
+    }
+      else ''
+      end
+    end
+    def head
+      %{#{doc_type}
+  
+    
+    #{@css.xhtml_epub}
+  
+  #{@vz.color_body}}
+    end
+    def concordance
+      if @md.concord_make
+      %{#{@vz.margin_css}
+      

+        
+          Concordance
+        
+      

+#{@vz.table_close}}
+      else
+      %{#{@vz.margin_css}
+#{@vz.table_close}}
+      end
+    end
+    def links_guide_open(type='horizontal')
+      (type=='vertical') \
+      ? links_guide_vertical_open
+      : links_guide_horizontal_open
+    end
+    def prefix_a
+    end
+    def rights
+      def all
+        rights=SanitizeXML.xml(@md.rights.all)
+        %{
Rights: #{rights}
}
+      end
+      self
+    end
+    def prefix_b
+      %{
Prefix: #{@md.prefix_b}}
+    end
+    def make_seg
+      concord=concordance_link(@vz.nav_txt_concordance)
+      %{
+
+

+  #{@vz.nav_txt_toc_link}
+
+  
+   #{concord}
+#{@vz.table_close}}
+    end
+    def manifest #check structure
+      manifest=manifest_link(@vz.nav_txt_manifest)
+      %{#{@vz.margin_txt_3}
+  #{@vz.paragraph_font_small}
+   #{manifest}
+    
+#{@vz.table_close}}
+    end
+    def concordance #check structure
+      concord=concordance_link(@vz.nav_txt_concordance)
+      %{#{@vz.margin_txt_3}
+  #{@vz.paragraph_font_small}
+   #{concord}
+    
+#{@vz.table_close}}
+    end
+    def metadata
+      %{#{@vz.margin_css}
+  

+    
+      MetaData
+    
+  

+#{@vz.table_close}}
+    end
+  end
+  class HeadSeg < HeadInformation
+    def initialize(md)
+      super(md)
+    end
+    def head
+      %{#{doc_type}
+  
+    
+      #{@seg_name_xhtml[@seg_name_xhtml_tracker]} -
+      #{@md.html_title}
+    
+    
+    #{@css.xhtml_epub}
+  
+  #{@vz.color_body}}
+    end
+    def endnote_mark
+%{
+  

+}
+    end
+  end
+  class HeadScroll < HeadToc
+    def initialize(md)
+      super(md)
+    end
+    def toc_owner_details
+      %{#{@vz.margin_txt_3}
+#{@vz.paragraph_font_small}
+  
+    Owner Details
+    
+      #{$ep[:hsp]*3}
+    
+  
+  
+#{@vz.table_close}}
+    end
+  end
+  class FormatTextObject
+    @vz=SiSU_Viz::Defaults.new
+    attr_accessor :md,:t_o,:txt,:ocn,:format,:table,:link,:linkname,:paranum,:p_num,:headname,:banner,:url
+    def initialize(md,t_o)
+      @md,@t_o=md,t_o
+      if t_o.is_a?(Hash)
+        @txt            =t_o[:txt]            || nil
+        @ocn            =t_o[:ocn]            || nil
+        @ocn_display    =t_o[:ocn_display]    || nil
+        @headname       =t_o[:headname]       || nil
+        @trailer        =t_o[:trailer]        || nil
+        @endnote_part_a =t_o[:endnote_part_a] || nil
+        @endnote_part_b =t_o[:endnote_part_b] || nil
+        @lnk_url        =t_o[:lnk_url]        || nil
+        @lnk_txt        =t_o[:lnk_txt]        || nil
+        @format         =t_o[:format]         || nil
+        @target         =t_o[:target]         || nil #occasionally passed but not used
+        if @format and not @format.empty?
+          if @format=~/^\d:(\S+)/ #need more reliable marker #if @format =~ /#{Rx[:lv]}/
+            headname=$1 #format[/\d~(\S+)/m,1]
+            @headname=(headname =~/^[a-zA-Z]/) \
+            ? %{}
+            : %{}
+            @headname=(headname =~/^[a-zA-Z]/) \
+            ? %{}
+            : %{}
+          end
+        end
+      elsif t_o.class.inspect =~/Object/
+        @dob=t_o if defined? t_o.is
+        @named=nametags_seg(@dob)
+        @txt=((defined? t_o.obj) ? t_o.obj : nil)
+        @ocn=((defined? t_o.ocn) ? t_o.ocn.to_s : nil)
+        @headname=((t_o.is==:heading and defined? t_o.name) ? t_o.name : nil)
+      else
+        if @md.opt.cmd =~/M/
+          p __FILE__ +':'+ __LINE__.to_s
+          p t_o.class
+          p caller
+        end
+      end
+      if @txt and not @txt.empty?
+        @txt=@txt.gsub(/#{Mx[:mk_o]}[-~]##{Mx[:mk_c]}/,'')
+      end
+      @p_num=ParagraphNumber.new(@md,@ocn)
+      @vz=SiSU_Viz::Defaults.new
+    end
+    def nametags_seg(dob) #FIX
+      tags=''
+      if defined? dob.tags \
+      and dob.tags.length > 0 # insert tags "hypertargets"
+        dob.tags.each do |t|
+          tags=tags +%{}
+        end
+      end
+      tags
+    end
+    def endnote_body
+      %{
+

+  #{@txt}
+

+}
+    end
+    def endnote_body_indent
+      %{
+  

+    #{@txt}
+  

+}
+    end
+    def no_paranum
+      %{
+

+  
+  

+    #{@txt}
+  

+

+}
+    end
+    def para_form_css(tag,attrib,txt)                                                    # regular paragraphs shaped here
+      ul=ulc=''
+      ul,ulc="
    \n  ","\n  
" if @tag =~/li/
+      %{
+

+  #{@p_num.ocn_display}
+  #{ul}<#{tag} class="#{attrib}" #{@p_num.id}>
+    #{@named}#{txt}
+  #{ulc}
+

+}
+    end
+    def para
+      para_form_css('p','norm',@txt)
+    end
+    def group
+      para_form_css('p','group',@txt)
+    end
+    def block
+      para_form_css('p','block',@txt)
+    end
+    def alt
+      para_form_css('p','alt',@txt)
+    end
+    def verse
+      para_form_css('p','verse',@txt)
+    end
+    def code
+      para_form_css('p','code',@txt)
+    end
+    def center
+      para_form_css('p','center',@txt)
+    end
+    def bold
+      para_form_css('p','bold',@txt)
+    end
+    def bullet
+      para_form_css('li','bullet',@txt)
+    end
+    def table
+      @txt=if @t_o.obj !~/^
').
+        gsub(/#{Mx[:br_obj]}/,'

')
+      para_form_css('p','norm',@txt)
+    end
+    def format(tag,attrib)
+      para_form_css(tag,attrib,@txt)
+    end
+    def title_heading(tag,attrib)
+      %{
+

+<#{tag} class="#{attrib}">
+    #{@named}#{@txt}
+  
+

+}
+    end
+    def title_heading1
+      id_u=DISABLE[:epub][:per_section_title] \
+      ? ''
+      : title_heading('h1','tiny')
+    end
+    def title_heading2
+      id_u=DISABLE[:epub][:per_section_title] \
+      ? ''
+      : title_heading('h2','tiny')
+    end
+    def title_heading3
+      id_u=DISABLE[:epub][:per_section_title] \
+      ? ''
+      : title_heading('h3','tiny')
+    end
+    def title_heading4
+      ''
+    end
+    def seg_heading_sub(tag,attrib,txt)
+      txt=txt.gsub(/(?:#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})\s*/m,' ')
+      %{
+

+  #{@p_num.ocn_display}
+  <#{tag} class="#{attrib}" #{@p_num.id}>#{@p_num.name}
+    #{@named}#{@txt}
+  
+

+}
+    end
+    def seg_heading4
+      %{
+

+  #{@p_num.ocn_display}
+  

+    #{@txt}
+  

+

+}
+    end
+    def seg_heading5
+      seg_heading_sub('p','bold',@txt)
+    end
+    def seg_heading6
+      seg_heading_sub('p','bold',@txt)
+    end
+    def dl #check :trailer
+      "
#{@txt} #{@trailer}
"
+    end
+    def table_css_end
+      '

+    

+  '
+    end
+    def gsub_body #unused
+      @txt=case @txt
+      when /^(?:#{Mx[:pa_o]}i[1-9]#{Mx[:pa_c]}\s*)?\((i+|iv|v|vi+|ix|x|xi+)\)/
+        @txt.gsub(/^\((i+|iv|v|vi+|ix|x|xi+)\)/,'(\1)').
+          gsub(/^(#{Mx[:pa_o]}i[1-9]#{Mx[:pa_c]})\s*\((i+|iv|v|vi+|ix|x|xi+)\)/,'\1(\2)')
+      when /^(?:#{Mx[:pa_o]}i[1-9]#{Mx[:pa_c]}\s*)?\(?(\d|[a-z])+\)/
+        @txt.gsub(/^\((\d+|[a-z])+\)/,'(\1)').
+          gsub(/^(#{Mx[:pa_o]}i[1-9]#{Mx[:pa_c]})\s*\((\d+|[a-z])+\)/,'\1(\2)')
+      when /^\s*\d{1,3}\.\s/
+        @txt.gsub(/^\s*(\d+\.)/,'\1')
+      when /^\s*[A-Z]\.\s/
+        @txt.gsub(/^\s*([A-Z]\.)/,'\1')
+      else @txt
+      end
+    end
+    def bold_para
+      %{#{@vz.margin_txt_0}
+  

+    #{@txt}
+  

+#{@vz.margin_num_css}
+  #{$ep[:hsp]*3}
+#{@vz.table_close}}
+    end
+    def bold_heading #unused
+      @txt=@txt.gsub(/[1-9]~\S+/,'').
+        gsub(/[1-9]~/,'')
+      %{

+    #{@txt}
+  

+#{@vz.margin_num_css}
+  #{$ep[:hsp]*3}
+#{@vz.table_close}}
+    end
+    def toc_head_copy_at
+      @txt=SanitizeXML.xml(@txt)
+      %{
#{@txt}
\n}
+    end
+    def center
+      @txt=SanitizeXML.xml(@txt)
+      %{
#{@txt}
\n}
+    end
+    def bold
+      @txt=SanitizeXML.xml(@txt)
+      %{
#{@txt}
\n}
+    end
+    def center_bold
+      @txt=SanitizeXML.xml(@txt)
+      %{
#{@txt}
\n}
+    end
+  end
+  class FormatScroll < FormatTextObject
+    def initialize(md,txt)
+      super(md,txt)
+      @vz=SiSU_Viz::Defaults.new
+    end
+  end
+  class FormatSeg < FormatTextObject
+    def initialize(md,txt)
+      super(md,txt)
+    end
+    def endnote_seg_body(fn='')  #FIX                                                #url construction keep within single line... BUG WATCH 200408
+      fn='doc' if fn.to_s.empty? #you may wish to reconsider, sends to 'doc' where no segment info
+      %{
+  

+    #{@endnote_part_a}#{fn}#{Sfx[:epub_xhtml]}#{@endnote_part_b}
+  

+}
+    end
+    def clean(txt)
+      txt=txt.gsub(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/,'').
+        gsub(/#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/,'')
+    end
+    def subtoc_lev(tag,attrib)
+      @txt=clean(@txt)
+      txt=if @txt \
+      and @txt =~/<\/?i>|/mi
+        @txt.gsub(/<\/?i>|/mi,'') #removes name markers from subtoc, go directly to substantive text
+      else @txt
+      end
+      note=''
+      if txt =~/(#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})/m # had \s* at end
+        note=$1
+        note=note.gsub(/[\n\s]+/m,' ')
+        txt=txt.gsub(/(?:#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})\s*/m,' ').
+          gsub(/#{$ep[:hsp]}\d+<\/sup>#{$ep[:hsp]}/m,'').
+          gsub(/#{$ep[:hsp]}\d+<\/sup>#{$ep[:hsp]}/m,'') #remove
+      end
+      %{<#{tag} class="#{attrib}">
+    #{txt} #{note}
+  }
+    end
+    def subtoc_lev5
+      subtoc_lev('h5','subtoc') if @txt
+    end
+    def subtoc_lev6
+      subtoc_lev('h6','subtoc') if @txt
+    end
+    def heading_sub(tag,attrib,txt)
+      txt=txt.gsub(/(?:#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})\s*/m,' ')
+      %{
+

+  #{@p_num.ocn_display}
+  <#{tag} class="#{attrib}" #{@p_num.id}> #{@headname}
+    #{@txt}
+  
+

+}
+    end
+    def heading4
+      %{
+

+  #{@p_num.ocn_display}
+  

+    #{@t_o[:format]}
+    #{@txt}
+  

+

+}
+    end
+    def heading5
+      heading_sub('p','bold',@txt)
+    end
+    def heading6
+      heading_sub('p','bold',@txt)
+    end
+    def navigation_heading4
+      %{
+
+}
+    end
+    def scroll(text)
+      if @md.fns =~ /\.(?:-|ssm\.)?sst$/
+        scroll=%{
+}
+      end
+    end
+    def seg(text)
+      %{
+}
+    end
+    def search
+      if @make.build.html_search_form?
+        env=SiSU_Env::InfoEnv.new(@md.fns,@md)
+        env.widget.search_form('sisusearch',nil,nil,true)
+      else ''
+      end
+    end
+    def manifest(page=:seg)
+      if @make.build.links_to_manifest? \
+      and not @o_str.dump_or_redirect?
+        manifest_lnk=if @file.output_dir_structure.by_language_code? \
+        or @file.output_dir_structure.by_filetype?
+          "#{Xx[:html_relative1]}manifest/#{@file.base_filename.manifest}"
+        else @file.base_filename.manifest
+        end
+        if page==:manifest
+          manifest_lnk="#{@md.file.output_path.manifest.url}/#{@file.base_filename.manifest}"
+          brace_url=SiSU_Viz::Defaults.new.url_decoration
+          %{}
+        else
+          %{}
+        end
+      else ''
+      end
+    end
+    def pdf #retired 2.7.9
+      pdf=if @md.programs[:pdf] \
+      and @cf_defaults.cf_0 =~/p/
+        %{
+
+
+}
+      else ''
+      end
+    end
+  end
+  class XML
+  end
+  class HeadToc < HeadInformation
+    def initialize(md)
+      super(md)
+      @md=md
+      @o_str ||=SiSU_Env::ProcessingSettings.new(md).output_dir_structure
+      @make=SiSU_Env::ProcessingSettings.new(@md)
+      @file=SiSU_Env::FileOp.new(@md)
+    end
+    def scroll_head_navigation_band
+      if @make.build.html_top_band?
+        search_and_manifest=<
+  #{make_scroll_search_form_and_manifest_link}
+
+WOK
+        %{

+

+  #{@txt}
+

+#{@vz.table_close}}
+    end
+    def navigation_heading5
+      %{

+  #{@txt}
+
}
+    end
+    def navigation_heading6
+      %{

+  #{@txt}
+
}
+    end
+    def navigation_center
+      %{
#{@txt}
}
+    end
+  end
+  class FormatToc < FormatTextObject
+    def initialize(md,txt)
+      super(md,txt)
+    end
+    def links_guide
+      %{  
  • 
+    
+      #{@lnk_txt}
+    
+  
    • 
+}
+    end
+    def lev(tag,attrib)
+      if @txt
+        %{<#{tag} class="#{attrib}">
+    #{@txt}
+  
+}
+      else ''
+      end
+    end
+    def lev1
+      lev('h1','toc')
+    end
+    def lev2
+      lev('h2','toc')
+    end
+    def lev3
+      lev('h3','toc')
+    end
+    def lev4
+      lev('h4','toc')
+    end
+    def lev5
+      lev('h5','toc')
+    end
+    def lev6
+      lev('h6','toc')
+    end
+    def lev0 #docinfo
+      lev('h0','toc')
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/epub_segments.rb b/lib/sisu/v5/epub_segments.rb
new file mode 100644
index 00000000..64bea759
--- /dev/null
+++ b/lib/sisu/v5/epub_segments.rb
@@ -0,0 +1,539 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: epub segment generation, processing
+
+=end
+module SiSU_EPUB_Seg
+  require_relative 'shared_xhtml'                       # shared_xhtml.rb
+  require_relative 'epub'                               # epub.rb
+  require_relative 'shared_metadata'                    # shared_metadata.rb
+  class Output
+    def initialize(md,outputfile,seg,type='')
+      @md,@output_epub_cont_seg,@seg,@type=md,outputfile,seg,type
+    end
+    def output #CONSIDER
+      if @seg[:title] =~/\S/
+        filename_seg=[]
+        filename_seg << @seg[:title] << @seg[:nav]
+        if @type=='endnotes'
+          @seg[:headings]=[] #watch
+          txt_obj={ txt: 'Endnotes', ocn_display: ''}
+          format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+          @seg[:headings] << format_seg.title_heading1
+          filename_seg << @seg[:heading_endnotes] << @seg[:headings] << %{\n
    \n} << @seg[:endnote_all] << '
    '
+        elsif @type=='idx'
+          @seg[:headings]=[]
+          format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+          @seg[:headings] << format_seg.title_heading1
+          filename_seg << @seg[:heading_idx] << @seg[:headings] << %{\n
    \n} << @seg[:idx] << '
    '
+        elsif @type=='metadata'
+          metadata=SiSU_Metadata::Summary.new(@md).xhtml_display.metadata
+          @seg[:headings]=[]
+          txt_obj={ txt: 'Metadata', ocn_display: ''}
+          format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+          @seg[:headings] << format_seg.title_heading1
+          filename_seg << @seg[:heading_idx] << @seg[:headings] << %{\n
    \n} << metadata << '
    '
+        elsif @type=='sisu_manifest'
+          env=SiSU_Env::InfoEnv.new(@md.fns)
+          path_and_name,url_and_name="#{env.path.output}/#{@md.fnb}/sisu_manifest.html","#{env.url.root}/#{@md.fnb}/sisu_manifest.html"
+          manifest=if FileTest.file?("#{path_and_name}")==true
+            <A list of available output types may be available at the following url:
    
+
    #{url_and_name}
    
+WOK
+          else ''
+          end
+          @seg[:headings]=[]
+          txt_obj={ txt: 'Manifest', ocn_display: ''}
+          format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+          @seg[:headings] << format_seg.title_heading1
+          filename_seg << @seg[:heading_idx] << @seg[:headings] << %{\n
    \n} << manifest << '
    '
+        else
+          filename_seg << @seg[:headings] << @seg[:main] << "\n\n"
+        end
+        filename_seg << @seg[:tail] << @seg[:nav] << @seg[:close]
+        filename_seg=filename_seg.flatten.compact #watch
+        filename_seg.each do |str|
+          unless str =~/\A\s*\Z/
+            @output_epub_cont_seg << str.strip
+          end
+        end
+        @output_epub_cont_seg.close
+      end
+    end
+  end
+  class Seg
+    @@seg,@@seg_subtoc,@@seg_endnotes,@@seg_ad={},{},{},{}
+    @@seg_name,@@seg_name_xhtml=[],[]
+    @@seg_url=@@fn=@@get_hash_to=@@get_hash_fn=''
+    @@loop_count=@@seg_total=@@tracker=0
+    @@is4=@@is3=@@is2=@@is1=0
+    @@heading1=@@heading2=@@heading3=@@heading4=0
+    @@seg[:headings],@@seg[:main],@@seg[:idx],@@seg[:tail],@@seg_subtoc_array,@@seg_endnotes_array,@@seg[:endnote_all]=Array.new(7){[]}
+    @@seg[:heading_endnotes]=''
+    @@tablehead,@@number_of_cols=0,0
+    @@fns_previous=''
+    attr_reader :seg_name_xhtml,:seg_name_xhtml_tracker
+    def initialize(md='',data='')
+      @md,@data=md,data
+      @vz=SiSU_Viz::Defaults.new
+      @seg_name_xhtml=@@seg_name_xhtml || nil
+      @seg_name_xhtml_tracker=@@tracker || nil
+      @make=SiSU_Env::ProcessingSettings.new(@md) if @md
+    end
+    def songsheet
+      begin
+        data=get_subtoc_endnotes(@data)
+        data=articles(data)
+        SiSU_EPUB_Seg::Seg.new.cleanup # (((( added ))))
+        #### (((( END )))) ####
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+        @@seg_name=[]
+      end
+    end
+  protected
+    def articles(data)
+      track,tracking,newfile=0,0,0
+      @@is4=@@is3=@@is2=@@is1=0
+      printed_endnote_seg='n'
+      idx_xhtml=nil
+      if @md.book_idx
+        idx_xhtml=SiSU_Particulars::CombinedSingleton.instance.get_idx_xhtml(@md).xhtml_idx
+        idx_xhtml.each {|x| @@seg[:idx] << x }
+        @@seg[:heading_idx]=''
+      end
+      data.each do |dob|
+        if (dob.is==:heading \
+        || dob.is==:heading_insert) \
+        and dob.ln==4
+          @@seg_name << dob.name
+          seg_name=dob.name
+        end
+      end
+      @@seg_name_xhtml=@@seg_name
+      @@seg_total=@@seg_name.length
+      testforartnum=@@seg_name_xhtml
+      SiSU_Screen::Ansi.new(@md.opt.cmd,@@seg_name.length) if @md.opt.cmd =~/[MVv]/
+      map_nametags=SiSU_Particulars::CombinedSingleton.instance.get_map_nametags(@md).nametags_map #p map_nametags
+      data.each do |dob|
+        #if defined? dob.obj \
+        #and dob.obj =~/href="#{Xx[:segment]}#+\S+?"/
+        #  ##Consider: remove, reinstate earlier?
+        #  #while dob.obj =~/href="#{Xx[:segment]}#+(\S+?)"/
+        #  #  m=$1
+        #  #  if map_nametags[m][:segname]
+        #  #    dob.obj=dob.obj.sub(/href="#{Xx[:segment]}#+(\S+?)"/,%{href="#{map_nametags[m][:segname]}#{Sfx[:html]}#\\1"})
+        #  #  else
+        #  #    p "NOT FOUND name_tags: #{m}"
+        #  #    dob.obj=dob.obj.sub(/href="#{Xx[:segment]}#+(\S+?)"/,%{href="#\\1"}) # not satisfactory
+        #  #  end
+        #  #end
+        #end
+        if (dob.is==:heading \
+        || dob.is==:heading_insert) \
+        and dob.ln==4
+          if dob.ocn==0
+            @@heading4=dob.obj
+          else @@heading4=dob.obj
+          end
+          @@is4=newfile=1
+        end
+        if (dob.is==:heading \
+        || dob.is==:heading_insert) \
+        and dob.ln==3
+          @@heading3=dob.obj
+          @@is4,@@is3=0,1
+        end
+        if (dob.is==:heading \
+        || dob.is==:heading_insert) \
+        and dob.ln==2
+          @@heading2=dob.obj
+          @@is4,@@is3,@@is2=0,0,1
+        end
+        if (dob.is==:heading \
+        || dob.is==:heading_insert) \
+        and dob.ln==1
+          @@heading1=dob.obj
+          @@is4,@@is3,@@is2,@@is1=0,0,0,1
+        end
+        if (@@is1 && !@@is2 && !@@is3 && !@@is4)
+          if not (dob.is==:heading \
+          || dob.is==:heading_insert) \
+          and dob.ln==1
+            head1=$_ #; check
+          end
+        end
+        if @@is4==1
+          dir_epub_cont="#{@md.env.processing_path.epub}/#{Ep[:d_oebps]}"
+          if newfile==1 \
+          or dob.obj =~/^#{Mx[:br_endnotes]}|^#{Mx[:br_eof]}/
+            newfile=0
+            if (dob.is==:heading \
+            || dob.is==:heading_insert) \
+            and dob.ln==4
+              if tracking != 0
+                SiSU_EPUB_Seg::Seg.new(@md).tail
+                segfilename="#{dir_epub_cont}/#{@@seg_name_xhtml[tracking-1]}#{Sfx[:epub_xhtml]}"
+                output_epub_cont_seg=File.new(segfilename,'w') if @@seg_name_xhtml[tracking-1]
+                if dob.is==:heading \
+                or @@seg_name_xhtml[tracking-1] !~/endnotes|book_index|metadata/
+                  SiSU_EPUB_Seg::Output.new(@md,output_epub_cont_seg,@@seg).output
+                elsif dob.is==:heading_insert
+                  if @@seg_name_xhtml[tracking-1]=='endnotes'
+                    SiSU_EPUB_Seg::Output.new(@md,output_epub_cont_seg,@@seg,'endnotes').output
+                  elsif @@seg_name_xhtml[tracking-1]=='book_index'
+                    SiSU_EPUB_Seg::Output.new(@md,output_epub_cont_seg,@@seg,'idx').output
+                    @@seg[:idx]=[]
+                  elsif @@seg_name_xhtml[tracking-1]=='metadata' # navigation bug FIX
+                    SiSU_EPUB_Seg::Output.new(@md,output_epub_cont_seg,@@seg,'metadata').output
+                  else puts "#{__FILE__}::#{__LINE__}"
+                  end
+                else puts "#{__FILE__}::#{__LINE__}"
+                end
+                SiSU_EPUB_Seg::Seg.new.reinitialise
+                heading_art(dob)
+                head(dob)
+                if @@seg_name_xhtml[tracking] =='metadata'
+                  segfilename="#{dir_epub_cont}/#{@@seg_name_xhtml[tracking]}#{Sfx[:epub_xhtml]}"
+                  output_epub_cont_seg=File.new(segfilename,'w')
+                  SiSU_EPUB_Seg::Output.new(@md,output_epub_cont_seg,@@seg,'metadata').output
+                  SiSU_EPUB_Seg::Seg.new.reinitialise #BUG navigation bug with items following metadata, and occurring before manifest, this becomes a bug ... work area for book index, FIX
+                end
+               #@output_epub_cont_seg.close                                         #%(((( EOF )))) -->
+              end
+              if tracking==0
+                heading_art(dob)
+                head(dob)
+              end
+            end
+            tracking=tracking+1
+          end
+          if (dob.is==:heading \
+          || dob.is==:heading_insert) \
+          and dob.ln==4 \
+          and dob.name
+            @@get_hash_to=dob.name
+            @@get_hash_fn=dob.name
+          end
+          if dob.obj.is_a?(String)
+            markup(dob)
+          elsif dob.obj.is_a?(Array)
+            dob.obj.each do |pg|
+              markup(pg)
+            end
+          end
+          if testforartnum[tracking-1] =~/endnote/
+            if printed_endnote_seg=='n'
+              printed_endnote_seg='y'
+            end
+          end
+        end
+      end
+      data
+    end
+    def heading_art(dob)
+      @@seg[:title]=SiSU_EPUB_Format::HeadSeg.new(@md).head
+    end
+    def head(dob)
+      clean=/|<:.*?>$/
+      @p_num ||= ''
+      if @@is1==1
+        if defined? @md.creator.author \
+        and @md.creator.author
+          @author=%{#{@md.creator.author}\n}
+        end
+        ocn=(@@heading1[/.+?#{Mx[:id_o]}~(\d+);(?:[oh]|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}#{@dp}:#{@dp}#{Mx[:id_c]}$/]) \
+        ? $1
+        : ''
+        @p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        txt_obj={ txt: @@heading1, ocn_display: @p_num.ocn_display }
+        format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+        @@seg[:headings] << format_seg.title_heading1.gsub(clean,'')
+        @@heading1=@@heading1.gsub(/#{$ep[:hsp]}#{$ep[:hsp]}[\d*+]+<\/sup>#{$ep[:hsp]}<\/a>/,'')
+      end
+      if @@is2==1
+        heading2=@@heading2
+        ocn=(heading2[/.+?#{Mx[:id_o]}~(\d+);(?:[oh]|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}#{@dp}:#{@dp}#{Mx[:id_c]}$/]) \
+        ? $1
+        : ''
+        @p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        txt_obj={ txt: heading2, ocn_display: @p_num.ocn_display }
+        format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+        @@seg[:headings] << format_seg.title_heading2.gsub(clean,'')
+        @@heading2=@@heading2.gsub(/#{$ep[:hsp]}#{$ep[:hsp]}[\d*+]+<\/sup>#{$ep[:hsp]}<\/a>/,'')
+      end
+      if @@is3==1
+        heading3=@@heading3
+        ocn=(heading3[/.+?#{Mx[:id_o]}~(\d+);(?:[oh]|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}#{@dp}:#{@dp}#{Mx[:id_c]}$/]) \
+        ? $1
+        : ''
+        @p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        txt_obj={ txt: heading3, ocn_display: @p_num.ocn_display }
+        format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+        @@seg[:headings] << format_seg.title_heading3.gsub(clean,'')
+        @@heading3=@@heading3.gsub(/#{$ep[:hsp]}#{$ep[:hsp]}[\d*+]+<\/sup>#{$ep[:hsp]}<\/a>/,'')
+      end
+      if @@is4==1
+        heading4=@@heading4
+        ocn=(heading4[/.+?#{Mx[:id_o]}~(\d+);(?:[oh]|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}#{@dp}:#{@dp}#{Mx[:id_c]}$/]) \
+        ? $1
+        : ''
+        @p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,ocn)
+        txt_obj={ txt: heading4, ocn_display: @p_num.ocn_display }
+        format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+        @@seg[:headings] << format_seg.title_heading4.gsub(clean,'')
+      end
+      @@tracker=@@tracker+1
+    end
+    def markup(dob)
+      @debug=[]
+      if dob.is ==:heading \
+      || dob.is ==:heading_insert \
+      || dob.is ==:para
+        #extend as necessary FIX
+        @p_num=SiSU_EPUB_Format::ParagraphNumber.new(@md,dob.ocn)
+      end
+      sto=SiSU_EPUB_Format::FormatTextObject.new(@md,dob)
+      dob_xhtml=if dob.is==:heading \
+      || dob.is==:heading_insert \
+      || dob.is==:para
+        dob_xhtml=if dob.is==:heading \
+        or dob.is==:heading_insert
+          if dob.ln==4
+            sto.seg_heading4 # work on see SplitTextObject
+          elsif dob.ln==5
+            sto.seg_heading5
+          elsif dob.ln==6
+            sto.seg_heading6
+          end
+        elsif dob.is==:para
+          if dob.indent \
+          and dob.hang \
+          and dob.indent =~/[0-9]/ \
+          and dob.hang =~/[0-9]/
+            if dob.bullet_
+              (dob.indent =~/[1-9]/) \
+              ? sto.format('li',"i#{dob.indent}")
+              : sto.format('li','bullet')
+            elsif dob.indent == dob.hang
+              sto.format('p',"i#{dob.indent}")
+            elsif dob.indent != dob.hang
+              sto.format('p',"h#{dob.hang}i#{dob.indent}")
+            else sto.para
+            end
+          else sto.para
+          end
+        end
+        elsif dob.is ==:block \
+        || dob.is ==:group \
+        || dob.is ==:alt
+          sto.para #fix this should be block type specific #FIX
+        elsif dob.is==:verse
+          sto.verse
+        elsif dob.is==:code
+          sto.code
+        elsif dob.is==:table
+          sto.table
+        elsif dob.is==:break
+          sto.break
+      end
+      if @md.flag_separate_endnotes # may need to revisit, check
+        dob.obj=dob.obj.gsub(/"\s+href="#note_ref(\d+)">/,%{" href=\"endnotes#{Sfx[:epub_xhtml]}#note_ref\\1">})       #endnote- twice #removed file type
+      end
+      if (dob.is ==:heading \
+      || dob.is==:heading_insert \
+      || dob.is==:para) \
+      && (not dob.ocn or dob.ocn.to_s.empty?)
+        format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,dob)
+      end
+      if (dob.is==:heading \
+      || dob.is==:heading_insert \
+      || dob.is==:para) \
+      and dob.note_ #dob.obj =~/ \n}
+        @@seg[:main] << dob_xhtml
+        if @make.build.segsubtoc?
+          @@seg[:main] << @@seg_subtoc[@@get_hash_fn]                       #% insertion of sub-toc
+        end
+      else
+        @@seg[:main] << dob_xhtml
+      end
+    end
+    def tail
+      format_head_seg=SiSU_EPUB_Format::HeadSeg.new(@md)
+      if @md.flag_auto_endnotes \
+      and @@seg_endnotes[@@get_hash_fn]
+        @@seg[:tail] <<  %{\n
    \n
    \n}
+        if @@seg_endnotes[@@get_hash_fn].flatten.length > 0
+          @@seg[:tail] << format_head_seg.endnote_mark
+          @@seg[:tail] << @@seg_endnotes[@@get_hash_fn].flatten #endnotes deposited at end of individual segments ||@|EXTRACTION OF ENDNOTES|
+        end
+        @@seg[:tail] << '
    '
+        @@seg[:tail] << '
    ' #this div closes div class content
+      end
+      @@seg[:close]=[]
+      @@seg[:close] << format_head_seg.xhtml_close
+    end
+    def reinitialise
+      @@seg[:headings],@@seg[:main],@@seg[:tail],@@seg[:credits]=Array.new(4){[]}
+    end
+    def cleanup
+      reinitialise
+      @@seg_total,@@tracker=0,0
+      @@seg_endnotes,@@seg_subtoc={},{}
+      @@seg_endnotes_array,@@seg_subtoc_array=[],[]
+      @@seg[:endnote_all]=[]
+    end
+    def get_subtoc_endnotes(data) #get endnotes & sub-table of contents subtoc
+      data.each do |dob|
+        dob.obj=dob.obj.gsub(/    (.+?)<\/a>/mi,'\1')
+        if @md.flag_auto_endnotes
+          if (dob.is==:heading \
+          || dob.is==:heading_insert) \
+          and dob.ln.to_s =~/^[1234]/ \
+          and not @@fn.to_s.empty?
+            @@seg_endnotes[@@fn]=[]
+            @@seg_endnotes[@@fn] << @@seg_endnotes_array
+            @@seg_endnotes_array=[] if dob.ln==4
+            @@fns_previous=@md.fns if dob.ln==4 and dob.name =~/^meta/
+          end
+          if (dob.is==:heading \
+          || dob.is==:heading_insert) \
+          and dob.ln==4                                              #%  EXTRACTION OF SUB-TOCs & SEGMENT NAME, after EXTRACTION OF ENDNOTES & SUB-TOCs
+            @@seg_subtoc[@@fn]=@@seg_subtoc_array
+            @@seg_subtoc_array=[]
+            if dob.name \
+            and dob.obj
+              @@fn=dob.name
+            else
+              @@fn=(dob.name =~/\S+/) \
+              ? dob.name
+              : ''
+            end
+          end
+        end
+        if dob.is==:heading \
+        and dob.ln.to_s =~/^[56]/
+          case dob.ln
+          when 5
+            format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,dob)
+            subtoc=format_seg.subtoc_lev5 #keep and make available, this is the subtoc
+          when 6
+            format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,dob)
+            subtoc=format_seg.subtoc_lev6 #keep and make available, this is the subtoc
+          end
+          @@seg_subtoc_array << subtoc
+        end
+        if @md.flag_auto_endnotes
+          ast,pls='*','+'
+          if dob.obj =~/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})(?:\d|#{ast}|#{pls})+ / \
+          and dob.is !=:code # endnote-
+            endnote_array=[]
+            if dob.obj=~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/m
+              endnote_array << dob.obj.scan(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/m)
+            end
+            if dob.obj=~/#{Mx[:en_b_o]}#{ast}\d+\s.+?#{Mx[:en_b_c]}/m
+              endnote_array << dob.obj.scan(/#{Mx[:en_b_o]}#{ast}\d+\s.+?#{Mx[:en_b_c]}/m)
+            end
+            if dob.obj=~/#{Mx[:en_b_o]}#{pls}\d+\s.+?#{Mx[:en_b_c]}/m
+              endnote_array << dob.obj.scan(/#{Mx[:en_b_o]}#{pls}\d+\s.+?#{Mx[:en_b_c]}/m)
+            end
+            endnote_array=endnote_array.flatten #.compact #check compacting
+            endnote_array.each do |note|
+              note_match=note.dup
+              note_match_seg=note.dup
+              e_n=note_match_seg[/(?:#{Mx[:en_a_o]}(?:\d|#{ast}|#{pls})+|#{Mx[:en_b_o]}(?:#{ast}|#{pls})\d+)\s+(.+?)(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/m,1]
+              try=e_n.split(/
    /)
+              try.each do |e|
+                txt_obj={ txt: e }
+                format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+                note_match=if e =~/#{Mx[:pa_o]}i[1-9]#{Mx[:pa_c]}/
+                  format_seg.endnote_body_indent
+                else format_seg.endnote_body
+                end
+                @@seg_endnotes_array << note_match
+              end
+              try.join('
    ')
+              #% creation of separate end segment/page of all endnotes referenced back to reference segment
+              m=/(?:#{Mx[:en_a_o]}(?:\d|#{ast}|#{pls})+|#{Mx[:en_b_o]}(?:#{ast}|#{pls})\d+)\s+(.+?href=")(#note_ref(?:\d|_a|_b)+".+)(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/mi
+              endnote_part_a=note_match_seg[m,1]
+              endnote_part_b=note_match_seg[m,2]
+              txt_obj={ endnote_part_a: endnote_part_a, endnote_part_b: endnote_part_b }
+              format_seg=SiSU_EPUB_Format::FormatSeg.new(@md,txt_obj)
+              note_match_all_seg=format_seg.endnote_seg_body(@@fn) #BUG WATCH 200408
+              @@seg[:endnote_all] << note_match_all_seg
+            end
+            dob.obj=dob.obj.gsub(/(?:#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]})\s*/m,' ')
+          end
+        end
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/epub_tune.rb b/lib/sisu/v5/epub_tune.rb
new file mode 100644
index 00000000..c0005365
--- /dev/null
+++ b/lib/sisu/v5/epub_tune.rb
@@ -0,0 +1,329 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: epub generation, epub pre-processing
+
+=end
+require_relative 'param'
+module SiSU_EPUB_Tune
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env; include SiSU_Screen
+  require_relative 'epub_format'                        # epub_format.rb #watch
+  @@line_mode=''
+  @@endnote_array=[]
+  @@endnote_call_counter=1
+  @@table_align='
+
+
+
    
+ \;'
+  @@table_align_close='
+
+    
    '
+  @@counter,@@column,@columns=0,0,0
+  class Output
+    def initialize(data,md)
+      @data,@md=data,md
+      @file=SiSU_Env::FileOp.new(@md)
+      @cX=SiSU_Screen::Ansi.new(@md.opt.cmd).cX
+    end
+    def hard_output
+      @filename_tune=@file.write_file_processing.html_tune
+      data=[]
+      @data.each {|x| x.obj.strip; data << x if not x.obj.empty?} #1.9 array?
+      data.each do |dob|
+        @filename_tune.puts dob, "\n" #check
+      end
+    end
+    def marshal
+      File.open(@file.marshal.html_tune,'w') {|f| Marshal.dump(@data.to_a,f)}
+    end
+  end
+  class CleanXHTML
+    def initialize(html='')
+      @html=html
+    end
+    def clean
+      html=@html
+      str=if html.is_a?(String)
+        html
+      else html.obj
+      end
+      str=str.gsub(/#{Mx[:gl_o]}(#[0-9]{3})#{Mx[:gl_c]}/u,'&\1;').
+        gsub(/#{Mx[:gl_o]}#([a-z]{2,4})#{Mx[:gl_c]}/u,'&\1;').
+        gsub(/
    /u,'
    ').
+        gsub(/#{Mx[:nbsp]}/u,$ep[:hsp])
+    end
+  end
+  class Tune
+    def initialize(data,md)
+      @data,@md=data,md
+      @vz=SiSU_Viz::Defaults.new
+      @env=SiSU_Env::InfoEnv.new(@md.fns)
+      @sys=SiSU_Env::SystemCall.new
+      @env=SiSU_Env::InfoEnv.new(@md.fns)
+      @brace_url=SiSU_Viz::Defaults.new.url_decoration
+    end
+    def songsheet
+      begin
+        @cX=SiSU_Screen::Ansi.new(@md.opt.cmd).cX
+        SiSU_Screen::Ansi.new(@md.opt.cmd,'Tune').txt_grey if @md.opt.cmd =~/[MVv]/
+        data=SiSU_EPUB_Tune::Tune.new(@data,@md).amp_angle_brackets
+        data=SiSU_EPUB_Tune::Tune.new(data,@md).endnotes_html
+        data=SiSU_EPUB_Tune::Tune.new(data,@md).url_markup
+        data=SiSU_EPUB_Tune::Tune.new(data,@md).markup
+        if @md.opt.cmd =~/M/  #Hard Output Tune Optional on/off here
+          data=SiSU_EPUB_Tune::Output.new(data,@md).hard_output
+          SiSU_EPUB_Tune::Output.new(data,@md).marshal
+        end
+        tuned=SiSU_EPUB_Tune::Tune.new(@data,@md).output
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    def markup
+      @tuned_file=[]
+      @data.each do |dob|
+        dob.obj=dob.obj.gsub(/#{Mx[:mk_o]}#([a-zA-Z]+)#{Mx[:mk_c]}/,'&\1;').
+          gsub(/#{Mx[:mk_o]}(#[0-9]+)#{Mx[:mk_c]}/,'&\1;')
+        dob.obj=dob.obj.gsub(/#{Mx[:br_line]}|#{Mx[:br_nl]}/,'
    ') unless dob.is==:table
+        dob.obj=dob.obj.gsub(/#{Mx[:fa_bold_o]}(.+?)#{Mx[:fa_bold_c]}/,'\1').
+          gsub(/#{Mx[:fa_italics_o]}(.+?)#{Mx[:fa_italics_c]}/,'\1').
+          gsub(/#{Mx[:fa_underscore_o]}(.+?)#{Mx[:fa_underscore_c]}/,'\1').
+          gsub(/#{Mx[:fa_superscript_o]}(.+?)#{Mx[:fa_superscript_c]}/,'\1').
+          gsub(/#{Mx[:fa_subscript_o]}(.+?)#{Mx[:fa_subscript_c]}/,'\1').
+          gsub(/#{Mx[:fa_insert_o]}(.+?)#{Mx[:fa_insert_c]}/,'\1').
+          gsub(/#{Mx[:fa_cite_o]}(.+?)#{Mx[:fa_cite_c]}/,'\1').
+          gsub(/#{Mx[:fa_strike_o]}(.+?)#{Mx[:fa_strike_c]}/,'\1').
+          gsub(/#{Mx[:fa_monospace_o]}(.+?)#{Mx[:fa_monospace_c]}/,'\1'). # tt, kbd
+          gsub(/#{Mx[:mk_o]}:name#(\S+?)#{Mx[:mk_c]}/,'').
+          gsub(/#{Mx[:gl_bullet]}/m,"●#{$ep[:hsp]*2}").
+          gsub(/#{Dx[:url_o]}/,Dx[:url_o_xml]).gsub(/#{Dx[:url_c]}/,Dx[:url_o_xml]).
+          gsub(/#{Mx[:nbsp]}/,$ep[:hsp]).
+          gsub(/<(p|br)>/,'<\1 />')
+        dob.obj=SiSU_EPUB_Tune::CleanXHTML.new(dob.obj).clean
+        @tuned_file << dob
+      end
+    end
+    def urls(data)
+      @words=[]
+      map_nametags=SiSU_Particulars::CombinedSingleton.instance.get_map_nametags(@md).nametags_map #p map_nametags
+      data.each do |word|
+        @words << if word=~/#{Mx[:lnk_o]}(.+?)#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}|image)/
+          http_=true
+          if word =~/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/
+            m,u=/#{Mx[:lnk_o]}(.+?)#{Mx[:lnk_c]}#{Mx[:url_o]}(\S+?)#{Mx[:url_c]}/.match(word).captures
+          elsif word =~/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}/
+            http_=false
+            m,u=/#{Mx[:lnk_o]}(.+?)#{Mx[:lnk_c]}#{Mx[:rel_o]}(\S+?)#{Mx[:rel_c]}/.match(word).captures
+          elsif word =~/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}image/
+            m,u=/#{Mx[:lnk_o]}(.+?)#{Mx[:lnk_c]}(image)/.match(word).captures
+          end
+          case m
+          when /\.png|\.jpg|\.gif|c=|\s\d+x\d+/
+            w,h=/\s(\d+)x(\d+)/.match(m).captures if m =~/\s\d+x\d+/
+            w=%{width="#{w}"} if w
+            h=%{height="#{h}"} if h
+            c=m[/"(.+?)"/m,1]
+            caption=%{
    #{c}
    } if c
+            png=m.scan(/\S+/)[0]
+            image_path=@md.file.output_path.epub.rel_image #image_path=@env.url.images_epub
+            ins=if u \
+            and u.strip !~/^image$/
+              %{    #{caption}}
+            else %{#{caption}}
+            end
+            word=word.gsub(/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/,ins)
+          else
+            link=m[/(.+)/m]
+            png=m.scan(/\S+/)[0].strip
+            link=link.strip
+            u=u.sub(/^#*/,'') #make neater
+            if map_nametags[u] \
+            and map_nametags[u][:segname]
+              u=unless http_
+                u=if u=~/^\d+$/
+                  u.gsub(/^(\d+)$/,"#{map_nametags[u][:segname]}#{Sfx[:xhtml]}#o\\1") if u !~/\//
+                else
+                  u.gsub(/(\S+)/,"#{map_nametags[u][:segname]}#{Sfx[:xhtml]}#\\1") if u !~/\//
+                end
+              else u
+              end
+            elsif u =~/^:/
+              u=u.gsub(/^:/,"#{@env.url.root}/")
+            elsif u =~/^\.\.\//
+              u=u.gsub(/^\.\.\//,"#{@env.url.root}/")
+            elsif u =~/https?:\/\//
+            else p "NOT FOUND name_tags: #{u}"
+            end
+            ins=%{#{link}}
+            word=word.gsub(/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/,ins).
+              gsub(/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}/,ins)
+          end
+          word
+        else word
+        end
+        word
+      end
+      @words=@words.join(' ')
+    end
+    def url_markup
+      data=@data
+      @tuned_file=[]
+      data.each do |dob|
+        unless dob.is==:code
+          if dob.obj =~/<::\s+/ #watch
+            dob.obj=dob.obj.gsub(/<::\s+(\S+?)\s+!>/,
+              %{\\1})
+          end
+          if dob.obj =~/<:image\s+/
+            dob.obj=dob.obj.gsub(/<:image\s+(http\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+>/,
+                %{}).
+              gsub(/<:image\s+(http\S+)\s+(\S+)\s+>/,
+                %{}).
+              gsub(/<:image\s+(\S+)\s+(\S+)\s+(\S+)\s+>/,
+                %{}).
+              gsub(/<:image\s+(\S+)\s+>/,
+                %{})
+          end
+          if dob.obj =~/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}|image)/
+            @word_mode=dob.obj.scan(/#{Mx[:lnk_o]}.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}|image)[()\[\]]*[,.;:!?'"]{0,2}|(?:#{Mx[:gl_o]}\S+?#{Mx[:gl_c]})+|[^#{Mx[:lnk_o]}#{Mx[:lnk_c]}]+/mu)
+            words=urls(@word_mode)
+            dob.obj=dob.obj.gsub(/.+/m,words)
+          end
+          dob.obj=dob.obj.gsub(/\\copyright/i,%{©})
+          dob.obj=if (dob.obj !~/\<:ad\s+\.\.\//)
+            dob.obj.gsub(/\<:ad\s+(\S+)?\s+(\S+\.png)\s+(.+)?\;\s+(.+)?\;\s*!\>/,
+              %{\n
    \\3
    \n})
+          else
+            dob.obj.gsub(/\<:ad\s+(\S+)?\s+(\S+\.png)\s+(.+)?\;\s+(.+)?\;\s*\>/,
+              %{\n
    \\3
    \n})
+          end
+          dob.obj=dob.obj.gsub(/!pick/,%{stellar}).
+            gsub(/!new/,%{#{$ep[:hsp]}new}).
+            gsub(/<:h(.{1,7}?)>/,'\1').
+            gsub(/<:to(\d{1,7}?)>/,%{to#{$ep[:hsp]}\{#{$ep[:hsp]}\\1#{$ep[:hsp]}\} }).
+            gsub(/#{Mx[:url_o]}_(\S+?)#{Mx[:url_c]}/,'\1'). #http ftp matches escaped, no decoration
+            gsub(/#{Mx[:url_o]}([a-zA-Z0-9._-]+\@\S+?\.[a-zA-Z0-9._-]+)#{Mx[:url_c]}/,%{#{@brace_url.xml_open}\\1#{@brace_url.xml_close}}).
+            gsub(/#{Mx[:url_o]}(\S+?)#{Mx[:url_c]}/,%{#{@brace_url.xml_open}\\1#{@brace_url.xml_close}}) #http ftp matches with decoration
+          if dob.obj =~/..\/\S+/ \
+          and dob.obj !~/(\"..\/\S+?\"|>\s*..\/\S+<)/
+            dob.obj=dob.obj.gsub(/(\.\.\/\S+)/,'\1')
+          end
+          dob.obj=dob.obj.gsub(//m,'>')
+        end
+        @tuned_file << dob
+      end
+    end
+    def amp_angle_brackets
+      data,data_new=@data,[]
+      data.each do |dob|
+        dob.obj=dob.obj.
+          gsub(/&/u,'&').
+          gsub(//u,'>')
+        data_new << dob
+      end
+      data_new
+    end
+    def endnotes_html
+      data=@data
+      @tuned_file=[]
+      a,s='_a','_s'
+      ast,pls='*','+'
+      data.each do |dob|
+        unless dob.is ==:code
+          dob.obj=dob.obj.gsub(/(#{Mx[:en_a_o]}|#{Mx[:en_b_o]})(\d+)\s+(.+?)(#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/,
+              %{#{Mx[:nbsp]}#{Mx[:nbsp]}\\2#{Mx[:nbsp]} } +  #note- endnote-
+              %{\\1\\2 #{Mx[:nbsp]}\\2. \\3 \\4}). #endnote- note- (careful may have switched)
+            gsub(/(#{Mx[:en_b_o]})[*](\d+)\s+(.+?)(#{Mx[:en_b_c]})/,
+              %{#{Mx[:nbsp]}#{Mx[:nbsp]}#{ast}\\2#{Mx[:nbsp]} } +  #note- endnote-
+              %{\\1#{ast}\\2 #{Mx[:nbsp]}#{ast}\\2. \\3 \\4}). #endnote- note- (careful may have switched)
+            gsub(/(#{Mx[:en_b_o]})[+](\d+)\s+(.+?)(#{Mx[:en_b_c]})/,
+              %{#{Mx[:nbsp]}#{Mx[:nbsp]}#{pls}\\2#{Mx[:nbsp]} } +  #note- endnote-
+              %{\\1#{pls}\\2 #{Mx[:nbsp]}#{pls}\\2. \\3 \\4}) #endnote- note- (careful may have switched) # double-check there may here be a bug
+          if dob.obj =~/#{Mx[:en_a_o]}([*+]+)\s+.+?#{Mx[:en_a_c]}/
+            m=$1.length.to_i
+            dob.obj=dob.obj.gsub(/(#{Mx[:en_a_o]})[*]+\s+(.+?)(#{Mx[:en_a_c]})/,
+                %{#{Mx[:nbsp]}#{Mx[:nbsp]}#{ast*m}#{Mx[:nbsp]} } +  #note- endnote-
+                %{\\1#{ast*m} #{Mx[:nbsp]}#{ast*m} \\2 \\3}). #endnote- note- (careful may have switched)
+              gsub(/(#{Mx[:en_a_o]})([+]+)\s+(.+?)(#{Mx[:en_a_c]})/,
+                %{#{Mx[:nbsp]}#{Mx[:nbsp]}#{pls*m}#{Mx[:nbsp]} } +  #note- endnote-
+                %{\\1#{pls*m} #{Mx[:nbsp]}#{pls*m} \\2 \\3}) #endnote- note- (careful may have switched)
+          end
+        end
+        @tuned_file << dob
+      end
+    end
+    def output
+      data=@data
+      @tuned_file=[]
+      data.each do |dob|
+        dob.obj=dob.obj.strip.chomp
+        @tuned_file << dob
+      end
+      @tuned_file << "\n" if (@md.fns =~/\.sst0/) #remove
+      @tuned_file
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/errors.rb b/lib/sisu/v5/errors.rb
new file mode 100644
index 00000000..9721bf72
--- /dev/null
+++ b/lib/sisu/v5/errors.rb
@@ -0,0 +1,89 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, error screen reporting
+
+=end
+module SiSU_Errors
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env; include SiSU_Screen
+  class InfoError .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_Git
+  require_relative 'param'                              # param.rb
+  require_relative 'sysenv'                             # sysenv.rb
+  require_relative 'dal'                                # dal.rb
+  class Source
+    def initialize(opt)
+      @opt=opt
+      @env=SiSU_Env::InfoEnv.new
+      @md=SiSU_Param::Parameters.new(@opt).get
+      @file=SiSU_Env::FileOp.new(@md)
+      l=SiSU_Env::StandardiseLanguage.new(@md.opt.lng).language
+      unless @opt.lng==l[:c] # @md.i18n[0]==l[:c]
+        p "using: #{@opt.lng} (@make: :language:); filename #{@md.fns} filename language: #{l[:c]}, mismatch"
+      end
+      if @env.output_dir_structure.multilingual?
+        m=/((.+?)(?:\~\w{2,3})?)\.((?:-|ssm\.)?sst|ssm)$/ #watch added match for sss
+        fnn,fnb,fnt=@opt.fns[m,1],@opt.fns[m,2],@opt.fns[m,3]
+      else m=/(.+?)\.((?:-|ssm\.)?sst|ssm)$/
+        fnb=@fnn=@opt.fns[m,1]
+        fnt=@opt.fns[m,2]
+      end
+      git_path_fnb=@env.processing_path.git + '/' + fnb
+      #unless @opt.cmd =~/q/
+      #  @opt.cmd=~/[MVvz]/ \
+      #  ? SiSU_Screen::Ansi.new(@opt.cmd,'Git path',@git_path[:fnb]).green_hi_blue \
+      #  : SiSU_Screen::Ansi.new(@opt.cmd,'Git path',@git_path[:fnb]).green_title_hi
+      #  SiSU_Screen::Ansi.new(@opt.cmd,"Git path","#{@opt.fns} -> #{@git_path[:fnb]}").warn if @opt.cmd =~/[MVv]/
+      #end
+      lng=(@md.opt.lng) ? (@md.opt.lng) : (@md.i18n[0])
+      @git_path={
+        fnb:       git_path_fnb,
+        doc:       git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:doc] + '/' + lng,
+        po:        git_path_fnb + '/' + Gt[:po] + '/' + lng,
+        pot:       git_path_fnb + '/' + Gt[:pot],
+        conf:      git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:conf],
+        image:     git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:image],
+        audio:     git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:audio],
+        video:     git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:video],
+        conf:      git_path_fnb + '/' + Gt[:sisupod] + '/' + Gt[:conf]
+      }
+      SiSU_DAL::Source.new(@opt).read                                         # -m
+    end
+    def create_file_structure_git
+      make_dir_fnb
+      if program_found?
+        git_init
+      end
+    end
+    def read
+      create_file_structure_git
+      populate.sisusrc_files
+      #if program_found?
+      #  git_commit
+      #end
+      unless @opt.cmd =~/q/
+        @opt.cmd=~/[MVvz]/ \
+        ? SiSU_Screen::Ansi.new(@opt.cmd,'Git path',@git_path[:fnb]).green_hi_blue
+        : SiSU_Screen::Ansi.new(@opt.cmd,'Git path',@git_path[:fnb]).green_title_hi
+        SiSU_Screen::Ansi.new(@opt.cmd,"Git path","#{@opt.fns} -> #{@git_path[:fnb]}").warn if @opt.cmd =~/[MVv]/
+      end
+    end
+    def program_found?
+      found=`whereis git`
+      (found =~/bin\/git\b/) ? true : false
+    end
+    def make_dir_fnb
+      FileUtils::mkdir_p(@git_path[:fnb]) unless FileTest.directory?(@git_path[:fnb])
+      FileUtils::mkdir_p(@git_path[:doc]) unless FileTest.directory?(@git_path[:doc])
+      FileUtils::mkdir_p(@git_path[:po]) unless FileTest.directory?(@git_path[:po])
+      FileUtils::mkdir_p(@git_path[:pot]) unless FileTest.directory?(@git_path[:pot])
+      FileUtils::mkdir_p(@git_path[:conf]) unless FileTest.directory?(@git_path[:conf])
+      FileUtils::mkdir_p(@git_path[:image]) unless FileTest.directory?(@git_path[:image])
+      #FileUtils::mkdir_p(@git_path[:audio]) unless FileTest.directory?(@git_path[:audio])
+      #FileUtils::mkdir_p(@git_path[:video]) unless FileTest.directory?(@git_path[:video])
+    end
+    def git_init
+      unless FileTest.directory?("#{@git_path[:fnb]}/.git")
+        pwd=Dir.pwd
+        Dir.chdir(@git_path[:fnb])
+        system("git init ")
+        Dir.chdir(pwd)
+      end
+    end
+    def git_commit
+      if program_found?
+        if FileTest.directory?("#{@git_path[:fnb]}")
+          pwd=Dir.pwd
+          Dir.chdir(@git_path[:fnb])
+          system("
+            git add . \
+            && git commit -a
+          ")
+          Dir.chdir(pwd)
+        end
+      end
+    end
+    def populate
+      def identify_language_versions
+        print __FILE__ + ':'
+        p __LINE__
+      end
+      def copy_src_head
+        if @opt.f_pth[:lng]
+          FileUtils::cp_r("#{@env.path.pwd}/#{@opt.f_pth[:lng]}/#{@opt.fns}",@git_path[:doc])
+        elsif @opt.fns =~/\.ssm\.sst/
+          ssm=@opt.fns.gsub(/\.ssm\.sst/,'.ssm')
+          FileUtils::cp_r("#{@env.path.pwd}/#{ssm}",@git_path[:doc])
+        else
+          FileUtils::cp_r("#{@env.path.pwd}/#{@opt.fns}",@git_path[:doc])
+        end
+      end
+      def copy_related_sst_ssi
+        doc_import=[]
+        @rgx_doc_import=/^<<\s(\S+?\.ss[ti])/
+        file_array=IO.readlines(@opt.fns,'')
+        file_array.each do |f|
+          if f =~@rgx_doc_import
+            doc_import = doc_import + f.scan(@rgx_doc_import).uniq.flatten
+          end
+        end
+        doc_import.each do |f|
+          if @opt.f_pth[:lng]
+            FileUtils::cp_r("#{@env.path.pwd}/#{@opt.f_pth[:lng]}/#{f}",@git_path[:doc])
+          else
+            FileUtils::cp_r("#{@env.path.pwd}/#{f}",@git_path[:doc])
+          end
+        end
+      end
+      def locate_parse_file
+        composite_src=@opt.fns=~/\.ssm$/ ? true : false
+        parse_file=if composite_src \
+        and @opt.cmd.inspect !~/m/
+          ##SiSU_Assemble::Composite.new(@opt).read
+          #SiSU_DAL::Source.new(@opt).read                                         # -m
+          "#{@env.processing_path.composite_file}/#{@opt.fnb}.ssm.sst"
+        elsif composite_src
+          "#{@env.processing_path.composite_file}/#{@opt.fnb}.ssm.sst"
+        else "#{@env.path.pwd}/#{@opt.fns}"
+        end
+      end
+      def read_composite
+        #print __FILE__ + ':'
+        #p __LINE__
+      end
+      def sisuyaml_rc
+        sisurc=@env.path.sisurc_path
+        if FileTest.file?(sisurc)
+          FileUtils::cp_r(sisurc,@git_path[:conf])
+        end
+      end
+      def read_src
+        print __FILE__ + ':'
+        p __LINE__
+      end
+      def composite_src?
+        @opt.fns=~/\.ssm$/ ? true : false
+      end
+      def sisusrc_files
+        populate.copy_src_head
+        if composite_src?
+          populate.copy_related_sst_ssi
+        end
+        #parse_file_name=locate_parse_file
+        #parse_file=IO.readlines(parse_file_name,'')
+        populate.sisuyaml_rc #(parse_file)
+        #populate.extract_composite_source
+        #populate.read_composite # or read_each_composite
+        populate.identify_language_versions
+      end
+      self
+    end
+  end
+end
+__END__
+@file.output_path.sisugit
diff --git a/lib/sisu/v5/harvest.rb b/lib/sisu/v5/harvest.rb
new file mode 100644
index 00000000..ffcb6aa3
--- /dev/null
+++ b/lib/sisu/v5/harvest.rb
@@ -0,0 +1,116 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+   harvest metadata from document corpus (suitable for medium sized sites)
+   (concept example, [to remove size constraint: implement SQL equivalent])
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: system environment, resource control and configuration details
+
+=end
+module SiSU_Harvest
+  @@the_idx_topics,@@the_idx_authors={},{}
+  class Source
+    require_relative 'options'                              # options.rb
+    require_relative 'harvest_topics'                       # harvest_topics.rb
+    require_relative 'harvest_authors'                      # harvest_authors.rb
+    require_relative 'sysenv'                               # sysenv.rb
+      include SiSU_Env
+    def initialize(opt)
+      @opt=opt
+      @env=SiSU_Env::InfoEnv.new
+    end
+    def read
+      begin
+        harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}"
+        FileUtils::mkdir_p(harvest_pth) unless FileTest.directory?(harvest_pth)
+        cases(@opt,@env)
+      rescue
+      ensure
+        SiSU_Env::CreateSite.new(@opt.cmd).cp_css
+      end
+    end
+    def help
+      puts <.
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: simple xml representation (sax style)
+
+=end
+module SiSU_HarvestAuthors
+  require_relative 'author_format'                      # author_format.rb
+  class Songsheet
+    @@the_idx_authors={}
+    def initialize(opt,env)
+      @opt,@env=opt,env
+      @file_list=opt.files
+    end
+    def songsheet
+      idx_array={}
+      @opt.f_pths.each do |y|
+        lang_hash_file_array={}
+        name=y[:f]
+        filename=y[:pth] + '/' + y[:f]
+        File.open(filename,'r') do |file|
+          file.each_line("\n\n") do |line|
+            if line =~/^@(?:title|creator|date):(?:\s|$)/m
+              lang_hash_file_array[y[:lng_is]] ||= []
+              lang_hash_file_array[y[:lng_is]] << line
+            elsif line =~/^@\S+?:(?:\s|$)/m \
+            or line =~/^(?:\s*\n|%+ )/
+            else break
+            end
+          end
+        end
+        lang_hash_file_array.each_pair do |lang,a|
+          idx_array[lang] ||= []
+          idx_array=SiSU_HarvestAuthors::Harvest.new(@opt,@env,a,filename,name,idx_array,lang).extract_harvest
+        end
+      end
+      the_idx=SiSU_HarvestAuthors::Index.new(idx_array,@@the_idx_authors).construct_book_author_index
+      SiSU_HarvestAuthors::OutputIndex.new(@opt,the_idx).html_print.html_songsheet
+    end
+  end
+  class Harvest
+    def initialize(opt,env,data,filename,name,idx_array,lang)
+      @opt,@env,@data,@filename,@name,@idx_array,@lang=opt,env,data,filename,name,idx_array,lang
+    end
+    def extract_harvest
+      data,filename,name,idx_array,lang=@data,@filename,@name,@idx_array,@lang
+      @title,@subtitle,@fulltitle,@author,@author_format,@date=nil,nil,nil,nil,nil,nil
+      @authors=[]
+      rgx={}
+      rgx[:author]=/^@creator:(?:[ ]+|.+?:author:[ ]+)(.+?)(?:\||\n)/m
+      rgx[:title]=/^@title:[ ]+(.+)/
+      rgx[:subtitle]=/^@title:.+?:subtitle:[ ]+(.+?)\n/m
+      rgx[:date]=/^@date:(?:[ ]+|.+?:published:[ ]+)(\d{4})/m
+      data.each do |para|
+        if para=~ rgx[:title]
+          @title=rgx[:title].match(para)[1]
+        end
+        if para=~ rgx[:subtitle]
+          @subtitle=rgx[:subtitle].match(para)[1]
+        end
+        if para=~ rgx[:author]
+          @author_format=rgx[:author].match(para)[1]
+        end
+        if para=~ rgx[:date]
+          @date=rgx[:date].match(para)[1]
+        end
+        break if @title && @subtitle && @author && @date
+      end
+      @fulltitle=@subtitle ? (@title + ' - ' + @subtitle) : @title
+      if @title \
+      and @author_format
+        creator=SiSU_FormatAuthor::Author.new(@author_format.strip).author_details
+        @authors,@authorship=creator[:authors],creator[:authorship]
+        file=if name=~/~[a-z]{2,3}\.ss[mt]$/
+          name.sub(/~[a-z]{2,3}\.ss[mt]$/,'')
+        else
+          name.sub(/\.ss[mt]$/,'')
+        end
+        page=if @env.output_dir_structure.by? == :language
+          "#{lang}/sisu_manifest.html"
+        else
+          "sisu_manifest.#{lang}.html"
+        end
+        idx_array[lang] <<= { filename: filename, file: file, date: @date, title: @fulltitle, author: creator, page: page, lang: lang }
+      else
+        #p "missing author field: #{@filename} title: #{@title}; author: #{@author_format}"
+      end
+      idx_array[lang]=idx_array[lang].flatten
+      idx_array
+    end
+  end
+  class Index
+    def initialize(idx_array,the_idx)
+      @idx_array,@the_idx=idx_array,the_idx
+      @@the_idx_authors=@the_idx
+    end
+    def capital(txt)
+      txt[0].chr.capitalize + txt[1,txt.length]
+    end
+    def construct_book_author_index
+      idx_array=@idx_array
+      idx_array.each_pair do |lang,idx_array|
+        @@the_idx_authors[lang] ||= {}
+        idx_array.each do |idx|
+          idx[:author][:last_first_format_a].each do |author|
+            author=author.strip
+            if @@the_idx_authors[lang][author].is_a?(NilClass)
+              @@the_idx_authors[lang][author]={ md: [] }
+            end
+            @@the_idx_authors[lang][author][:md] << { filename: idx[:filename], file: idx[:file], author: idx[:author], title: idx[:title], date: idx[:date], page: idx[:page], lang: idx[:lang] }
+          end
+        end
+      end
+      @the_idx=@@the_idx_authors
+    end
+  end
+  class OutputIndex
+    require_relative 'i18n'                               # i18n.rb
+    def initialize(opt,the_idx)
+      @opt,@the_idx=opt,the_idx
+      @env=SiSU_Env::InfoEnv.new
+      @rc=SiSU_Env::GetInit.new.sisu_yaml.rc
+      @alphabet_list=%W[9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+      @alph=@alphabet_list.dup
+      @letter=@alph.shift
+      @vz=SiSU_Viz::Defaults.new
+    end
+    def html_file_open
+      @the_idx.keys.each do |lng|
+        @output ||={}
+        @output[lng] ||={}
+        harvest_pth,file='',''
+        if @env.output_dir_structure.by? == :language
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}/#{lng}/manifest"
+          file="#{harvest_pth}/authors.html"
+        elsif @env.output_dir_structure.by? == :filetype
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}/manifest"
+          file="#{harvest_pth}/authors.#{lng}.html"
+        elsif @env.output_dir_structure.by? == :filename
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}"
+          file="#{harvest_pth}/authors.#{lng}.html"
+        end
+        FileUtils::mkdir_p(harvest_pth) unless FileTest.directory?(harvest_pth)
+        fileinfo=(@opt.cmd =~/[vVM]/) ? ("file://#{file}") : ''
+        SiSU_Screen::Ansi.new(@opt.cmd,"harvest authors (#{@opt.files.length} files)",fileinfo).dark_grey_title_hi unless @opt.cmd =~/q/
+        @output[lng][:html]=File.new(file,'w')
+      end
+    end
+    def html_file_close
+      @the_idx.keys.each do |lng|
+        @output[lng][:html].close
+        @output[lng][:html_mnt].close if @output[lng][:html_mnt].is_a?(File)
+      end
+    end
+    def html_print
+      def html_songsheet
+        html_file_open
+        html_head
+        html_alph
+        html_body
+        html_tail
+        html_file_close
+      end
+      def html_head_adjust(lng,type='')
+        css_path,topics='',''
+        if @env.output_dir_structure.by? == :language
+          css_path=(type !~/maintenance/) \
+          ? '../../_sisu/css/harvest.css'
+          : 'harvest.css'
+          topics='topics.html'
+        elsif @env.output_dir_structure.by? == :filetype
+          css_path=(type !~/maintenance/) \
+          ? '../_sisu/css/harvest.css'
+          : 'harvest.css'
+          topics="topics.#{lng}.html"
+        elsif @env.output_dir_structure.by? == :filename
+          css_path=(type !~/maintenance/) \
+          ? './_sisu/css/harvest.css'
+          : 'harvest.css'
+          topics="topics.#{lng}.html"
+        end
+        ln=SiSU_i18n::Languages.new.language.list
+        harvest_languages=''
+        @the_idx.keys.each do |lng|
+          if @env.output_dir_structure.by? == :language
+            harvest_pth="../../#{lng}/manifest"
+            file="#{harvest_pth}/authors.html"
+          elsif @env.output_dir_structure.by? == :filetype
+            harvest_pth='.'
+            file="#{harvest_pth}/authors.#{lng}.html"
+          elsif @env.output_dir_structure.by? == :filename
+            harvest_pth='.'
+            file="#{harvest_pth}/authors.#{lng}.html"
+          end
+          l=ln[lng][:t]
+          harvest_languages += %{#{l}   }
+        end
+        sv=SiSU_Env::InfoVersion.instance.get_version
+        if @env.output_dir_structure.by? == :language
+          home_pth='../..'
+          output_structure_by='(output organised by language & filetype)'
+        elsif @env.output_dir_structure.by? == :filetype
+          home_pth='..'
+          output_structure_by='(output organised by filetype)'
+        elsif @env.output_dir_structure.by? == :filename
+          home_pth='.'
+          output_structure_by='(output organised by filename)'
+        else
+          home_pth='.'
+          output_structure_by='(output organised by ?)'
+        end
+        <
+
+
+SiSU Metadata Harvest - Authors
+
+
+
+
+
+
+
+
+
+
+
+
+
    SiSU Metadata Harvest - Authors #{output_structure_by}
    
+
    [ HOME ] also see SiSU Metadata Harvest - Topics
    
+
    #{@env.widget_static.search_form}
    
+
    
+
    #{harvest_languages}
    
+
    
+WOK
+      end
+      def html_head
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << html_head_adjust(lng,'maintenance') if @opt.cmd.inspect =~/M/
+          @output[lng][:html] << html_head_adjust(lng)
+        end
+      end
+      def html_alph
+        a=[]
+        a << '
    '
+        @alph.each do |x|
+          a << ((x =~/[0-9]/) \
+          ? ''
+          : %{#{x}, })
+        end
+        a=a.join
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << a if @opt.cmd.inspect =~/M/
+          @output[lng][:html] << a
+        end
+      end
+      def html_tail
+        a =<
+
+
+
+
+
+
+#{@vz.credits_sisu}
+
+
+WOK
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << a if @output[lng][:html_mnt].is_a?(File)
+          @output[lng][:html] << a
+        end
+      end
+      def do_html(lng,html)
+        @output[lng][:html_mnt] << html if @output[lng][:html_mnt].is_a?(File)
+        @output[lng][:html] << html
+      end
+      def do_string_name(lng,attrib,string)
+        f=/^(\S)/.match(string[0])[1]
+        if @lng != lng
+          @alph=@alphabet_list.dup
+          @letter=@alph.shift
+          @lng = lng
+        end
+        if @letter < f
+          while @letter < f
+            if @alph.length > 0
+              @letter=@alph.shift
+              if @output[lng][:html_mnt].is_a?(File)
+                @output[lng][:html_mnt] << %{\n
    #{@letter}
    }
+              end
+              @output[lng][:html] << %{\n
    #{@letter}
    }
+            else break
+            end
+          end
+        end
+      end
+      def html_body
+        the_idx=@the_idx
+        the_idx.each_pair do |lng,lng_array|
+          lng_array.sort.each do |a|
+            do_string_name(lng,'',a)
+            name=a[0].sub(/(.+?)(?:,.+|$)/,'\1').gsub(/\s+/,'_')
+            x = %{
    #{a[0]}
    }
+            if @output[lng][:html_mnt].is_a?(File)
+              @output[lng][:html_mnt] << x
+            end
+            @output[lng][:html] << x
+            works=[]
+            a[1][:md].each do |x|
+              manifest_at=if @env.output_dir_structure.by? == :language
+                manifest_pth="#{@env.path.output}/#{x[:file]}"
+                x[:file] + '.html'
+              elsif @env.output_dir_structure.by? == :filetype
+                manifest_name=x[:file]
+                x[:file] + '.' + lng +  '.html'
+              elsif @env.output_dir_structure.by? == :filename
+                "./#{x[:file]}/#{x[:page]}"
+              else '' #error
+              end
+              work=[ "#{x[:date]} #{x[:title]}", %{
    #{x[:date]} #{x[:title]}, #{x[:author][:authors_s]}
    } ]
+              works<<=(@output[lng][:html_mnt].is_a?(File)) \
+              ? (work.concat([%{
    [src]  #{x[:date]} #{x[:title]}, #{x[:author][:authors_s]} -- [#{x[:file]}.sst]
    }]))
+              : work
+            end
+            works.sort_by {|x| x[0]}.each do |x|
+              @output[lng][:html] << x[1]
+              @output[lng][:html_mnt] << x[2] if @output[lng][:html_mnt].is_a?(File)
+            end
+          end
+        end
+      end
+      self
+    end
+    def screen_print
+      def cycle
+        the_idx=@the_idx
+        the_idx.sort.each do |a|
+          puts a[0]
+          a[1][:md].each do |x|
+            puts "\t" + x[:file]
+          end
+        end
+      end
+      self
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/harvest_topics.rb b/lib/sisu/v5/harvest_topics.rb
new file mode 100644
index 00000000..7375e191
--- /dev/null
+++ b/lib/sisu/v5/harvest_topics.rb
@@ -0,0 +1,825 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+   metadata harvest, extract topics and associated writings from document set
+   (topics use topic_register header)
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: simple xml representation (sax style)
+
+=end
+module SiSU_HarvestTopics
+  require_relative 'author_format'                      # author_format.rb
+  include SiSU_Viz
+  class Songsheet
+    @@the_idx_topics={}
+    def initialize(opt,env)
+      @opt,@env=opt,env
+      @file_list=opt.files
+    end
+    def songsheet
+      idx_array={}
+      @opt.f_pths.each do |y|
+        lang_hash_file_array={}
+        name=y[:f]
+        filename=y[:pth] + '/' + y[:f]
+        File.open(filename,'r') do |file|
+          file.each_line("\n\n") do |line|
+            if line =~/^@(?:title|creator|classify):(?:\s|$)/m
+              lang_hash_file_array[y[:lng_is]] ||= []
+              lang_hash_file_array[y[:lng_is]] << line
+            elsif line =~/^@\S+?:(?:\s|$)/m \
+            or line =~/^(?:\s*\n|\s*$|%+ )/
+            else break
+            end
+          end
+        end
+        lang_hash_file_array.each_pair do |lang,a|
+          idx_array[lang] ||=[]
+          idx_array=SiSU_HarvestTopics::Harvest.new(@opt,@env,a,filename,name,idx_array,lang).extract_harvest
+        end
+      end
+      the_hash=SiSU_HarvestTopics::Index.new(@opt,@env,idx_array,@@the_idx_topics).song
+      SiSU_HarvestTopics::OutputIndex.new(@opt,the_hash).html_print.html_songsheet
+    end
+  end
+  class Harvest
+    def initialize(opt,env,data,filename,name,idx_array,lang)
+      @opt,@env,@data,@filename,@name,@idx_array,@lang=opt,env,data,filename,name,idx_array,lang
+    end
+    def extract_harvest
+      data,filename,name,idx_array,lang=@data,@filename,@name,@idx_array,@lang
+      @idx_lst,@title,@subtitle,@fulltitle,@author,@author_format=nil,nil,nil,nil,nil,nil
+      rgx={}
+      rgx[:author]=/^@creator:(?:[ ]+|.+?:author:[ ]+)(.+?)(?:\||\n)/m
+      rgx[:title]=/^@title:[ ]+(.+)/
+      rgx[:subtitle]=/^@title:.+?:subtitle:[ ]+(.+?)\n/m
+      rgx[:idx]=/^@classify:.+?:topic_register:[ ]+(.+?)(?:\n\n|\n\s+:\S|\n%)/m
+      data.each do |para|
+        if para=~ rgx[:idx]
+          @idx_list=(rgx[:idx].match(para)[1]).split(/\s*\n\s*/).join
+        end
+        if para=~ rgx[:title]
+          @title=rgx[:title].match(para)[1]
+        end
+        if para=~ rgx[:subtitle]
+          @subtitle=rgx[:subtitle].match(para)[1]
+        end
+        if para=~ rgx[:author]
+          @author_format=rgx[:author].match(para)[1]
+        end
+        break if @title && @subtitle && @author && @idx_lst
+      end
+      @fulltitle=@subtitle ? (@title + ' - ' + @subtitle) : @title
+      if @title \
+      and @author_format \
+      and @idx_list
+        creator=SiSU_FormatAuthor::Author.new(@author_format.strip).author_details
+        @authors,@authorship=creator[:authors],creator[:authorship]
+        file=if name=~/~[a-z]{2,3}\.ss[mt]$/
+          name.sub(/~[a-z]{2,3}\.ss[mt]$/,'')
+        else
+          name.sub(/\.ss[mt]$/,'')
+        end
+        page=if @env.output_dir_structure.by? == :language
+          "#{lang}/sisu_manifest.html"
+        else
+          "sisu_manifest.#{lang}.html"
+        end
+        idx_array[lang] <<=if @idx_list =~/;/
+          g=@idx_list.scan(/[^;]+/)
+          idxl=[]
+          g.each do |i|
+            i=i.strip
+            idxl << { filename: filename, file: file, rough_idx: i, title: @fulltitle, author: creator, page: page, lang: lang  }
+          end
+          idxl
+        else { filename: filename, file: file, rough_idx: @idx_list, title: @fulltitle, author: creator, page: page, lang: lang  }
+        end
+      else
+        p "missing required field in #{@filename} - [title]: <<#{@title}>>; [author]: <<#{@author_format}>>; [idx]: <<#{@idx_list}>>" if @opt.cmd.inspect =~/[VM]/
+      end
+      idx_array[lang]=idx_array[lang].flatten
+      idx_array
+    end
+  end
+  class Index
+    def initialize(opt,env,idx_array,the_idx)
+      @opt,@env,@idx_array,@the_idx=opt,env,idx_array,the_idx
+      @@the_idx_topics=@the_idx
+    end
+    def song
+      the_idx=construct_book_topic_keys
+      the_hash=construct_book_topic_hash(the_idx)
+    end
+    def capital(txt)
+      txt_a=txt.scan(/\S+/)
+      tx=''
+      txt_a.each do |txt|
+        tx += txt[0].chr.capitalize + txt[1,txt.length] + ' '
+      end
+      tx.strip
+    end
+    def capital_(txt)
+      txt[0].chr.capitalize + txt[1,txt.length]
+    end
+    def contents(idx,lang)
+      names=''
+      idx[:author][:last_first_format_a].each do |n|
+        s=n.sub(/(.+?)(?:,.+|$)/,'\1').gsub(/\s+/,'_')
+        names=if @env.output_dir_structure.by? == :language
+          names += %{#{n}, }
+        else
+          names += %{#{n}, }
+        end
+      end
+      md={ filename: idx[:filename], file: idx[:file], author: names, title: idx[:title], page: idx[:page] }
+    end
+    def capital_(txt)
+      txt[0].chr.capitalize + txt[1,txt.length]
+    end
+    def key_create(c,alt)
+      x=nil
+      x=if c.length==6
+        c[0].to_s + '|' +
+          capital(c[1][0].to_s) + '|' +
+          capital(c[2][0].to_s) + '|' +
+          capital(c[3][0].to_s) + '|' +
+          capital(alt.to_s)
+      elsif c.length==5
+        c[0].to_s + '|' +
+          capital(c[1][0].to_s) + '|' +
+          capital(c[2][0].to_s) + '|' +
+          capital(alt.to_s)
+      elsif c.length==4
+        c[0].to_s + '|' +
+          capital(c[1][0].to_s) + '|' +
+          capital(alt.to_s)
+      elsif c.length==3
+        c[0].to_s + '|' +
+          capital(alt.to_s)
+      end
+    end
+    def construct_book_topic_keys
+      idx_array=@idx_array
+      @idx_a=[]
+      @the_a=[]
+      idx_array.each_pair do |lang,idx_array|
+        @@the_idx_topics[lang] ||= {}
+        idx_array.each do |idx|
+          if idx[:rough_idx]
+            idx_lst=idx[:rough_idx].scan(/[^:]+/)
+          else
+            puts "no topic register in: << #{idx[:filename]} >>"
+            next
+          end
+          idx_a=[]
+          idx_lst.each do |c|
+            idx_a << c.scan(/[^|\n]+/m)
+          end
+          idx_a << contents(idx,lang)
+          @idx_a << [lang] + idx_a
+        end
+      end
+      @idx_a.each do |c|
+        if c.length > 1 \
+        and c.is_a?(Array)
+          if c[2].is_a?(Hash)
+            c[1].each do |alt|
+              v=key_create(c,alt)
+              @the_a << [v, c[2]] if v
+            end
+          end
+        end
+        if c.length > 2 \
+        and c.is_a?(Array)
+          if c[3].is_a?(Hash)
+            c[2].each do |alt|
+              v=key_create(c,alt)
+              @the_a << [v, c[3]] if v
+            end
+          end
+        end
+        if c.length > 3 \
+        and c.is_a?(Array)
+          if c[4].is_a?(Hash)
+            c[3].each do |alt|
+              v=key_create(c,alt)
+              @the_a << [v, c[4]] if v
+            end
+          end
+        end
+        if c.length > 4 \
+        and c.is_a?(Array)
+          if c[5].is_a?(Hash)
+            c[4].each do |alt|
+              v=key_create(c,alt)
+              @the_a << [v, c[5]] if v
+            end
+          end
+        end
+        if c.length > 5 \
+        and c.is_a?(Array)
+          if c[6].is_a?(Hash)
+            c[5].each do |alt|
+              v=key_create(c,alt)
+              @the_a << [v, c[6]] if v
+            end
+          end
+        end
+      end
+      y=@the_a.sort_by { |x| x[0] } #; y.each {|z| puts z}
+    end
+    def construct_book_topic_hash(y)
+      @the_h={}
+      y.each do |z|
+        x=z[0].scan(/[^|]+/)
+        depth=x.length
+        extract=(depth-1)
+        k=case extract
+        when 4
+          { x[0] => { x[1] => { x[2] => { x[3] => { x[4] => z[1] } } } } }
+        when 3
+          { x[0] => { x[1] => { x[2] => { x[3] => z[1] } } } }
+        when 2
+          { x[0] => { x[1] => { x[2] => z[1] } } }
+        when 1
+          { x[0] => { x[1] => z[1] } }
+        when 0
+          { x[0] => z[1] }
+        end
+        if extract >= 0
+          k.each_pair do |x0,y|
+            if extract == 0
+              @the_h[x0] ||={ md: [] }
+              @the_h[x0][:md] << y
+            else
+              @the_h[x0] ||={}
+            end
+            #puts '  '*0 + x0
+            if extract >= 1
+              y.each_pair do |x1,y|
+                if extract == 1
+                  @the_h[x0][x1] ||={ md: [] }
+                  @the_h[x0][x1][:md] << y
+                else
+                  @the_h[x0][x1] ||={}
+                end
+                #puts '  '*1 + x1
+                if extract >= 2
+                  y.each_pair do |x2,y|
+                    if extract == 2
+                      @the_h[x0][x1][x2] ||={ md: [] }
+                      @the_h[x0][x1][x2][:md] << y
+                    else
+                      @the_h[x0][x1][x2] ||={}
+                    end
+                    #puts '  '*2 + x2
+                    if extract >= 3
+                      y.each_pair do |x3,y|
+                        if extract == 3
+                          @the_h[x0][x1][x2][x3] ||={ md: [] }
+                          @the_h[x0][x1][x2][x3][:md] << y
+                        else
+                          @the_h[x0][x1][x2][x3] ||={}
+                        end
+                        #puts '  '*3 + x3
+                        if extract == 4
+                          y.each_pair do |x4,y|
+                            if extract == 4
+                              @the_h[x0][x1][x2][x3][x4] ||={ md: [] }
+                              @the_h[x0][x1][x2][x3][x4][:md] << y
+                            else
+                              @the_h[x0][x1][x2][x3][x4] ||={}
+                            end
+                            #puts '  '*4 + x4
+                            if extract == 5
+                              y.each_pair do |x5,y|
+                                if extract == 5
+                                  @the_h[x0][x1][x2][x3][x4][x5] ||={ md: [] }
+                                  @the_h[x0][x1][x2][x3][x4][x5][:md] << y
+                                end
+                                #puts '  '*5 + x5
+                              end
+                            end
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      #@the_h.each_pair { |x,y| p x; p y }
+      @the_h
+    end
+    def traverse_base
+      @the_h.each_pair do |x0,y|
+        puts '  '*0 + x0 if x0.is_a?(String)
+        if y.is_a?(Hash)
+          y.each_pair do |x1,y|
+            puts '  '*1 + x1 if x1.is_a?(String)
+            if y.is_a?(Hash)
+              y.each_pair do |x2,y|
+                puts '  '*2 + x2 if x2.is_a?(String)
+                if y.is_a?(Hash)
+                  y.each_pair do |x3,y|
+                    puts '  '*3 + x3 if x3.is_a?(String)
+                    if y.is_a?(Hash)
+                      y.each_pair do |x4,y|
+                        puts '  '*4 + x4 if x4.is_a?(String)
+                        if y.is_a?(Hash)
+                          y.each_pair do |x5,y|
+                            puts '  '*5 + x5 if x5.is_a?(String)
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+    def traverse
+      @the_h.each_pair do |x0,y|
+        puts '  '*0 + x0 if x0.is_a?(String)
+        if y.is_a?(Hash)
+          if y.has_key?(:md)
+            y[:md].each { |x| puts '  '*5 + x[:title] }
+          end
+          y.each_pair do |x1,y|
+            puts '  '*1 + x1 if x1.is_a?(String)
+            if y.is_a?(Hash)
+              if y.has_key?(:md)
+                y[:md].each { |x| puts '  '*5 + x[:title] }
+              end
+              y.each_pair do |x2,y|
+                puts '  '*2 + x2 if x2.is_a?(String)
+                if y.is_a?(Hash)
+                  if y.has_key?(:md)
+                    y[:md].each { |x| puts '  '*5 + x[:title] }
+                  end
+                  y.each_pair do |x3,y|
+                    puts '  '*3 + x3 if x3.is_a?(String)
+                    if y.is_a?(Hash)
+                      if y.has_key?(:md)
+                        y[:md].each { |x| puts '  '*5 + x[:title] }
+                      end
+                      y.each_pair do |x4,y|
+                        puts '  '*4 + x4 if x4.is_a?(String)
+                        if y.is_a?(Hash)
+                          if y.has_key?(:md)
+                            y[:md].each { |x| puts '  '*5 + x[:title] }
+                          end
+                          y.each_pair do |x5,y|
+                            puts '  '*5 + x4 if x4.is_a?(String)
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+  class OutputIndex
+    require_relative 'i18n'                               # i18n.rb
+    def initialize(opt,the_idx)
+      @opt,@the_idx=opt,the_idx
+      @env=SiSU_Env::InfoEnv.new
+      @rc=SiSU_Env::GetInit.new.sisu_yaml.rc
+      @alphabet_list=%W[9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+      @alph=@alphabet_list.dup
+      @letter=@alph.shift
+      @vz=SiSU_Viz::Defaults.new
+    end
+    def html_file_open
+      @the_idx.keys.each do |lng|
+        @output ||={}
+        @output[lng] ||={}
+        harvest_pth,file='',''
+        if @env.output_dir_structure.by? == :language
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}/#{lng}/manifest"
+          file="#{harvest_pth}/topics.html"
+        elsif @env.output_dir_structure.by? == :filetype
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}/manifest"
+          file="#{harvest_pth}/topics.#{lng}.html"
+        elsif @env.output_dir_structure.by? == :filename
+          harvest_pth="#{@env.path.webserv}/#{@opt.base_stub}"
+          file="#{harvest_pth}/topics.#{lng}.html"
+        end
+        FileUtils::mkdir_p(harvest_pth) unless FileTest.directory?(harvest_pth)
+        fileinfo=(@opt.cmd =~/[vVM]/) ? ("file://#{file}") : ''
+        SiSU_Screen::Ansi.new(@opt.cmd,"harvest topics(#{@opt.files.length} files)",fileinfo).dark_grey_title_hi unless @opt.cmd =~/q/
+        @output[lng][:html]=File.new(file,'w')
+        if @opt.cmd.inspect =~/[M]/
+          @output[lng][:html_mnt]=File.new("#{@env.path.pwd}/topics.html",'w')
+        end
+      end
+    end
+    def html_file_close
+      @the_idx.keys.each do |lng|
+        @output[lng][:html].close
+        @output[lng][:html_mnt].close if @output[lng][:html_mnt].is_a?(File)
+      end
+    end
+    def html_print
+      def html_songsheet
+        #traverse
+        html_file_open
+        html_head
+        html_alph
+        html_body_traverse
+        html_tail
+        html_file_close
+      end
+      def html_body_traverse
+        @the_idx.each_pair do |x0,y|
+          lng=x0
+          if x0.is_a?(String)
+            #do_string_name(lng,'lev0',x0)
+            #puts '  '*0 + x0
+          end
+          if y.is_a?(Hash)
+            if y.has_key?(:md)
+              y[:md].each do |x|
+                #do_hash(lng,attrib,x) #lv==0 ?
+                #puts '  '*5 + x[:title]
+              end
+            end
+            y.each_pair do |x1,y|
+              if x1.is_a?(String)
+                do_string_name(lng,'lev0',x1)
+                #puts '  '*1 + x1
+              end
+              if y.is_a?(Hash)
+                if y.has_key?(:md)
+                  y[:md].each do |x|
+                    do_hash(lng,0,x)
+                    #puts '  '*5 + x[:title]
+                  end
+                end
+                y.each_pair do |x2,y|
+                  if x2.is_a?(String)
+                    do_string(lng,'lev1',x2)
+                    #puts '  '*2 + x2
+                  end
+                  if y.is_a?(Hash)
+                    if y.has_key?(:md)
+                      y[:md].each do |x|
+                        do_hash(lng,1,x)
+                        #puts '  '*5 + x[:title]
+                      end
+                    end
+                    y.each_pair do |x3,y|
+                      if x3.is_a?(String)
+                        do_string(lng,'lev2',x3)
+                        #puts '  '*3 + x3
+                      end
+                      if y.is_a?(Hash)
+                        if y.has_key?(:md)
+                          y[:md].each do |x|
+                            do_hash(lng,2,x)
+                            #puts '  '*5 + x[:title]
+                          end
+                        end
+                        y.each_pair do |x4,y|
+                          if x4.is_a?(String)
+                            do_string(lng,'lev3',x4)
+                            #puts '  '*4 + x4
+                          end
+                          if y.is_a?(Hash)
+                            if y.has_key?(:md)
+                              y[:md].each do |x|
+                                do_hash(lng,3,x)
+                                #puts '  '*5 + x[:title]
+                              end
+                            end
+                            y.each_pair do |x5,y|
+                              if x5.is_a?(String)
+                                do_string(lng,'lev4',x5)
+                                #puts '  '*5 + x5
+                              end
+                            end
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      def html_head_adjust(lng,type='')
+        css_path,authors='',''
+        if @env.output_dir_structure.by? == :language
+          css_path=(type !~/maintenance/) \
+          ? '../../_sisu/css/harvest.css'
+          : 'harvest.css'
+          authors='authors.html'
+        elsif @env.output_dir_structure.by? == :filetype
+          css_path=(type !~/maintenance/) \
+          ? '../_sisu/css/harvest.css'
+          : 'harvest.css'
+          authors="authors.#{lng}.html"
+        elsif @env.output_dir_structure.by? == :filename
+          css_path=(type !~/maintenance/) \
+          ? './_sisu/css/harvest.css'
+          : 'harvest.css'
+          authors="authors.#{lng}.html"
+        end
+        ln=SiSU_i18n::Languages.new.language.list
+        harvest_languages=''
+        @the_idx.keys.each do |lng|
+          if @env.output_dir_structure.by? == :language
+            harvest_pth="../../#{lng}/manifest"
+            file="#{harvest_pth}/topics.html"
+          elsif @env.output_dir_structure.by? == :filetype
+            harvest_pth='.'
+            file="#{harvest_pth}/topics.#{lng}.html"
+          elsif @env.output_dir_structure.by? == :filename
+            harvest_pth='.'
+            file="#{harvest_pth}/topics.#{lng}.html"
+          end
+          l=ln[lng][:t]
+          harvest_languages += %{#{l}   }
+        end
+        sv=SiSU_Env::InfoVersion.instance.get_version
+        if @env.output_dir_structure.by? == :language
+          home_pth='../..'
+          output_structure_by='(output organised by language & filetype)'
+        elsif @env.output_dir_structure.by? == :filetype
+          home_pth='..'
+          output_structure_by='(output organised by filetype)'
+        elsif @env.output_dir_structure.by? == :filename
+          home_pth='.'
+          output_structure_by='(output organised by filename)'
+        else
+          home_pth='.'
+          output_structure_by='(output organised by ?)'
+        end
+        <
+
+
+SiSU Metadata Harvest - Topics
+
+
+
+
+
+
+
+
+
+
+
+
+
    SiSU Metadata Harvest - Topics #{output_structure_by}
    
+
    [ HOME ] also see SiSU Metadata Harvest - Authors
    
+
    #{@env.widget_static.search_form}
    
+
    
+
    #{harvest_languages}
    
+
    
+WOK
+      end
+      def html_head
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << html_head_adjust(lng,'maintenance') if @opt.cmd.inspect =~/M/
+          @output[lng][:html] << html_head_adjust(lng)
+        end
+      end
+      def html_alph
+        a=[]
+        a << '
    '
+        @alph.each do |x|
+          a << ((x =~/[0-9]/) \
+          ? ''
+          : %{#{x}, })
+        end
+        a=a.join
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << a if @opt.cmd.inspect =~/M/
+          @output[lng][:html] << a
+        end
+      end
+      def html_tail
+        a =<
+
+
+
+
+
+
+#{@vz.credits_sisu}
+
+
+WOK
+        @the_idx.keys.each do |lng|
+          @output[lng][:html_mnt] << a if @output[lng][:html_mnt].is_a?(File)
+          @output[lng][:html] << a
+        end
+      end
+      def do_html(lng,html)
+        @output[lng][:html] << html
+      end
+      def do_html_maintenance(lng,html)
+        @output[lng][:html_mnt] << html if @output[lng][:html_mnt].is_a?(File)
+      end
+      def do_string(lng,attrib,string)
+        html=%{
    #{string}
    }
+        do_html(lng,html)
+        do_html_maintenance(lng,html) if @output[lng][:html_mnt].is_a?(File)
+      end
+      def do_string_default(lng,attrib,string)
+        html=%{
    #{string}
    }
+        do_html(lng,html)
+      end
+      def do_string_maintenance(lng,attrib,string)
+        html=%{
    #{string}
    }
+        do_html_maintenance(lng,html) if @output[lng][:html_mnt].is_a?(File)
+      end
+      def do_string_name(lng,attrib,string)
+        f=/^(\S)/.match(string)[1]
+        if @lng != lng
+          @alph=@alphabet_list.dup
+          @letter=@alph.shift
+          @lng = lng
+        end
+        if @letter < f
+          while @letter < f
+            if @alph.length > 0
+              @letter=@alph.shift
+              if @output[lng][:html_mnt].is_a?(File)
+                @output[lng][:html_mnt] << %{\n
    #{@letter}
    }
+              end
+              @output[lng][:html] << %{\n
    #{@letter}
    }
+            else break
+            end
+          end
+        end
+        name=string.strip.gsub(/\s+/,'_')
+        html=%{
    #{string}
    }
+        do_html(lng,html)
+        do_html_maintenance(lng,html) if @output[lng][:html_mnt].is_a?(File)
+      end
+      def do_array(lng,lv,array)
+        lv+=1
+        array.each do |b|
+          do_case(lng,lv,b)
+        end
+      end
+      def do_hash_md(lng,attrib,hash)
+        manifest_at=if @env.output_dir_structure.by? == :language
+          hash[:file] + '.html'
+        elsif @env.output_dir_structure.by? == :filetype
+          hash[:file] + '.' + lng +  '.html'
+        elsif @env.output_dir_structure.by? == :filename
+          "./#{hash[:file]}/#{hash[:page]}"
+        else '' #error
+        end
+        html=%{#{hash[:title]} - #{hash[:author]}}
+        do_string_default(lng,attrib,html)
+      end
+      def do_hash_md_maintenance(lng,attrib,hash)
+        if @output[lng][:html_mnt].is_a?(File) #should not be run for presentation output
+          html=%{[src]  #{hash[:title]} - #{hash[:author]}}
+          do_string_maintenance(lng,attrib,html)
+        end
+      end
+      def do_hash(lng,lv,hash)
+        lv+=1
+        key=[]
+        hash.each_key do |m|
+          if m == :md
+            do_case(lng,lv,hash[m])
+          elsif m != :title \
+          and m != :author \
+          and m != :filename \
+          and m != :file \
+          and m != :rough_idx \
+          and m != :page
+            key << m
+          elsif m == :title
+            do_hash_md(lng,'work',hash)
+            do_hash_md_maintenance(lng,'work',hash)
+          end
+        end
+        if key.length > 0
+          key.sort.each do |m|
+            attrib="lev#{lv}"
+            lv==0 ? do_string_name(lng,attrib,m) : do_string(lng,attrib,m)
+            do_case(lng,lv,hash[m])
+          end
+        end
+      end
+      def do_case(lng,lv,a)
+        case a
+        when String
+          attrib="lev#{lv}"
+          if a=~/S/
+            lv==0 ? do_string_name(lng,attrib,a) : do_string(lng,attrib,a)
+          end
+        when Array
+          do_array(lng,lv,a)
+        when Hash
+          do_hash(lng,lv,a)
+        end
+      end
+      #def html_body
+      #  the_idx=@the_idx
+      #  the_idx.each_pair do |lng,lng_array|
+      #    lng_array.sort.each do |a|
+      #      do_case(lng,-1,a)
+      #    end
+      #  end
+      #end
+      self
+    end
+  end
+end
+__END__
+terms -|_  t{tl1} -|_ {fa}[fa]{filenames and other details}
+       |           |_ {tl2} -|_ {fa}[fa]{filenames and other details}
+       |           |         |_{tl3} -|_ {fa}[fa]{filenames and other details}
+       |           |         |        |_{tl4} - {fa}[fa]{filenames and other details}
+       |           |         |        |
+       |           |         |        |_{tl4a} - {fa}[fa]{filenames and other details}
+       |           |         |        |
+       |           |         |        |_{tl4b} - {fa}[fa]{filenames and other details}
+       |           |         |        |
+       |           |         |        |_ ...
+       |           |         |
+       |           |         |_{tl3a} - {fa}[fa]{filenames and other details}
+       |           |
+       |           |_{tl2a} - {fa}[fa]{filenames and other details}
+       |
+       |_ t{tl1a} -|_ {fa}[fa]{filenames and other details}
+                   |_ ...
diff --git a/lib/sisu/v5/help.rb b/lib/sisu/v5/help.rb
new file mode 100644
index 00000000..b73cd138
--- /dev/null
+++ b/lib/sisu/v5/help.rb
@@ -0,0 +1,1881 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: interactive infomation/help
+
+=end
+module SiSU_Help
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Screen
+  require_relative 'param'                              # param.rb
+  class Help
+     require_relative 'i18n'                             # i18n.rb
+    def initialize(request='',color='')
+      @request,@color=request,color
+      @cX=((color =~/color_off/) \
+      || (defined? color.act && color.act[:color_state][:set]==:off)) \
+      ? (SiSU_Screen::Ansi.new('k').cX)
+      : (SiSU_Screen::Ansi.new('yes').cX)
+      fns='help_example_dummy_file_name.sst'
+      @env=SiSU_Env::InfoEnv.new(fns)
+      @db=SiSU_Env::InfoDb.new
+      m=/.+\/(?:src\/)?(\S+)/im # m=/.+?\/(?:src\/)?([^\/]+)$/im # m=/.+\/(\S+)/m
+      @output_stub=Dir.pwd[m,1]
+    end
+    def help_request
+      begin
+        gotten=nil
+        regx=/^(list|com(?:mands)?|mod(?:ifiers)|markup|syntax|example(?:_v1|_v2)?|head(?:ers?)?|(?:heading|title|level|structure)s?|endnotes|footnotes|tables?|customise|skin|dir(?:ectories)?|paths?|lang(?:uage)?|modules|setup|conf(?:ig(?:ure)?)?|standards?|li[cs]en[sc]e|scratch|install|termsheet|dublin(?:core)?|dc|customise|styles?|appearance|theme|env(ironment)?|dir(?:ector(?:y|ies))?|metaverse|abstract|features|summary|(?:short)?cuts?|sisu|about|ext(?:ernal)?(?:_?prog(?:rams)?)?)|utf-?8|plaintext|html|xml|xhtml|epub|odf|odt|opendocument|css|pdf|latex|tex|(?:tex)?info|search|(?:hyper)?est(?:raier)?|searchform|cgi|sql|db|pg|postgresql|pg?sql|sqlite|convert|php|webrick|sitemaps?|ya?ml|ansi|colors|-[AabcDdEeFHhIMmNnopqrRSstUuVvwXxyZz0-9]|-[Ddcv]|-[CcFLSVvW]/
+      help_info=%{#{@cX.blue_hi}SiSU help#{@cX.off} #{@cX.ruby}~#{@cX.off} #{@request}}
+      help_list=%{#{@cX.blue}sisu --help#{@cX.off} #{@cX.cyan}type keyword else "enter" to exit help:\n\tkeywords include:#{@cX.off} #{@cX.brown}list, (com)mands, short(cuts), (mod)ifiers, (env)ironment, markup, syntax, headers, headings, endnotes, tables, example, customise, skin, (dir)ectories, path, (lang)uage, db, install, setup, (conf)igure, convert, termsheet, search, sql, features, license#{@cX.off} \n}
+      help_prompt=%{#{@cX.fuschia}exit, [or carriage return to exit help] #{@cX.off}\n#{@cX.blue_hi}SiSU help#{@cX.off} #{@cX.ruby}~#{@cX.off} }
+        until gotten =~/exit|quit|bye|q|^\s*$/ \
+        and ( @request.nil? or @request.empty? )
+          @help=Help.new(@request,@color)
+          if @request
+            puts help_info
+            gotten=@request
+            @request=nil
+          end
+          case gotten
+          when /h((?:elp)| )|~/i
+            @help.summary
+            help_@request
+          when /list/;                                     @help.summary
+          when /com(mands)?/;                              @help.commands
+          when /mod(ifiers)?/;                             @help.modifiers
+          when /markup|syntax/;                            @help.markup
+          when /example\b/;                                @help.example_v2
+          when /example_v1/;                               @help.example_v1
+          when /example(_v2)?/;                            @help.example_v2
+          when /(?:heading|title|level)s?|structure/;      @help.headings
+          when /head(ers?)?/;                              @help.headers
+          when /dublin(core)?|dc/;                         @help.dublin_core
+          when /(?:foot|end)notes/;                        @help.endnotes
+          when /tables?/;                                  @help.tables
+          when /customise|skin/;                           @help.customise
+          when /modules/;                                  @help.modules
+          when /env(ironment)?/;                           @help.environment
+          when /dir(ector(y|ies))?/;                       @help.directories
+          when /paths?/;                                   @help.path
+          when /setup/;                                    @help.setup
+          when /conf(?:ig(?:ure)?)?/;                      @help.configure
+          when /standards?/;                               @help.standards
+          when /lang(?:uage)?/;                            @help.languages
+          when /li[cs]en[sc]e/;                            @help.license
+          when /scratch/;                                  @help.scratch
+          when /install/;                                  @help.install
+          when /(?:--)?termsheet/;                         @help.termsheet
+          when /customise|styles?|appearance|theme/;       @help.customise
+          when /metaverse/;                                @help.dal
+          when /(?:--)?plaintext|(?:--)?te?xt|-[aAeE]/;    @help.plaintext
+          when /utf-?8/i;                                  @help.utf8
+          when /(?:--)?html|-[hH]/;                        @help.html
+          when /css/;                                      @help.css
+          when /(?:--)?xhtml|-b/;                          @help.xhtml
+          when /(?:--)?xml|-[xX]/;                         @help.xml
+          when /(?:--)?odf|(?:--)?odt|opendocument|-o/;    @help.odf
+          when /(?:--)?epub|-e/;                           @help.epub
+          when /php/;                                      @help.php
+          when /(?:--)?pdf|-p/;                            @help.pdf
+          when /latex|tex/;                                @help.latex
+          when /(tex)?info/;                               @help.texinfo
+          when /lout/;                                     @help.lout
+          when /concordance|index|-w/;                     @help.concordance
+          when /search\b/;                                 @help.help_search
+          when /(?:hyper)?est(?:raier)?/;                  @help.hyperestraier
+          when /db|database|sql|postgresql|(?:--)?sqlite|(?:--)?pg|pg?sql|-[dD]/; @help.sql
+          when /searchform|cgi/;                           @help.cgi
+          when /convert/;                                  @help.convert
+          when /(?:--)?webrick|-W/;                        @help.webrick
+          when /abstract|features|summary|about|sisu/;     @help.abstract
+          when /ext(?:ernal)?(?:_?prog(?:rams)?)?/;        @help.external_programs
+          when /ya?ml/;                                    @help.yaml
+          when /sitemaps?/;                                @help.sitemap
+          when /(?:short)?cuts?/;                          @help.shortcuts
+          when /ansi|colors?/;                             SiSU_Screen::Ansi.new('c').colors
+          else                                             @help.summary
+          end
+          print help_list
+          print help_prompt
+          gotten=nil
+          gotten=gets
+        end
+      rescue
+        #STDERR.puts Ansi.new($!, $@).rescue
+        # dies silently... for now, silence of use in connection with "sisu ~ commands" etc.
+      ensure
+      end
+    end
+    def summary
+      print <<-WOK
+    SiSU, Copyright (C) 1997 - 2013 Ralph Amissah
+    License GPL version 3 or Later. This program comes with ABSOLUTELY NO WARRANTY;
+    This is free software, and you are welcome to redistribute it under the conditions of the GPL3 or later.
+    For more license detail type/enter: "sisu --help license"
+
+for help type 'sisu --help', 'sisu --help [help request]', 'man sisu', (or see the system or online documentation)
+
+typing "sisu" on its own or "sisu --help", should give you this sisu help summary and the sisu (interactive help mode) help prompt, from which help on each keyword can be obtained.
+
+alternatively typing #{@cX.orange}sisu --help#{@cX.off} #{@cX.green}[keyword]#{@cX.off} at the command prompt will provide the sisu help page requested and return to the command prompt (if nothing is found it will print this page and return to the command prompt)
+
+  Keywords (related to using SiSU)
+      #{@cX.green}help#{@cX.off} or #{@cX.green}list#{@cX.off}   this sisu help summary
+      #{@cX.green}commands#{@cX.off}       sisu --help commands
+      #{@cX.green}environment#{@cX.off}    sisu --help env
+    ------------------------------------------
+    Preparing Documents for SiSU
+      #{@cX.green}markup#{@cX.off}         sisu --help markup     (an incomplete overview)
+      #{@cX.green}headers#{@cX.off}        sisu --help headers    (document-wide instructions, meta-data)
+      #{@cX.green}structure#{@cX.off}      sisu --help structure  (document structure, headings, tables of contents)
+      #{@cX.green}endnotes#{@cX.off}       sisu --help endnotes
+      #{@cX.green}tables#{@cX.off}         sisu --help tables
+      #{@cX.green}example 1.0#{@cX.off}    sisu --help example_v1
+      #{@cX.green}example 2.0#{@cX.off}    sisu --help example
+    ------------------------------------------
+      #{@cX.green}search#{@cX.off}         sisu --help search
+    ------------------------------------------
+      #{@cX.green}customise#{@cX.off}      sisu --help customise
+    ------------------------------------------
+    SiSU's License
+      #{@cX.green}license#{@cX.off}        sisu --help license
+
+for help type 'sisu --help', 'sisu --help [help request]', 'man sisu', (or see the system or online documentation)
+     WOK
+  end
+  def abstract
+    print <<-WOK
+Features:
+#{@cX.cyan}(i)#{@cX.off} minimal markup requirement
+#{@cX.cyan}(ii)#{@cX.off} single file marked up for multiple outputs
+#{@cX.cyan}(iii)#{@cX.off} markup is simpler than html
+#{@cX.cyan}(iv)#{@cX.off} the simple syntax is mnemonic, influenced by mail/messaging/wiki markup practices
+#{@cX.cyan}(v)#{@cX.off} human readable, and easily writable
+#{@cX.cyan}(vi)#{@cX.off} multiple outputs include amongst others: html; pdf via LaTeX; (structured) XML; sql - currently PostgreSQL and sqlite; plaintext, (also texinfo)
+#{@cX.cyan}(vii)#{@cX.off} all text objects (headings and paragraphs) are numbered identically, for citation purposes, in all outputs (html, pdf, sql etc.)
+#{@cX.cyan}(viii)#{@cX.off} creates organised directory/file structure for output
+#{@cX.cyan}(ix)#{@cX.off} easily mapped with its clearly defined structure, with all text objects numbered, you know in advance where in each document output type, a bit of text will be found (eg. from an sql search, you know where to go to find the prepared html output or pdf etc.)... there is more
+#{@cX.cyan}(x)#{@cX.off} use of Dublin Core and other meta-tags to permit the addition of some semantic information on documents, and making easy integration of rdf/rss feeds etc.
+#{@cX.cyan}(xi)#{@cX.off} very easily skinnable, document appearance on a project/site wide, or document instance level easily controlled/changed
+#{@cX.cyan}(xii)#{@cX.off} in many cases a regular expression may be used (once in the document header) to define all or part of a documents structure obviating or reducing the need to provide structural markup within the document
+#{@cX.cyan}(xiii)#{@cX.off} is a batch processor for handling large document sets, ... though once generated they need not be re-generated, unless changes are made to the desired presentation of a particular output type
+#{@cX.cyan}(xiv)#{@cX.off} possible to pre-process, which permits the easy creation of standard form documents, and templates/term-sheets
+#{@cX.cyan}(xv)#{@cX.off} extremely modular, (thanks in no small part to Ruby) another output format required, write another module....
+#{@cX.cyan}(xvi)#{@cX.off} easy to update output formats (eg html, xhtml, latex/pdf produced can be updated in program and run against whole document set)
+#{@cX.cyan}(xvii)#{@cX.off} easy to add, modify, or have alternative syntax rules for input, should you need to
+#{@cX.cyan}(xviii)#{@cX.off} "Concordance" wordmap, consisting of all the words in a document and their (text object) locations within the text
+#{@cX.cyan}(xix)#{@cX.off} tied to revision control system, only code and marked up file need be backed up, to be sure of the much larger document set
+#{@cX.cyan}(xx)#{@cX.off} syntax highlighting files for markup, primarily (g)vim so far.
+
+SiSU was developed in relation to legal documents, and so is strong across a wide variety of texts (law, literature...), though weak on formulae/statistics, it does handle images. An assumption has been document sets that are to be preserved and maintained over time (also a result of the legal text origin). SiSU has been developed and used over a number of years, and the requirements to cover a wide range of documents have been thoroughly explored.
+
+There is more detailed information available on it from:
+  #{@cX.blue}http://www.jus.uio.no/sisu#{@cX.off}
+ *  plaintext
+ *  HTML
+ *  ODT (Open Document Format text)
+ *  EPUB
+ *  XML (structured)
+ *  LaTeX
+ *  PDF (via LaTeX)
+ *  TeXinfo
+ *  SQL (at postgreSQL & SQLite)
+      WOK
+    end
+    def commands
+      print <<-WOK
+
+ #{@cX.ruby}-a#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces plaintext with Unix linefeeds. Without markup, (object numbers are omitted), has footnotes at end of each para‐ graph that contains them. Modifier options available: --footnotes (default) or --endnotes and for linefeeds --unix (default) or --msdos
+
+ #{@cX.ruby}-b#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces xhtml/XML output for browser viewing (sax parsing)
+
+ #{@cX.ruby}-C#{@cX.off} initialise shared output directory (config files such as css and dtd files are not updated if they already exist unless modifier is used) #{@cX.ruby}-C --init=site#{@cX.off} configure/initialise site more extensive than -C on its own, shared output directory files/force update, existing shared output config files such as css and dtd files are updated if this modifier is used. in a new markup document working directory should initialise the corresponding output directory, though SiSU will automatically do this, the first time it is run (for processing) in a given directory.
+
+ #{@cX.ruby}-c#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} screen toggle ansi screen colour on or off depending on default set (unless -c flag is used: if sisurc colour default is set to ´true´, output to screen will be with colour, if sisurc colour default is set to ´false´ or is undefined screen output will be without colour)
+
+ #{@cX.ruby}-D#{@cX.off} #{@cX.green}[instruction]#{@cX.off} #{@cX.green}[filename]#{@cX.off} database instruction, see database section below
+
+ #{@cX.ruby}-d#{@cX.off} #{@cX.green}[instruction]#{@cX.off} #{@cX.green}[filename]#{@cX.off} database instruction, see database section below [only -D currently available]
+
+ #{@cX.ruby}-F#{@cX.off} generate examples of (naive) cgi search form for sqlite and pgsql depends on your already having used sisu to populate an sqlite and/or pgsql database, (the sqlite version scans the output directories for existing sisu_sqlite databases, so it is first necessary to create them, before generating the search form) see -d -D and the database section below. If the optional parameter webrick is passed, the cgi examples created will be set up to use the default port set for use by the webrick server, (otherwise the port is left blank and the system setting used, usually 80). The samples are dumped in the present work directory which must be writable, (with screen instructions given that they be copied to the cgi-bin directory).
+
+ #{@cX.ruby}-H#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces html (css version) (creates html (using css)), with url link suffixes (.html .pdf etc.) omitted ("Hide"). For web servers that are confireud so as not to require file extensions to locate and serve files. [behaviour switched see -h]
+
+ #{@cX.ruby}-h#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces html (hardlinks i.e. with name suffixes in links/local urls). html, with internal document links that include the document suffix, ie whether it is .html or .pdf (required for browsing directly off a file system, and works with most web servers). [behaviour switched see -H]
+
+ #{@cX.ruby}-I#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces texinfo file with its myriad of possibilities
+
+ #{@cX.ruby}-L#{@cX.off} prints license information
+
+ #{@cX.ruby}-M#{@cX.off} #{@cX.green}[filename/wildcard/url]#{@cX.off} maintenance mode, files created for processing are not deleted, and their locations are indicated (also see -V)
+
+ #{@cX.ruby}-m#{@cX.off} #{@cX.green}[filename/wildcard/url]#{@cX.off} create (new)metaVerse (used in all subsequent processing). Produce a meta file, the first step in processing, and the file all subsequent processing utilize. (Should usually be run together with other commands to ensure that the lated version of markup source document is used, i.e. add -m flag to other flags required).
+
+ #{@cX.ruby}-N#{@cX.off} #{@cX.green}[filename/wildcard/url]#{@cX.off} document content certificate as md5 digest tree of document produced (as digest.txt), the digest for the document, and digests for each object contained within the document (together with information on software versions that produced it). Try -mNV for verbose digest output to screen
+
+ #{@cX.ruby}-n#{@cX.off} #{@cX.green}[filename/wildcard/url]#{@cX.off} skip meta-markup (building of "metaverse"), this skips the equivalent of -m
+
+ #{@cX.ruby}-p#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces LaTeX pdf (portrait & landscape). Default paper size is set in config file, or document header, or provided with additional command line parameter, e.g. --papersize='a4' preset sizes include: 'A4', U.S. 'letter' and 'legal' and book sizes 'A5' and 'B5' (system defaults to A4).
+
+ #{@cX.ruby}-q#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} quiet, less output to screen
+
+ #{@cX.ruby}-r#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} copies sisu output files to remote host using scp (default). This requires that sisurc.yml has been provided with information on hostname and user name, and that you have your "keys" and ssh agent in place.
+
+ #{@cX.ruby}-S#{@cX.off} #produces a sisupod, a zipped sisu directory of markup files including sisu markup source files and the directories local configuration file, images and skins. Note: this only includes the configuration files or skins contained in ./_sisu not those in ~/.sisu The resulting tar gzip file has a .zip suffix added to the markup source directory name. To tar and gzip individual files see the -Z [filename/wildcard] option. Note: (this option is tested only with zsh)
+
+ #{@cX.ruby}-S#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces a sisupod a zipped sisu of the content assocated with the specified sisu markup documnt, i.e. including sisu markup source file, (and associated documents if a master file, or available in multilingual versions), together with related images and skin. The resulting zipped file has a .zip suffix added to the markup source file name by default, though a .ssp suffix is also recognised. The directory structure of the unzipped file is understood by sisu, and sisu commands can be run within it. SiSU commands can be run against a sisupod contained in a local directory, or provided as a url on a remote site. As there is a security issue with skins provided by other users, they are not applied unless the flag --trust or --trusted is added to the command instruction, it is recommended that file that are not your own are treated as untrusted. This provides a convenient way of packing documents files for sending Note: if you wish to send multiple files, it quickly becomes more space efficient to tar and gzip the sisu markup directory, (without the _sisu_processing subdirectory) rather than the individual files for sending). See the -S option without [filename/wildcard]
+
+ #{@cX.ruby}-T#{@cX.off} #{@cX.green}[filename/wildcard (*.termsheet.rb)]#{@cX.off} standard form document builder, preprocessing feature
+
+ #{@cX.ruby}-t#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces plaintext with Unix linefeeds. Without markup, (object numbers are omitted), has footnotes at end of each para‐ graph that contains them. Modifier options available: --footnotes (default) or --endnotes and for linefeeds --unix (default) or --msdos
+
+ #{@cX.ruby}-U#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} prints url list/map for the available processing flags options and resulting files that could be requested, (can be used to get a list of processing options in relation to a file, together with information on the output that would be produced), -u provides url mapping for those flags requested for processing. The default assumes sisu_webrick is running and provides webrick url mappings where appropriate, but these can be switched to file system paths in sisurc.yml
+
+ #{@cX.ruby}-u#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} provides url mapping of output files for the flags requested for processing, also see -U
+
+ #{@cX.ruby}-V#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} on its own provides SiSU version and environment information (sisu --help env)
+
+ #{@cX.ruby}-V#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} even more verbose than -v the -V flag provides some additional information, also see -M
+
+ #{@cX.ruby}-v#{@cX.off} on its own, provides SiSU version information.
+
+ #{@cX.ruby}-v#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} provides more verbose output of what is being built, where it is being built (and error messages if any), as with -u flag provides a url mapping of files created for each of the processing flag requests
+
+ #{@cX.ruby}-X#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces XML output with deep document structure, in the nature of dom
+
+ #{@cX.ruby}-x#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces XML output (sax parsing)
+
+ #{@cX.ruby}-W#{@cX.off} #{@cX.green}[port]#{@cX.off} starts ruby´s webrick webserver, points at sisu output directories (default port is set)
+
+ #{@cX.ruby}-w#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces concordance, a rudimentary index of all the words in a document
+
+ #{@cX.ruby}-Z#{@cX.off} Zap, if used with other processing flags #{@cX.green}deletes output files#{@cX.off} of the type about to be processed, prior to processing. If -Z is used as the lone processing related flag (or in conjunction with a combination of -[mMvVq]), will remove the related document output directory.
+
+ #{@cX.ruby}-z#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} produces php (zend) [feature disabled, depreciated]
+
+ #{@cX.ruby}--harvest#{@cX.off} #{@cX.green}*.ss[tm]#{@cX.off} makes two lists of sisu output based on the sisu markup documents in a directory: list of author and authors works (year and titles), and; list by topic with titles and author. Makes use of header metadata fields (author, title, date, topic_register). Can be used with -M and -R flags.
+
+ #{@cX.ruby}databases#{@cX.off}
+
+ #{@cX.ruby}dbi - database interface -D or --pgsql set for postgresql -d or --sqlite set for sqlite#{@cX.off}
+
+ #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--create#{@cX.off} creates empty postgresql db and required tables & indexes (rb.dbi) [#{@cX.ruby}-d --create#{@cX.off} sqlite equivalent] it may be necessary to first run sisu #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--createdb#{@cX.off}
+
+ #{@cX.ruby}-Di#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} or #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--import#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} imports data specified to postgresql db (rb.dbi) [#{@cX.ruby}-d --import#{@cX.off} sqlite equivalent]
+
+ #{@cX.ruby}-Du#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} or #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--update#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} updates/imports specified data to postgresql db (rb.dbi) [#{@cX.ruby}-d --update#{@cX.off} sqlite equivalent]
+
+ #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--remove#{@cX.off} #{@cX.green}[filename/wildcard]#{@cX.off} removes specified data to postgresql db (rb.dbi) [#{@cX.ruby}-d --remove#{@cX.off} sqlite equivalent]
+
+ #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--dropall#{@cX.off} kills data" and drops (postgresql) db, tables & indexes [#{@cX.ruby}-d --dropall#{@cX.off} sqlite equivalent]
+
+ #{@cX.ruby}-D#{@cX.off} #{@cX.ruby}--recreate#{@cX.off} kills data" and drops (postgresql or sqlite) db, tables & indexes, then creates an empty db with tables and indexes [#{@cX.ruby}-d --recreate#{@cX.off} sqlite equivalent]
+
+  also see command #{@cX.green}shortcuts#{@cX.off}, and shorthand mappings for multiple flags
+      WOK
+    end
+    def shortcuts
+      cf_defaults=SiSU_Env::InfoProcessingFlag.new
+      print <<-WOK
+
+ #{@cX.ruby}Shorthand for multiple flags#{@cX.off}
+
+        #{@cX.ruby}--update#{@cX.off}  #{@cX.green}[filename/wildcard]#{@cX.off} Checks  existing file output and runs the flags required to update this output. This means that if only html and pdf output  was  requested  on previous  runs, only the -hp files will be applied, and only these will be generated this time, together with the summary.  This  can  be  very convenient, if you offer different outputs of different files, and just want to do the same again.
+
+       #{@cX.ruby}-0#{@cX.off} to #{@cX.ruby}-5#{@cX.off} #{@cX.green}[filename or wildcard]#{@cX.off}
+       #{@cX.green}Default shorthand mappings#{@cX.off} (note that the defaults can be changed in the #{@cX.green}sisurc.yml#{@cX.off} file):
+
+                                                  (these can be turned off if unavailable in sisurc.yml under program_set:)
+  #{@cX.green}processing shortcut defaults set to:#{@cX.off}
+  color defaut set (on==true)                    #{@cX.blue}#{cf_defaults.color}#{@cX.off}
+  sisu -0 (also just "sisu")                     #{@cX.blue}#{cf_defaults.cf_0}#{@cX.off}
+  sisu -1                                        #{@cX.blue}#{cf_defaults.cf_1}#{@cX.off}
+  sisu -2                                        #{@cX.blue}#{cf_defaults.cf_2}#{@cX.off}
+  sisu -3                                        #{@cX.blue}#{cf_defaults.cf_3}#{@cX.off}
+  sisu -4                                        #{@cX.blue}#{cf_defaults.cf_4}#{@cX.off}
+  sisu -5                                        #{@cX.blue}#{cf_defaults.cf_5}#{@cX.off}
+  defaults may be changed in active sisurc.yml file under 'flag:'
+
+       add -v for verbose mode and -c (color toggle), e.g.
+      WOK
+    end
+    def modifiers
+      print <<-WOK
+
+ #{@cX.ruby}Command flag modifiers#{@cX.off}
+
+  #{@cX.ruby}--no-ocn#{@cX.off} [with -h -H or -p] switches off object citation numbering. Produce  output without identifying numbers in margins of html or LaTeX/pdf output.
+
+  #{@cX.ruby}--no-annotate#{@cX.off} strips output text of editor endnotes~[* square brackets ]~ denoted by asterisk or dagger/plus sign
+
+  #{@cX.ruby}--no-asterisk#{@cX.off} strips output text of editor endnotes~[* square brackets ]~ denoted by asterisk
+
+  #{@cX.ruby}--no-dagger#{@cX.off} strips output text of editor endnotes~[+ square brackets ]~ denoted by dagger/plus sign
+
+      WOK
+    end
+    def misc
+      <<-WOK
+  #{@cX.cyan}misc#{@cX.off}
+  #{@cX.green}-s#{@cX.off} [filename or wildcard]          #{@cX.green}spellcheck#{@cX.off} (aspell previously ispell
+  \t#{@cX.green}mailer examples#{@cX.off}
+  from vim mail.er10 (not gvim) issue command
+  \t:! ruby -S mailer.rb instruction/landscape|a4l|portrait|a4p/ email@address/alias subject line
+  \t:! ruby -S mailer.rb a4l ralph@amissah.com testing continues
+  from vim mail.er10 use your vim alias \\mail or \\mutt (modify command line as required)
+  #{@cX.green}feeds rss/rdf#{@cX.off}
+  #{@cX.blue}-R #{@cX.off} (yaml|rss)  extraction of semantic data into yaml file for auto build of xml feeds (rss, rdf) #{@cX.fuschia}[work area]#{@cX.off}
+  #{@cX.green}-R #{@cX.off} yaml  extraction of semantic data into yaml file for auto build of xml feeds (rss, rdf)
+  #{@cX.green}-R #{@cX.off} rss   creates rss2.0 feed
+      WOK
+    end
+    def markup
+      print <<-WOK
+sisu
+  Note: files for SiSU should be in UTF-8 character encoding.
+
+  #{@cX.cyan}Data text markup#{@cX.off} (alternative to available html subset)
+  #{@cX.green}% SiSU 4.00#{@cX.off}          [statement on first line of document, declared file-type identifier, SiSU markup document
+
+  #{@cX.green}A~#{@cX.off} heading/title    [levels A to C available (and beneath that 1 to 6)]
+
+  #{@cX.green}1~#{@cX.off}filename heading   [segmentation level, levels 1 to 6 available]
+
+  #{@cX.green}!{#{@cX.off}emphasis#{@cX.green}}!#{@cX.off}
+  #{@cX.green}*{#{@cX.off}bold text#{@cX.green}}*#{@cX.off}
+  #{@cX.green}_{#{@cX.off}underscore#{@cX.green}}_#{@cX.off}
+  #{@cX.green}/{#{@cX.off}italics#{@cX.green}}/#{@cX.off}
+  #{@cX.green}"{#{@cX.off}citation#{@cX.green}}"#{@cX.off}
+  #{@cX.green}^{#{@cX.off}superscript#{@cX.green}}^#{@cX.off}
+  #{@cX.green},{#{@cX.off}subscript#{@cX.green}},#{@cX.off}
+  #{@cX.green}+{#{@cX.off}inserted text#{@cX.green}}+#{@cX.off}
+  #{@cX.green}-{#{@cX.off}strikethrough#{@cX.green}}-#{@cX.off}
+
+  ------------------------------------------
+  #{@cX.cyan}Indentation and bullets#{@cX.off}
+
+  #{@cX.green}_1#{@cX.off}                     indent paragraph one level
+
+  #{@cX.green}_2#{@cX.off}                     indent paragraph two steps
+
+  #{@cX.green}_*#{@cX.off}                     bullet text
+
+  #{@cX.green}_1*#{@cX.off}                    bullet text, first indent
+
+  ------------------------------------------
+  #{@cX.cyan}Numbered List#{@cX.off} (not to be confused with headings/titles, (document structure))
+
+  #{@cX.green}##{@cX.off} numbered list        numbered list 1., 2., 3, etc.
+
+  #{@cX.green}_##{@cX.off} numbered list       numbered list indented second level a., b., c., d., etc.
+
+  ------------------------------------------
+  #{@cX.cyan}Endnotes#{@cX.off}
+
+  #{@cX.green}~{#{@cX.off}footnote/endnote#{@cX.green}}~#{@cX.off}    endnote#{@cX.green}~{#{@cX.off}self contained endnote marker & endnote in one#{@cX.green}}~#{@cX.off}
+  #{@cX.green}~{*#{@cX.off}asterisk footnote/endnote#{@cX.green}}~#{@cX.off}
+  editor's annotations, square bracket notes
+  #{@cX.green}~[*#{@cX.off}numbered asterisk footnote/endnote series#{@cX.green}]~#{@cX.off}
+  #{@cX.green}~[+#{@cX.off}numbered dagger/plus sign footnote/endnote series#{@cX.green}]~#{@cX.off}
+  ---
+  alternative endnote pair notation
+  #{@cX.green}~^#{@cX.off}                                           endnote marker
+  #{@cX.green}^~#{@cX.off} endnote text following the paragraph in which the marker occurs
+
+  ------------------------------------------
+  #{@cX.cyan}Links#{@cX.off}
+
+    http://url.org                           on its own would be automatically marked up and hyperlinked to itself
+    #{@cX.green}{#{@cX.off} [text to link] #{@cX.green}}#{@cX.off}http://url.org
+    #{@cX.green}{#{@cX.off}image.png#{@cX.green}}#{@cX.off}http://url.org
+    #{@cX.green}{#{@cX.off}image.png#{@cX.green}}#{@cX.off}image    #{@cX.green}{#{@cX.off}tux.png 64#{@cX.green}x#{@cX.off}80#{@cX.green}}#{@cX.off}image
+  Linked image example
+    #{@cX.green}{#{@cX.off} SiSU Geek Writer #{@cX.green}}#{@cX.off}http://www.jus.uio.no/sisu/                   url example
+    #{@cX.green}{#{@cX.off}tux.png 64#{@cX.green}x#{@cX.off}80 "a better way" #{@cX.green}}#{@cX.off}http://www.jus.uio.no/sisu/    image example with all options (width x height)
+    Note: png and jpg support only (no gif support)
+
+  shortcut - hyper-linked text with endnote providing the url information
+    #{@cX.green}{~^#{@cX.off} [text to link] #{@cX.green}}#{@cX.off}http://url.org         maps to   #{@cX.green}{#{@cX.off} [text to link] #{@cX.green}}#{@cX.off}http://url.org #{@cX.green}~{#{@cX.off} http://url.org #{@cX.green}}~#{@cX.off}
+     produces hyper-linked text within a document/paragraph, with an endnote providing the url for the text location used in the hyperlink
+
+  shortcut:
+    #{@cX.green}{#{@cX.off} [text to link] #{@cX.green}[3sS]}#{@cX.off}markup_source_filename.sst
+    if  a  server host name has been provided/configured, will provide a list of available output types that would be generated using the shortcut command and the markup file provided, i.e. output generated using the command (as configured):
+       "sisu -3sS markup_source_filename.sst"
+    using server host, directory stub, filename to compose the link.
+
+  ------------------------------------------
+  adding fixed names in html, manual location marker/tagging
+  #{@cX.green}*~[name]#{@cX.off}           
+
+  ------------------------------------------
+  #{@cX.green}~##{@cX.off}         unnumbered paragraph (place marker at end of paragraph)
+  #{@cX.green}-##{@cX.off}         unnumbered paragraph, delete when not required (place marker at end of paragraph) [used in dummy headings, eg. for segmented html]
+
+  ------------------------------------------
+  manual page breaks (LaTeX/pdf)
+  #{@cX.green}<:pb>#{@cX.off}  page  break,  which  breaks a page, starting a new page in single column text and a new column in double column text
+  #{@cX.green}<:pn>#{@cX.off} page new, which starts a new page, in both single and double column text (leaving an empty column in double column text if necessary).
+  Note: page breaks are usually introduced to pdfs either as header instructions, indicating that pages should break at given levels
+
+  ------------------------------------------
+  #{@cX.cyan}Composite documents#{@cX.off}
+
+    It is possible to build a document by creating a master document that requires other documents. The documents required may complete documents that could be generated independently, or they could be markup snippets, prepared so as to be easily available to be placed within another text. If the calling document is a master document (built mainly from other documents), it should be named with the suffix #{@cX.blue}.ssm#{@cX.off} Within this document you would provide information on the other documents that should be included within the text. These may be other documents that would be processed in a regular way, or markup bits prepared only for inclusion within a master document #{@cX.blue}.sst#{@cX.off} regular markup file, or #{@cX.blue}.ssi#{@cX.off} (insert/information) A secondary file of the composite document is built prior to processing with the same prefix and the suffix #{@cX.blue}.ssm.sst#{@cX.off}
+
+    #{@cX.cyan}#basic sisu markup alternatives#{@cX.off}
+      #{@cX.green}{#{@cX.off}filename.ssi#{@cX.green}}require#{@cX.off}
+      #{@cX.green}<< {#{@cX.off}filename.ssi#{@cX.green}}#{@cX.off}
+
+    #{@cX.cyan}#using textlink alternatives#{@cX.off}
+      #{@cX.green}|#{@cX.off}filename.ssi#{@cX.green}|@|^|require#{@cX.off}
+      #{@cX.green}<< |#{@cX.off}filename.ssi#{@cX.green}|@|^|#{@cX.off}
+
+    #{@cX.cyan}#using thlnk alternatives#{@cX.off}
+      #{@cX.green}require#{@cX.off}
+      #{@cX.green}<< #{@cX.off}
+
+  #{@cX.cyan}Composite documents - remote parts#{@cX.off}
+    Composite documents may be built from remote parts, by using the composite document syntax with a url. This makes sense using either sisu regular syntax (which is just a convenient way of marking up), or thlnk syntax, which also recognises remote urls, and permits hyperlinking ascii to the url location.
+
+  #{@cX.cyan}Remote documents#{@cX.off}
+    SiSU will download and process remote locations if a url is provided instead of a filename. [this at present works only for sisu markup files without images]
+
+  ------------------------------------------
+  #{@cX.green}%#{@cX.off}#{@cX.off}       add a comment to text, that will be removed prior to processing (place marker at beginning of line)
+  #{@cX.green}\\#{@cX.off}#{@cX.off}       escape a sepcial character, whether general: { } < > or contextual special characters, (in combination with  other  characters) ~  - _ / % ^ and occasionally ! # + ,
+  #{@cX.green}%%#{@cX.off}#{@cX.off}      same as above but recognised by vim folds for placing fold in document text, in addition to headers and headings
+  ------------------------------------------
+
+  #{@cX.ruby}More HELP on Markup#{@cX.off} markup help is available on:
+    document wide instructions: headers (document structure)
+    general text markup: headings; endnotes; tables (which also includes a note on preformatted text)
+  configuration and customisation
+    document or site wide customisation: customise; skin
+      WOK
+      help_markup
+    end
+    def example
+      help_markup
+    end
+    def example_v1
+      print <<-WOK
+% SiSU 1.0
+
+@title: Working Sample Document
+
+@subtitle: Demonstrating markup
+
+@creator: Ralph Amissah
+
+@date:
+
+@markup: num_top=4
+
+@bold: [regular expression of words/phrases to be made bold]
+
+@italics: [regular expression of words/phrases to italicise]
+
+@links: { SiSU }http://www.jus.uio.no/sisu { FSF }http://www.fsf.org
+
+A~ A Sample Document
+
+B~ just for fun
+
+1~ This is Chapter One or Article One
+
+Ordinary Text follows here. The Title would be a Chapter or Article depending on the type of document you were working to produce.
+
+1~ This would be Chapter Two or Article Two
+
+And so on.
+
+Assuming sisu is configured properly so it has been instructed where to put the work files and ouput files, you would generate this text once saved, with the suffix .sst if saved as example.sst, by typing sisu -mhwxp example.sst while in the directory in which the file is saved.
+
+_1 -m initial processing, -h html (css based), -w concordance for html, -x xml, -p pdf output, generated via latex, there are of course additional options
+
+_1 for a listing type: sisu ~ commands
+
+_1 for an outline of sisu markup type: sisu ~ markup
+
+The example ends here.
+    WOK
+    help_markup
+  end
+    def example_v2
+      print <<-WOK
+% SiSU 2.0
+
+@title: Working Sample Document
+ :subtitle: Demonstrating markup
+
+@creator:
+ :author: Amissah, Ralph
+
+@date:
+ :published: 2010-09-18
+
+@rights:
+ :copyright: Ralph Amissah
+
+@make:
+ :num_top: 1
+ :bold: [regular expression of words/phrases to be made bold]
+ :italics: [regular expression of words/phrases to italicise]
+
+@links:
+ { SiSU }http://www.jus.uio.no/sisu { FSF }http://www.fsf.org
+
+A~ @title @author
+
+B~ just for fun
+
+1~ This is Chapter One or Article One
+
+Ordinary Text follows here. The Title would be a Chapter or Article depending on the type of document you were working to produce.
+
+1~ This would be Chapter Two or Article Two
+
+And so on.
+
+Assuming sisu is configured properly so it has been instructed where to put the work files and ouput files, you would generate this text once saved, with the suffix .sst if saved as example.sst, by typing sisu -mhwxp example.sst while in the directory in which the file is saved.
+
+_1 --dal initial processing, --html (css based), --concordance (html), --epub, --odt, --pdf output, generated via LaTeX, there are of course additional options
+
+_1 for a listing type: sisu ~ commands
+
+_1 for an outline of sisu markup type: sisu ~ markup
+
+The example ends here.
+    WOK
+    help_markup
+  end
+  def headers
+    print <<-WOK
+Header tags appear at the beginning of a document and provide meta information on the document (such as the Dublin Core), or information as to how the document as a whole is to be processed.
+All header instructions take either the form #{@cX.green}@headername:#{@cX.off} followed on the next line by an indented sub-category header if any #{@cX.green}:sub-headername:#{@cX.off}.
+#{@cX.green}@indentifier:#{@cX.off} information or instructions
+where the #{@cX.green}"identifier"#{@cX.off} is a tag recognised by the program, and the #{@cX.green}"information"#{@cX.off} or #{@cX.green}"instructions"#{@cX.off} belong to the tag/indentifier specified
+ Note: a header where used should only be used once; all  headers  apart from #{@cX.ruby}@title:#{@cX.off} are optional.
+
+#{@cX.ruby}@title:#{@cX.off} My Title - This is now the Title of the Document and used as such
+  #{@cX.cyan}:subtitle:#{@cX.off} The Subtitle if any
+
+#{@cX.ruby}@creator:#{@cX.off}
+  #{@cX.cyan}:author:#{@cX.off} Surname, Other names (if more than one author separate author names with a semi colon, if name is of an institution just write name or the name contains a comma enclose in quotation marks)
+  #{@cX.cyan}:contributor:#{@cX.off}
+  #{@cX.cyan}:translator:#{@cX.off} [or :translated_by:]
+  #{@cX.cyan}:illustrator:#{@cX.off} [or :illustrated_by:]
+  #{@cX.cyan}:prepared_by:#{@cX.off} [or :digitized_by:]
+
+#{@cX.ruby}@date:#{@cX.off}
+  #{@cX.cyan}:published:#{@cX.off}
+  #{@cX.cyan}:created:#{@cX.off}
+  #{@cX.cyan}:issued:#{@cX.off}
+  #{@cX.cyan}:available:#{@cX.off}
+  #{@cX.cyan}:valid:#{@cX.off}
+  #{@cX.cyan}:modified:#{@cX.off}
+
+#{@cX.ruby}@rights:#{@cX.off}
+  #{@cX.cyan}:copyright:#{@cX.off} Author's name, all rights reserved
+  #{@cX.cyan}:license:#{@cX.off} public domain, copyleft, creative commons variant, etc.
+
+#{@cX.ruby}@classify:#{@cX.off}
+  #{@cX.cyan}:topic_register:#{@cX.off} [e.g.:] text markup language; application:text processing;output:html|xml|latex|pdf|sql
+  #{@cX.cyan}:subject:#{@cX.off} (whatever your subject)
+  #{@cX.cyan}:keywords:#{@cX.off}
+  #{@cX.cyan}:loc:#{@cX.off} [Library of Congress classification]
+  #{@cX.cyan}:dewey:#{@cX.off} [Dewey classification]
+
+#{@cX.ruby}@identifier:#{@cX.off}
+  #{@cX.cyan}:isbn:#{@cX.off}
+  #{@cX.cyan}:oclc:#{@cX.off}
+
+#{@cX.ruby}@notes:#{@cX.off}
+  #{@cX.cyan}:description:#{@cX.off}
+  #{@cX.cyan}:abstract:#{@cX.off}
+  #{@cX.cyan}:comment:#{@cX.off}
+  #{@cX.cyan}:coverage:#{@cX.off}
+  #{@cX.cyan}:relation:#{@cX.off}
+  #{@cX.cyan}:source:#{@cX.off}
+  #{@cX.cyan}:history:#{@cX.off}
+  #{@cX.cyan}:type:#{@cX.off}
+  #{@cX.cyan}:format:#{@cX.off}
+  #{@cX.cyan}:prefix:#{@cX.off}
+  #{@cX.cyan}:suffix:#{@cX.off}
+
+#{@cX.ruby}@publisher:#{@cX.off}
+
+#{@cX.ruby}@language:#{@cX.off} [or @language.document:] [country code for language if available, or language, English, en is the default setting] (en - English, fr - French, de - German, it - Italian, es - Spanish, pt - Portuguese, sv - Swedish, da - Danish, fi - Finnish, no - Norwegian, is - Icelandic, nl - Dutch, et - Estonian, hu - Hungarian, pl - Polish, ro - Romanian, ru - Russian, el - Greek, uk - Ukranian, tr - Turkish, sk - Slovak, sl - Slovenian, hr - Croatian, cs - Czech, bg - Bulgarian ) [however, encodings are not available for all of the languages listed.]
+
+#{@cX.ruby}@language.original:#{@cX.off}
+original language in which the work was published
+
+#{@cX.ruby}@links:#{@cX.off}
+  { SiSU }http://www.jus.uio.no/sisu/
+  { FSF }http://www.fsf.org
+
+#{@cX.ruby}@make:#{@cX.off}
+  #{@cX.cyan}:breaks:#{@cX.off} new=:B; break=1
+  #{@cX.cyan}:bold:#{@cX.off} [regular expression of words/phrases to be made bold]
+  #{@cX.cyan}:italics:#{@cX.off} [regular expression of words/phrases to italize]
+  #{@cX.cyan}:home_button_image:#{@cX.off} {free_as_in_freedom.png }http://stallman.org/
+  #{@cX.cyan}:home_button_text:#{@cX.off} {Free Culture}http://www.free-culture.cc; {Lawrence Lessig}http://www.lessig.org
+  #{@cX.cyan}:footer:#{@cX.off} {Free Culture}http://www.free-culture.cc; {Lawrence Lessig}http://www.lessig.org
+  #{@cX.cyan}:headings:#{@cX.off} PART; CHAPTER; Section; Article;
+     #{@cX.grey}% optional, document structure can be defined by words to match or regular expression (the regular expression is assumed to start at the beginning of a line of text i.e. ^) default markers A~ to C~ and 1~ to 6~ can be used within text instead, without this header tag, and may be used to supplement the instructions provided in this header tag if provided#{@cX.off}
+  #{@cX.cyan}:papersize:#{@cX.off} (A4|US_letter|book_B5|book_A5|US_legal)
+  #{@cX.cyan}:markup:#{@cX.off} num_top=4 [various markup instructions, eg: num_top=4 headings tobe numbered, starting at heading level 4... the default is to provide 3 levels, as in 1 level 4, 1.1 level 5, 1.1.1 level 6, markup to be merged within level]
+  #{@cX.cyan}:promo:#{@cX.off} sisu, ruby, search_libre_docs, open_society [places content in right pane in html, makes use of list.yml and promo.yml, commented out sample in  document  sample:  free_as_in_freedom.richard_stallman_crusade_for_free_software.sam_williams.sst]
+
+#{@cX.grey}% header ends here, NB only @title: is mandatory [this would be a comment]#{@cX.off}
+
+#{@cX.blue}A~#{@cX.off} Top level heading [this is usually the same as the title @title: ]
+
+#{@cX.blue}B~#{@cX.off} Second level heading [this is a heading level divider]
+
+#{@cX.blue}C~#{@cX.off} Third level heading [this is a heading level divider]
+
+#{@cX.blue}1~#{@cX.off} Top level heading preceding substantive text of document or sub-heading 5, the heading level that would normally be marked 1. or 2. or 3. etc. in a document
+
+#{@cX.blue}2~#{@cX.off} Second level heading preceding substantive text of document or sub-heading 6, the heading level that would normally be marked 1.1 or 1.2 or 1.3 or 2.1 etc in a document
+
+#{@cX.blue}3~#{@cX.off} Third level heading preceding substantive text of document, that would normally be marked 1.1.1 or 1.1.2 or 1.2.1 or  2.1.1 etc. in a document
+
+      WOK
+      help_markup
+    end
+    def customise
+      print <<-WOK
+    There are a number of files that control the appearance of a site or a document i.e.
+
+    #{@cX.blue}(i)#{@cX.off} files that control #{@cX.green}site-wide appearance#{@cX.off}:
+
+        #{@cX.green}defaults.rb#{@cX.off}  initial values used by program, not necessary to change
+
+        skins are placed in a configuration directory beneath the markup directory ./_sisu/skin/doc or in ~/.sisu/skin/doc
+        if a skin is also to be used for a directory or site wide presentations, rather than being called for a single document then a softlink is placed from ~/.sisu/skin/dir or ~/.sisu/skin/site respectively to the relevant skin within ~/.sisu/skin/doc
+
+        rules for skin loading are in #{@cX.green}sysenv.rb#{@cX.off}
+        the skin loading heuristics / hierarchy is currently as follows:
+
+          a. if there is a skin requested within a document, thent that skin is used
+            0~skin skin_lm (would load skin ~/.sisu/skin/doc/skin_lm.rb)
+
+          b. use it if there is a "dir" skin with the same name as the source directory "stub" i.e.
+            if working in a directory called /home/myhome/workdir/lm
+            then if ~/.sisu/skin/dir/skin_lm.rb were found it would be used and skin_lm.rb would be a softlink to the relevant skin in the document skin directory (where all the actual skins are kept)
+
+          x. [additionally though discouraged a special rule can be set up in sysenv  AddSkin.select (which could override b if desired)]
+
+        #{@cX.green}skin_site_#{@cX.off}#{@cX.cyan}[name of site skin in use]#{@cX.off}#{@cX.green}.rb#{@cX.off}  this file is called by skin.rb
+          if there are several such files, site appearance may be changed by
+          requesting the skin_site desired from skin.rb
+    and
+
+    #{@cX.blue}(ii)#{@cX.off} presentation maintenance:
+          if you have a body of documents the easiest way to give them a particular appearance/skin, is to associate that directory with a skin, all files in that directory take on the specified appearance, changing appearance of documents once directories are set up is as simple as copying the files from one directory to another (or renaming the directory); or associating the directory with a different skin.
+
+    see also directories
+      WOK
+    end
+    def configure
+      print <<-WOK
+      see the following topics
+        install
+        setup
+        and note for initial configuration it is necessary to run:
+          sisu -C
+          (this places the default CSS files and DTDs in place)
+      WOK
+    end
+    def path
+      help_env
+      puts <<-WOK
+
+If you have problems check permissions (and if in home directory ownership).
+
+#{@cX.green}directory paths as currently set#{@cX.off}:
+  output docs:                 #{@cX.blue}#{@env.path.webserv}#{@cX.off}
+  cgi scripts:                 #{@cX.blue}#{@env.path.cgi}#{@cX.off}
+  processing:                  #{@cX.blue}#{@env.processing_path.processing}#{@cX.off}
+      sisu meta markup:        #{@cX.blue}#{@env.processing_path.dal}#{@cX.off}
+      html tuning:             #{@cX.blue}#{@env.processing_path.tune}#{@cX.off}
+      latex:                   #{@cX.blue}#{@env.processing_path.tex}#{@cX.off}
+      texinfo:                 #{@cX.blue}#{@env.processing_path.texi}#{@cX.off}
+  images:
+    source:                    #{@cX.blue}#{@env.path.image_source}#{@cX.off}
+    latex source:              #{@cX.blue}#{@env.path.image_source_include}#{@cX.off}
+    note images are also sourced from within your pwd - #{@cX.blue}#{Dir.pwd}/_sisu/image#{@cX.off} if it exists
+  #{@cX.grey}[ texinfo:                     #{@env.processing_path.texinfo} - check duplication ]#{@cX.off}
+
+ #{@cX.green}resource configuraton files#{@cX.off} to change the paths specified above, are searched for in the following order:
+   under the current SiSU markup data directory: #{@cX.blue}#{Dir.pwd}/_sisu/sisurc.yml#{@cX.off}
+   under the home directory ~/.sisu:             #{@cX.blue}#{@env.path.home}/.sisu/sisurc.yml#{@cX.off}
+   in the "/etc" directory:                      #{@cX.blue}#{@env.path.etc}/sisurc.yml#{@cX.off}
+   default file paths are set by the program SiSU
+
+ #{@cX.green}skins#{@cX.off} for document appearances on a site, directory or per document basis are located in subdirectories #{@cX.blue}doc/#{@cX.off}  #{@cX.blue}dir/#{@cX.off} and #{@cX.blue}site/#{@cX.off} within:
+   #{@cX.blue}#{Dir.pwd}/_sisu/skin#{@cX.off}
+   #{@cX.blue}#{@env.path.home}/.sisu/skin#{@cX.off}
+   #{@cX.blue}#{@env.path.etc}/skin#{@cX.off}
+   default appearances are set by the program SiSU in the absence of skins
+
+   #{@cX.green}Note on subdirectories for output documents#{@cX.off}
+   The last part of the name of the directory you choose to work from is used as the major sub-directory in which output files are placed,
+   i.e. if you are working in a directory called #{@cX.blue}#{@env.path.home}/ebook#{@cX.off}
+        the output files will be placed in a sub-directory named after the processed text within #{@cX.blue}#{@env.path.webserv}/ebook#{@cX.off}
+   ( Within this major sub-directory, a sub-directory is made with the name of each document processed, into which output files - html, pdf, xml, plaintext etc. are placed (texinfo being an exception at present) )
+
+    #{@cX.green}sisu -C#{@cX.off} [#{@cX.green}--init=site#{@cX.off}] configure/initialise shared output directory files initialize shared output directory (config files such as css and dtd files are not updated if they already exist unless modifier is used).  -C --init=site configure/initialise site more extensive than -C on its own, shared output directory files/force update, existing shared output config files such as css and dtd files  are updated if this modifier is used.  in a new markup document working directory should initialise the corresponding output directory, though SiSU will automatically do this, the first time it is run (for processing) in a given directory.
+
+   There are additional details, ... this should get you started.
+
+   See also
+      sisu --help directory
+      WOK
+      help_env
+    end
+    def directories
+      help_env
+      print <<-WOK
+
+  the directory structure used by sisu is controlled by the configuration files #{@cX.blue}sisurc.yml#{@cX.off}
+
+   there are separate directories for the following:
+
+    (a) #{@cX.green}data directories#{@cX.off}
+      the directories in which you place the SiSU marked-up data files that are to be processed
+      there may be as many directories and files as you choose to have,
+      you are currently in:                                    #{@cX.blue}#{Dir.pwd}#{@cX.off}
+
+    (b) #{@cX.green}output directory#{@cX.off}
+      by default files in the data directory are output to
+      a sub-directory within the output directory (usually the web document directory), or to an sql database
+      of the same name as the stub or last portion of the data directory name
+      by way of example
+      you are currently in:                                    #{@cX.blue}#{Dir.pwd}#{@cX.off}
+      the document output directory is set to:                 #{@cX.blue}#{@env.path.webserv}#{@cX.off}
+      documents from your current directory will be placed in: #{@cX.blue}#{@env.path.output}#{@cX.off}
+
+      the final output is placed in subdirectories either as configured by default in the program or as modified by SiSU configuration files
+      subdirectories are created within the main output directory, based on the name of the data directory
+      subsubdirectories are created the sub-directory contained in the main output directory based on the name of the file
+      (subdirectories are created in this (output) sub-directory named after the data file)
+      for this reason it is a convention to give descriptive names to the data file.
+      The default output directory is #{@cX.green}~/sisu_www#{@cX.off}
+      The output directory is currently set to:
+        #{@cX.blue}#{@env.path.webserv}#{@cX.off}
+      Output files, are currently set to be produced in:
+        processed document output:           #{@cX.blue}#{@env.path.output}#{@cX.off}
+        a document in the current directory  #{@cX.blue}#{Dir.pwd}#{@cX.off}
+            will have its output placed in:  #{@cX.blue}#{@env.path.output}/#{@output_stub}#{@cX.off}
+
+      NB: the verbose flag v included in the generate command string, should
+      result in a list of output filenames together with their paths.
+      The flag U on its own (e.g. sisu -U gpl3.fsf.sst) should provide a list
+      of output files that could be generated together with their paths.
+
+    (c) #{@cX.green}configuration files#{@cX.off}
+       SiSU program defaults are set within the program, and may be adjusted in the yml file #{@cX.blue}sisurc.yml#{@cX.off} which is searched for in the following paths, which are prioritized as listed (the first one found is loaded):
+###
+       under the current SiSU markup data directory: #{@cX.blue}#{@env.path.pwd}/_sisu/sisurc.yml#{@cX.off}
+       under the home directory ~/.sisu:             #{@cX.blue}#{@env.path.home}/.sisu/sisurc.yml#{@cX.off}
+       in the "/etc" directory:                      #{@cX.blue}#{@env.path.etc}/sisurc.yml#{@cX.off}
+
+    (d) #{@cX.green}processing directories#{@cX.off}     sisu creates a number of processing directories,
+      where these should be located can be modified in #{@cX.green}~/.sisu/sisurc.yml#{@cX.off}
+      work directories include the following:
+          root working directory                               #{@cX.blue}#{@env.processing_path.processing}#{@cX.off}
+          metaverse       intermediate markup                  #{@cX.blue}#{@env.processing_path.dal}#{@cX.off}
+          tune html       (for special html/navigation pages)  #{@cX.blue}#{@env.processing_path.tune}#{@cX.off}
+          tex             for latex and pdf                    #{@cX.blue}#{@env.processing_path.tex}#{@cX.off}
+          texinfo         for texinfo and info files           #{@cX.blue}#{@env.processing_path.texi}#{@cX.off}
+
+      These files are usually used only for processing and removed. There is a maintenance flag to keep them.
+
+    (e) #{@cX.green}images#{@cX.off}
+      there are a number of categories of images,
+      NB the document markup directory is initialised by issuing the command
+        sisu -C --init=site
+      this creates the output sub-directory, and makes necessary image links, and
+      copies images specific to the markup directory if there are any.
+
+      general images for the sisu program that come with the package,
+      that are the defaults used by sisu, these are installed with the program
+
+      images that the author wishes to include within documents,
+      these should be placed in a sub-directory
+      within the current document markup directory called #{@cX.green}_sisu/image#{@cX.off}
+
+      if an instruction is given to process a remote document which contains
+      downloadable images, they are included in a sub-directory of the current
+      markup directory #{@cX.green}_sisu/sisu/image_external#{@cX.off} that is created
+      if necessary for the purpose.
+
+      finally skins may specify/indicate other image directories. see sisu --help skin
+      any site images required by the skin instruction must be copied in to the
+      site image directory (it may be necessary to do manual configuration depending
+      on what you are trying to achieve).
+
+    (f) #{@cX.green}program directories#{@cX.off}
+      sisu --help install
+      sisu ~ install
+
+ See also
+    sisu --help path
+      WOK
+      help_env
+    end
+    def program_found?(program)
+      if program
+        rc=if SiSU_Env::InfoSettings.new.program?(program)
+          SiSU_Env::InfoSettings.new.program?(program)
+        else ''
+        end
+        if program =='rmagick' then program='identify' #rmagick is ruby lib uses imagemagick's identify
+        end
+        bin=if SiSU_Env::SystemCall.new.program_found?(program)
+          SiSU_Env::SystemCall.new.program_found?(program)
+        else 'false'
+        end
+      else bin,rc='false','false'
+      end
+      (program) \
+      ? ("#{@cX.blue}#{program}#{@cX.off}  bin: #{@cX.brown}#{bin}#{@cX.off} rc: #{@cX.brown}#{rc}#{@cX.off}")
+      : ("bin: #{@cX.brown}#{bin}#{@cX.off} rc: #{@cX.brown}#{rc}#{@cX.off}")
+    end
+    def sisu_version
+      version=SiSU_Env::InfoVersion.instance.get_version
+      rb_ver=SiSU_Env::InfoVersion.instance.rbversion
+      if version[:version]
+        opt_cmd=if defined? @color.cmd \
+        and @color.cmd =~/[ck]/
+          @color.cmd
+        else '-v'
+        end
+        SiSU_Screen::Ansi.new(opt_cmd,version[:project],version[:version],version[:date_stamp],version[:date],rb_ver).version
+      else puts 'SiSU version information not available'
+      end
+    end
+    def rhost
+      @ls=leading_spaces=' '*49
+      @rhost=SiSU_Env::InfoRemoteHost.new.rhost
+      def r1
+        (@rhost.r1) \
+        ? @rhost.r1 + "\n"
+        : ''
+      end
+      def r2
+        (@rhost.r2) \
+        ? @ls + @rhost.r2 + "\n"
+        : ''
+      end
+      def r3
+        (@rhost.r3) \
+        ? @ls + @rhost.r3 + "\n"
+        : ''
+      end
+      def r4
+        (@rhost.r4) \
+        ? @ls + @rhost.r4 + "\n"
+        : ''
+      end
+      def r5
+        (@rhost.r5) \
+        ? @ls + @rhost.r5 + "\n"
+        : ''
+      end
+      def r6
+        (@rhost.r6) \
+        ? @ls + @rhost.r6 + "\n"
+        : ''
+      end
+      def note
+        msg='(remote settings user and host set in sisurc.yml under remote:)'
+        (@rhost.r1) \
+        ? @ls + msg
+        : msg
+      end
+      self
+    end
+    def environment
+      cf_defaults=SiSU_Env::InfoProcessingFlag.new
+      sisu_version
+      x =<<-WOK
+  #{@cX.green}current and output directories#{@cX.off}
+  user:                                          #{@cX.blue}#{@env.user}#{@cX.off}
+  home:                                          #{@cX.blue}#{@env.path.home}#{@cX.off}
+  remote set [remote user]@[remote host]:        #{@cX.blue}#{rhost.r1}#{rhost.r2}#{rhost.r3}#{rhost.r4}#{rhost.r5}#{rhost.r6}#{@cX.off}#{rhost.note}
+  locale (encoding, UTF-8 desired):              #{@cX.blue}#{@env.locale}#{@cX.off}
+  current directory:                             #{@cX.blue}#{@env.path.pwd}#{@cX.off}
+  document output root directory set to:         #{@cX.blue}#{@env.path.webserv}#{@cX.off}
+  documents from current directory placed in:    #{@cX.blue}#{@env.path.output}#{@cX.off}
+  webrick url:                                   #{@cX.blue}#{@env.url.webserv_base_cgi}#{@cX.off}
+                                                 (to start webrick server 'sisu -W')
+  sqlite db for present directory:               #{@cX.blue}sqlite #{@env.path.output}/sisu_sqlite.db#{@cX.off}
+  postgresql port set to:                        #{@cX.blue}#{@db.psql.port}#{@cX.off}
+  postgresql db for present directory:           #{@cX.blue}#{@db.psql.db}#{@cX.off}
+                                                 [first create manually if necessary: 'createdb #{@db.psql.db}']
+
+  [generated sqlite cgi search form]:            #{@cX.blue}#{@env.url.webserv_base_cgi}/cgi-bin/sisu_sqlite.cgi#{@cX.off}
+  [generated postgresql cgi search form]:        #{@cX.blue}#{@env.url.webserv_base_cgi}/cgi-bin/sisu_pgsql.cgi#{@cX.off}
+                                                 (to generate 'sisu -F' or 'sisu -F webrick')
+  #{@cX.green}configuration files#{@cX.off}
+  sisurc.yml used:                              #{@cX.blue}#{@env.path.yamlrc}#{@cX.off}
+  configuration information search path:        #{@cX.blue}#{@env.path.rc.join(', ')}#{@cX.off}
+                                                (directory also relevant for skins and images)
+  digest (md5 or sha256):                        #{@cX.blue}#{@env.digest.type}#{@cX.off}
+  papersize set (LaTeX/pdf):                     #{@cX.blue}#{@env.papersize}#{@cX.off}
+                                                 (digest and papersize can be changed in sisurc.yml under default:)
+  #{@cX.green}intermediate processing#{@cX.off}
+  processing directory:                          #{@cX.blue}#{@env.processing_path.processing}#{@cX.off}
+                                                  (to keep processing output, use -M flag)
+  #{@cX.green}programs selected for viewing output#{@cX.off}
+  text editor:                                   #{@cX.blue}#{@env.program.text_editor}#{@cX.off}
+  web browser:                                   #{@cX.blue}#{@env.program.web_browser}#{@cX.off}
+  console web browser:                           #{@cX.blue}#{@env.program.console_web_browser}#{@cX.off}
+  pdf viewer:                                    #{@cX.blue}#{@env.program.pdf_viewer}#{@cX.off}
+  xml viewer:                                    #{@cX.blue}#{@env.program.xml_editor}#{@cX.off}
+  odf viewer:                                    #{@cX.blue}#{@env.program.odf_viewer}#{@cX.off}
+  epub viewer:                                   #{@cX.blue}#{@env.program.epub_viewer}#{@cX.off}
+                                                  (default selections can be changed in sisurc.yml under program_select:)
+  #{@cX.green}programs used if available#{@cX.off}
+  word count:                                    #{program_found?(@env.program.wc)}
+  imagemagick/rmagick:                           #{program_found?(@env.program.rmagick)}
+  tidy:                                          #{program_found?(@env.program.tidy)}
+  rexml:                                         #{program_found?(@env.program.rexml)}
+  latex to pdf:                                  #{program_found?(@env.program.pdflatex)}
+  postgresql:                                    #{program_found?(@env.program.postgresql)}
+  sqlite:                                        #{program_found?(@env.program.sqlite)}
+                                                  (these can be turned off if unavailable in sisurc.yml under program_set:)
+  #{@cX.green}processing shortcut defaults set to:#{@cX.off}
+  color defaut set (on==true)                    #{@cX.blue}#{cf_defaults.color}#{@cX.off}
+  sisu -0                                        #{@cX.blue}#{cf_defaults.cf_0}#{@cX.off} [set as default]
+  sisu -1                                        #{@cX.blue}#{cf_defaults.cf_1}#{@cX.off}
+  sisu -2                                        #{@cX.blue}#{cf_defaults.cf_2}#{@cX.off}
+  sisu -3                                        #{@cX.blue}#{cf_defaults.cf_3}#{@cX.off}
+  sisu -4                                        #{@cX.blue}#{cf_defaults.cf_4}#{@cX.off}
+  sisu -5                                        #{@cX.blue}#{cf_defaults.cf_5}#{@cX.off}
+                                                  (defaults may be changed in active sisurc.yml file under flag:)
+  #{@cX.green}special powers, risky operations set:#{@cX.off}
+  zap (delete output directories)                #{@cX.blue}#{SiSU_Env::InfoSettings.new.permission?('zap')}#{@cX.off}
+  css copy (copy over css files)                 #{@cX.blue}#{SiSU_Env::InfoSettings.new.permission?('css_modify')}#{@cX.off}
+                                                  (true/false defaults may be changed in active sisurc.yml file under permissions_set:)
+
+NOTE: for HELP type 'sisu --help', 'sisu --help [help request]', 'man sisu', (or see the system or online documentation)
+      WOK
+      print x
+    end
+    def dublin_core
+    print <<-WOK
+@title:
+
+@subtitle: [is added to title for purposes of Dublin Core description]
+
+@creator:
+
+@type:
+
+@subject:
+
+@date: [ccyy-mm-dd]
+
+@date.created:
+
+@date.issued:
+
+@date.available:
+
+@date.valid:
+
+@date.modified:
+
+@source:
+
+@language: en
+
+@papersize: A4
+
+@relation:
+
+@coverage:
+
+@rights:
+
+      WOK
+    end
+    def headings
+      print <<-WOK
+These are not required, a header is quicker to prepare if a documents structure can be defined by matching words or a regular expression, see headers).
+
+@structure: PART; CHAPTER; SECTION; ARTICLE; none; none;
+
+structure can be defined by a match words or regular expression (the regular expression is assumed to start at the beginning of a line of text i.e. ^)
+
+The following heading or level (structuring) defaults are available (for use instead of or together with @structure: header):
+
+  A~  B~  C~  1~  2~  3~
+
+Heading tags take either of the forms above, ranging from 1-6
+
+They appear at the beginning of the line on which a heading appears,
+the number indicates the level of the heading with level 1 being a title,
+
+segments (in html output) are by default created on level 4
+(segmented text is split/segmented on level 4, and
+assigned a file name automatically according to the title number,
+unless you explicitly specify otherwise)
+
+A~ Document Title
+
+B~ Document Subtitle whatever it is
+
+C~ Part
+
+1~ Chapter
+
+2~ Heading
+
+3~ sub-heading
+
+Normal text would follow each heading level as appropriate, though it is most usual to start with ordinary writing beneath level 4 as it is at this level that segments are created.
+
+Automatic numbering of paragraphs is usually set to start at level 4 trough level 6 and takes the form 1. then 1.1 then 1.1.1 this being given as a Header tag 0~ There may be up to 6 levels in a document,
+
+If auto-numbering is on, then for html output the segments created  (on level 4 headings) are automatically assigned the name of the title number.
+However, you may indicate an alternative set of key/title words, with one of the following instructions:
+4~filename This is a Section or Subject Heading
+
+If there is a strictly discernable word appearing at the beginning of the line in a document that identifies the level,
+the words can be used to identify the levels in a header tag:
+0~toc Part; Chapter; Section; Article; none; none
+instead of providing individual heading tags
+
+sisu structure, notation, A,B,C,1,2,3
+
+ SiSU has in effect two sets of levels to be considered
+
+ A-C pre-ordinary paragraphs /pre-substantive text, and
+
+ 1-3 levels which are followed by ordinary text.
+
+ This may be conceptualised as levels A,B,C, 1,2,3, and using such letter number notation, in effect:
+
+ A must exist, optional B and C follow in the sequence
+
+ 1 must exist, optional 2 and 3 follow in the sequence
+
+ i.e. there are two independent heading level sequences A,B,C and 1,2,3 or using the standard notation 1,2,3 and 4,5,6
+
+on the positive side: (a) the A,B,C,1,2,3 alternative makes explicit an aspect of structuring documents in SiSU that is not otherwise obvious to the newcomer (though it appears more complicated, is more in your face and likely to be understood fairly quickly); (b) the substantive text follows levels 1,2,3 and it is 'nice' to do most work in those levels
+
+      WOK
+    end
+    def languages
+      lang=SiSU_i18n::Languages.new.language.table
+      lang.each_value do |lng|
+        if lng[:t] == lng[:n]
+          puts "#{lng[:c]}:  #{lng[:t]}"
+        else
+          puts "#{lng[:c]}:  #{lng[:t]} (#{lng[:n]})"
+        end
+      end
+      puts <<-WOK
+
+ Multi-language Document File Naming and Directory Mapping
+
+ A list of language codes and languages that may be available for most output formats. These are for the most part ISO 639/1 and where there is a departure it follows those made by po4a, and the languages supported will be limited (at this time) to those supported by polyglossia. (SiSU markup text is stored in Unicode so some output formats are served by this, for example, html and sql had no problem with Chinese, pdfs would be a problem as this is not (yet?) suported by xetex and polyglossia. SiSU parses left to right and would have problems with right to left languages.)
+
+ If the same document exists in different language versions, a directory structure, or a filenaming convention may be followed. Using Spannish as the sample language code (es) [filename]~[language code].sst
+
+ The simplest now is to place each document in a sub-directory which has the language code as a name, if this convention is used it is preferable that all files are placed by language in the appropriate language sub-directory ... e.g.
+   ./en/filename.sst
+   ./es/filename.sst
+
+ Alternatively files may all be placed in the same directory with the language and named using a tilde and the language code as shown:
+  filename~es.sst
+
+  within sisurc.yml under the heading
+    default:
+      language file: [at 1, 2 or 3]
+
+  determines the output filenaming convention used, as follows:
+
+    (1) [output directory path]/filename/es.index.html
+
+    (2) [output directory path]/filename/index.es.html
+
+    (3) [output directory path]/filename/index.html.es (which Apache for example can be configured to use to automatically serve each users preference)
+
+    filename~fr.sst
+    filename~de.sst
+
+  etc. would be placed in the same directory using the same convention as indeed would:
+    filename.sst
+  using the default convention mapping convention.
+
+  Selecting this form of filename will overide other language settings including the language header within a document.
+
+      WOK
+    end
+    def endnotes
+      print <<-WOK
+
+(1) Footnote/endnotes tags take a number of possible forms, the simplest being to embed an endnote within your text~{ this would appear as an endnote, and would have an automatically assigned number }~ Embedded endnotes~{this is an endnote}~ and at the end of the paragraph~{* an asterisk marked note }~ writing the endnote:~{another endnote}~
+
+  ------------------------------------------
+#{@cX.ruby}All you need to know about endnotes appears above this line...#{@cX.off} apart from the fact that you cannot mix endnote markup styles
+
+(2) The other ways of inserting an endnote involve placing a tag within the text as to where the endnote reference number should appear like so~^ and at the end of the paragraph writing the endnote:~^
+
+^~ like so, this is an endnote
+
+^~ another endnote
+
+      WOK
+    end
+    def tables
+      print <<-WOK
+
+#{@cX.green}table{ [number of columns] [column width %];[column width %]#{@cX.off}
+
+[table content, line breaks are important see example below]
+
+#{@cX.green}}table#{@cX.off}
+#{@cX.grey}----#{@cX.off}
+This is a sample table:
+-----------------------
+
+#{@cX.green}table{ c3; 40; 30; 30;#{@cX.off}
+
+This is a table
+this would become column two of row one
+column three of row one is here
+
+And here begins another row
+column two of row two
+column three of row two, and so on
+
+#{@cX.green}}table#{@cX.off}
+
+there is an alternative way to markup tables, a sample document is provided in the file
+#{@cX.green}sisu_output_overview.sst#{@cX.off} located in
+#{@cX.green}/usr/share/doc/sisu#{@cX.off} or equivalent directory
+
+preformatted text
+-----------------
+
+#{@cX.green}poem{#{@cX.off}
+
+  [Text here]
+
+#{@cX.green}}poem#{@cX.off}
+#{@cX.grey}----#{@cX.off}
+
+#{@cX.green}block{#{@cX.off}
+
+  [Text here]
+
+#{@cX.green}}block#{@cX.off}
+#{@cX.grey}----#{@cX.off}
+
+#{@cX.green}group{#{@cX.off}
+
+  [Text here]
+
+#{@cX.green}}group#{@cX.off}
+#{@cX.grey}----#{@cX.off}
+
+#{@cX.green}code{#{@cX.off}
+
+  [Text here]
+
+#{@cX.green}}code#{@cX.off}
+
+      WOK
+    end
+    def modules
+      print <<-WOK
+
+      #{@cX.ruby}IGNORE#{@cX.off}
+
+      WOK
+    end
+    def install                                                        #% system configuration
+      print <<-WOK
+  #{@cX.green}Install SiSU#{@cX.off}
+
+  Presumably if you are reading this interactively you have a copy of SiSU already installed, nevertheless here are a few notes.
+
+  SiSU does require setup, the executable file #{@cX.blue}sisu#{@cX.off} is placed in #{@cX.blue}#{Config::CONFIG['bindir']}#{@cX.off} or #{@cX.blue}#{Config::CONFIG['sitelibdir']}#{@cX.off} and the library files, in #{@cX.blue}#{Config::CONFIG['rubylibdir']}/#{SiSU_lib}#{@cX.off}
+ or in #{@cX.blue}#{Config::CONFIG['sitelibdir']}/#{SiSU_lib}#{@cX.off}
+
+  SiSU comes with a number of installers, including #{@cX.blue}setup.rb#{@cX.off}, #{@cX.blue}install#{@cX.off} and a #{@cX.blue}Rantfile#{@cX.off} if rant is installed on your system, you may need to be root to install sisu on your system. After unpacking the tarball, in the top directory of the tarball which contains the named files type, one of:
+      sudo ./sisu-install base
+      sudo ./sisu-install setup
+    if that does not work try
+      sudo ruby ./sisu-install setup
+    if rant is installed on your system you may instead run:
+      sudo rant base
+
+    for further options:
+      ./sisu-install -T
+
+  SiSU is pre-packaged for some GNU/Linux distributions such as Debian.
+
+  For information on download and installation, see #{@cX.blue}http://www.jus.uio.no/sisu/SiSU/download#{@cX.off}
+
+  For post installation help it is best you refer to '#{@cX.blue}man 8 sisu#{@cX.off}'
+
+  Host
+    host:             #{@cX.blue}#{@env.hostname}#{@cX.off}
+    arch:             #{@cX.blue}#{@env.arch}#{@cX.off}
+
+  Directories for installation
+    bin:                                     #{@cX.blue}#{Config::CONFIG['bindir']}#{@cX.off} or #{@cX.blue}#{Config::CONFIG['sitelibdir']}#{@cX.off}
+    lib (site-ruby):                         #{@cX.blue}#{Config::CONFIG['rubylibdir']}/#{SiSU_lib}#{@cX.off} or #{@cX.blue}#{Config::CONFIG['sitelibdir']}/#{SiSU_lib}#{@cX.off}
+    conf [etc]:                              #{@cX.blue}#{@env.path.etc}/sisu#{@cX.off}
+    data (document samples, images, README): #{@cX.blue}#{@env.path.sample_data}#{@cX.off}
+    processing:                              #{@cX.blue}#{@env.processing_path.processing}#{@cX.off}
+    output www:                              #{@cX.blue}#{@env.path.output}#{@cX.off}
+
+    Output files, are currently set to be produced in:
+      processed document output:             #{@cX.blue}#{@env.path.output}#{@cX.off}
+      a document in the current directory    #{@cX.blue}#{Dir.pwd}#{@cX.off}
+          will have its output placed in:    #{@cX.blue}#{@env.path.output}/#{@output_stub}#{@cX.off}
+
+  Install any additional programs of interest (that SiSU makes use of, that are not already on your system)
+
+    Programs SiSU makes use of include:
+      LaTeX
+      texinfo
+      pdfetex aka. pdflatex
+      sqlite
+      postgresql
+
+  Again, refer to '#{@cX.blue}man 8 sisu#{@cX.off}'
+
+  For additional help on using SiSU once installed type:
+
+    sisu --help
+
+      WOK
+    end
+    def setup
+      print <<-WOK
+
+     #{@cX.green}configuration files#{@cX.off}
+     sisurc.yml used:
+       #{@cX.blue}#{@env.path.yamlrc}#{@cX.off}
+     configuration information search path:
+       #{@cX.blue}#{@env.path.rc.join(', ')}#{@cX.off}
+                                                (directory also relevant for skins and images)
+
+      to initialise the mapped output directory, from within the markup document directory type:
+        #{@cX.blue}sisu -CC#{@cX.off}
+
+      for information on your current sisu configuration settings, type:
+        #{@cX.blue}sisu --help env#{@cX.off}
+      or
+        #{@cX.blue}sisu -V#{@cX.off}
+
+      see also
+        #{@cX.blue}man 8 sisu#{@cX.off}
+      and
+        #{@cX.blue}http://www.jus.uio.no/sisu/SiSU#{@cX.off}
+      WOK
+    end
+    def termsheet
+      print <<-WOK
+
+  #{@cX.green}sisu --termsheet [termsheetname].termsheet.rb#{@cX.off}
+    will produce the collection of documents associated with [termsheetname.termsheet.rb]
+
+  #{@cX.green}sisu -t [termsheetname].termsheet.rb#{@cX.off}
+    will produce the collection of documents associated with [termsheetname.termsheet.rb]
+
+  #{@cX.green}termsheet.rb#{@cX.off} files:
+  (i) are named after the facility
+  (ii) contain instructions as to which standard forms to use for the agreement (standard_form.rb)
+     there may be several, eg. the termsheet, and the resulting:
+     main agreement; collection account charge; and deed of assignment
+  (iii) contain the variable terms of the agreement, borrower, interest etc.
+
+  for a new loan agreement fill out a new termsheet specifying
+  what standard forms are to be used,
+  and the terms of the agreement.
+
+  #{@cX.green}standard_form.rb#{@cX.off} files:
+  contain the standard terms of the agreement
+  there is a standard form for each variation of agreement
+    (so eg. there are as many facility agreements as there are variations in standard facility)
+  these may be prepared for any agreement that is to be reused.
+  (preparation takes the form of formating and
+  placing variable holders for the variables that are to be provided by the termsheet.rb file)
+
+  it is necessary to purge the directory ~facilityData when old files are removed
+
+      WOK
+    end
+    def help_commands
+      print <<-WOK
+
+    also see:
+      sisu --help commands
+      man sisu
+      WOK
+    end
+    def help_env
+      print <<-WOK
+
+    for sisu environment information see:
+      sisu --help env
+      WOK
+    end
+    def help_general
+      help_env
+      help_commands
+    end
+    def help_markup
+      print <<-WOK
+
+    for help with sisu markup see:
+      sisu --help markup
+      sisu --help header
+      sisu --help structure [sisu --help heading]
+
+      to check markup version in file:
+        sisu --identify [filename].sst
+
+      for brief descriptive summary of markup history
+        sisu --query-history
+
+      for markup:
+        sisu --help example
+
+      sample marked up documents are provided in directory:
+        #{@cX.green}/usr/share/doc/sisu/markup-samples#{@cX.off}
+      and online
+        #{@cX.green}www.jus.uio.no/sisu#{@cX.off}
+        WOK
+      end
+    def convert
+      print <<-WOK
+
+      for information on the markup version used within a sisu markup file:
+        sisu --inspect [filename]
+
+      convert an sst file with footnotes following text to (preferred) inline footnotes
+        sisu --convert-footnotes  [filename/wildcard]
+
+      to convert from sst to simple xml representations (sax, dom and node):
+        sisu --to-sax [filename/wildcard]
+        sisu --to-sxs [filename/wildcard]
+
+        sisu --to-dom [filename/wildcard]
+        sisu --to-sxd [filename/wildcard]
+
+        sisu --to-node [filename/wildcard]
+        sisu --to-sxn [filename/wildcard]
+
+      to convert to sst from simple xml representations (sax, dom and node):
+        sisu --from-xml2sst [filename/wildcard [.sxs.xml,.sxd.xml,sxn.xml]]
+        sisu --from-sxml [filename/wildcard [.sxs.xml,.sxd.xml,sxn.xml]]
+
+      to attempt to convert a kdissert (.kdi) file to sisu markup:
+        sisu --from-kdi  [kdissert filename]
+        (very basic and experimental)
+      WOK
+    end
+    def dal
+      print <<-WOK
+
+    sisu --dal [filename/wildcard]    creates the metaverse, used by all other modules for downstream processing
+    sisu -m [filename/wildcard]    creates the metaverse, used by all other modules for downstream processing
+      WOK
+      help_commands
+    end
+    def utf8
+      print <<-WOK
+
+      documents prepared for sisu processing should be stored in utf8
+      WOK
+    end
+    def plaintext
+      print <<-WOK
+
+    sisu --txt [filename/wildcard]    plaintext with Unix linefeeds (footnotes follow paragraphs)
+    sisu -t [filename/wildcard]    plaintext with Unix linefeeds (footnotes follow paragraphs)
+
+      WOK
+      help_commands
+    end
+    def html
+      print <<-WOK
+
+    sisu --html [filename/wildcard]    html document type suffixes included
+    sisu -h [filename/wildcard]    html document type suffixes included
+
+      WOK
+      help_commands
+    end
+    def xhtml
+      print <<-WOK
+
+    sisu --xhtml [filename/wildcard]    xhtml document
+    sisu -b [filename/wildcard]    xhtml document
+      WOK
+      help_commands
+    end
+    def xml
+      print <<-WOK
+
+    sisu --xml-sax [filename/wildcard]    xml document (sax type parsing)
+    sisu -x [filename/wildcard]    xml document (sax type parsing)
+
+    sisu --xml-dom [filename/wildcard]    xml document (dom type parsing)
+    sisu -X [filename/wildcard]    xml document (dom type parsing)
+
+    sisu --odt [filename/wildcard]    odt document, (odf open document format)
+    sisu -o [filename/wildcard]    odt document, (odf open document format)
+
+    sisu --epub [filename/wildcard]    odt document, (epub document)
+    sisu -e [filename/wildcard]    odt document, (epub document)
+      WOK
+      help_commands
+    end
+    def odf
+      print <<-WOK
+
+    sisu --odt [filename/wildcard]    odt document, (odf open document format)
+
+    sisu -o [filename/wildcard]    odt document, (odf open document format)
+      WOK
+      help_commands
+    end
+    def epub
+      print <<-WOK
+
+    sisu --epub [filename/wildcard]    epub document, (open format for e-books)
+    sisu -e [filename/wildcard]    epub document, (open format for e-books)
+      WOK
+      help_commands
+    end
+    def php
+      print <<-WOK
+
+    not supported
+    [php output has been removed]
+      WOK
+      help_commands
+    end
+    def pdf
+      print <<-WOK
+
+    sisu --pdf [filename/wildcard]     produces pdf files from LaTeX output
+    sisu -p [filename/wildcard]     produces pdf files from LaTeX output
+      WOK
+      help_commands
+    end
+    def latex
+      print <<-WOK
+
+    sisu --pdf [filename/wildcard]     produces pdf files from LaTeX output
+    sisu -p [filename/wildcard]     produces pdf files from LaTeX output
+      WOK
+      help_commands
+    end
+    def texinfo
+      print <<-WOK
+
+    sisu --texinfo [filename]     produces texinfo and info files
+    sisu -I [filename]     produces texinfo and info files
+
+    info and texinfo files are currently left in a separate work/output directory... have not decided what to do with them
+
+    on my system info works fine point at file with info command
+
+    pinfo, requires you to be within the work/output directory
+      cd [work/output directory]
+
+    and then to point at the file using
+      pinfo ./[filename]
+      WOK
+      help_commands
+    end
+    def lout
+      print <<-WOK
+
+    sisu -l [filename/wildcard]
+
+    not currently supported, revisit someday?
+      WOK
+      help_commands
+    end
+    def concordance
+      print <<-WOK
+
+    sisu --concordance        starts the sisu webrick server, default port 8081
+    sisu --wordmap        starts the sisu webrick server, default port 8081
+    sisu -W        starts the sisu webrick server, default port 8081
+      WOK
+      help_commands
+    end
+    def help_search
+      print <<-WOK
+
+      SiSU searches,
+      depending on how you wish to implement search,
+      the following may bre of interest:
+        sisu --help sql
+        sisu --help searchform    (or 'sisu --help cgi')
+        sisu --help hyperestraier (or 'sisu --help est')
+        sisu --help webrick
+      WOK
+    end
+    def cgi
+      print <<-WOK
+
+      sisu --sample-search-form                 generates a sample search form
+      sisu -F                generates a sample search form
+
+      sisu -F --webserv=webrick  generates a sample search form for use with the webrick server
+      sisu -Fv               as above, and provides some information on setting up hyperestraier
+      sisu -W                starts the webrick server
+
+      the generated search form must be copied to the webserver directory as instructed
+      WOK
+      help_search
+      help_general
+    end
+    def sql
+      print <<-WOK
+
+    Mappings to two databases are provided by default,
+    postgresql (--pg or -D) and sqlite (--sqlite or -d),
+    the same modifier commands are used within sisu to construct and populate
+    databases the examples here will use --sqlite
+
+    sisu --sqlite --createdb         creates database where no database existed before
+    sisu --sqlite --create           creates database tables where no database tables existed before
+    sisu --sqlite --dropall          destroys database (including all its content)!!
+    sisu --sqlite --recreate         destroys existing database and builds a new empty database structure
+
+    sisu --sqlite --import [filename/wildcard]   populates database with the contents of the file
+    sisu --sqlite --update [filename/wildcard]   updates file contents in database
+
+    sisu --sample-search-form --webserv=webrick    builds a cgi web search frontend for the database created
+
+  Postgresql
+    user:             #{@cX.blue}#{@db.psql.user}#{@cX.off}
+    current db set:   #{@cX.blue}#{@db.psql.db}#{@cX.off}
+    port:             #{@cX.blue}#{@db.psql.port}#{@cX.off}
+    dbi connect:      #{@cX.blue}#{@db.psql.dbi}#{@cX.off}
+
+  sqlite
+    current db set:   #{@cX.blue}#{@db.sqlite.db}#{@cX.off}
+    dbi connect       #{@cX.blue}#{@db.sqlite.dbi} #{@cX.off}
+
+  Note on databases built
+   By default, [unless otherwise specified] databases are built on a directory basis, from collections of documents within that directory.
+   The name of the directory you choose to work from is used as the database name,
+   i.e. if you are working in a directory called #{@cX.blue}#{@env.path.home}/ebook#{@cX.off} the database #{@cX.blue}#{Db[:name_prefix]}ebook#{@cX.off} is used. [otherwise a manual mapping for the collection is necessary]
+      WOK
+      help_search
+      help_general
+    end
+    def webrick
+      print <<-WOK
+
+    sisu --webrick  starts the sisu webrick server, default port 8081
+    sisu -W         starts the sisu webrick server, default port 8081
+      WOK
+      help_commands
+    end
+    def hyperestraier
+      out_dir='(' + `ls #{@env.path.webserv}`.split("\n").join('|') + ')'
+      print <<-WOK
+  Not tested recently, not up-to-date
+
+  NOTE: The examples that follow assume that sisu output is placed in the directory
+  /home/ralph/sisu_www
+
+ (A)  to generate the index
+   within the webserver directory to be indexed:
+      #{@cX.orange}estcmd gather -sd [index name] [directory path to index]#{@cX.off}
+   the following are examples that will need to be tailored according to your needs:
+     #{@cX.green}
+     cd #{@env.path.webserv}
+     estcmd gather -sd casket #{@env.path.webserv}
+     #{@cX.off}
+     you may use the 'find' command together with 'egrep' to limit
+     indexing to particular document collection directories within
+     the web server directory:#{@cX.green}
+
+     find /home/ralph/sisu_www -type f | egrep '#{@env.path.output}/.+?\.html$' |estcmd gather -sd casket - #{@cX.off}
+
+     check which directories in the webserver/output directory #{@cX.green}#{@env.path.webserv}#{@cX.off}
+     you wish to include in the search index, these appear to be:
+
+     #{@env.path.webserv}/#{@cX.green}#{out_dir}#{@cX.off}
+
+     as sisu duplicates output in multiple file formats,
+     it it is probably preferable to limit the estraier index
+     to html output, and as it may also be desirable to
+     exclude files 'doc.html' and 'concordance.html', as these
+     duplicate information held in other html output e.g. #{@cX.green}
+
+     find /home/ralph/sisu_www -type f | egrep '/sisu_www/(sisu|bookmarks)/.+?\.html$' | egrep -v '(doc|concordance)\.html$' |estcmd gather -sd casket - #{@cX.off}
+
+     from your current document preparation/markup directory, you would construct a rune along the following lines: #{@cX.green}
+
+     find /home/ralph/sisu_www -type f | egrep '#{@env.path.webserv}/([specify first directory for inclusion]|[specify second directory for inclusion]|[another directory for inclusion? ...])/.+?\.html$' | egrep -v '(doc|concordance)\.html$' |estcmd gather -sd #{@env.path.webserv}/casket - #{@cX.off}
+
+ (B) to set up the search form
+   (i) copy #{@cX.green}estseek.cgi#{@cX.off} to your cgi directory and set file permissions to 755: #{@cX.green}
+
+     sudo cp -vi /usr/lib/estraier/estseek.cgi /usr/lib/cgi-bin
+     sudo chmod -v 755 /usr/lib/cgi-bin/estseek.cgi
+     sudo cp -v /usr/share/hyperestraier/estseek.* /usr/lib/cgi-bin #{@cX.off}
+     [see estraier documentation for paths]
+
+   (ii) edit #{@cX.green}estseek.conf#{@cX.off}, with attention to the lines starting 'indexname:' and 'replace:': #{@cX.green}
+
+     indexname: #{@env.path.webserv}/casket
+     replace: ^file://#{@env.path.webserv}{{!}}#{@env.url.webserv_host_base}
+     replace: /index\.html?${{!}}/ #{@cX.off}
+
+ (C) to test using webrick, start webrick: #{@cX.green}
+     sisu -W #{@cX.off}
+
+   and try open the url: #{@cX.blue}
+     #{@env.url.webserv_host_base}/cgi-bin/estseek.cgi #{@cX.off}
+
+      WOK
+    end
+    def yaml
+      print <<-WOK
+   Yaml sisurc files may be used to configure sisu, these are searched for in the following locations:
+
+   under the current SiSU markup data directory: #{@cX.blue}#{Dir.pwd}/_sisu/sisurc.yml#{@cX.off}
+   under the home directory ~/.sisu:             #{@cX.blue}#{@env.path.home}/.sisu/sisurc.yml#{@cX.off}
+   in the "/etc" directory:                      #{@cX.blue}#{@env.path.etc}/sisurc.yml#{@cX.off}
+
+   The Yaml files #{@cX.blue}promo.yml#{@cX.off} and #{@cX.blue}list.yml#{@cX.off} may be used to build a minor right pane in html, they may be placed in the following locations:
+
+   under the current SiSU markup data directory: #{@cX.blue}#{Dir.pwd}/_sisu/skin/yml/#{@cX.off}
+   under the home directory ~/.sisu:             #{@cX.blue}#{@env.path.home}/.sisu/skin/yml/#{@cX.off}
+   in the "/etc" directory:                      #{@cX.blue}#{@env.path.etc}/skin/yml/#{@cX.off}
+
+   these may be called by the sisurc.yml, skins for document, directory or site, or from individual document headers in the header @@promo: [a commented out example header may be found in  document  sample: free_as_in_freedom.richard_stallman_cru‐ sade_for_free_software.sam_williams.sst, it calls for the specified lists in list.yml, which in turn calls the widgets named in promo.yml which are used to build content in the right pane of html output]
+
+      WOK
+    end
+    def sitemap
+      print <<-WOK
+
+      SiSU sitemaps,
+      an experimental feature (following g,y,m announcement to use them this week)
+        sisu -Y [filename/wildcard]
+      it may be necessary run -m first (generate the the metaverse)
+        sisu -mY [filename/wildcard]
+
+      to generate/update the index of sitemaps
+        sisu --sitemaps
+      WOK
+    end
+    def license
+
+      print <<-WOK
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) 1997 - 2013 Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see <#{@cX.blue}http://www.gnu.org/licenses/#{@cX.off}>.
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+     <#{@cX.blue}http://www.fsf.org/licenses/gpl.html#{@cX.off}>
+     <#{@cX.blue}http://www.gnu.org/licenses/gpl.html#{@cX.off}>
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   <#{@cX.blue}http://www.sisudoc.org#{@cX.off}>
+   <#{@cX.blue}http://www.jus.uio.no/sisu#{@cX.off}>
+
+ * Download:
+   <#{@cX.blue}http://sources.sisudoc.org#{@cX.off}>
+   <#{@cX.blue}http://www.jus.uio.no/sisu/en/html/SiSU/download.html#{@cX.off}>
+
+   Ralph Amissah
+   <#{@cX.blue}ralph@amissah.com#{@cX.off}>
+   <#{@cX.blue}ralph.amissah@gmail.com#{@cX.off}>
+
+      WOK
+    end
+    def standards
+      print <<-WOK
+
+ * SiSU uses:
+   *  Standard SiSU markup syntax,
+   *  Standard SiSU meta-markup syntax, and the
+   *  Standard SiSU object citation numbering and system
+
+Copyright (C) Ralph Amissah 1997, current 2013.
+All Rights Reserved.
+
+Information on these may be obtained from:
+  <#{@cX.blue}http://sisudoc.org#{@cX.off}>
+
+More information to be provided later.
+
+* however note also the License section
+
+* Ralph Amissah ralph@amissah.com
+  Ralph Amissah ralph.amissah@gmail.com
+
+      WOK
+    end
+    def conversion
+      print <<-WOK
+sisu_convert does the initial conversion from a couple of file formats to SiSU file format, currently only html and word97
+  #{@cX.cyan}sisu_convert#{@cX.off} [keyword]
+  sisu [keyword]
+    #{@cX.green}--html#{@cX.off}       convert from html
+  ------------------------------------------
+  Preparing Documents for SiSU
+    #{@cX.green}--word97#{@cX.off}       sisu --help markup     (an incomplete overview)
+      WOK
+    end
+    def external_programs
+      puts <<-WOK
+
+    external ruby programs
+
+    external programs
+      #{@cX.cyan}pdf output - tex/latex#{@cX.off}
+        #{@cX.orange}required#{@cX.off}
+          tex-base/latex
+          pdfetex aka. pdflatex
+        #{@cX.brown}suggested/recommended#{@cX.off}
+      #{@cX.cyan}db/sql output#{@cX.off}
+        #{@cX.orange}required#{@cX.off}
+          postgresql
+        #{@cX.brown}suggested/recommended#{@cX.off}
+          sqlite
+      #{@cX.cyan}xml/xhtml/html output#{@cX.off}
+        #{@cX.orange}required#{@cX.off}
+        #{@cX.brown}suggested/recommended#{@cX.off}
+          tidy  (xml, xhtml well formed check)
+          trang (relaxng, rnc to dtd conversion)
+      WOK
+    end
+    def scratch
+      print <<-WOK
+Types of tag,
+
+then there are various tags which occur within the document.
+Structural tags, which consist of:
+  heading tags that identify headings within text, and;
+  footnote/endnote tags ...
+
+Markup instructions: giving information as to what is to be done to the presentation of the text
+
+Markup tags
+
+These have been kept to a minimum. A number of text html markup tags can be used to bold to indent for superscript for subscript text
+
+_1 at the beginning of a line indents the paragraph
+
+_2 at the beginning of a line double indents the paragraph
+
+Others include
+
+Other things to note:
+
+By default paragraphs are automatically numbered... and is the same across all output formats
+This makes citation a lot easier... regardless of the form of output that is being looked at
+It also permits the building of various addons, like the concordance feature which identifies each word and the paragraphs in which the word appears with links to the paragraph...
+
+Urls are automatically turned to live links in the html and pdf files created...
+      WOK
+    end
+  end
+end
+__END__
+
diff --git a/lib/sisu/v5/html.rb b/lib/sisu/v5/html.rb
new file mode 100644
index 00000000..d3122670
--- /dev/null
+++ b/lib/sisu/v5/html.rb
@@ -0,0 +1,693 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: html generation, processing
+
+=end
+module SiSU_HTML
+  require 'pstore'
+  require_relative 'particulars'                        # particulars.rb
+    include SiSU_Particulars
+  require_relative 'defaults'                           # defaults.rb
+    include SiSU_Viz
+  require_relative 'html_table'                         # html_table.rb
+  require_relative 'html_format'                        # html_format.rb
+    include SiSU_HTML_Format
+  require_relative 'html_segments'                      # html_segments.rb
+    include SiSU_HTML_Seg
+  require_relative 'html_scroll'                        # html_scroll.rb
+  require_relative 'html_promo'                         # html_promo.rb
+    include SiSU_HTML_Promo
+  require_relative 'html_tune'                          # html_tune.rb
+    include SiSU_HTML_Tune
+  class Source
+    def initialize(opt)
+      @opt=opt
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+    end
+    def read
+      begin
+        songsheet
+      ensure
+        SiSU_Env::CreateSite.new(@opt.cmd).cp_css
+      end
+    end
+    def songsheet
+      begin
+        @md=@particulars.md
+        @fnb=@md.fnb
+        @env=@particulars.env
+        loc=@env.url.output_tell
+        primary_output_file=(@opt.act[:html_seg][:set]==:on) \
+        ? (@md.file.output_path.html_seg.dir + '/' + @md.file.base_filename.html_segtoc)
+        : (@md.file.output_path.html_scroll.dir + '/' + @md.file.base_filename.html_scroll)
+        unless @opt.cmd =~/q/
+          tool=(@opt.cmd =~/[MVvz]/) \
+          ? ("#{@env.program.web_browser} file://#{primary_output_file}")
+          : ("[#{@opt.f_pth[:lng_is]}] #{@opt.fno}")
+          @opt.cmd=~/[MVvz]/ \
+          ? SiSU_Screen::Ansi.new(@opt.cmd,'HTML',tool).green_hi_blue
+          : SiSU_Screen::Ansi.new(@opt.cmd,'HTML',tool).green_title_hi
+          SiSU_Screen::Ansi.new(@opt.cmd,@opt.fns,"file://#{primary_output_file}").flow if @opt.cmd =~/[MV]/
+        end
+        data=nil
+        tuned_file_array=SiSU_HTML::Source::HTML_Environment.new(@particulars).tuned_file_instructions
+        data=tuned_file_array
+        if @opt.act[:html_scroll][:set]==:on
+          scr_endnotes=SiSU_HTML::Source::Endnotes.new(data,@md).scroll
+        end
+        toc=SiSU_HTML::Source::Toc.new(@md,data).songsheet
+        links_guide=SiSU_HTML::Source::LinksGuide.new(data,@md).toc
+        data=tuned_file_array
+        scr_toc=SiSU_HTML::Source::ScrollHeadAndSegToc.new(@md,toc,links_guide).in_common #watch
+        if @opt.act[:html_seg][:set]==:on
+          SiSU_HTML::Source::Seg.new(@md,data).songsheet
+        end
+        data=tuned_file_array
+        if @opt.act[:html_scroll][:set]==:on
+          scr=SiSU_HTML::Source::Scroll.new(@md,data,scr_endnotes).songsheet
+          scroll=SiSU_HTML::Source::ScrollOutput.new(scr_toc,scr[:body],scr[:metadata],scr[:owner_details],scr[:tails],@md).publish
+          SiSU_HTML::Source::Output.new(scroll,@md).scroll
+        end
+      rescue
+        SiSU_Errors::InfoError.new($!,$@,@opt.cmd,@opt.fns).error do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+        unless @opt.cmd =~/[MV]/ #check maintenance flag
+          texfiles=Dir["#{@env.processing_path.tune}/#{@opt.fns}*"]
+          texfiles.each do |f|
+            if FileTest.file?(f)
+              File.unlink(f)
+            end
+          end
+        end
+        SiSU_Env::Clear.new(@opt.cmd,@opt.fns,@md).param_instantiate
+        @@flag,@@scr,@@seg,@@seg_endnotes,@@seg_subtoc,@@seg_ad={},{},{},{},{},{}
+        @@seg_total,@@tracker,@@loop_count,@@tablehead,@@number_of_cols=0,0,0,0,0
+        @@seg_name,@@seg_name_html,@@seg_subtoc_array,@@seg_endnotes_array,@@segtocband,@@tablefoot=Array.new(7){[]}
+        @@filename_seg,@@seg_url,@@fn,@@to_lev4,@@get_hash_to,@@get_hash_fn='','','','','','',''
+        @@is4=@@is3=@@is2=@@is1=@@heading1=@@heading2=@@heading3=@@heading4=0
+      end
+    end
+    private
+    class HTML_Environment
+      def initialize(particulars)
+        @particulars=particulars
+        @md,@env=particulars.md,particulars.env
+        @vz=SiSU_Viz::Defaults.new
+        @env,@css=particulars.env,SiSU_Style::CSS.new
+      end
+      def tuned_file_instructions
+        @tell=SiSU_Screen::Ansi.new(@md.opt.cmd)
+        dal_array=@particulars.dal_array # dal file drawn here
+        tuned_file_array=SiSU_HTML_Tune::Tune.new(dal_array,@md).songsheet
+        tuned_file_array
+      end
+    end
+    class LinksGuide
+      @links_guide_toc=[]
+      def initialize(data,md)
+        @data,@md=data,md
+        @links_guide_=SiSU_Env::CreateSite.new(@md.opt.cmd).html_quick_ref?
+      end
+      def toc
+        @links_guide_toc=[]
+        if @links_guide_
+          format_head_toc=SiSU_HTML_Format::HeadToc.new(@md)
+          guide_type='horzontal' #values: horizontal or vertical
+          @links_guide_toc << format_head_toc.links_guide_open(guide_type)
+          if defined? @md.lnk \
+          and @md.lnk
+            @md.lnk.each do |l|
+              if defined? l[:say]
+                target=(l[:url] !~/^\.(\.)?\//) \
+                ? 'external'
+                : '_top'
+                s_lnk_url,s_lnk_lnk=l[:url],l[:say]
+                txt_obj={ lnk_url: s_lnk_url, lnk_txt: s_lnk_lnk, target: target }
+                lev_dob_ocn=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+                @links_guide_toc << lev_dob_ocn.links_guide if s_lnk_lnk
+              end
+            end
+          end
+          format_head_toc=SiSU_HTML_Format::HeadToc.new(@md)
+          @links_guide_toc << format_head_toc.links_guide_close #(guide_type)
+          @links_guide_toc
+        else ''
+        end
+      end
+    end
+    class Endnotes
+      include SiSU_HTML_Format
+      def initialize(data,md)
+        @data,@md=data,md
+      end
+      def scroll
+        @scr_endnotes=[]
+        format_head_scroll=SiSU_HTML_Format::HeadScroll.new(@md)
+        @data.each do |dob|
+          pg=dob.dup
+          unless pg.is ==:code
+            if pg.obj =~/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[\d*+]+ /
+              endnote_array=[]
+              if pg.obj=~/#{Mx[:en_a_o]}[\d*+].+?#{Mx[:en_a_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_a_o]}[\d*+]+(.+?)#{Mx[:en_a_c]}/m)
+              end
+              if pg.obj=~/#{Mx[:en_b_o]}[\d*]+\s.+?#{Mx[:en_b_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_b_o]}[\d*]+(.+?)#{Mx[:en_b_c]}/m)
+              end
+              if pg.obj=~/#{Mx[:en_b_o]}[\d+]+\s.+?#{Mx[:en_b_c]}/m
+                endnote_array = pg.obj.scan(/#{Mx[:en_b_o]}[\d+]+(.+?)#{Mx[:en_b_c]}/m)
+              end
+              endnote_array.flatten.each do |note|
+                txt_obj={ txt: note }
+                format_scroll=SiSU_HTML_Format::FormatScroll.new(@md,txt_obj)
+                @scr_endnotes << format_scroll.endnote_body
+              end
+            end
+          end
+        end
+        @scr_endnotes
+      end
+    end
+    class Toc ' + minitoc + ''
+      end
+    protected
+      def rss #sort all wrong, disabled but kept
+        @@toc[:seg] <<<
+
    
+
    
+(relatively static) RSS feeds for DOCUMENTS:
    
+RSS feed http://www.jus.uio.no/lm/rssfeed/documents.xml
    
+RSS feed http://www.jus.uio.no/lm/rssfeed/tradelaw.xml
    
+RSS feed http://www.jus.uio.no/lm/rssfeed/environmental.xml
    
+
    info@address.com
    
+
    
+
    
+WOK
+      end
+#not used -->
+      def level_endnotes
+        if @md.flag_endnotes
+          format_head_scroll=SiSU_HTML_Format::HeadScroll.new(@md)
+          @@toc[:scr] << format_head_scroll.toc_endnote
+        end
+      end
+      def level_concordance
+        format_head_toc=SiSU_HTML_Format::HeadToc.new(@md)
+        @@toc[:seg_mini] << format_head_toc.mini_seg_concordance
+      end
+      def level_metadata
+        format_head_toc=SiSU_HTML_Format::HeadToc.new(@md)
+        @@toc[:scr] << format_head_toc.metadata
+        @@toc[:seg] << format_head_toc.seg_metadata
+        @@toc[:seg_mini] << format_head_toc.mini_seg_metadata
+      end
+      def level_word_index
+        format_head_toc=SiSU_HTML_Format::HeadToc.new(@d0c)
+        @@toc[:scr] << format_head_toc.concordance
+        @@toc[:seg] << format_head_toc.concordance
+        @@toc[:seg_mini] << format_head_toc.mini_concordance
+      end
+# <-- not used
+      def level_1
+        dob=@data
+        linkname,link=dob.obj.strip,dob.ocn
+        if link \
+        and link !~/#/ #% keep eye on link
+          p_num=SiSU_HTML_Format::ParagraphNumber.new(@md,link)
+        end
+        title=if dob.obj !~/Metadata/ then linkname
+        else
+          link='metadata'
+          %{    #{linkname}}
+        end
+        toc={}
+        txt_obj={ txt: title }
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc[:seg]=if dob.name =~/^meta/ \
+        and dob.obj =~/Document Information/
+          format_toc.lev0
+        else format_toc.lev1
+        end
+        toc[:seg_mini]=if dob.name =~/^meta/ \
+        and dob.obj =~/Document Information/
+          x=if @md.concord_make
+            format_toc.mini_concord_tail
+          else format_toc.mini_tail
+          end
+        else format_toc.mini_lev1
+        end
+        title=if dob.ocn ==0
+          if dob.name =~/^meta/ \
+          and dob.obj =~/Document Information/
+            %{#{linkname}}
+          else linkname
+          end
+        else
+          @@toc[:scr] <<  '
    '
+          link=(dob.ln) \
+          ? dob.ln
+          : ''
+          %{#{linkname}}
+        end
+        txt_obj={ txt: title }
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc[:scr]=if dob.name =~/^meta/ \
+        and dob.obj =~/Document Information/
+          format_toc.lev0
+        else format_toc.lev1
+        end
+        toc
+      end
+      def level_2
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        p_num=if ocn \
+        and ocn !~/#/
+          SiSU_HTML_Format::ParagraphNumber.new(@md,ocn)
+        else nil
+        end
+        txt_obj={ txt: linkname }
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev2
+        toc[:seg_mini]=format_toc.mini_lev2
+        if p_num
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev2
+        end
+        toc
+      end
+      def level_3
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        p_num=if ocn \
+        and ocn !~/#/
+          SiSU_HTML_Format::ParagraphNumber.new(@md,ocn)
+        else nil
+        end
+        txt_obj={ txt: linkname }
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev3
+        toc[:seg_mini]=format_toc.mini_lev3
+        if p_num
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev3
+        end
+        toc
+      end
+      def level_4
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        p_num=SiSU_HTML_Format::ParagraphNumber.new(@md,ocn) if ocn
+        if dob.ln ==4
+          seg_link=%{  
+    #{dob.obj}
+   }
+          @@seg_url=dob.name
+        elsif dob.obj =~/\d+.\d+.\d+.\d+|\d+.\d+.\d+|\d+.\d+|\d+/
+          seg_link=dob.obj.gsub(/^(\d+.\d+.\d+.\d+|\d+.\d+.\d+|\d+.\d+|\d+)(.*)/,
+            %{\\1 \\2 })
+        end
+        p_num=SiSU_HTML_Format::ParagraphNumber.new(@md,ocn) if ocn
+        @file=SiSU_Env::FileOp.new(@md) if @md
+        txt_obj=if seg_link=~/sisu_manifest\.html/
+          man_link=if @file.output_dir_structure.by_language_code? \
+          or @file.output_dir_structure.by_filetype?
+            seg_link.gsub(/sisu_manifest\.html/,"../../manifest/#{@file.base_filename.manifest}")
+          else seg_link
+          end
+          { txt: man_link }
+        else { txt: seg_link }
+        end
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc={}
+        toc[:seg]=format_toc.lev4
+        toc[:seg_mini]=format_toc.mini_lev4
+        title=%{#{p_num.goto}#{linkname}} if p_num
+        txt_obj=if title=~/sisu_manifest.html/
+          man_link=title.gsub(/sisu_manifest.html/,"../manifest/#{@file.base_filename.manifest}")
+          { txt: man_link }
+        else { txt: title }
+        end
+        format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+        toc[:scr]=format_toc.lev4
+        toc
+       #end
+      end
+      def level_5
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        toc={}
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_HTML_Format::ParagraphNumber.new(@md,ocn)
+          lnk_n_txt=%{  
+    #{linkname}
+  }
+          txt_obj={ txt: lnk_n_txt }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:seg]=format_toc.lev5
+          toc[:seg_mini]=format_toc.mini_lev5
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev5
+        end
+        toc
+      end
+      def level_6
+        dob=@data
+        linkname,ocn=dob.obj.strip,dob.ocn
+        toc={}
+        if ocn \
+        and ocn !~/#/
+          p_num=SiSU_HTML_Format::ParagraphNumber.new(@md,ocn)
+          lnk_n_txt=%{  
+  #{linkname}
+}
+          txt_obj={ txt: lnk_n_txt }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:seg]=format_toc.lev6
+          toc[:seg_mini]=format_toc.mini_lev6
+          title=%{#{p_num.goto}#{linkname}}
+          txt_obj={ txt: title }
+          format_toc=SiSU_HTML_Format::FormatToc.new(@md,txt_obj)
+          toc[:scr]=format_toc.lev6
+        end
+        toc
+      end
+      def level_crosslink
+        dob=@data
+        dob=if dob !~/^4~!/
+          dob.gsub(/^4~!\s+(\S+)\s+(.+)/,
+            %{
+  
    
+    #{@png.crosslink_ext}
+        \\2
+    <\/a>
+  
    
+})
+        else
+          dob.gsub(/^4~!\s+(\S+)\s+(.+)/,
+            %{
    
+  
+    
+      #{@png.crosslink}
+          \\2
+    <\/a>
+  
    
+})
+        end
+      end
+    end
+    class ScrollHeadAndSegToc < Toc
+      def initialize(md='',toc='',links_guide_toc='')
+        @md,@toc,@links_guide_toc=md,toc,links_guide_toc
+        @vz=SiSU_Viz::Defaults.new
+        @make=SiSU_Env::ProcessingSettings.new(@md)
+      end
+      def in_common
+        toc_shared=[]
+        @segtoc=[]
+        SiSU_Screen::Ansi.new(@md.opt.cmd,'Scroll & Segtoc').txt_grey if @md.opt.cmd =~/[MVv]/
+        format_head_toc=SiSU_HTML_Format::HeadToc.new(@md)
+        dochead=format_head_toc.head
+        dochead=dochead.gsub(/toc\.(html)/,'doc.\1') #kludge
+        ads=SiSU_HTML_Promo::Ad.new(@md)
+        toc_shared << dochead << ads.div.major
+        @segtoc << format_head_toc.head << ads.div.major
+        if @make.build.html_top_band?
+          toc_shared << format_head_toc.scroll_head_navigation_band
+        end
+        if defined? @md.rights.all
+          rights=format_head_toc.rights.all
+          rights=SiSU_HTML_Tune::CleanHTML.new(rights).clean
+        end
+        if @md.prefix_b
+          prefix_b=format_head_toc.notes.prefix_b
+          prefix_b=SiSU_HTML_Tune::CleanHTML.new(prefix_b).clean
+        end
+        if @make.build.html_top_band?
+          @segtoc << format_head_toc.seg_head_navigation_band
+        end
+        toc_shared << format_head_toc.scroll_head_title_banner_open
+        @segtoc << format_head_toc.seg_head_title_banner_open
+        tmp_head=nil
+        doc_title_endnote=@md.title.full.gsub(/(\*+)/,
+          '\1')
+        tmp_head=doc_title_endnote + "\n"
+        txt_obj={ txt: tmp_head }
+        format_txt_obj=SiSU_HTML_Format::FormatTextObject.new(@md,txt_obj)
+        toc_shared << format_txt_obj.center_bold
+        @segtoc << format_txt_obj.center_bold
+        if defined? @md.creator.author
+          creator_endnote=@md.creator.author.gsub(/(\*+)/,
+            %{ \\1})
+          tmp_head=creator_endnote + "\n"
+          txt_obj={ txt: tmp_head }
+          format_txt_obj=SiSU_HTML_Format::FormatTextObject.new(@md,txt_obj)
+          toc_shared << format_txt_obj.center_bold
+          @segtoc << format_txt_obj.center_bold
+        end
+        toc_shared << "#{@vz.table_close*1}\n"
+        @segtoc << "#{@vz.table_close*1}\n"
+        tmp_head=nil
+        if @md.prefix_a
+          tmp_head ||= %{
    #{@md.prefix_a}\n}
+          toc_shared << tmp_head.dup
+          @segtoc << tmp_head.dup
+        end
+        tmp_head=nil
+        toc_shared << @links_guide_toc
+        if defined? @md.rights.all #and ? @md.rights.all
+          toc_shared << rights
+        end
+        if defined? @md.prefix_b
+          toc_shared << prefix_b
+        end
+        scr_toc=if @make.build.toc? #Table of Contents added/appended here
+          toc_shared << @toc[:scr]
+        end
+        @segtoc << @links_guide_toc
+        @segtoc << @toc[:seg]
+        if defined? @md.rights.all \
+        and not @md.rights.all.empty?
+          @segtoc << rights
+        end
+        @segtoc << prefix_b if @md.prefix_b
+        #Segtoc tail added here
+        @segtoc << "
    \n" #bugfix sort later DEBUGNOW
+        @segtoc << @seg_toc_band_bottom
+        ads=SiSU_HTML_Promo::Ad.new(@md)
+        @segtoc << format_head_toc.seg_navigation_tail << ads.div.close << ads.display << format_head_toc.html_close
+        @segtoc=@segtoc.flatten.compact #watch
+        if @md.opt.act[:html_seg][:set]==:on
+          SiSU_HTML::Source::Output.new(@segtoc,@md).segtoc
+        end
+        @segtoc=[]
+        @toc[:scr],@toc[:seg]=[],[]
+        toc_shared
+      end
+    end
+    class Table < SiSU_HTML_Table::TableHTML
+    end
+    class Scroll < SiSU_HTML_Scroll::Scroll
+    end
+    class ScrollOutput
+      def initialize(scr_toc,scr_body,scr_metadata,scr_owner_details,scr_tails,md)
+        @scr_toc,@scr_body,@scr_metadata,@scr_owner_details,@scr_tails,@md=scr_toc,scr_body,scr_metadata,scr_owner_details,scr_tails,md
+      end
+      def publish
+        scroll=[]
+        scroll << @scr_toc << '
    ' << @scr_body << @scr_endnotes << @scr_owner_details << '
    ' << @scr_tails
+        scroll=scroll.flatten.compact #watch
+      end
+    end
+    class Seg < SiSU_HTML_Seg::Seg
+    end
+    class Output
+      def initialize(data='',md='')
+        @data,@md=data,md
+        @file=SiSU_Env::FileOp.new(md)
+        @o_str ||=SiSU_Env::ProcessingSettings.new(md).output_dir_structure
+      end
+      def scroll
+        if @md.opt.act[:html_scroll][:set]==:on
+          begin
+            @filename_html_scroll=@file.write_file.html_scroll
+            @data.each do |para|
+              para=para.strip.
+                gsub(/<:.+?>/,'').
+                gsub(Xx[:html_relative2],@file.path_rel_links.html_scroll_2).
+                gsub(Xx[:html_relative1],@file.path_rel_links.html_scroll_1).
+               #gsub(/#{Xx[:html_relative]}/,@file.path_rel_links.html_scroll).
+                gsub(/#{Rx[:mx_fa_clean]}/,'')
+              unless para =~/\A\s*\Z/
+                @filename_html_scroll.puts para,"\n"
+              end
+            end
+          rescue
+            SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+              __LINE__.to_s + ':' + __FILE__
+            end
+          ensure
+            @filename_html_scroll.close
+          end
+        end
+      end
+      def segtoc
+        if @md.opt.act[:html_seg][:set]==:on
+          begin
+            @filename_html_segtoc=@file.write_file.html_segtoc
+            @data.each do |para|
+              para=para.strip.
+                gsub(//,'').
+                gsub(Xx[:html_relative2],@file.path_rel_links.html_seg_2).
+                gsub(Xx[:html_relative1],@file.path_rel_links.html_seg_1)
+              unless para =~/\A\s*\Z/
+                @filename_html_segtoc.puts para,"\n"
+              end
+            end
+          rescue
+            SiSU_Errors::InfoError.new($!,$@,@md.opt.cmd,@md.fns).error do
+              __LINE__.to_s + ':' + __FILE__
+            end
+          ensure
+            @filename_html_segtoc.close
+            pwd_set=Dir.pwd
+            idx_lnk=(@o_str.dump_or_redirect?) \
+            ? @file.base_filename.manifest
+            : @file.base_filename.html_segtoc
+            mlnk=@file.base_filename.html_seg_index
+            Dir.chdir(@file.output_path.html_seg.dir)
+            FileUtils::rm_f(mlnk)
+            FileUtils::ln_s(idx_lnk,mlnk)
+            Dir.chdir(pwd_set)
+          end
+        end
+      end
+    end
+  end
+end
+__END__
diff --git a/lib/sisu/v5/html_format.rb b/lib/sisu/v5/html_format.rb
new file mode 100644
index 00000000..3c8bc465
--- /dev/null
+++ b/lib/sisu/v5/html_format.rb
@@ -0,0 +1,1428 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+   2007, 2008, 2009, 2010, 2011, 2012, 2013 Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see .
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   
+   
+
+   
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   
+   
+
+ * Download:
+   
+
+ * Git
+   
+   
+
+ * Ralph Amissah
+   
+   
+
+ ** Description: html formating, css template
+
+=end
+module SiSU_HTML_Format
+  include SiSU_Viz
+  class ParagraphNumber
+    def initialize(md,ocn)
+      @md,@ocn=md,ocn.to_s
+      @ocn ||=''
+    end
+    def ocn_display
+      make=SiSU_Env::ProcessingSettings.new(@md)
+      if make.build.ocn?
+        ocn_class='ocn'
+        if @ocn.to_i==0 \
+        or @ocn.empty?
+          %{}
+        else
+          @ocn.gsub(/^(\d+|)$/,
+            %{})
+        end
+      else
+        %{}
+      end
+    end
+    def name
+      (@ocn==nil || @ocn.empty?) ? '' : %{}
+    end
+    def id #w3c? "tidy" complains about numbers as identifiers ! annoying
+      (@ocn==nil || @ocn.empty?) ? '' : %{id="o#{@ocn}"}
+    end
+    def goto
+      (@ocn==nil || @ocn.empty?) ? '' : %{}
+    end
+  end
+  class HeadInformation
+    require_relative 'css'                              # css.rb
+    require_relative 'shared_xml'                       # shared_xml.rb
+    include SiSU_Viz
+    attr_reader :md,:rdf,:vz
+    def initialize(md)
+      @md=md
+      # DublinCore 1 - title
+      @vz=SiSU_Viz::Defaults.new
+      @seg_name_html=(SiSU_HTML::Source::Seg.new.seg_name_html || [])
+      @seg_name_html_tracker=(SiSU_HTML::Source::Seg.new.seg_name_html_tracker || [])
+      @tocband_scroll,@tocband_segtoc=nil,nil
+      @stylesheet=SiSU_Style::CSS_HeadInfo.new(md).stylesheet
+      @o_str ||=SiSU_Env::ProcessingSettings.new(md).output_dir_structure
+      @index,@metalink='index','#metadata'
+      @toc=@md.file.base_filename.html_segtoc
+    end
+    def url_path_image_sys
+      (@o_str.dump_or_redirect?) \
+      ? './image'
+      : "#{Xx[:html_relative2]}_sisu/image_sys"
+    end
+    def icon
+      def up
+        'arrow_up_red.png'
+      end
+      def next
+        'arrow_next_red.png'
+      end
+      def previous
+        'arrow_prev_red.png'
+      end
+      def dot_clear
+        'dot_clear.png'
+      end
+      def dot_white
+        'dot_white.png'
+      end
+      def dot
+        dot_white
+      end
+      self
+    end
+    def png_nav
+      def toc
+        %{toc}
+      end
+      def pre
+        %{<< previous}
+      end
+      def nxt
+        %{next >>}
+      end
+      def dot_toc
+        %{^}
+      end
+      def dot_pre
+        %{<}
+      end
+      def dot_nxt
+        %{>}
+      end
+      self
+    end
+    def doc_types(page=:seg) #used in toc & seg_nav_band
+      wgt=SiSU_HTML_Format::Widget.new(@md)
+      %{
+
+
+
    
+  #{wgt.manifest(page)}
+  #{wgt.search}
+
    }
+    end
+    def rdf
+      SiSU_XML_Tags::RDF.new(md)
+    end
+    def doc_type
+      %{
+\n}
+    end
+    def table_close
+      %{  
+#{@vz.table_close}}
+    end
+    def button_home(page=:seg)
+      button=%{ \n \n 
    \n}
+      if @md.make.home_button_image.is_a?(Hash)
+        image_path=if page==:manifest
+          @md.file.output_path.manifest.rel_image
+        elsif  page==:scroll
+          @md.file.output_path.html_scroll.rel_image
+        else
+          @md.file.output_path.html_seg.rel_image
+        end
+        dir=SiSU_Env::FileOp.new(@md)
+        button +=%{   
    home icon -->
    \n}
+      elsif @md.home_button_links.is_a?(Array)
+        @md.home_button_links.each do |links|
+          button +=%{  
    \n    #{links[:say]}\n  
    \n}
+        end
+      end
+      button +=%{ 
    }
+      button
+    end
+    def html_close #moved
+    %{
+}
+    end
+  end
+  class Widget < HeadInformation
+    def initialize(md)
+      super(md)
+      @md=md
+      @cf_defaults=SiSU_Env::InfoProcessingFlag.new
+      @env=SiSU_Env::InfoEnv.new(md.fns)
+      @file=SiSU_Env::FileOp.new(md)
+      @o_str ||=SiSU_Env::ProcessingSettings.new(md).output_dir_structure
+      @make=SiSU_Env::ProcessingSettings.new(@md)
+    end
+    def home
+      %{    		
+  
+  #{@vz.nav_txt_homepage}
+
+  
+    #{text}
+  
+
+  
+    #{text}
+  
+
+  
+  #{brace_url.xml_open}#{@md.file.output_path.manifest.url}/#{@file.base_filename.manifest}#{brace_url.xml_close}
+  
+
+  
+    #{@vz.nav_txt_manifest}
+  
+
+  
+    #{@vz.nav_txt_pdf_portrait}
+  
+
+  
+    #{@vz.nav_txt_pdf_landscape}
+  
+
    
+
+
+
    
+  #{button_home(:scroll)}
+
+  #{doc_types}
+
+   
+#{@vz.table_close}
+
    }
+      else ''
+      end
+    end
+    def concordance_navigation_band
+      if @make.build.html_top_band?
+        %{
+
+
+
+}
+      %{
    
+  #{button_home}
+
+  #{doc_types}
+
+   
+      #{png_nav.toc}
+     
+#{@vz.table_close}
+
    }
+      else ''
+      end
+    end
+    def seg_head_navigation_band(page=:seg)
+      if page==:manifest
+        nxt=(@file.output_dir_structure.by_language_code? \
+        || @file.output_dir_structure.by_filetype?) \
+        ? "../html/#{@md.fnb}/toc#{@md.lang_code_insert}#{Sfx[:html]}"
+        : "toc#{@md.lang_code_insert}#{Sfx[:html]}"
+        firstseg=%{
+        #{png_nav.nxt}}
+      elsif @md.firstseg =~/\S+/
+        firstseg=%{
+      #{png_nav.nxt}}
+      end
+      %{
+
+
+
+}
+      %{
    
+#{button_home(page)}
+
+  #{doc_types(page)}
+
+   #{firstseg} 
+#{@vz.table_close}
+
    }
+    end
+    def manifest_link(text)
+#     @file=SiSU_Env::FileOp.new(@md) if @md
+  %{
+    #{text}
+  }
+    end
+    def concordance_link(text)
+      if @md.concord_make
+  %{
+    
+      #{text}
+    
+  }
+      else ''
+      end
+    end
+    def make_scroll_search_form_and_manifest_link
+      wgt=SiSU_HTML_Format::Widget.new(@md)
+      scroll=%{
    		
+  #{@vz.nav_txt_doc_link}
+
    
+
+  #{wgt.manifest}
+  #{wgt.search}
+
    }
+    end
+    def make_scroll_seg_pdf
+      manifest=scroll=seg=''
+      wgt=SiSU_HTML_Format::Widget.new(@md)
+      seg=%{
    		
+  #{@vz.nav_txt_toc_link}
+
    
+
+
    
+  #{wgt.manifest}
+  #{wgt.search}
+
    }
+    end
+    def make_concordance
+      manifest=scroll=seg=''
+      wgt=SiSU_HTML_Format::Widget.new(@md)
+      %{
+
+
    
+  #{wgt.manifest}
+  #{wgt.search}
+
    }
+    end
+    def head
+      rdf=SiSU_XML_Tags::RDF.new(@md)
+      %{#{doc_type}
+
+  
+    #{@md.html_title}
+  
+
+#{rdf.metatag_html}
+#{@stylesheet.css_head}
+
+#{@vz.color_body}
+
+
+}
+    end
+    def concordance
+      if @md.concord_make
+      %{#{@vz.margin_css}
+  
    
+    
+      Concordance
+    
+  
    
+#{@vz.table_close}}
+      else
+      %{#{@vz.margin_css}
+#{@vz.table_close}}
+      end
+    end
+    def links_guide_vertical_open
+#     @file=SiSU_Env::FileOp.new(@md) if @md
+      url=((defined? @vz.url_hp) && @vz.url_hp =~/^https?:\/\/\S+$/ ? @vz.url_hp : @vz.url_home)
+      %{
+