v3dv, add dev branch (use to make some changes to module & class names & test)

* (intended as) short term branch, merge back into v3 once tested
* sisu --dev (to invoke)

1 files changed, 506 insertions, 0 deletions

diff --git a/lib/sisu/v3dv/dal.rb b/lib/sisu/v3dv/dal.rb
new file mode 100644
index 00000000..50cf94f9
--- /dev/null
+++ b/lib/sisu/v3dv/dal.rb
@@ -0,0 +1,506 @@
+# encoding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997 - 2012, Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+   SiSU, a framework for document structuring, publishing and search
+
+   Copyright (C) Ralph Amissah
+
+   This program is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the Free
+   Software Foundation, either version 3 of the License, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+   more details.
+
+   You should have received a copy of the GNU General Public License along with
+   this program. If not, see <http://www.gnu.org/licenses/>.
+
+   If you have Internet connection, the latest version of the GPL should be
+   available at these locations:
+   <http://www.fsf.org/licensing/licenses/gpl.html>
+   <http://www.gnu.org/licenses/gpl.html>
+
+   <http://www.jus.uio.no/sisu/gpl.fsf/toc.html>
+   <http://www.jus.uio.no/sisu/gpl.fsf/doc.html>
+   <http://www.jus.uio.no/sisu/gpl.fsf/plain.txt>
+
+ * SiSU uses:
+   * Standard SiSU markup syntax,
+   * Standard SiSU meta-markup syntax, and the
+   * Standard SiSU object citation numbering and system
+
+ * Hompages:
+   <http://www.jus.uio.no/sisu>
+   <http://www.sisudoc.org>
+
+ * Download:
+   <http://www.jus.uio.no/sisu/SiSU/download.html>
+
+ * Ralph Amissah
+   <ralph@amissah.com>
+   <ralph.amissah@gmail.com>
+
+ ** Description: preprocessing, (document abstraction), data abstraction used
+   in subsequent processing
+
+=end
+module SiSU_DAL
+  require_relative 'defaults'                           # defaults.rb
+    include SiSU_Viz
+  require_relative 'sysenv'                             # sysenv.rb
+    include SiSU_Env
+  require_relative 'param'                              # param.rb
+    include SiSU_Param
+  require_relative 'dal_doc_objects'                    # dal_doc_objects.rb
+  require_relative 'dal_syntax'                         # dal_syntax.rb
+    include SiSU_Syntax
+  require_relative 'dal_doc_str'                        # dal_doc_str.rb
+  require_relative 'dal_idx'                            # dal_idx.rb
+  require_relative 'dal_numbering'                      # dal_numbering.rb
+  require_relative 'dal_hash_digest'                    # dal_hash_digest.rb
+  require_relative 'dal_endnotes'                       # dal_endnotes.rb
+  require_relative 'dal_images'                         # dal_images.rb
+  require_relative 'dal_metadata'                       # dal_metadata.rb
+  require_relative 'dal_character_check'                # dal_character_check.rb
+  require_relative 'dal_substitutions_and_insertions'   # dal_substitutions_and_insertions.rb
+  require_relative 'dal_expand_insertions'              # dal_expand_insertions.rb
+  require_relative 'prog_text_translation'              # prog_text_translation.rb
+  require_relative 'shared_sem'                         # shared_sem.rb
+  class Instantiate < SiSU_Param::Parameters::Instructions
+    def initialize
+      @@flag_vocab=0
+      @@line_mode=''
+    end
+  end
+  class Source <Instantiate
+    @@dal_array=[]
+    @@idx_arr={ sst: [], tex: [], html: [], xhtml: [] }
+    @@map_arr={ nametags: [], ocn_htmlseg: [] }
+    @@fns=nil
+    def initialize(opt,fnx=nil)
+      @opt,@fnx=opt,fnx
+      @@fns||@opt.fns
+      @make_fns=if @fnx and @fnx =~/\.ss[tmi]$/
+        SiSU_Env::Info_file.new(@fnx)
+      else
+        SiSU_Env::Info_file.new(@opt.fns)
+      end
+      @fnm=@make_fns.marshal.dal_metadata
+      @fnc=@make_fns.marshal.dal_content
+      @idx_sst=@make_fns.marshal.dal_idx_sst_rel_html_seg
+      @idx_tex=@make_fns.marshal.dal_idx_sst_rel
+      @idx_html=@make_fns.marshal.dal_idx_html
+      @idx_xhtml=@make_fns.marshal.dal_idx_xhtml
+      @map_nametags=@make_fns.marshal.dal_map_nametags
+      @map_ocn_htmlseg=@make_fns.marshal.dal_map_ocn_htmlseg
+      SiSU_Env::Create_system_link.new.images
+      @env=SiSU_Env::Info_env.new
+    end
+    def read                                                                   #creates dal
+      begin
+        dal=[]
+        @@dal_array=[]
+        @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+        ? @fnx
+        : @opt.fns
+        create_dal
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@@fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get                                                                    #reads dal, unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@dal_array=[]
+        end
+        dal=(@@dal_array.empty?) ? read_fnc : @@dal_array.dup
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_idx_sst                                                            #reads dal idx.sst, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@idx_arr[:sst]=[]
+        end
+        dal=(@@idx_arr[:sst].empty?) ? read_idx_sst : @@idx_arr[:sst].dup #check
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_idx_tex                                                            #reads dal idx.tex, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@idx_arr[:tex]=[]
+        end
+        dal=(@@idx_arr[:tex].empty?) ? read_idx_tex : @@idx_arr[:tex].dup #check
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_idx_html                                                           #reads dal idx.html, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@idx_arr[:html]=[]
+        end
+        dal=(@@idx_arr[:html].empty?) ? read_idx_html : @@idx_arr[:html].dup
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_idx_xhtml                                                          #reads dal idx.xhtml, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@idx_arr[:xthml]=[]
+        end
+        dal=(@@idx_arr[:xhtml].empty?) ? read_idx_xhtml : @@idx_arr[:xhtml].dup
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_map_nametags                                                       #reads dal map.nametags, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@map_arr[:nametags]=[]
+        end
+        dal=(@@map_arr[:nametags].empty?) ? read_map_nametags : @@map_arr[:nametags].dup
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+    def get_map_ocn_htmlseg                                                    #reads dal map.ocn_htmlseg, #unless does not exist then creates first
+      begin
+        dal=[]
+        unless @@fns==@opt.fns \
+        or @@fns==@fnx
+          @@fns=(@fnx && @fnx =~/\.ss[tmi]$/) \
+          ? @fnx
+          : @opt.fns
+          @@map_arr[:ocn_htmlseg]=[]
+        end
+        dal=(@@map_arr[:ocn_htmlseg].empty?) ? read_map_ocn_htmlseg : @@map_arr[:ocn_htmlseg].dup
+      rescue; SiSU_Errors::Info_error.new($!,$@,@opt.cmd,@opt.fns).error
+      ensure
+        Instantiate.new
+      end
+    end
+  protected
+    def create_dal
+      dal_array=[]
+      unless @opt.cmd =~/q/
+        tell=(@opt.cmd=~/[vVM]/) \
+        ? SiSU_Screen::Ansi.new(@opt.cmd,'Document Abstraction')
+        : SiSU_Screen::Ansi.new(@opt.cmd,'Document Abstraction',"[#{@opt.f_pth[:lng_is]}] #{@opt.fns}")
+        tell.green_title_hi
+      end
+      fn=(@fnx && @fnx =~/\.ss[tmi]$/) \
+      ? @fnx
+      : @opt.fns
+      file_array=@env.read_source_file(fn)
+      file_array.each do |l|
+        if l =~/\r\n/; l.gsub!(/\r\n/,"\n")
+        end
+      end
+      meta=file_array.dup
+      meta=meta.join.split("\n\n") #check whether can be eliminated, some of these are large objects to have twice
+      @md=SiSU_Param::Parameters::Instructions.new(meta,@opt).extract
+      meta=nil
+      dal=SiSU_DAL::Make.new(fn,@md,file_array).song
+      if @opt.cmd =~/[vM]/
+        cf=SiSU_Env::Create_file.new(fn)
+        SiSU_Screen::Ansi.new(@opt.cmd,@opt.fns,"~meta/#{@opt.fns}.meta").output if @opt.cmd =~/v/i
+        SiSU_Screen::Ansi.new(@opt.cmd,"dal -> #{cf.meta}").txt_grey if @opt.cmd =~/M/
+      end
+      dal.each {|s| dal_array << s}
+      dal_array
+    end
+    def read_fnm
+      dal=[]
+      dal=if FileTest.file?(@fnm)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@fnm){ |f| dal=Marshal.load(f)})
+        : (File.open(@fnm,'r:utf-8'){ |f| dal=Marshal.load(f)})
+      else SiSU_DAL::Source.new(@opt).create_dal
+      end
+    end
+    def read_fnc
+      dal=[]
+      dal=if FileTest.file?(@fnc)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@fnc){ |f| dal=Marshal.load(f)})
+        : (File.open(@fnc,'r:utf-8'){ |f| dal=Marshal.load(f)})
+      else SiSU_DAL::Source.new(@opt).create_dal
+      end
+    end
+    def read_idx_sst
+      m=[]
+      m=if FileTest.file?(@idx_sst)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@idx_sst){ |f| m=Marshal.load(f)})
+        : (File.open(@idx_sst,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+    def read_idx_tex
+      m=[]
+      m=if FileTest.file?(@idx_tex)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@idx_tex){ |f| m=Marshal.load(f)})
+        : (File.open(@idx_tex,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+    def read_idx_html
+      m=[]
+      m=if FileTest.file?(@idx_html)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@idx_html){ |f| m=Marshal.load(f)})
+        : (File.open(@idx_html,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+    def read_idx_xhtml
+      m=[]
+      m=if FileTest.file?(@idx_xhtml)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@idx_xhtml){ |f| m=Marshal.load(f)})
+        : (File.open(@idx_xhtml,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+    def read_map_nametags
+      m=[]
+      m=if FileTest.file?(@map_nametags)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@map_nametags){ |f| m=Marshal.load(f)})
+        : (File.open(@map_nametags,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+    def read_map_ocn_htmlseg
+      m=[]
+      m=if FileTest.file?(@map_ocn_htmlseg)
+        (RUBY_VERSION < '1.9') \
+        ? (File.open(@map_ocn_htmlseg){ |f| m=Marshal.load(f)})
+        : (File.open(@map_ocn_htmlseg,'r:utf-8'){ |f| m=Marshal.load(f)})
+      else nil
+      end
+    end
+  end
+  class Output
+    def initialize(fn,md,data)
+      @fn,@md,@data=fn,md,data
+      @cf=SiSU_Env::Create_file.new(@fn)
+      @make=SiSU_Env::Info_file.new(@fn)
+      @dir=SiSU_Env::Info_env.new(@fn)
+    end
+    def screen_dump(o)
+      if defined? o.of
+        print %{OF: #{o.of}; }
+      end
+      if defined? o.is
+        print %{IS: #{o.is}; }
+      end
+      if defined? o.ocn
+        print %{OCN: #{o.ocn}; }
+      end
+      if defined? o.node
+        print %{NODE: #{o.node}; }
+      end
+      if defined? o.parent
+        print %{Parent: #{o.parent}; }
+      end
+      if defined? o.obj and not o.obj.empty?
+        puts %{\n#{o.obj}; }
+      else "\n"
+      end
+    end
+    def screen_print(t_o)
+      if defined? t_o
+        print ' ' + t_o.to_s
+      end
+    end
+    def screen_output(data)
+      data.each do |o|
+        print o.class
+        screen_print(o.ocn)
+        screen_print(o.obj)
+        puts "\n"
+      end
+    end
+    def hard_output
+      if @md.opt.cmd =~/M/
+        filename_meta=@cf.metaverse.file_meta
+        @data.each {|o| filename_meta.puts o.inspect.sub(/:0x[0-9a-f]{8}\s/,': ')} #to make diffing easier
+        filename_txt=@cf.metaverse.file_txt
+        @data.each do |o|
+          if defined? o.ocn
+            filename_txt.puts case o.is
+            when 'heading'
+              "[#{o.is} #{o.lv}~#{o.name} [#{o.ocn}]] #{o.obj}"
+            else "[#{o.is} [#{o.ocn}]] #{o.obj}"
+            end
+          else
+            filename_txt.puts case o.is
+            when 'meta'
+              "[m~#{o.tag}] #{o.obj}"
+            else "[#{o.is}] #{o.obj}"
+            end
+          end
+        end
+        filename_debug=@cf.file_debug
+        @data.each do |o|
+          if defined? o.ocn
+            case o.is
+            when 'heading'
+              filename_debug.puts "#{o.is} #{o.lv}~#{o.name} odv=#{o.odv} osp=#{o.osp} [#{o.ocn}] -->\n\t#{o.obj}"
+            end
+          end
+        end
+      else
+        hard="#{@dir.processing_path.dal}/#{@md.fns}.meta"
+        File.unlink(hard) if FileTest.file?(hard)
+        hard="#{@dir.processing_path.dal}/#{@md.fns}.txt"
+        File.unlink(hard) if FileTest.file?(hard)
+        hard="#{@dir.processing_path.dal}/#{@md.fns}.debug.txt"
+        File.unlink(hard) if FileTest.file?(hard)
+      end
+    end
+    def make_marshal_content
+      marshal_dal=@make.marshal.dal_content
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def make_marshal_metadata
+      marshal_dal=@make.marshal.dal_metadata
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def idx_html_hard_output
+      if @md.book_idx \
+      and @md.opt.cmd =~/M/
+        filename_meta=@cf.file_meta_idx_html
+        unless @data.nil? #REMOVE earliest possible
+          @data.each {|s| p s.inspect + "\n" unless s.class==String}
+          @data.each {|s| filename_meta.puts s.strip + "\n" unless s.strip.empty?}
+        end
+      else
+        hard_idx_html="#{@dir.processing_path.dal}/#{@md.fns}.idx.html"
+        File.unlink(hard_idx_html) if FileTest.file?(hard_idx_html)
+      end
+    end
+    def make_marshal_idx_sst_html_seg
+      marshal_dal=@make.marshal.dal_idx_sst_rel_html_seg
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def make_marshal_idx_sst_rel
+      marshal_dal=@make.marshal.dal_idx_sst_rel
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def make_marshal_idx_html
+      marshal_dal=@make.marshal.dal_idx_html
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def make_marshal_idx_xhtml
+      marshal_dal=@make.marshal.dal_idx_xhtml
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Array
+    end
+    def make_marshal_map_nametags
+      marshal_dal=@make.marshal.dal_map_nametags
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Hash
+    end
+    def make_marshal_map_name_ocn_htmlseg
+      marshal_dal=@make.marshal.dal_map_ocn_htmlseg
+      File.open(marshal_dal,'w'){|f| Marshal.dump(@data,f)} if @data.class==Hash
+    end
+  end
+  class Make
+    def initialize(fn,md,data)
+      @fn,@md,@data=fn,md,data
+      @env=SiSU_Env::Info_env.new(@md.fns)
+    end
+    def reset
+      @@flag_vocab=0
+      @@line_mode=''
+    end
+    def song
+      reset
+      data=@data
+      data=data.join.split("\n\n")
+      data=SiSU_insertions::Insertions.new(@md,data).expand_insertions?                                                # dal_expand_insertions.rb
+      data=SiSU_substitute_and_insert::SI.new(@md,data).substitutions_and_insertions?                                  # dal_substitutions_and_insertions.rb
+      data,metadata=SiSU_document_structure_extract::Build.new(@md,data).identify_parts                                # dal_doc_str.rb
+      data=SiSU_Syntax::Markup.new(@md,data).songsheet                                                                 # dal_syntax.rb
+      data,endnote_array=SiSU_character_check::Check.new(data).character_check_and_oldstyle_endnote_array              # dal_character_check.rb
+      data=SiSU_images::Images.new(@md,data).images                                                                    # dal_images.rb
+      data,tags_map,ocn_html_seg_map=SiSU_numbering::Numbering.new(@md,data).numbering_song                            # dal_numbering.rb
+      data,book_index_rel,book_index_rel_html_seg,html_idx,xhtml_idx=SiSU_book_index::Book_index.new(@md,data,@env).indexing_song if @md.book_idx  # dal_idx.rb
+      data=SiSU_endnotes::Endnotes.new(@md,data,endnote_array).endnotes                                                # dal_endnotes.rb
+      outputdata=data
+      if @md.opt.cmd =~/[mM]/
+        SiSU_DAL::Output.new(@fn,@md,outputdata).hard_output
+        SiSU_DAL::Output.new(@fn,@md,outputdata).make_marshal_content
+        SiSU_DAL::Output.new(@fn,@md,metadata).make_marshal_metadata
+        SiSU_DAL::Output.new(@fn,@md,html_idx).idx_html_hard_output
+        SiSU_DAL::Output.new(@fn,@md,book_index_rel_html_seg).make_marshal_idx_sst_html_seg
+        SiSU_DAL::Output.new(@fn,@md,book_index_rel).make_marshal_idx_sst_rel
+        SiSU_DAL::Output.new(@fn,@md,html_idx).make_marshal_idx_html
+        SiSU_DAL::Output.new(@fn,@md,xhtml_idx).make_marshal_idx_xhtml
+        SiSU_DAL::Output.new(@fn,@md,tags_map).make_marshal_map_nametags
+        SiSU_DAL::Output.new(@fn,@md,ocn_html_seg_map).make_marshal_map_name_ocn_htmlseg
+      end
+      reset
+      outputdata
+    end
+  protected
+  end
+end
+__END__