path: root/lib/sisu/v5/concordance.rb

# encoding: utf-8
=begin

 * Name: SiSU

 * Description: a framework for document structuring, publishing and search

 * Author: Ralph Amissah

 * Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
   2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Ralph Amissah,
   All Rights Reserved.

 * License: GPL 3 or later:

   SiSU, a framework for document structuring, publishing and search

   Copyright (C) Ralph Amissah

   This program is free software: you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by the Free
   Software Foundation, either version 3 of the License, or (at your option)
   any later version.

   This program is distributed in the hope that it will be useful, but WITHOUT
   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
   more details.

   You should have received a copy of the GNU General Public License along with
   this program. If not, see <http://www.gnu.org/licenses/>.

   If you have Internet connection, the latest version of the GPL should be
   available at these locations:
   <http://www.fsf.org/licensing/licenses/gpl.html>
   <http://www.gnu.org/licenses/gpl.html>

   <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>

 * SiSU uses:
   * Standard SiSU markup syntax,
   * Standard SiSU meta-markup syntax, and the
   * Standard SiSU object citation numbering and system

 * Hompages:
   <http://www.jus.uio.no/sisu>
   <http://www.sisudoc.org>

 * Download:
   <http://www.sisudoc.org/sisu/en/SiSU/download.html>

 * Git
   <http://sources.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
   <http://sources.sisudoc.org/?p=code/sisu.git;a=blob;f=lib/sisu/v5/concordance.rb;hb=HEAD>

 * Ralph Amissah
   <ralph@amissah.com>
   <ralph.amissah@gmail.com>

 ** Description: concordance file (html concordance, wordmap, linked index of
    words in document)

=end
module SiSU_Concordance
  require_relative 'particulars'                        # particulars.rb
    include SiSU_Particulars
  require_relative 'sysenv'                             # sysenv.rb
    include SiSU_Env
  require_relative 'defaults'                           # defaults.rb
    include SiSU_Viz
  require_relative 'html_format'                        # html_format.rb
    include SiSU_HTML_Format
  require_relative 'html_minitoc'                       # html_minitoc.rb
  class Source
    def initialize(opt)
      @opt=opt
      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
    end
    def read
      begin
        @env,@md=@particulars.env,@particulars.md
        @env.url.output_tell
        unless @md.opt.act[:quiet][:set]==:on
          tool=(@md.opt.act[:verbose][:set]==:on \
          || @md.opt.act[:verbose_plus][:set]==:on \
          || @md.opt.act[:maintenance][:set]==:on) \
          ? "#{@env.program.web_browser} #{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}"
          : "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}"
          (@md.opt.act[:verbose][:set]==:on \
          || @md.opt.act[:verbose_plus][:set]==:on \
          || @md.opt.act[:maintenance][:set]==:on) \
          ? SiSU_Screen::Ansi.new(@opt.act[:color_state][:set],"Concordance",tool).green_hi_blue
          : SiSU_Screen::Ansi.new(@opt.act[:color_state][:set],'Concordance',tool).green_title_hi
        end
        wordmax=@env.concord_max
        unless @md.wc_words.nil?
          if @md.wc_words < wordmax
            SiSU_Concordance::Source::Words.new(@particulars).songsheet
          else
            SiSU_Screen::Ansi.new(@md.opt.act[:color_state][:set],"concordance skipped, large document has over #{wordmax} words (#{@md.wc_words})").warn unless @md.opt.act[:quiet][:set]==:on
          end
        else
          SiSU_Screen::Ansi.new(@md.opt.act[:color_state][:set],"wc (word count) is off, concordance will be processed for all files including those over the max set size of: #{wordmax} words").warn unless @md.opt.act[:quiet][:set]==:on
          SiSU_Concordance::Source::Words.new(@particulars).songsheet
        end
      rescue
        SiSU_Errors::Rescued.new($!,$@,@md.opt.cmd,@md.fns).location do
          __LINE__.to_s + ':' + __FILE__
        end
      ensure
        SiSU_Env::CreateSite.new(@opt.cmd).cp_css
      end
    end
    private
    class DocTitle
      include SiSU_Viz
      #revisit, both requires (html & xml_shared) needed for stand alone operation (sisu -w [filename])
      require_relative 'xml_shared'                     # xml_shared.rb
      require_relative 'html'                           # html.rb
      def initialize(particulars)
        @particulars,@md=particulars,particulars.md
        @data=SiSU_HTML::Source::HTML_Environment.new(particulars).tuned_file_instructions
        @file=SiSU_Env::FileOp.new(@md)
        @fnb=@md.fnb
        @lex_button=%{<a href="http://www.jus.uio.no/sisu/" target="_top"><img border="0" height="44" width="144" valign="center" src="#{@file.path_rel_links.html_seg_2}_sisu/image/sisu.png" alt="SiSU home --&gt;"></a>}
        @doc_details =<<WOK
<table summary="links to text related to this rudimentary index" width="96%" border="0" bgcolor="white" cellpadding="0" align="center"><tr><td width="2%" align="right">&nbsp;</td><td width="94%" valign="top" align="justify"><h1 class="small"><a href="#{@md.file.base_filename.html_segtoc}"><b>#{@md.title.full}</b></a></h1><p class="bold">#{@md.author}</p></td></tr></table>
WOK
        @make=SiSU_Env::ProcessingSettings.new(@md)
      end
      def create
        head_banner=SiSU_HTML_Format::HeadToc.new(@md)
        minitoc=SiSU_HTML_MiniToc::TocMini.new(@md,@data).songsheet.join("\n")
        stylesheet=SiSU_Style::CSS_HeadInfo.new(@md).stylesheet
        if @make.build.manifest_minitoc?
          toc='<div class="toc">' + minitoc + '</div>'
          div_class='content'
        else
          toc=''
          div_class='content0'
        end
        top_band=if @make.build.html_top_band?
          head_banner.concordance_navigation_band
        else ''
        end
        <<WOK
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
  <title>
    SiSU created WordIndex for: #{@md.title.full}
  </title>
  <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
  <meta name="Description" content="&nbsp;SiSU created">
  <meta name="keywords" content="word index for #{@md.title.full}">
  <meta name="generator" content="SiSU (Linux &amp; Ruby!)">
  <link rel="generator" href="http://www.jus.uio.no/sisu" />
  <link rel="shortcut icon" href="../_sisu/image/rb7.ico" />
  #{stylesheet.css_head_seg}
</head>
<body>
  #{top_band}
  #{toc}
<div class="#{div_class}">
 #{@doc_details}
<p>Word index links are to html versions of the text the segmented version followed by the scroll (single document) version.<br />[For segmented text references [T1], [T2] or [T3] appearing without a link, indicates that the word appears in a title (or subtitle) of the text (that is identifiable by the appended object citation number).]</p>
<p>(The word listing/index is Case sensitive: Capitalized words appear before lower case)</p>
  <p>
    <b>word</b> (number of occurences)<br />linked references to word within document <br />
    [if number of occurences exceed number of references - word occurs more than once in at least one reference. Footnote/endnotes are either assigned to the paragraph from which they are referenced or ignored, so it is relevant to check the footnotes referenced from within a paragraph as well.]
  </p>
  <p>
    (After the page is fully loaded) you can jump directly to a word by appending a hash (#) and the word to the url for this text, (do not forget that words are case sensitive, and may be listed twice (starting with and without an upper case letter)), #your_word # [&nbsp;http://[web host]/#{@fnb}/concordance.html#your_word&nbsp;]
  </p>
WOK
      end
    end
    class Word
      @@word_previous=''
      def initialize(word,freq)
        @word,@freq=word,freq
      end
      def html
        w=if @word.capitalize==@@word_previous
          %{\n<p class="concordance_word">#{@word}</p><p class="concordance_count">(#{@freq})</p>\n\t<p class="concordance_object"> }
        else n=@word.strip.gsub(/\s+/,'_') #also need to convert extended character set to html
          %{\n<p class="concordance_word"><a name="#{n}">#{@word}</a></p><p class="concordance_count">(#{@freq})</p>\n\t<p class="concordance_object"> }
        end
        @@word_previous=@word.capitalize
        w
      end
    end
    class Words
      require_relative 'i18n'                           # i18n.rb
        include SiSU_i18n
      require_relative 'defaults'                       # defaults.rb
        include SiSU_Viz
      require_relative 'html_format'                    # html_format.rb
        include SiSU_HTML_Format
      require_relative 'sysenv'                         # sysenv.rb
        include SiSU_Screen
      @@dp=nil
      def initialize(particulars)
        @particulars=particulars
        begin
          @env,@md,@ao_array=particulars.env,particulars.md,particulars.ao_array
          @file=SiSU_Env::FileOp.new(@md)
          @freq=Hash.new(0)
          @dp=@@dp ||=SiSU_Env::InfoEnv.new.digest.pattern
          @rxp_lv1=/^#{Mx[:lv_o]}1:/ #fix @rxp_lv #  Mx[:lv_o]
          @rxp_lv2=/^#{Mx[:lv_o]}2:/ #fix @rxp_lv #  Mx[:lv_o]
          @rxp_lv3=/^#{Mx[:lv_o]}3:/ #fix @rxp_lv #  Mx[:lv_o]
          @rxp_title=Regexp.new("^#{Mx[:meta_o]}title#{Mx[:meta_c]}\s*(.+?)\s*$")
          @rxp_t1=Regexp.new('^T1')
          @rxp_t2=Regexp.new('^T2')
          @rxp_t3=Regexp.new('^T3')
          @rxp_excluded1=/#{Mx[:url_o]}\S+?#{Mx[:url_c]}|#{Mx[:rel_o]}\S+?#{Mx[:rel_c]}/
          @rxp_excluded0=/^(?:#{Mx[:fa_bold_o]}|#{Mx[:fa_italics_o]})?(?:to\d+|\d+|&nbsp;|#{Mx[:br_endnotes]}|EOF|#{Mx[:br_eof]}|thumb_\S+|snap_\S+|_+|-+|[(]?(?:ii+|iv|vi+|ix|xi+|xiv|xv|xvi+|xix|xx)[).]?|\S+?_\S+|[\d_]+\w\S+|[\w\d]{1,2}|\d{1,3}\w?|#{@dp}|[0-9a-f]{16,64}|\d{2,3}x\d{2,3}|\S{0,2}sha\d|\S{0,3}\d{4}w\d\d|\b\w\d+|\d_all\b|e\.?g\.?)(?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})?$/mi #this regex causes and cures a stack dump in ruby 1.9 !!!
          @rgx_splitlist=%r{[—.,;:#{Mx[:nbsp]}-]}mi
          @alph=SiSU_i18n::Alphabet.new(@md.opt.lng).hash_arrays
          @alphlst=SiSU_i18n::Alphabet.new(@md.opt.lng).hash_strings
          @rgx_scanlist=%r{#{Mx[:fa_italics_o]}[#{@alphlst[:l]}#{@alphlst[:u]}0-9"\s]{2,12}#{Mx[:fa_italics_c]}|#{Mx[:fa_bold_o]}[#{@alphlst[:l]}#{@alphlst[:u]}0-9"\s]{2,12}#{Mx[:fa_bold_c]}|(?:https?|file)://\S+|<\S+?>|[#{@alphlst[:l]}#{@alphlst[:u]}]+|\w+}mi
        rescue
          SiSU_Errors::Rescued.new($!,$@,@md.opt.cmd,@md.fns).location do
            __LINE__.to_s + ':' + __FILE__
          end
        end
      end
      def songsheet
        begin
          FileUtils::mkdir_p(@file.output_path.html_concordance.dir) unless FileTest.directory?(@file.output_path.html_concordance.dir)
          @file_concordance=File.open(@file.place_file.html_concordance.dir,'w')
          map_para
        rescue
          SiSU_Errors::Rescued.new($!,$@,@md.opt.cmd,@md.fns).location do
            __LINE__.to_s + ':' + __FILE__
          end
        ensure
          @file_concordance.close
        end
      end
    protected
      def location_scroll(wordlocation,show)                    # not used
        %{<a href="doc#{@md.lang_code_insert}#{Sfx[:html]}\##{Mx[:ocn_id_char]}#{wordlocation}">#{wordlocation}</a>;  }
      end
      def location_seg(wordlocation,show)
        unless wordlocation.nil?
          wl=wordlocation.gsub(/(.+?)\#(\d+)/,
            "\\1#{@md.lang_code_insert}#{Sfx[:html]}##{Mx[:ocn_id_char]}\\2")     # id="o\d+" always available; a name="\d+" not available if html strict used
          case wordlocation
          when /#{@rxp_t1}|@rxp_t2}|#{@rxp_t3}/
            %{[<a href="doc#{@md.lang_code_insert}#{Sfx[:html]}##{show}">H</a>]#{show},  }
          when /(.+?)\#(\d+)/
             %{<a href="#{wl}">#{show}</a>,  }
          end
        end
      end
      def map_para
        @seg,ocn=nil,nil
        @word_map={}
        @ao_array.each do |line|
          if defined? line.ocn \
          and line.ocn.to_s =~/\d/
            if (line.is ==:heading \
            || line.is ==:heading_insert) \
            && line.ln==4
              @seg=line.name
            end
            ocn=line.ocn.to_s
            if ocn =~/\d+/ \
            and ocn !~/^0$/
              line.obj=line.obj.gsub(/#{@rxp_excluded1}/,' ')
              line.obj=line.obj.split(@rgx_splitlist).join(' ') #%take in word or other match
              for word in line.obj.scan(@rgx_scanlist) #%take in word or other match
                if word =~ /^([#{@alphlst[:l]}])/
                  firstletter=$1
                  flu=firstletter.tr(@alphlst[:l],@alphlst[:u])
                  word=word.gsub(/^#{firstletter}/,flu )
                end
                word=word.gsub(/#{Mx[:lnk_o]}|#{Mx[:lnk_c]}|#{Mx[:url_o]}|#{Mx[:url_c]}/,'').
                  gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'').
                  gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'').
                  gsub(/#{Mx[:gl_o]}#[a-z]+#{Mx[:gl_c]}/,'').
                  gsub(/#{Mx[:gl_o]}#[0-9]+#{Mx[:gl_c]}/,'')
                word=word.gsub(/[0-9a-f]{10,}/,' ') if word =~/[0-9]/
                word=word.gsub(/#{Mx[:br_line]}/,' ').
                  gsub(/^ +/,'').
                  gsub(/^\S$/,'')
                word=nil if word.empty?
                word=nil if word =~@rxp_excluded0 #watch
                word=nil if word =~/^\S$/
                if word
                  word=word.gsub(/#{Mx[:br_nl]}|#{Mx[:br_line]}/,' ').
                    gsub(/#{Mx[:fa_o]}[a-z]{1,7}#{Mx[:fa_o_c]}|#{Mx[:fa_c_o]}[a-z]{1,7}#{Mx[:fa_c]}/,'').
                    gsub(/#{Mx[:en_a_o]}(?:\d|[*+])*|#{Mx[:en_b_o]}(?:\d|[*+])*|#{Mx[:en_a_c]}|#{Mx[:en_b_c]}/mi,'').
                    gsub(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'').
                    gsub(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'').
                    gsub(/<\/?\S+?>/,'').
                    gsub(/^\@+/,'').
                    strip.
                    gsub(/#{Mx[:tc_p]}.+/,'').
                    gsub(/[\.,;:"]$/,'').
                    gsub(/["]/,'').
                    gsub(/^\s*[\(]/,'').
                    gsub(/[\(]\s*$/,'').
                    gsub(/^(?:See|e\.?g\.?).+/,'').
                    gsub(/^\s*[.,;:]\s*/,'').
                    strip.
                    gsub(/^\(?[a-zA-Z]\)$/,'').
                    gsub(/^\d+(st|nd|rd|th)$/,'').
                    gsub(/^(\d+\.?)+$/, '').
                    gsub(/#{Mx[:mk_o]}|#{Mx[:mk_c]}/,'').
                    gsub(/:name#\S+/,'').
                    gsub(/^\S$/,'')
                  word=nil if word =~/^\S$/
                  word=nil if word =~/^\s*$/ #watch
                  if word
                    unless word =~/[A-Z][A-Z]/ \
                    or word =~/\w+\s\w+/
                      word=word.capitalize
                    end
                    @freq[word] +=1
                    @word_map[word] ||= []
                    if line !~ /#{@rxp_lv1}|#{@rxp_lv2}|#{@rxp_lv3}/
                      loc_=%{#{location_seg("#{@seg}\##{ocn}",ocn).to_s}}
                      unless loc_.empty?
                        @word_map[word] << loc_
                      end
                    else
                      @word_map[word] << case line
                      when @rxp_lv1 then location_seg('T1',ocn) #fix @rxp_lv #  Mx[:lv_o]
                      when @rxp_lv2 then location_seg('T2',ocn) #fix @rxp_lv #  Mx[:lv_o]
                      when @rxp_lv3 then location_seg('T3',ocn) #fix @rxp_lv #  Mx[:lv_o]
                      end
                    end
                  end
                end
              end
            end
          end
        end
        seg=''
        head=SiSU_Concordance::Source::DocTitle.new(@particulars).create
        head=head.gsub(/#{Xx[:html_relative2]}/m,@file.path_rel_links.html_seg_2).
          gsub(/#{Xx[:html_relative1]}/m,@file.path_rel_links.html_seg_1)
        @file_concordance << head
        @file_concordance << '<p>'
        alph=@alph[:u]
        alph.each {|x| @file_concordance << %{<a href="##{x}">#{x}</a>,&nbsp;}}
        @file_concordance << '</p>'
        letter=alph.shift
        @file_concordance << %{\n<p class="letter"><a name="A">A</a></p>}
        for word in @freq.keys.sort! {|a,b| a.downcase<=>b.downcase}
          f=/^(\S)/.match(word)[1]
          if letter < f.upcase
            while letter < f.upcase
              if alph.length > 0
                letter=alph.shift
                @file_concordance << %{\n<p class="letter"><a name="#{letter}">#{letter}</a></p>}
              else break
              end
            end
          end
          keyword=SiSU_Concordance::Source::Word.new(word,@freq[word]).html
          if keyword !~ @rxp_excluded0
            if @word_map[word][0] =~ /\d+/
              @file_concordance << %{#{keyword}#{seg}#{@word_map[word].uniq.compact.join}}
            end
            @file_concordance << '</p>'
          end
          # special cases endnotes and header levels 1 - 3
        end
        @file_concordance << %{</div></body>\n</html>} # footer
        if @md.opt.act[:verbose_plus][:set]==:on \
        || @md.opt.act[:maintenance][:set]==:on
          SiSU_Screen::Ansi.new(@md.opt.act[:color_state][:set],@md.fns,"#{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}").flow
        end
      end
    end
  end
end
__END__