aboutsummaryrefslogtreecommitdiffhomepage
path: root/lib/sisu/v2/concordance.rb
diff options
context:
space:
mode:
Diffstat (limited to 'lib/sisu/v2/concordance.rb')
-rw-r--r--lib/sisu/v2/concordance.rb340
1 files changed, 340 insertions, 0 deletions
diff --git a/lib/sisu/v2/concordance.rb b/lib/sisu/v2/concordance.rb
new file mode 100644
index 00000000..06cc3056
--- /dev/null
+++ b/lib/sisu/v2/concordance.rb
@@ -0,0 +1,340 @@
+# coding: utf-8
+=begin
+
+ * Name: SiSU
+
+ * Description: a framework for document structuring, publishing and search
+
+ * Author: Ralph Amissah
+
+ * Copyright: (C) 1997 - 2010, Ralph Amissah, All Rights Reserved.
+
+ * License: GPL 3 or later:
+
+ SiSU, a framework for document structuring, publishing and search
+
+ Copyright (C) Ralph Amissah
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by the Free
+ Software Foundation, either version 3 of the License, or (at your option)
+ any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+
+ If you have Internet connection, the latest version of the GPL should be
+ available at these locations:
+ <http://www.fsf.org/licensing/licenses/gpl.html>
+ <http://www.gnu.org/licenses/gpl.html>
+
+ <http://www.jus.uio.no/sisu/gpl.fsf/toc.html>
+ <http://www.jus.uio.no/sisu/gpl.fsf/doc.html>
+ <http://www.jus.uio.no/sisu/gpl.fsf/plain.txt>
+
+ * SiSU uses:
+ * Standard SiSU markup syntax,
+ * Standard SiSU meta-markup syntax, and the
+ * Standard SiSU object citation numbering and system
+
+ * Hompages:
+ <http://www.jus.uio.no/sisu>
+ <http://www.sisudoc.org>
+
+ * Download:
+ <http://www.jus.uio.no/sisu/SiSU/download.html>
+
+ * Ralph Amissah
+ <ralph@amissah.com>
+ <ralph.amissah@gmail.com>
+
+ ** Description: concordance file (html concordance, wordmap, linked index of
+ words in document)
+
+=end
+module SiSU_Concordance
+ require "#{SiSU_lib}/particulars" # particulars.rb
+ include SiSU_Particulars
+ require "#{SiSU_lib}/sysenv" # sysenv.rb
+ include SiSU_Env
+ require "#{SiSU_lib}/defaults" # defaults.rb
+ include SiSU_Viz
+ require "#{SiSU_lib}/html_format" # html_format.rb
+ include SiSU_HTML_Format
+ require "#{SiSU_lib}/html_minitoc" # html_minitoc.rb
+ class Source
+ def initialize(opt)
+ @opt=opt
+ @particulars=SiSU_Particulars::Combined_singleton.instance.get_all(opt)
+ end
+ def read
+ begin
+ @env,@md=@particulars.env,@particulars.md
+ loc=@env.url.output_tell
+ tool=(@md.cmd =~/[MVv]/) ? "#{@env.program.web_browser} #{loc}/#{@md.fnb}/#{@md.fn[:concordance]}" : ''
+ tell=SiSU_Screen::Ansi.new(@md.cmd,"Concordance",tool)
+ tell.grey_title_hi unless @md.cmd =~/q/
+ wordmax=@env.concord_max
+ unless @md.wc_words.nil?
+ if @md.wc_words < wordmax
+ SiSU_Concordance::Source::Words.new(@particulars).songsheet
+ else
+ tell=SiSU_Screen::Ansi.new(@md.cmd,"concordance skipped, large document has over #{wordmax} words (#{@md.wc_words})")
+ tell.warn unless @md.cmd =~/q/
+ end
+ else
+ tell=SiSU_Screen::Ansi.new(@md.cmd,"wc (word count) is off, concordance will be processed for all files including those over the max set size of: #{wordmax} words")
+ tell.warn unless @md.cmd =~/q/
+ SiSU_Concordance::Source::Words.new(@particulars).songsheet
+ end
+ rescue; SiSU_Errors::Info_error.new($!,$@,@md.cmd,@md.fns).error
+ ensure
+ end
+ end
+ private
+ class Doc_title
+ include SiSU_Viz
+ #revisit, both requires (html & shared_xml) needed for stand alone operation (sisu -w [filename])
+ require "#{SiSU_lib}/shared_xml" # shared_xml.rb
+ require "#{SiSU_lib}/html" # html.rb
+ def initialize(particulars)
+ @particulars,@md=particulars,particulars.md
+ @data=SiSU_HTML::Source::Html_environment.new(particulars).tuned_file_instructions
+ @vz=SiSU_Env::Get_init.instance.skin
+ txt_path=%{#{@md.dir_out}}
+ SiSU_Env::Info_skin.new(@md).select
+ @fnb=@md.fnb
+ @lex_button=%{<a href="http://www.jus.uio.no/sisu/" target="_top"><img border="0" height="44" width="144" valign="center" src="../_sisu/image/sisu.png" alt="SiSU home --&gt;"></a>}
+ @doc_details =<<WOK
+<table summary="links to text related to this rudimentary index" width="96%" border="0" bgcolor="white" cellpadding="0" align="center"><tr><td width="2%" align="right">&nbsp;</td><td width="94%" valign="top" align="justify"><h1 class="small"><a href="#{@md.fn[:toc]}" #{@vz.js_toc}><b>#{@md.full_title}</b></a></h1><p class="bold">#{@md.author}</p></td></tr></table>
+WOK
+ end
+ def create
+ head_banner=SiSU_HTML_Format::Head_toc.new(@md)
+ minitoc=SiSU_HTML_minitoc::Toc_mini.new(@md,@data).songsheet.join("\n")
+ toc='<div class="toc">' + minitoc + '</div>'
+ <<WOK
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+ <title>
+ SiSU created WordIndex for: #{@md.full_title}
+ </title>
+ <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+ <meta name="Description" content="&nbsp;SiSU created">
+ <meta name="keywords" content="word index for #{@md.full_title}">
+ <meta name="generator" content="SiSU (Linux &amp; Ruby!)">
+ <link rel="generator" href="http://www.jus.uio.no/sisu" />
+ <link rel="shortcut icon" href="../_sisu/image/rb7.ico" />
+ <link rel="stylesheet" href="../_sisu/css/html.css" type="text/css" />
+ #{@vz.js_head}
+</head>
+<body>
+ #{@vz.js_top}
+ #{head_banner.concordance_navigation_band('pdf')}
+ #{toc}
+<div class="content">
+ #{@doc_details}
+<p>Word index links are to html versions of the text the segmented version followed by the scroll (single document) version.<br />[For segmented text references [T1], [T2] or [T3] appearing without a link, indicates that the word appears in a title (or subtitle) of the text (that is identifiable by the appended object citation number).]</p>
+<p>(The word listing/index is Case sensitive: Capitalized words appear before lower case)</p>
+ <p>
+ <b>word</b> (number of occurences)<br />linked references to word within document <br />
+ [if number of occurences exceed number of references - word occurs more than once in at least one reference. Footnote/endnotes are either assigned to the paragraph from which they are referenced or ignored, so it is relevant to check the footnotes referenced from within a paragraph as well.]
+ </p>
+ <p>
+ (After the page is fully loaded) you can jump directly to a word by appending a hash (#) and the word to the url for this text, (do not forget that words are case sensitive, and may be listed twice (starting with and without an upper case letter)), #your_word # [&nbsp;http://[web host]/#{@fnb}/concordance.html#your_word&nbsp;]
+ </p>
+WOK
+ end
+ end
+ class Word
+ @@word_previous=''
+ def initialize(word,freq)
+ @word,@freq=word,freq
+ end
+ def html
+ w=if @word.capitalize==@@word_previous
+ %{\n<p class="concordance_word">#{@word}</p><p class="concordance_count">(#{@freq})</p>\n\t<p class="concordance_object"> }
+ else n=@word.strip.gsub(/\s+/,'_') #also need to convert extended character set to html
+ %{\n<p class="concordance_word"><a name="#{n}">#{@word}</a></p><p class="concordance_count">(#{@freq})</p>\n\t<p class="concordance_object"> }
+ end
+ @@word_previous=@word.capitalize
+ w
+ end
+ end
+ class Words
+ require "#{SiSU_lib}/defaults" # defaults.rb
+ include SiSU_Viz
+ require "#{SiSU_lib}/html_format" # html_format.rb
+ include SiSU_HTML_Format
+ require "#{SiSU_lib}/sysenv" # sysenv.rb
+ include SiSU_Screen
+ @@dp=nil
+ def initialize(particulars)
+ @particulars=particulars
+ begin
+ @vz=SiSU_Env::Get_init.instance.skin
+ @env,@md,@dal_array=particulars.env,particulars.md,particulars.dal_array
+ @path="#{@env.path.output}/#{@md.fnb}"
+ @freq=Hash.new(0)
+ @dp=@@dp ||=SiSU_Env::Info_env.new.digest.pattern
+ @rxp_lv1=/^#{Mx[:lv_o]}1:/ #fix @rxp_lv # Mx[:lv_o]
+ @rxp_lv2=/^#{Mx[:lv_o]}2:/ #fix @rxp_lv # Mx[:lv_o]
+ @rxp_lv3=/^#{Mx[:lv_o]}3:/ #fix @rxp_lv # Mx[:lv_o]
+ @rxp_title=Regexp.new("^#{Mx[:meta_o]}title#{Mx[:meta_c]}\s*(.+?)\s*$")
+ @rxp_t1=Regexp.new('^T1')
+ @rxp_t2=Regexp.new('^T2')
+ @rxp_t3=Regexp.new('^T3')
+ @rxp_excluded1=/#{Mx[:url_o]}\S+?#{Mx[:url_c]}/
+ @rxp_excluded0=/^(?:#{Mx[:fa_bold_o]}|#{Mx[:fa_italics_o]})?(?:to\d+|\d+|&nbsp;|#{Mx[:br_endnotes]}|EOF|#{Mx[:br_eof]}|thumb_\S+|snap_\S+|_+|-+|[(]?(?:ii+|iv|vi+|ix|xi+|xiv|xv|xvi+|xix|xx)[).]?|\S+?_\S+|[\d_]+\w\S+|[\w\d]{1,2}|\d{1,3}\w?|#{@dp}|[0-9a-f]{16,64}|\d{2,3}x\d{2,3}|\S{0,2}sha\d|\S{0,3}\d{4}w\d\d|\b\w\d+|\d_all\b|e\.?g\.?)(?:#{Mx[:fa_bold_c]}|#{Mx[:fa_italics_c]})?$/mi #this regex causes and cures a stack dump in ruby 1.9 !!!
+ @rgx_splitlist=%r{[—.,;:#{Mx[:nbsp]}-]}mi
+ @rgx_scanlist=%r{#{Mx[:fa_italics_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_italics_c]}|#{Mx[:fa_bold_o]}[a-zA-Z0-9"\s]{2,12}#{Mx[:fa_bold_c]}|(?:https?|file)://\S+|<\S+?>|\w+|[a-zA-Z]+}mi
+ rescue; SiSU_Errors::Info_error.new($!,$@,@md.cmd,@md.fns).error
+ end
+ end
+ def songsheet
+ begin
+ mkdir_p(@path) unless FileTest.directory?(@path)
+ @file_concordance=File.open("#{@path}/#{@md.fn[:concordance]}",'w')
+ map_para
+ rescue; SiSU_Errors::Info_error.new($!,$@,@md.cmd,@md.fns).error
+ ensure
+ @file_concordance.close
+ end
+ end
+ protected
+ def location_scroll(wordlocation,show)
+ %{<a href="#{@md.fnl[:pre]}doc#{@md.fnl[:mid]}#{Sfx[:html]}#{@md.fnl[:post]}\##{wordlocation}">#{wordlocation}</a>; }
+ end
+ def location_seg(wordlocation,show) ##fix
+ @word_location_seg=wordlocation.gsub(/(.+?)\#(\d+)/,"#{@md.fnl[:pre]}\\1#{@md.fnl[:mid]}#{Sfx[:html]}#{@md.fnl[:post]}#\\2") unless wordlocation.nil?
+ case wordlocation
+ when @rxp_t1
+ %{[<a href="#{@md.fnl[:pre]}doc#{@md.fnl[:mid]}#{Sfx[:html]}#{@md.fnl[:post]}##{show}">H</a>]#{show}, }
+ when @rxp_t2
+ %{[<a href="#{@md.fnl[:pre]}doc#{@md.fnl[:mid]}#{Sfx[:html]}#{@md.fnl[:post]}##{show}">H</a>]#{show}, }
+ when @rxp_t3
+ %{[<a href="#{@md.fnl[:pre]}doc#{@md.fnl[:mid]}#{Sfx[:html]}#{@md.fnl[:post]}##{show}">H</a>]#{show}, }
+ else %{<a href="#{@word_location_seg}">#{show}</a>, }
+ end
+ end
+ def map_para
+ @seg,ocn=nil,nil
+ @word_map={}
+ @dal_array.each do |line|
+ if defined? line.ocn \
+ and line.ocn.to_s =~/\d/
+ if line.is =~/heading/ \
+ and line.ln==4
+ @seg=line.name
+ end
+ ocn=line.ocn.to_s
+ if ocn =~/\d+/ \
+ and ocn !~/^0$/
+ line.obj.gsub!(/#{@rxp_excluded1}/,' ')
+ line.obj=line.obj.split(@rgx_splitlist).join(' ') #%take in word or other match
+ for word in line.obj.scan(@rgx_scanlist) #%take in word or other match
+ word.gsub!(/#{Mx[:lnk_o]}|#{Mx[:lnk_c]}|#{Mx[:url_o]}|#{Mx[:url_c]}/,'')
+ word.gsub!(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,'')
+ word.gsub!(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'')
+ word.gsub!(/#{Mx[:gl_o]}#[a-z]+#{Mx[:gl_c]}/,'')
+ word.gsub!(/#{Mx[:gl_o]}#[0-9]+#{Mx[:gl_c]}/,'')
+ word.gsub!(/[0-9a-f]{10,}/,' ') if word =~/[0-9]/
+ word.gsub!(/#{Mx[:br_line]}/,' ')
+ word.gsub!(/^ +/,'')
+ word.gsub!(/^\S$/,'')
+ word=nil if word.empty?
+ word=nil if word =~@rxp_excluded0 #watch
+ word=nil if word =~/^\S$/
+ if word
+ word.gsub!(/#{Mx[:br_nl]}|#{Mx[:br_line]}/,' ')
+ word.gsub!(/#{Mx[:fa_o]}[a-z]{1,7}#{Mx[:fa_o_c]}|#{Mx[:fa_c_o]}[a-z]{1,7}#{Mx[:fa_c]}/,'')
+ word.gsub!(/#{Mx[:en_a_o]}(?:\d|[*+])*|#{Mx[:en_b_o]}(?:\d|[*+])*|#{Mx[:en_a_c]}|#{Mx[:en_b_c]}/mi,'')
+ word.gsub!(/#{Mx[:fa_o]}\S+?#{Mx[:fa_o_c]}/,''); word.gsub!(/#{Mx[:fa_c_o]}\S+?#{Mx[:fa_c]}/,'')
+ word.gsub!(/<\/?\S+?>/,'')
+ word.gsub!(/^\@+/,'')
+ word.strip!
+ word.gsub!(/#{Mx[:tc_p]}.+/,'')
+ word.gsub!(/[\.,;:"]$/,'')
+ word.gsub!(/["]/,'')
+ word.gsub!(/^\s*[\(]/,'')
+ word.gsub!(/[\(]\s*$/,'')
+ word.gsub!(/^(?:See|e\.?g\.?).+/,'')
+ word.gsub!(/^\s*[.,;:]\s*/,'')
+ word.strip!
+ word.gsub!(/^\(?[a-zA-Z]\)$/,'')
+ word.gsub!(/^\d+(st|nd|rd|th)$/,'')
+ word.gsub!(/^(\d+\.?)+$/, '')
+ word.gsub!(/#{Mx[:mk_o]}|#{Mx[:mk_c]}/,'')
+ word.gsub!(/:name#\S+/,'')
+ word.gsub!(/^\S$/,'')
+ word=nil if word =~/^\S$/
+ word=nil if word =~/^\s*$/ #watch
+ if word
+ unless word =~/[A-Z][A-Z]/ \
+ or word =~/\w+\s\w+/
+ word.capitalize!
+ end
+ @freq[word] +=1
+ @word_map[word] ||= []
+ if line !~@rxp_lv1 \
+ and line !~@rxp_lv2 \
+ and line !~@rxp_lv3 #fix @rxp_lv # Mx[:lv_o]
+ @word_map[word] << location_seg("#{@seg}\##{ocn}",ocn)
+ else
+ @word_map[word] << case line
+ when @rxp_lv1; location_seg('T1',ocn) #fix @rxp_lv # Mx[:lv_o]
+ when @rxp_lv2; location_seg('T2',ocn) #fix @rxp_lv # Mx[:lv_o]
+ when @rxp_lv3; location_seg('T3',ocn) #fix @rxp_lv # Mx[:lv_o]
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ scr='<font size="1" color="#777777" face=times new roman><img border="0" height="15" width="15" src="../_sisu/image/b_doc.png" alt="Full Text">&nbsp;scroll:&nbsp;</font><font size="1" color="#222222" face=times new roman>doc#&nbsp;</font> '
+ seg=''
+ @file_concordance << SiSU_Concordance::Source::Doc_title.new(@particulars).create
+ alph=%W[A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+ @file_concordance << '<p>'
+ alph.each {|x| @file_concordance << %{<a href="##{x}">#{x}</a>,&nbsp;}}
+ @file_concordance << '</p>'
+ letter=alph.shift
+ @file_concordance << %{\n<p class="letter"><a name="A">A</a></p>}
+ for word in @freq.keys.sort! {|a,b| a.downcase<=>b.downcase}
+ f=/^(\S)/.match(word)[1]
+ if letter < f.upcase
+ while letter < f.upcase
+ if alph.length > 0
+ letter=alph.shift
+ @file_concordance << %{\n<p class="letter"><a name="#{letter}">#{letter}</a></p>}
+ else break
+ end
+ end
+ end
+ keyword=SiSU_Concordance::Source::Word.new(word,@freq[word]).html
+ if keyword !~ @rxp_excluded0
+ if @word_map[word][0] =~ /\d+/
+ wm=[]
+ @file_concordance << %{#{keyword}#{seg}#{@word_map[word].uniq.compact.join}}
+ end
+ @file_concordance << '</p>'
+ end
+ # special cases endnotes and header levels 1 - 3
+ end
+ credits=@vz.credits_sisu
+ @file_concordance << %{</div><div class="content">#{credits}<div></body>\n</html>} # footer
+ tell=SiSU_Screen::Ansi.new(@md.cmd,@md.fns,"#{@env.path.output_tell}/#{@md.fn[:concordance]}")
+ tell.flow if @md.cmd =~/[MV]/
+ end
+ end
+ end
+end
+__END__