#!/usr/bin/env ruby
# encoding: utf-8

Version='4.01'
Myname = File.basename($0)

require 'optparse'

Help =
<<'DOC'
= words - list, and optionally count, words in a list of files	

= Synopsis
words [options] files	

-c,--count	Report the number of occurrences of each word.
-s,--sum	Report only the total number of words
-f,--fold	Convert input to lower case before detecting words.
-p,--pattern=RE	set pattern defining the word separators
		default: |[^[:alpha:]_]|
-V,--version	print version and exit
-h		print short help and exit
--help		print full documentation via less and exit

= Description
words lists, and optionally counts, words occurring in a list of files or, 
if no arguments are present, standard input.
Words are defined as character sequences separated by the regexp set by the
|--pattern| option. By default, any character other then underscore and
alphabetic characters (including accented characters) acts as a separator.

Without the |--count| option, the output comes in 1 column of words,
sorted in case insensitive order. With the |--count| option two
tab-separated columns appear with the counts in column 1 and the words
in column 2; the order will be reverse numerically sorted on column 1
and normally sub-sorted on column 2.

The |--fold| option converts all input to lowercase.

= Examples
Given an input file test containing:

  The Prêt-à-porter robe is priced at € 77.50,
  the shoes (ladies' only) at € 255.

To show the words in it:

    words test #=> 
    à
    at
    is
    ladies
    only
    porter
    priced
    Prêt
    robe
    shoes
    the
    The

To count the words, after folding upper to lower case:

    words --count --fold test #=>
    2	at
    2	the
    1	à
    1	is
    1	ladies
    1	only
    1	porter
    1	prêt
    1	priced
    1	robe
    1	shoes


to include |-| to be a possible word character, thus finding words
like |avant-garde|:

    words -p '[^[:alpha:]-]' test #=>
    at
    is
    ladies
    only
    priced
    Prêt-à-porter
    robe
    shoes
    the
    The

Note that the - must be at the end of the expression, in order not to be
interpreted as a range-character.

To count the number of backslashes in a TeX file:
    words --pattern='[^\\]' -c test #=>

but, of course, this is a lot faster:
    tr -dc '\\' <test |wc -c

= Author
[Wybo Dekker](wybodekker@me.com)

= Copyright
Released under the [GNU General Public License](www.gnu.org/copyleft/gpl.html)
DOC

def wrn(m)
   STDERR.puts(''+m+'')
end

def help
  nv="-#{Myname}-#{Version.sub(/\./,'\\.')} press h for help, q to quit"
  IO.popen("less -P'#{nv}'",'w') do |pipe| pipe.puts Help end
end

begin
   require 'sort_alphabetical'
   require 'unicode_utils'
   include UnicodeUtils
rescue LoadError
   wrn <<-EOF.gsub(/^ */,'')
      You need the sort_alphabetical and unicode_utils gems.
      One or both are missing; install with:
      sudo gem install sort_alphabetical unicode_utils.
   EOF
   exit 1
end

class Array
   def sum
      self.inject(0,:+)
   end
end

c = "\n" + " "*19
fold,count,sum,pat,patset = false,false,false,%r{[^[:alpha:]_]},false

OptionParser.new(
   "#{Myname} - list, and optionally count, words occurring in a list of files\n"+
   "Usage: #{Myname} [options] [file]\n\n"+
   "Options:",
   18, # width for option column
   ''  # indent for option column
) do |opts|
 
   opts.on('-c','--count','Report the number of occurrences of each word.') do
      count = true
   end
   opts.on('-s','--sum','Report only the total number of words') do
      sum = count = true
   end
   opts.on('-f','--fold','Convert input to lower case before detecting words.'
   ) do
      fold = true
   end
   opts.on('-p','--pattern=RE', String,
      [ "set pattern defining the word separators",
        "default: |[^[:alpha:]_]|"
      ].join(c)
   ) do |v|
      pat = %r{#{v}}
      patset = true
   end
   opts.on('-V','--version','print version and exit') do
      puts Version
      exit
   end
   opts.on('-H','--help','show full documentation (use less!) and exit') do
      help
      exit
   end
   opts.on('-h','print short help and exit') do
      puts opts.to_s.sub(/^ *-I\n/,'')
      exit
   end
   opts.on('-I') do
      system("instscript #{Myname}")
      exit 0
   end
   begin
      opts.parse!
   rescue 
      wrn $!.to_s
      puts opts.to_s.sub(/^ *-I\n/,'')
      exit
   end
end

# hier input file generen door alles te cat-ten en daarbij pdf's te converteren met 
# pdftotext. In alle files checken op -$

io = open("|cat #{ARGV.join(' ')}")

wrds={}
while l=io.gets
   # test for characters with invalid utf-8 codes
   # and replace them so that texlog_extract can go on:
   unless l.valid_encoding?
      l.encode!("utf-8",:invalid=>:replace)
      wrn("invalid encoding (#{Myname} expects utf-8) in:\n#{l}")
   end
   l = downcase(l) if fold
   l.split(pat).each do |word|
      wrds[word]||=0
      wrds[word]+=1
   end
end

wrds.delete('') if wrds['']
if count
   if sum
      puts wrds.values.sum
   else
      # need 1-v here, not -v, don't know why; the 1 may be any number but zero or nil
      wrds.sort_alphabetical_by{|k,v| [1-v,downcase(k)]}.each{ |k,v| puts "#{v}\t#{k}" }
   end
else
   wrds.keys.sort_alphabetical_by{|k| downcase(k)}.each{ |k| puts k }
end