#!/usr/bin/perl -w

# first order word frequencies and entropy

my $start=time();
my $log2 = log(2);
my %firstorder;
open(DARWIN, 'darwin.txt') or die(" ** OOPS **  couldn't open input");
print "Analyzing 'darwin.txt' for first order word counts ... \n";

while (my $line = <DARWIN>) {
  next if $line =~ /^\s*$/;
  $line = lc($line);
  for ($line) {
    s/[0-9.\-',;&\/()?!:]//g;                                #']; #
    s/\s+/ /g;
    s/\s*$/ /;   # make each line end with a space ( i.e. word separator)
  }

  my @words = split / /, $line;
  while (@words) {
    my $word = pop @words;
    $firstorder{$word}++;
  }
}

my $total;
foreach my $word (keys %firstorder) {
  $total += $firstorder{$word};
}
print "Total number of words = $total . \n";
my $entropy;
foreach my $word (keys %firstorder) {
  my $p = $firstorder{$word}/$total;
  $entropy -=  $p * log($p)/$log2;
}

print "Number of distinct words = " . (scalar keys %firstorder) . ".\n";

my $maxentropy = log($total)/$log2;
my $redundancy = 1.0 - $entropy/$maxentropy;

print "Shannon's 1st order entropy = -sum{p_i log2(p_i)} = $entropy .\n";

print "    (With $total words \n";
print "      max entropy = log base 2 ( $total ) = $maxentropy .)\n";
printf("So word redundancy (first order) = 1 - entropy/max = %5.2f %% \n", 100*$redundancy);

print "Elapsed time was " . (time() - $start) . " seconds. \n";
