#!/usr/bin/perl -w

# A look at letter frequencies and 
# shannon's entropy in darwin's origin of the species.
#  - Jim Mahoney, Dec 2001

my $start=time();
my $log2 = log(2);
my %firstorder;

open(DARWIN, 'darwin.txt') or die(" ** OOPS **  couldn't open input");
print "Analyzing 'darwin.txt' for first order letter counts ... \n";

while (my $line = <DARWIN>) {
  next if $line =~ /^\s*$/;
  $line = lc($line);
  for ($line) {
    s/[0-9.\-',;&\/()?!:]//g;                                #']; #
    s/\s+/ /g;
    s/\s*$/ /;   # make each line end with a space ( i.e. word separator)
  }

  my @chars = split //, $line;
  while (@chars) {
    my $char = pop @chars;
    $firstorder{$char}++;
  }
}

my $total;
foreach my $char (keys %firstorder) {
  $total += $firstorder{$char};
}
print "Total number of characters = $total . \n";

print " char |  count  | probability \n";
print " ---- +  -----  + ----------- \n";
my $entropy;
foreach my $char (keys %firstorder) {
  my $p = $firstorder{$char}/$total;
  $entropy -=  $p * log($p)/$log2;
  printf( " '$char'  | %7.0f | %8.6f \n", $firstorder{$char},$p);
}
my $maxentropy = log(27.0)/$log2;
my $redundancy = 1.0 - $entropy/$maxentropy;

print "Shannon's 1st order entropy = -sum{p_i log2(p_i)} = $entropy .\n";

print "    (With 27 symbols (a-z, space), \n";
print "      max entropy = log base 2 (27) = $maxentropy .)\n";
printf("So character redundancy (first order) = 1 - entropy/max = %5.2f %% \n", 100*$redundancy);

print "  (But since we're using 8 bits = 256 distinct symbols to encode \n";
print "   these 27 symbols, the file itself has of course a much higher \n";
print "   redundancy factor.) \n\n";

print "Elapsed time was " . (time() - $start) . " seconds. \n";


