Moby

Definitions:

Off[General::spell1];

\(\(\( (*\ 
      Grabs\ the\ next\ word\ *) \)\(
\)\(getWord[end_] := 
        Module[{cur, 
            word}, 
word = ""; 
While[
            i ≤ end, 

 (*\ 
              Keep\ looking\ at\ the\ next\ character\ until\ we\ find\ a\ letter\ *) 
cur = 
              StringTake[text, {i, i}]; 
If[\ 
              LetterQ[cur]\  == \ True, 
              Break[]]; 
\(i++\);

]; 

While[
            i ≤ end, 

 (*\ 
              Keep\ grabbing\ letters\ until\ we\ hit\ a\ character\ that\ isn' t\ a\ letter\ *) 
cur = 
              ToLowerCase[StringTake[text, {i, i}]]; 
If[\ 
              LetterQ[cur]\  == \ True, word\  = \ word\  <> \ cur, 
              Break[]]; 
\(i++\);

]; 
Return[
            word]
];\)\)\)

(* A variation of binary sort,  
      only now we add the word if we don' 
        t find it. *) 
place[word_] := 
        Module[{iLow, iHigh, i}, 
iLow = 
            1; 
iHigh = 
            Length[words]; 
While[
            True, 

 (* 
              If iLow  >  iHigh,  
              we' ve found the index where our new word should go *) 
If[iLow > iHigh, 
              Break[words  = 
                  Insert[words, {word, 1}, 
                    iLow]]]; 

 (* 
              Otherwise,  
              we check to see if we' 
                ve found the word in our list *) 
i = Floor[(iLow + iHigh)/2]; 
If[ 
              word  ==  words[[i, 1]],  
              Break[ words[[i, 
                      2]]++ ] ]; 

 (* If not,  we keep looking *) 
If[ 
              Order[word, words[[i, 1]]]  ==  1, iHigh = i - 1, 
              iLow = i + 
                  1];
];
];

\(\(countWords[] := 
        Module[{section, begin, end, startTime, totalTime, 
            wordCount}, 
section\  = \ 
            1; 
startTime\  = \ 
            SessionTime[]; 

While[
            section ≤ 100, 
i = 
              1; 
begin\  = \ 
              Round[\((section - 1)\) StringLength[fullText]/100]\  + \ 
                1; 
end\  = \ 
              Round[section\ StringLength[fullText]/
                    100]; 
text\  = \ 
              StringTake[fullText, {begin, end}]; 
len = 
              StringLength[text]; 

While[
              i < \ len, 

 (*grab\ the\ next\ word*) 
newWord\  = \ 
                getWord[len]; 

 (*\ 
                Make\ sure\ we\ really\ got\ a\ word\ *) 
If[newWord\  == \ "", 
                Break[]\ ]; 

 (*\ 
                Add\ the\ word\ to\ our\ list\ *) 
place[
                newWord];

]; 

totalTime\  = \ 
              SessionTime[] - startTime; 
times\  = \ 
              Append[times, \ totalTime]; 
wordCount\  = \ 
              Length[words]; 
wordGrowth\  = \ 
              Append[wordGrowth, 
                wordCount]; 
Print["Section = ", 
              section, \ ", Words = ", \ wordCount, ", Time = ", 
              totalTime]; 

\(section++\);
]; 

wordCounts\  = \ 
            Reverse[Sort[
                Transpose[{words[\([All, 2]\)], 
                    words[\([All, 
                        1]\)]}]]];
];\)\)

\(\(init[] := 
        Module[{}, 
SetDirectory["N:\ "]; 
fullText = 
            Import["moby.txt"]; 
words\  = \ {}; 
wordCounts\  = \ {}; 
wordGrowth\  = \ {}; 
times\  = \ {};
];\)\)

Running the program:

init[];
countWords[]

Results:

Total words:

Total[words[[All, 2]]]

Total unique words:

Length[wordCounts]

And here's the rate at which we find new words as our list grows bigger (number of unique words / sections of text analyzed):

ListPlot[progression, PlotJoined  ->  True]