#!/usr/bin/perl 
#######################################################################
#
# allMusicGuideSearch.pl
#
#	Program to search AllMusicGuide for artist information
#
# by Jacob Turino.
#
# See documentation for description and version history.
#
#######################################################################

use strict;
#use WWW::Mechanize::Sleepy;
use WWW::Mechanize;
use Encode;

my $VERSION = "0.7";
my $DEBUG = 0;

print "\n\nThis is *** allMusicGuideSearch *** (version $VERSION). \n\n" unless @ARGV;
print "Debug mode... \n" if $DEBUG;	

my $url = 'http://www.allmusic.com';

# Long list of strings that will be used in regex to parse information of allmusic pages:

#                                                   ******* HTML parsing variables *******
my $searchPageHead      = filterForRegex('<title>allmusic</title>');
my $beginArtistPageHead = filterForRegex('<TITLE>allmusic  (((');   
my $endArtistPageHead   = filterForRegex('> Overview ');
my $artistPageHead      = $beginArtistPageHead . '.*' . $endArtistPageHead;
#                                        ******* name search page parsing variables *******
# match line should look like this:
#break-word;"><strong><a href="/cg/amg.dll?p=amg&sql=11:bx1m961o3ep8">The Beatles</a> [...] break-word;">Rock</TD>
my $startMatchNS        = filterForRegex('break-word;">');
my $boldTagNS           = filterForRegex('<strong>');
my $beginArtistNameNS   = filterForRegex('<a href="/cg/amg.dll?p=amg&sql=');
$beginArtistNameNS     .= '..?:\w+">';     # we don't want to filter this part b/c it includes regex chars
my $endArtistNameNS     = filterForRegex('</a>');
my $startGenreNS        = filterForRegex('break-word;">');
my $endGenreNS          = filterForRegex('</TD');
#                                         ******* album search page parsing variables *******
my $startSearchResults  = filterForRegex('<div id="artistpage">');
my $beginArtistNameAS   = filterForRegex('172px;word-wrap:break-word;">');
my $endArtistNameAS     = filterForRegex('</TD>');
my $beginGenreNameAS    = filterForRegex('&sql=73:');
$beginGenreNameAS      .= '\w+">';     # we don't want to filter this either
my $endGenreNameAS      = filterForRegex('</a>');
#                                               ******* artist page parsing variables *******
my $startArtistInfo     = filterForRegex('<div id="artistpage">');
my $beginArtistName     = filterForRegex('class="titlebar"><span class="title">');
my $endArtistName       = filterForRegex('</span');
my $beginGenreName      = filterForRegex('amg.dll?p=amg&sql=73:');
$beginGenreName        .= '\w+">';     # we don't want to filter this either
my $endGenreName        = filterForRegex('</a>');
#                                                         *****Discography variables *****
my $startAlbumInfo      = filterForRegex('<div id="artistpage">');
my $beginAlbumName      = filterForRegex('href="/cg/amg.dll?p=amg&sql=10:');
$beginAlbumName        .= '\w+">';     # not filtered
my $endAlbumName        = filterForRegex('</a>');
#                                                                  ********************


my ($artistID, $searchTerm, $amgArtist, $amgGenre, @amgAlbums);
my $retrieveAlbums = 0;
my $printToFile = 0;
my $searchByAlbum = 0;

my %artistList = getSearchTerms();

if ($printToFile) {
	my $output = 'skippedArtists.txt'; # to record artists that can't be matched
	open(SKIPPED,">$output") || die "can't open file: $!";
}

my $agent = WWW::Mechanize->new( autocheck => 1 );
#my $agent = WWW::Mechanize::Sleepy->new( sleep => '2..8' );
$agent->get( $url );

my $i = 1; #counter to show progress (for long lists)

foreach my $id ( keys %artistList ) {
	$artistID = $id; # necessary because $id has limited scope
	my (@searchTerms, $opt1); # $opt1 will be used in the allmusic search form (1 for Name and 2 for Album) 
	my ($artist, $genre, $albumsRef) = @{$artistList{$id}};
	my @albums = @$albumsRef if $albumsRef;
	
	if (@albums) {
		$searchByAlbum = 1;
		@searchTerms = @albums;
		$opt1 = 2;
	} else {
		$searchByAlbum = 0;
		@searchTerms = ($artist);
		$opt1 = 1;
	}
	foreach $searchTerm ( @searchTerms ) {
		print "$i: " if $printToFile;
		debug ("Searching for '$searchTerm'.\n");
		$i++;
		
		# Search:
		$agent->form_name( 'SimpleSearchForm' );
		$agent->field('SQL', $searchTerm); # 'SQL' is the text input field on the search form 
		$agent->field('OPT1', $opt1); # 'OPT1' defines what we will search by: 1 = Name, 2 = Album, 55 = Classical
	
		$agent->submit() or die "Couldn't Submit Form!! : $! \n";
	
		my @content = responseAsArray($agent->content);
		
		redo if isSearchPage(@content) == 2; # try again if amg doesn't return a page we can recognize 
	
		if ( isSearchPage(@content) ) {
			debug("Search page returned...\n");
			($amgArtist, $amgGenre) = findArtistMatch($artist, @content);
			next unless ($amgArtist);
			unless ($amgGenre) {
				print OUTPUT "$artistID|$searchTerm|$amgArtist|\n" if $printToFile;
				print "\tName: $amgArtist; Too many genres\n";
				next;
			}
			if ($retrieveAlbums) {
				debug ("Following link: $amgArtist\n");
			
				$agent->follow_link( text => $amgArtist );
			
				@content = responseAsArray($agent->content);
			}			
		} else {
			next if $searchByAlbum;
			($amgArtist, $amgGenre) = findArtistGenre(@content);
		}
		# allmusic uses charset iso-8859-1, so we need to change it to utf8, which perl uses
		$amgArtist = decode("iso-8859-1", $amgArtist);
		print OUTPUT "$artistID|$searchTerm|$amgArtist|$amgGenre\n" if $printToFile;
		print "\tName: $amgArtist; Genre: $amgGenre\n";
		last if $searchByAlbum; # no need to search other albums by this artist, so skip to next artist
		
		if ($retrieveAlbums) {
	
			$agent->follow_link( text => 'Discography' );
	
			@content = responseAsArray($agent->content);
			@amgAlbums = getAlbums(@content);
			
			print "\tAlbums:\t", join("\n\t\t", @amgAlbums), "\n\n";
		}
	}
}
close SKIPPED;

#========================================= Subroutines =========================================#
# debug($debugStatement): print debug statement
# parameters: print statement
# returns: nothing

sub debug {
  my ($statement) = @_;
  print "\tDEBUG: $statement\n" if ($DEBUG);
}
#filterForRegex($regexString): escapes all characters that might interfere w/ regex
# parameters: string
# returns: filtered string
sub filterForRegex{
	my ($word) = @_;
	# change "/" to "\/", etc
	$word =~ s/([\/
				 \\
				 \*
				 \+
				 \.
				 \?
				 \{
				 \}
				 \(
				 \)
				 \$
				 \^
				 \|
				 \ ])/\\$1/gx;
	return $word; 
}
#filterForSearch($searchTerm): removes all characters that might cause similar terms to not match
# parameters: string
# returns: filtered string
sub filterForSearch{
	my ($word) = @_;
	$word =~ tr/- / /s; # remove multiple spaces
	$word =~ s/ & | \+ / and /g; # change " & " and " + " to " and "
	$word =~ s/[^\w, ]//g; # remove all non-word characters except comma and space
	$word =~ s/^ | $//g; # remove extra space at beginning or at end
	$word =~ tr/ / /s; # remove multiple spaces (in case the characters we removed caused more double spaces)
	return $word; 
}
# getSearchTerms( ): grabs search terms from @ARGV or asks the user for search term
# parameters: none
# returns: searchTerms as array
sub getSearchTerms {
	my %artists;
	while (my $arg = shift @ARGV) { 	# if there are any arguments
		if ($arg =~ m/^(-\w)/) {	# if arg is a flag
			if ($1 eq '-a') {
				$retrieveAlbums = 1;
			} elsif ($1 eq '-d') {
				$DEBUG = 1;
			} else {
				die "Flag '$arg' not recognized.\n";
			}
		} elsif ($arg =~ m/\S+\.txt$/) {	# if arg is a text file
			if (%artists) {
				# if we've already got search tems, this will be the file to write the results in
				open(OUTPUT,">$arg") || die "can't open file: $!";
				$printToFile = 1;
			} else {
				# otherwise, this file will have the search terms in it
				open (ARTISTS, $arg) || die "can't open file: $!";
				while ( <ARTISTS> ) {
					chomp;
					# Data structure of %artists = ( artistID => [artistName, genre, [album1, album2, ...] ] ) 
					my ($artistID,$artistName,$genre,$album) = split/\|/ or next;
					$artists{$artistID} = [$artistName,$genre] unless $artists{$artistID};
					push (@{${$artists{$artistID}}[2]}, $album) if $album;
				}
			}
		} else { # any other arguments will be treated as a search term
			if ($artists{a}) {
				# if we already grabbed an artist search term, this will be the album
				push (@{${$artists{a}}[2]}, $arg);
			} else {
				# if this is the first search term, it will be the artist name
				# the key is supposed to be the ArtistID from my database, which should always be an integer.
				# we use "a" for a key to distinguish it from an actual ArtistID
				$artists{a} = [$arg,''];
			}
		}
	} 
	unless (%artists) {
		# If no search terms in args, prompt user for search term
		print "\nArtist Name: ";
		chomp( my $term = <STDIN> );
		print "\n";
		$artists{a} = [$term];
	}
	return %artists;
}
# responseAsArray( $response ): turn agent response into array
# parameters: response
# returns: response in array
sub responseAsArray {
	my ($response) = @_;
	$response =~ s/ &amp; / & /g;  # sometimes they use '&amp;' instead of '&'
	return split(/\n/, $response); # put the result in an array
}
# isSearchPage(@htmlContent): check if amg returned a search page
# parameters:  html content as array (split on line breaks)
# returns: 1 if page is search page
sub isSearchPage {
	my @html = @_;
	foreach my $line ( @html ) {
		if ($line =~ m/$searchPageHead/ ) {
			return 1;
		}
		if ($line =~ m/$artistPageHead/){
			return 0;
		}	
	}
	# if it's not an artist page or a search page, we're stuck.
	if ($DEBUG) {
		print "\t\twoops - don't recognize this page.\n";
		my $output = "error_page_for_query_$i";
		open(HTML,">$output") || die "can't open file: $!";
		print HTML join("\n", @html);
		close HTML;
	}
	$agent->back(); # go back and try again
	return 2;
}

# findArtistMatch( $searchTerm, @amgSearchPage ): find best match for $searchTerm in @amgSearchPage
# parameters: search term, amg search page as html content in array (split on lines breaks)
# returns: link to artist page
sub findArtistMatch {
	my ($artistName, @searchPage) = @_;
	my %possibleArtists;
	if ($searchByAlbum) {
		%possibleArtists = parseAlbumSearchResults( @searchPage );
	} else {
		%possibleArtists = parseNameSearchResults( @searchPage );
	}
	debug (scalar(keys(%possibleArtists)) . " results found:\n");
	if ($DEBUG) {
		foreach my $name (keys %possibleArtists) {
			my @genres = keys(%{$possibleArtists{$name}}); 
			print "\t$name: " . join(", ",@genres) . "\n";
		}
	print "\n";
	}
	return bestMatch($artistName, %possibleArtists);
};

# parseNameSearchResults( @amgSearchPage ): find best match for $searchTerm in @amgSearchPage
# parameters: amg search page as html content in array (split on lines breaks)
# returns: list of likely matches, other results
sub parseNameSearchResults {
	my (@searchPageToBeParsed) = @_;
	my (%searchResults);		
	# likely matches will appear in bold
	foreach my $line ( @searchPageToBeParsed ) {
		if ($line =~ m/ $startMatchNS			# begin possible match
						($boldTagNS)?			# title in bold? = $1
						$beginArtistNameNS
						([^<]*)					# artist name = $2
						$endArtistNameNS
						.{50,100}?
						$startGenreNS	
						(.{2,50})				# genre = $3
						$endGenreNS/gcx){	
			my ($name, $genre, $isBold) = ($2, $3, $1);
			# Data structure of %searchResults = ( artistName => genre => (bold?->1, No?->0)) 
			if ($isBold){
				${${$searchResults{$name}}{$genre}} = 1;
			} else {
				${${$searchResults{$name}}{$genre}} += 0;
			}
			redo;
		}
	}
	return(%searchResults);
}
# parseAlbumSearchResults( @amgSearchPage ): find best match for $searchTerm in @amgSearchPage
# parameters: amg album search page as html content in array (split on lines breaks)
# returns: list of names and genres in search results
sub parseAlbumSearchResults {
	my (@searchPageToBeParsed) = @_;
	my (%searchResults);
	foreach my $line ( @searchPageToBeParsed ) {
		if ($line =~ m/ $beginArtistNameAS
						([^<]*)					# artist name = $1
						$endArtistNameAS
						.*?
						$beginGenreNameAS	
						(.{2,50})				# genre = $2
						$endGenreNameAS/gcx){	
			my ($name, $genre) = ($1, $2);
			# Data structure of %searchResults = ( artistName => genre => 1) 
			${${$searchResults{$name}}{$genre}} = 1;
			redo;
		}
	}
	return(%searchResults);
}
# bestMatch( $searchTerm, @likelyMatches ): find best match for $searchTerm in @likelyMatches
# parameters: search term, array containing possible matches
# returns: link to artist page
sub bestMatch {
	my ($artistToFind, %searchResults) = @_;
	my $skippedReason;
	my $filteredName = filterForSearch($artistToFind);
	# create a list of variants of our search term that will count as a match
	my @artistNameVariations = ($filteredName);
	if ( $filteredName =~ m/^([^,]*), ([^,]*)$/ ) {  # if the artist name has one (and only one) comma
		push(@artistNameVariations, $2 . " " . $1);  # switch the words around (smith, bob becomes bob smith)
		if ( $filteredName =~ m/^([^,]*), (\w+) ([^,]*)$/ ) { # if name has  multiple words right of comma
			push(@artistNameVariations, $2 . " " . $1 . " " . $3); # Marley, Bob and the Wailers
																   # becomes Bob Marley and the Wailers
		}
	} else {
		push(@artistNameVariations, "The " . $filteredName);
	}
	# test each name in results agains each variant and delete it if it doesn't match
	foreach my $name ( keys %searchResults ) {
		my $match = 0;
		foreach my $variant ( @artistNameVariations ) {
			my $a = filterForSearch($name);
			$match = 1 if ($variant =~ m/^$a$/i)
		}
		delete $searchResults{$name} unless $match;
	}
	# See if the list of results is specific enough now
	my $numMatches = scalar( keys(%searchResults) );
	if ($numMatches == 1){
		my ($name) = keys(%searchResults);
		debug ("Matched $name.\n");
		my $numGenres = scalar( keys(%{$searchResults{$name}}) );
		if ($numGenres == 1){
			my ($genre) = keys(%{$searchResults{$name}});
			return ($name, $genre);
		} else {
			return ($name, 0);
		}
	} elsif ($numMatches > 1) {
		$skippedReason = "Too many name matches.";
	} else {
		$skippedReason = "No match.";
	}

	# print unmatched artists to file to help analyze what names don't match and why so that I can make a better search algorithm some day.
	#	like maybe one that will return the bold result if possible (which is what it used to do, but doesn't anymore
	#	because I was worried that it wouldn't be acurate enough--better no data than bad data in this case)
	#	or maybe check the genre from my database against the amgGenre.
	print SKIPPED "$artistID|$artistToFind: $skippedReason\n" if ($printToFile);
	
	print "Can't find $artistToFind: $skippedReason\n";
	return 0;
}
#findArtistGenre(@amgArtistPage): find artist name and genre in @amgArtistPage
# parameters: amg artist page as html content in array (split on lines breaks)
# returns: artist, genre
sub findArtistGenre {
	my (@html) = @_;
	my ($name, $genre);
	foreach my $line ( @html ) {
		if ($line =~ m/$beginArtistPageHead \s*(.+?)\s* $endArtistPageHead/) {
			$name = $1;
		}
		if ($line =~ m/^$startArtistInfo/) {			# line that contains artist info
			if ($line =~ m/$beginArtistName 
					(.*?)					# artist name = $1
					$endArtistName
					.*?	
					$beginGenreName
					(.*?)					# genre = $2
					$endGenreName/x) {								
				($name, $genre) = ($1,$2);
				last;
			}
		}
	}
	$genre = "Classical" unless $genre;	# if page doesn't match regex above, must be a classical artist
	return ($name, $genre) if $name;
}

#getAlbums(@amgDiscographyPage): find album names in @amgDiscographyPage
# parameters: amg discography page as html content in array (split on lines breaks)
# returns: list of albums
sub getAlbums {
	my (@html) = @_;
	my @albumNames;
	foreach my $line ( @html ) {
		if ($line =~ m/^$startAlbumInfo/) {			# line that contains artist info
			if ($line =~ m/$beginAlbumName
					(.*?)							# album name = $1
					$endAlbumName/gcx){								
				my $albumName = $1;
				$albumName =~ filterForSearch($albumName);
				push(@albumNames, $albumName);
				redo;
			}
			return @albumNames;
		}
	}
}

__END__

=head1 NAME

  allMusicGuideSearch.pl - search AllMusicGuide's website for artist information

=head1 SYNOPSIS

 Run the program like this:
	>perl allMusicGuideSearch.pl Kraftwerk
   
   returns:
   
	Name: Kraftwerk; Genre: Electronica

 Use -a flag (for albums) to get main discography:
	>perl allMusicGuideSearch.pl -a Kraftwerk
	
   returns:
	
	Name: Kraftwerk; Genre: Electronica
	Albums: Kraftwerk 1
        Kraftwerk 2
        Ralf and Florian
        Autobahn
        Radio-Activity
        Trans-Europe Express
        The Man-Machine
        Computer World
        Techno Pop [Not Released]
        Electric Cafe
        Tour de France Soundtracks
        Minimum-Maximum [live]
        Ruckzuck! (Live 1971)
		
 Use quotes for names with multiple words:
	>perl allMusicGuideSearch.pl "The Cars"
   
 Optionally, enter album name as second search term:
	>perl allMusicGuideSearch.pl "The Cars" "Candy O"
   
 Or use a text file as argument and program will read in a list of arguments:
	>perl allMusicGuideSearch.pl artistList.txt
   
 Adding a second text file and program will write output to file:
	>perl allMusicGuideSearch.pl artistList.txt searchResults.txt
   
 Or leave out the arguments:
	>perl allMusicGuideSearch.pl
   and the program will prompt you for search terms.

=head1 DESCRIPTION

Program to search AllMusicGuide's website for artist name, genre, and albums (optional).

=head1 VERSION HISTORY

 * 0.1	2/5/06	Program created. Can perform simple search. Enter artist name and program will 
                search for it on the amg website. If amg finds a match and returns an artist 
                page, program will parse out artist name and genre from the results.
		
                Known bugs: 
                    Says "<input> outside <form> at C:/Perl/site/lib/WWW/Mechanize.pm line 1723"
                        twice when program runs. Doesn't affect performance.
				

 * 0.2	2/11/06	Much progress. Program now separated into subroutines. Added subroutine to look
                through search results to find a possible match. Basically, amg displays more 
                likely result in bold on the search results page, so the program just checks to
                see if there is one (and only one) bold link which matches the search terms
                exactly. If so, it follows the link. Also added subroutine to extract list of
                albums from discography page.
				
                Known bugs: 
                    Still says "<input> outside <form>..."
                    Sometimes gives error message for names that contain a comma (Dylan, Bob).
                        (But still functions correctly)
				
 
 * 0.3  2/14/06	Added the ability to read in artist names from a text file. Added subroutine,
                getSearchTerms() to deal with the many ways to get search criteria. Cleaned up
                code a lot. Found a way to turn off warnings for those parts of the code that
                caused "<input> outside <form>..." message.
				
                Known bugs: 
                    Still gives error message for names with commas.
                        (But still functions correctly)
					
* 0.4   2/15/06	I feel like I re-did everything, but program functions pretty much the same.
                Changed getSearchTerms() so that it will read a text file w/ artistID and
                artistName fields (and optionally genre and album fields) and will return a
                hash instead of an array. Changed the search results subroutines so that they
                create a hash instead of an array and return ($artist, $genre) instead of
                just a link to the artist page (this meant pretty much re-writing them
                comletely). Added a filter subroutine to remove unnecessary characters from
                searches. Discovered that the error message I was getting before had nothing
                to do with commas in names, but that those names were returning amg pages for
                classical artists, which this program doesn't yet recognize.
				
                Known bugs: 
                    Seems that the $DOWARN stuff doesn't work. It turns warnings off, but
                        doesn't turn them back on.
                    Doesn't understand certain characters from amg (turns "é" into theta)
										
* 0.5   2/16/06	Now I've got it set up so that it will search by album title. This only works
                with text file input, not with standard user input. It just depends
                on how many columns there are in the text file -- if there are four columns,
                it assumes that the fourth column is the album name and it will then search
                by album name, otherwise it will search the normal way. Also, I changed the 
                regular expressions that extract information from the web pages to try and 
                make them slightly easier to update if and when amg changes there site.
                Basically, I just created a long list of variables that I put at the beginning
                of the program.
				
                Known bugs: 
                    Same as last time.
										
* 0.6   2/22/06	Added a couple of command line flags: one to turn on the debugger and one to 
                control whether the program retrieves albums. Also, made it so that you can
                pass two arguments from the command line and the first will be taken as
                artist name and the second will be album. Also fixed a number of bugs that
                popped up after the last update.
				
                Known bugs: 
                    Same as last time.
										
                To Do:
                    Search by artist if album search fails. Or maybe the other way around?
                    Find a better way to deal with search results.
                    Perhaps set it up so that, if it returns multiple matches, it will record
                        the best match, but make a note that there were other possible matches.


=head1 SEE ALSO

The site that we search: http://www.allmusic.com
 
WWW::Mechanize man page: http://www.perl.com/pub/a/2003/01/22/mechanize.html

=head1 AUTHOR

Jacob Turino

2006

=cut