#!/usr/bin/perl -w
#####################
# Fetch a given web page and print out its links.
#   See HTML::LinkExtor documentation, 
#   eg http://www.perldoc.com/perl5.6/lib/HTML/LinkExtor.html
####################################################
use strict;
use LWP::UserAgent;
use HTML::LinkExtor;
use URI::URL;

my $url = "http://www.marlboro.edu/admissions/";  # for instance

my $userAgent = LWP::UserAgent->new;
my $parser    = HTML::LinkExtor->new( \&addLink );
my @links     = ();
sub addLink {
  my ($tag, %links) = @_;
  return if $tag ne 'a';			# only keep <a ...> tags
  push @links,$links{href} if $links{href};	# with href= attributes.
}

# Request the document, parsing it as it arrives
my $response = $userAgent->request(HTTP::Request->new(GET => $url),
				   sub {$parser->parse($_[0])});

# Expand the links to absolute urls.
my $base = $response->base;
@links = map { $_ = url($_, $base)->abs; } @links;

# print 'em out.
print join("\n", @links), "\n";

