3673l1.orig100664 1061 330 1111 6765256226 11127 0ustar mlrljedit#!/usr/bin/perl -w use strict; use diagnostics; use HTTP::Request; use HTTP::Response; use LWP::UserAgent; # Define the URL my $url = "http://www.lerner.co.il"; # Create the HTTP request my $request = new HTTP::Request("GET", $url); # Create a new useragent my $ua = new LWP::UserAgent; # Pass the request to the useragent, and get a # response my $response = $ua->request($request); # If it was successful, print the reversed # contents if ($response->is_success) { print scalar reverse $response->content, "\n"; } else { print qq{Error retrieving "$url": }, $response->message; } 3673l2.orig100664 1061 330 405 6765256222 11111 0ustar mlrljeditsub callback { # Get the arguments my ($tag, %attributes) = @_; # Indicate the tag print "\n\n$tag\n"; # Iterate through each attribute, printing it foreach my $name (sort keys %attributes) { print "$name = $attributes{$name}\n"; } } 3673l3.orig100664 1061 330 505 6765256250 11114 0ustar mlrljeditsub callback { # Get the arguments my ($tag, %attributes) = @_; # Ignore all but hyperlinks return unless ($tag eq "a"); # Indicate the tag print "\n\n$tag\n"; # Iterate through each attribute, printing it foreach my $name (sort keys %attributes) { print "$name = $attributes{$name}\n"; } } 3673l4.orig100664 1061 330 4117 6765256321 11137 0ustar mlrljedit#!/usr/bin/perl -w use strict; use diagnostics; use LWP::RobotUA; use HTML::LinkExtor; use URI::URL; # Where do we want to start? my $origin = "http://www.lerner.co.il/"; # Create two hashes, one for URLs we have seen and # another for those we haven't my %already_retrieved = (); my %to_be_retrieved = ($origin => 1); # Create a hash in which we will deposit URLs we # wish to use my %interesting_urls = (); # Create a new user agent my $ua = new LWP::RobotUA "ATF/1.0", 'reuven@lerner.co.il'; $ua->delay(0); while (my $url = (keys %to_be_retrieved)[0]) { # Create a new HTTP request my $request = new HTTP::Request('GET', $url); # Move this URL from %to_be_retrieved to # %already_retrieved delete $to_be_retrieved{$url}; $already_retrieved{$url} = 1; # Indicate what we are retrieving print "Retrieving $url..."; # Hand $request to $ua, and get an HTTP # response back in return my $response = $ua->request($request); # Complete printout print ".\n"; # If there was a problem, send a report to # STDERR if (!$response->is_success) { print STDERR qq{Error retrieving "$url":}, $response->status_line , "\n"; } # Create an instance of HTML::LinkExtor, # making links relative to $url my $parser = HTML::LinkExtor->new(\&callback, $url); # Parse the output $parser->parse($response->content); } #--------------------------------------------- # Define our callback, which is passed scalar and # a hash sub callback { # Get the tag and its associated attributes my ($tag, %attributes) = @_; # We only care about anchor tags return unless ($tag eq "a"); # Iterate through the attributes foreach my $name (sort keys %attributes) { # If this is a link, then put the URL in the # queue if ($name eq "href") { # Get the URL for this anchor my $url = $attributes{$name}; # Ignore non-HTTP URLs next unless ($url =~ m/^http:/); # Indicate that we should retrieve this # URL in the future $to_be_retrieved{$url} = 1 unless $already_retrieved{$url}; } } }