);
# Turn entities into text
decode_entities($contents);
# Turn HTML tags into whitespace
$contents =~ s|<.+?>| |sg;
# Get a list of words in the document
my @found =
grep {!$IGNORE{$_}} map {lc $_} ($contents =~ m/\b\S+\b/g);
# Index each word that was found
foreach my $word (@found)
{
# Print a heartbeat
print ".";
# Get around a problem with the tie interface
# by dereferencing and assigning in stages
my $hashref = $data{$word};
$hashref->{$url}++;
$data{$word} = $hashref;
}
print "\n";
close FILE;
}
else
{
warn qq{Unable to open "$filename": $! };
return;
}
}
3800l4.orig 100664 1061 330 3172 7023552716 11122 0 ustar mlr ljedit Listing 4: dbsearch.pl, a simple search program which retrieves
results from the MLDBM file created by index-files.pl
#!/usr/bin/perl -w
use strict;
use diagnostics;
use File::Find;
use HTML::Entities;
use Fcntl;
use MLDBM qw(DB_File);
use CGI;
use CGI::Carp qw(fatalsToBrowser);
# Create an instance of CGI
my $query = new CGI;
# Send a MIME header to the user's browser
print $query->header("text/html");
# In what file should the index be stored?
my $index_file = "/tmp/lj.db";
# Create the word index hash, and tie it to the file
my %data;
tie %data, "MLDBM", $index_file, O_RDONLY
or die "Error tying %data: $! ";
# Get the user's search term
my $term = lc $query->param("term");
# Make sure that the term contains non-whitespace
unless ($term =~ /\S/)
{
print $query->start_html(-title => "No search term!");
print "Please enter a search term.
\n";
print $query->end_html;
exit;
}
# Get the hash for this word
my $results = $data{$term};
# Results
my %results = %$results if (defined $results);
# Give a warning if there were no results
unless (%results)
{
print $query->start_html(-title => "No matches!");
print qq|Sorry, but there were no matches to "$term".
\n|;
print $query->end_html;
exit;
}
print $query->start_html(-title => "Search results");
print "There were ", scalar keys %results, " results to your query:
\n";
print "\n";
# If there were results, sort them by frequency
foreach my $url (sort {$results{$a} <=> $results{$b} } keys %results)
{
print qq|- $url\n|;
}
print "
\n";
print $query->end_html;
# Untie the data
untie %data;
3800l5.orig 100664 1061 330 4605 7023552723 11123 0 ustar mlr ljedit Listing 5: better-dbsearch.pl, which allows users to perform "or"
and "and" searches.
#!/usr/bin/perl -w
# filename: better-dbsearch.pl
use strict;
use diagnostics;
use File::Find;
use HTML::Entities;
use Fcntl;
use MLDBM qw(DB_File);
use CGI;
use CGI::Carp qw(fatalsToBrowser);
# Create an instance of CGI
my $query = new CGI;
# Send a MIME header to the user's browser
print $query->header("text/html");
# In what file should the index be stored?
my $index_file = "/tmp/lj.db";
# Create the word index hash, and tie it to the file
my %data;
tie %data, "MLDBM", $index_file, O_RDONLY
or die "Error tying %data: $! ";
# Get the user's search term
my $term = lc $query->param("term");
# Our algorithm now depends on the boolean
my $boolean = $query->param("boolean");
my %full_results = ();
my %match_counter = ();
# Split the words across whitespace
my @terms = split / /, $term;
# Get the results hash for each word
foreach my $word (@terms)
{
# Get a hashref to the data (key is filename, value
# is the number of times the word appears)
my $frequency = $data{$word};
# Add each key-value pair in $frequency to %full_results
foreach my $filename (keys %$frequency)
{
$full_results{$filename} += $frequency->{$filename};
# Indicate that this file matched, for "and" searches
$match_counter{$filename}++ if ($boolean eq "and");
}
}
# For "and" searches, remove files that don't contain all terms
if ($boolean eq "and")
{
# Remove filenames from %full_results unless all
# terms were found
foreach my $filename (keys %full_results)
{
delete $full_results{$filename}
unless ($match_counter{$filename} == @terms);
}
}
# Give a warning if there were no results
unless (%full_results)
{
print $query->start_html(-title => "No matches!");
print qq|Sorry, but there were no matches to "$term".
\n|;
print $query->end_html;
exit;
}
# Display the search results
print $query->start_html(-title => "Search results");
my $query_string = join " $boolean ", @terms;
print "There were ", scalar keys %full_results;
print qq| matches to "$query_string":
\n|;
print "\n";
# If there were results, sort them by frequency
foreach my $url
(sort {$full_results{$a} <=> $full_results{$b} } keys %full_results)
{
print qq|- $url\n|;
}
print "
\n";
print $query->end_html;
# Untie the data
untie %data;