File Coverage

blib/lib/HTML/Content/HTMLTokenizer.pm

Criterion	Covered	Total	%
statement	20	51	39.2
branch	2	6	33.3
condition	1	3	33.3
subroutine	5	8	62.5
pod	2	4	50.0
total	30	72	41.6

=head1 NAME

HTML::Content::HTMLTokenizer - Perl module to tokenize HTML documents.

=head1 SYNOPSIS

  use HTML::Content::HTMLTokenizer;

  my $tokenizer = new HTML::Content::HTMLTokenizer('TAG','WORD');

  open(HTML,"index.html");
  my $doc = join("",);
  close(HTML);

  my ($word_count_arr_ref,$tag_count_arr_ref,$token_type_arr_ref,$token_hash_ref) = $tokenizer->Tokenize($doc);

=head1 DESCRIPTION

HTML::Content::HTMLTokenizer has one main method, Tokenize, which tokenizes a HTML document into a sequence of 'TAG' and 'WORD' tokens.

=head2 Methods

=over 4

=item * my $tokenizer = new HTML::Content::HTMLTokenizer($tagMarker,$wordMarker)

Initializes HTML::Content::HTMLTokenizer.

$tagMarker - String that will represent tags in the token sequence returned from Tokenize.

$wordMarker - String that will represent words in the token sequence returned from Tokenize.

=item * my (\@WordCount,\@TokenCount,\@Sequence,\%Tokens) = $tokenizer->Tokenize(\$htmldocument);

$WordCount[$i] is the number of word tokens before or at the ith token in the input HTML document.

$TagCount[$i] is the number of tag tokens before or at the ith token in the input HTML document.

$Sequence[$i] is the type of token at the ith spot in the input HTML document. Either $tagMarker or $wordMarker.

$Tokens{$i} is the word at the ith spot in the input HTML document. This is defined only if there is a word at the ith spot in the document.

=back

=head1 AUTHOR

Jean Tavernier ([e-mail])

=head1 COPYRIGHT

Copyright 2005 Jean Tavernier.  All rights reserved.

This library is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.

=head1 SEE ALSO

ContentExtractorDriver.pl (1), HTML::Content::ContentExtractor (3), HTML::WordTagRatio::Ratio (3),HTML::WordTagRatio::WeightedRatio (3), HTML::WordTagRatio::SmoothedRatio (3), HTML::WordTagRatio::RelativeRatio (3), HTML::WordTagRatio::ExponentialRatio (3), HTML::WordTagRatio::NormalizedRatio (3).

=cut


package HTML::Content::HTMLTokenizer;

use strict;
use warnings;
use Carp;

use fields qw(TAGMARKER WORDMARKER);

# new - constructs HTMLTokenizer object
# - preconditions: 1st arg points to string to indicate tag
#		   2nd arg points to string to indicate word
# - postconditions: HTMLTokenizer is constructed
sub new
{
	my $invocant = shift;
    	my $class   = ref($invocant) || $invocant;
    	my $tagMarker = shift or croak "HTMLTokenizer: TagMarker missing \n\tex: HTMLTokenizer tok = new HTMLTokenizer('TAG','WORD');\n";
    	my $wordMarker = shift or croak "HTMLTokenizer: WordMarker missing \n\tex: HTMLTokenizer tok = new HTMLTokenizer('TAG','WORD');\n";

    	my $self = fields::new($invocant);
    	$self->{TAGMARKER} = $tagMarker;
    	$self->{WORDMARKER} = $wordMarker;

    	return bless($self, $class);
}
sub GetTagMarker
{
	my $self = shift;
	return $self->{TAGMARKER};
}
sub GetWordMarker
{
	my $self = shift;
	return $self->{WORDMARKER};
}
sub Tokenize
{
	my $self = shift;
	my $doc = shift;
	my @N = ();
	my @T = ();
	my %tokens = ();

	#Remove carriage returns and newlines
	$doc =~ s/[\n\r]+/ /g;

	#Eliminate comments
	$doc =~ s/()/ <> /gis;

	#Eliminate scripts
	$doc =~ s/(.+?<\/script>)/ <> /gis;

	#Eliminate scripts
	$doc =~ s/(.+?<\/style>)/ <> /gis;

	#Eliminate tag words
	$doc =~ s/(<.+?>)/ <> /gs;

	#Remove HTML spaces
	$doc =~ s/\Q \E/ /g;

	$doc =~ s/\Q"\E/\"/g;
	$doc =~ s/\Q—\E/-/g;

	#Remove HTML directives
	$doc =~ s/\Q&\E.*?\Q;\E/ /g;

	my @seq = split(/\s+/,$doc);

	my $tagcnt = 0;
	my $wordcnt = 0;
	for(my $i = 0; $i <= $#seq; $i++)
	{
		if ($seq[$i] eq '<>')
		{
			$seq[$i] = $self->{TAGMARKER};
			$tagcnt++;
		}
		else
		{
			$tokens{$i} = $seq[$i];
			$seq[$i] = $self->{WORDMARKER};
			$wordcnt++;
		}
		push(@N,$wordcnt);
		push(@T,$tagcnt);
	}
	return (\@N,\@T,\@seq,\%tokens);
}
1;