File Coverage

blib/lib/HTML/Parser/Simple.pm

Criterion	Covered	Total	%
statement	161	192	83.8
branch	63	84	75.0
condition	22	36	61.1
subroutine	18	19	94.7
pod	4	14	28.5
total	268	345	77.6

package HTML::Parser::Simple;

use strict;
use warnings;

use Moo;

use Tree::Simple;

has block =>
(
	default => sub {return {} },
	is      => 'rw',
);

has current_node =>
(
	default => sub {return ''},
	is      => 'rw',
);

has depth =>
(
	default => sub {return 0},
	is      => 'rw',
);

has empty =>
(
	default => sub {return {} },
	is      => 'rw',
);

has inline =>
(
	default => sub {return {} },
	is      => 'rw',
);

has input_file =>
(
	default => sub {return ''},
	is      => 'rw',
);

has node_type =>
(
	default => sub {return 'global'},
	is      => 'rw',
);

has output_file =>
(
	default => sub {return ''},
	is      => 'rw',
);

has result =>
(
	default => sub {return ''},
	is      => 'rw',
);

has root =>
(
	default => sub {return ''},
	is      => 'rw',
);

has self_close =>
(
	default => sub {return {} },
	is      => 'rw',
);

has tagged_attribute =>
(
	default => sub {return {} },
	is      => 'rw',
);

has verbose =>
(
	default => sub {return 0},
	is      => 'rw',
);

has xhtml =>
(
	default => sub {return 0},
	is      => 'rw',

	trigger =>
	sub
	{
		my($self, $new) = @_;

		$self -> _set_tagged_attribute($new);
	}
);

our $VERSION = '2.01';

# -----------------------------------------------

sub BUILD
{
	my($self) = @_;

	$self -> block
	({
	 address => 1,
	 applet => 1,
	 blockquote => 1,
	 button => 1,
	 center => 1,
	 dd => 1,
	 del => 1,
	 dir => 1,
	 div => 1,
	 dl => 1,
	 dt => 1,
	 fieldset => 1,
	 form => 1,
	 frameset => 1,
	 hr => 1,
	 iframe => 1,
	 ins => 1,
	 isindex => 1,
	 li => 1,
	 map => 1,
	 menu => 1,
	 noframes => 1,
	 noscript => 1,
	 object => 1,
	 ol => 1,
	 p => 1,
	 pre => 1,
	 script => 1,
	 table => 1,
	 tbody => 1,
	 td => 1,
	 tfoot => 1,
	 th => 1,
	 thead => 1,
	 'tr' => 1,
	 ul => 1,
	});

	$self -> empty
	({
	 area => 1,
	 base => 1,
	 basefont => 1,
	 br => 1,
	 col => 1,
	 embed => 1,
	 frame => 1,
	 hr => 1,
	 img => 1,
	 input => 1,
	 isindex => 1,
	 link => 1,
	 meta => 1,
	 param => 1,
	 wbr => 1,
	});

	$self -> inline
	({
	 a => 1,
	 abbr => 1,
	 acronym => 1,
	 applet => 1,
	 b => 1,
	 basefont => 1,
	 bdo => 1,
	 big => 1,
	 br => 1,
	 button => 1,
	 cite => 1,
	 code => 1,
	 del => 1,
	 dfn => 1,
	 em => 1,
	 font => 1,
	 i => 1,
	 iframe => 1,
	 img => 1,
	 input => 1,
	 ins => 1,
	 kbd => 1,
	 label => 1,
	 map => 1,
	 object => 1,
	 'q' => 1,
	 's' => 1,
	 samp => 1,
	 script => 1,
	 select => 1,
	 small => 1,
	 span => 1,
	 strike => 1,
	 strong => 1,
	 sub => 1,
	 sup => 1,
	 textarea => 1,
	 tt => 1,
	 u => 1,
	 var => 1,
	});

	$self -> self_close
	({
	 colgroup => 1,
	 dd => 1,
	 dt => 1,
	 li => 1,
	 options => 1,
	 p => 1,
	 td => 1,
	 tfoot => 1,
	 th => 1,
	 thead => 1,
	 'tr' => 1,
	});

	$self -> current_node($self -> create_new_node('root', '', Tree::Simple -> ROOT) );
	$self -> root($self -> current_node);

	if ($self -> xhtml)
	{
		# Compared to the non-XHTML re, this has an extra  ':' in the first [].

		$self -> tagged_attribute
		(
			q#^(<(\w+)((?:\s+[-:\w]+(?:\s*=\s*(?:(?:"[^"]*")|(?:'[^']*')|[^>\s]+))?)*)\s*(\/?)>)#
		);
	}
	else
	{
		$self -> tagged_attribute
		(
			q#^(<(\w+)((?:\s+[-\w]+(?:\s*=\s*(?:(?:"[^"]*")|(?:'[^']*')|[^>\s]+))?)*)\s*(\/?)>)#
		);
	}

}	# End of BUILD.

# -----------------------------------------------
# Create a new node to store the new tag.
# Each node has metadata:
# o attributes: The tag's attributes, as a string with N spaces as a prefix.
# o content:    The content before the tag was parsed.
# o name:       The HTML tag.
# o node_type:  This holds 'global' before '' and between ''
#               and '', and after ''. It holds 'head' from
#               '' to ', and holds 'body' from '' to
#               ''. It's just there in case you need it.

sub create_new_node
{
	my($self, $name, $attributes, $parent) = @_;
	my($metadata) =
	{
		attributes => $attributes,
		content    => [],
		depth      => $self -> depth,
		name       => $name,
		node_type  => $self -> node_type,
	};

	return Tree::Simple -> new($metadata, $parent);

} # End of create_new_node.

# -----------------------------------------------

sub handle_comment
{
	my($self, $s) = @_;

	$self -> handle_content($s);

} # End of handle_comment.

# -----------------------------------------------

sub handle_content
{
	my($self, $s)                 = @_;
	my($count)                    = $self -> current_node -> getChildCount;
	my($metadata)                 = $self -> current_node -> getNodeValue;
	$$metadata{'content'}[$count] .= $s;

	$self -> current_node -> setNodeValue($metadata);

} # End of handle_content.

# -----------------------------------------------

sub handle_doctype
{
	my($self, $s) = @_;

	$self -> handle_content($s);

} # End of handle_doctype.

# -----------------------------------------------

sub handle_end_tag
{
	my($self, $tag_name) = @_;

	$self -> node_type('global') if ( ($tag_name eq 'head') || ($tag_name eq 'body') );

	if (! ${$self -> empty}{$tag_name})
	{
		$self -> current_node($self -> current_node -> getParent);
		$self -> depth($self -> depth - 1);
	}

} # End of handle_end_tag.

# -----------------------------------------------

sub handle_start_tag
{
	my($self, $tag_name, $attributes, $unary) = @_;

	$self -> depth($self -> depth + 1);

	if ($tag_name eq 'head')
	{
		$self -> node_type('head');
	}
	elsif ($tag_name eq 'body')
	{
		$self -> node_type('body');
	}

	my($node) = $self -> create_new_node($tag_name, $attributes, $self -> current_node);

	$self -> current_node($node) if (! ${$self -> empty}{$tag_name});

} # End of handle_start_tag.

# -----------------------------------------------

sub handle_xml_declaration
{
	my($self, $s) = @_;

	$self -> handle_content($s);

} # End of handle_xml_declaration.

# -----------------------------------------------

sub log
{
	my($self, $msg) = @_;

	print STDERR "$msg\n" if ($self -> verbose);

} # End of log.

# -----------------------------------------------

sub parse
{
	my($self, $html) = @_;
	my($original)    = $html;
	my(%special)     =
	(
	 script => 1,
	 style  => 1,
	);
	my($tagged_attribute) = $self -> tagged_attribute;

	my($in_content);
	my($offset);
	my(@stack, $s);

	for (; $html;)
	{
		$in_content = 1;

		# Make sure we're not in a script or style element.

		if (! $stack[$#stack] || ! $special{$stack[$#stack]})
		{
			# Rearrange order of testing so rarer possiblilites are further down.
			# Is it an end tag?

			$s = substr($html, 0, 2);

			if ($s eq '
			{
				if ($html =~ /^(<\/(\w+)[^>]*>)/)
				{
					substr($html, 0, length $1) = '';
					$in_content                 = 0;

					$self -> parse_end_tag($2, \@stack);
				}
			}

			# Is it a start tag?

			if ($in_content)
			{
				if (substr($html, 0, 1) eq '<')
				{
					# Use lc() since tags are stored in this module in lower-case.

					if (lc($html) =~ /$tagged_attribute/)
					{
						# Since the regexp matched, save matches in lower-case.
						# Then, re-match to get attributes in original case.
						# In each case:
						# o $1 => The whole string which matched.
						# o $2 => The tag name.
						# o $3 => The attributes.
						# o $4 => The trailing / if any (aka $unity).
						# But we have to lower-case the prefix '<$tag' of the string
						# to ensure the 2nd regexp actually matches.

						my(@match)                       = ($2, $3, $4);
						substr($html, 0, length($2) + 1) = lc substr($html, 0, length($2) + 1);

						if ($html =~ /$tagged_attribute/)
						{
							substr($html, 0, length $1) = '';
							$in_content                 = 0;

							# Here we use $3 from the 2nd match to get the attributes in the original case.
							$self -> parse_start_tag($match[0], $3, $match[2], \@stack);
						}
					}
				}
			}

			# Is it a comment?

			if ($in_content)
			{
				$s = substr($html, 0, 4);

				if ($s eq '');

					if ($offset >= 0)
					{
						$self -> handle_comment(substr($html, 0, ($offset + 3) ) );

						substr($html, 0, $offset + 3) = '';
						$in_content                   = 0;
					}
				}
			}

			# Is it a doctype?

			if ($in_content)
			{
				$s = substr($html, 0, 9);

				if ($s eq '
				{
					$offset = index($html, '>');

					if ($offset >= 0)
					{
						$self -> handle_doctype(substr($html, 0, ($offset + 1) ) );

						substr($html, 0, $offset + 1) = '';
						$in_content                   = 0;
					}
				}
			}

			# Is is an XML declaration?

			if ($self -> xhtml && $in_content)
			{
				$s = substr($html, 0, 5);

				if ($s eq '
				{
					$offset = index($html, '?>');

					if ($offset >= 0)
					{
						$self -> handle_xml_declaration(substr($html, 0, ($offset + 2) ) );

						substr($html, 0, $offset + 2) = '';
						$in_content                   = 0;
					}
				}
			}

			if ($in_content)
			{
				$offset = index($html, '<');

				if ($offset < 0)
				{
					$self -> handle_content($html);

					$html = '';
				}
				else
				{
					$self -> handle_content(substr($html, 0, $offset) );

					substr($html, 0, $offset) = '';
				}
			}
		}
		else
		{
			my($re) = "(.*)<\/$stack[$#stack]\[^>]*>";

			# lc() is needed because only lc tag names are pushed onto the stack.

			if (lc($html) =~ /$re/s)
			{
				my($text) = $1;
				$text     =~ s//$1/g;
				$text     =~ s//$1/g;

				$self -> handle_content($text);
			}

			$self -> parse_end_tag($stack[$#stack], \@stack);
		}

		if ($html eq $original)
		{
			my($msg)    = 'Parse error. ';
			my($parent) = $self -> current_node -> getParent;

			my($metadata);

			if ($parent && $parent -> can('getNodeValue') )
			{
				$metadata = $parent -> getNodeValue;
				$msg      .= "Parent tag: <$$metadata{'name'}>. ";
			}

			$metadata = $self -> current_node -> getNodeValue;
			$msg      .= "Current tag: <$$metadata{'name'}>. Next 100 chars: " . substr($html, 0, 100);

			die "$msg\n";
		}

		$original = $html;
	}

	# Clean up any remaining tags.

	$self -> parse_end_tag('', \@stack);

	# Return the invocant to allow method chaining.

	return $self;

} # End of parse.

# -----------------------------------------------

sub parse_end_tag
{
	my($self, $tag_name, $stack) = @_;
	$tag_name = lc $tag_name;

	# Find the closest opened tag of the same name.

	my($pos);

	if ($tag_name)
	{
		for ($pos = $#$stack; $pos >= 0; $pos--)
		{
			last if ($$stack[$pos] eq $tag_name);
		}
	}
	else
	{
		$pos = 0;
	}

	if ($pos >= 0)
	{
		# Close all the open tags, up the stack.

		my($count) = 0;

		for (my($i) = $#$stack; $i >= $pos; $i--)
		{
			$count++;

			$self -> handle_end_tag($$stack[$i]);
		}

		# Remove the open elements from the stack.
		# Does not work: $#$stack = $pos. Could use splice().

		pop @$stack for ($count);
	}

} # End of parse_end_tag.

# -----------------------------------------------

sub parse_file
{
	my($self, $input_file_name, $output_file_name) = @_;
	$input_file_name  ||= $self -> input_file;
	$output_file_name ||= $self -> output_file;

	$self -> input_file($input_file_name);
	$self -> output_file($output_file_name);
	$self -> log("Reading $input_file_name");

	open(INX, $input_file_name) || die "Can't open($input_file_name): $!\n";
	my($html);
	read(INX, $html, -s INX);
	close INX;

	die "Can't read($input_file_name): $!\n" if (! defined $html);

	$self -> log('Parsing');

	$self -> parse($html);

	$self -> log('Traversing');

	$self -> traverse($self -> root);

	$self -> log("Writing $output_file_name");

	open(OUT, "> $output_file_name") || die "Can't open(> $output_file_name): $!\n";
	print OUT $self -> result;
	close OUT;

	# Return the invocant to allow method chaining.

	return $self;

} # End of parse_file.

# -----------------------------------------------

sub parse_start_tag
{
	my($self, $tag_name, $attributes, $unary, $stack) = @_;
	$tag_name = lc $tag_name;

	if (${$self -> block}{$tag_name})
	{
		for (; $#$stack >= 0 && ${$self -> inline}{$$stack[$#$stack]};)
		{
			$self -> parse_end_tag($$stack[$#$stack], $stack);
		}
	}

	if (${$self -> self_close}{$tag_name} && ($$stack[$#$stack] eq $tag_name) )
	{
		$self -> parse_end_tag($tag_name, $stack);
	}

	$unary = ${$self -> empty}{$tag_name} || $unary;

	push @$stack, $tag_name if (! $unary);

	$self -> handle_start_tag($tag_name, $attributes, $unary);

} # End of parse_start_tag.

# -----------------------------------------------

sub _set_tagged_attribute
{
	my($self, $new, $old) = @_;

	if ($new)
	{
		$self -> tagged_attribute
		(
			# Compared to the non-XHTML re, this has an extra  ':' in the first [].

			q#^(<(\w+)((?:\s+[-:\w]+(?:\s*=\s*(?:(?:"[^"]*")|(?:'[^']*')|[^>\s]+))?)*)\s*(\/?)>)#
		);
	}
	else
	{
		$self -> tagged_attribute
		(
			q#^(<(\w+)((?:\s+[-\w]+(?:\s*=\s*(?:(?:"[^"]*")|(?:'[^']*')|[^>\s]+))?)*)\s*(\/?)>)#
		);
	}

} # End of _set_tagged_attribute.

# -----------------------------------------------

sub traverse
{
	my($self, $node) = @_;
	my(@child)       = $node -> getAllChildren;
	my($metadata)    = $node -> getNodeValue;
	my($content)     = $$metadata{'content'};
	my($name)        = $$metadata{'name'};

	# Special check to avoid printing '' when we still need to output
	# the content of the root, e.g. the DOCTYPE.

	$self -> result($self -> result . "<$name$$metadata{'attributes'}>") if ($name ne 'root');

	my($index);
	my($s);

	for $index (0 .. $#child)
	{
		$self -> result($self -> result . ($index <= $#$content && defined($$content[$index]) ? $$content[$index] : '') );
		$self -> traverse($child[$index]);
	}

	# Output the content after the last child node has been closed,
	# but before the current node is closed.

	$index = $#child + 1;

	$self -> result($self -> result . ($index <= $#$content && defined($$content[$index]) ? $$content[$index] : '') );
	$self -> result($self -> result . "") if (! ${$self -> empty}{$name} && ($name ne 'root') );

	# Return the invocant to allow method chaining.

	return $self;

} # End of traverse.

# -----------------------------------------------

1;

=head1 NAME

HTML::Parser::Simple - Parse nice HTML files without needing a compiler

=head1 Synopsis

	#!/usr/bin/env perl

	use strict;
	use warnings;

	use HTML::Parser::Simple;

	# -------------------------

	# Method 1:

	my($p) = HTML::Parser::Simple -> new
	(
		input_file  => 'data/s.1.html',
		output_file => 'data/s.2.html',
	);

	$p -> parse_file;

	# Method 2:

	my($p) = HTML::Parser::Simple -> new;

	$p -> parse_file('data/s.1.html', 'data/s.2.html');

	# Method 3:

	my($p) = HTML::Parser::Simple -> new;

	print $p -> parse('...') -> traverse($p -> root) -> result;

Of course, these can be abbreviated by using method chaining. E.g. Method 2 could be:

	HTML::Parser::Simple -> new -> parse_file('data/s.1.html', 'data/s.2.html');

See scripts/parse.html.pl and scripts/parse.xhtml.pl.

=head1 Description

C is a pure Perl module.

It parses HTML V 4 files, and generates a tree of nodes, with 1 node per HTML tag.

The data associated with each node is documented in the L.

See also L and L.

=head1 Distributions

This module is available as a Unix-style distro (*.tgz).

See L for details.

See L for
help on unpacking and installing.

=head1 Constructor and initialization

new(...) returns an object of type C.

This is the class contructor.

Usage: C<< HTML::Parser::Simple -> new >>.

This method takes a hash of options.

Call C<< new() >> as C<< new(option_1 => value_1, option_2 => value_2, ...) >>.

Available options (each one of which is also a method):

=over 4

=item o input_file => $a_file_name

This takes the file name, including the path, of the input file.

Default: '' (the empty string).

=item o output_file => $a_file_name

This takes the file name, including the path, of the output file.

Default: '' (the empty string).

=item o verbose => $Boolean

This takes either a 0 or a 1.

Write more or less progress messages.

Default: 0.

=item o xhtml => $Boolean

This takes either a 0 or a 1.

0 means do not accept an XML declaration, such as
at the start of the input file, and some other XHTML features, explained next.

1 means accept XHTML input.

Default: 0.

The only XHTML changes to this code, so far, are:

=over 4

=item o Accept the XML declaration

E.g.: .

=item o Accept attribute names containing the ':' char

E.g.: .

=back

=back

=head1 Methods

=head2 block()

Returns a hashref where the keys are the names of block-level HTML tags.

The corresponding values in the hashref are just 1.

Typical keys: address, form, p, table, tr.

Note: Some keys, e.g. tr, are also returned by L.

=head2 current_node()

Returns the L object which the parser calls the current node.

=head2 depth()

Returns the nesting depth of the current tag.

The method is just here in case you need it.

=head2 empty()

Returns a hashref where the keys are the names of HTML tags of type empty.

The corresponding values in the hashref are just 1.

Typical keys: area, base, input, wbr.

=head2 inline()

Returns a hashref where the keys are the names of HTML tags of type inline.

The corresponding values in the hashref are just 1.

Typical keys: a, em, img, textarea.

=head2 input_file($in_file_name)

Gets or sets the input file name used by L.

Note: The parameters passed in to L, take precedence over the
I and I parameters passed in to C<< new() >>, and over the internal values set with
C<< input_file($in_file_name) >> and C<< output_file($out_file_name) >>.

'input_file' is a parameter to L. See L for details.

=head2 log($msg)

Print $msg to STDERR if C<< new() >> was called as C<< new(verbose => 1) >>, or if C<< $p -> verbose(1) >>
was called.

Otherwise, print nothing.

=head2 new()

This is the constructor. See L for details.

=head2 node_type()

Returns the type of the most recently created node, I, I, or I.

See the first question in the L for details.

=head2 output_file($out_file_name)

Gets or sets the output file name used by L.

Note: The parameters passed in to L, take precedence over the
I and I parameters passed in to C<< new() >>, and over the internal values set with
C<< input_file($in_file_name) >> and C<< output_file($out_file_name) >>.

'output_file' is a parameter to L. See L for details.

=head2 parse($html)

Returns the invocant. Thus C<< $p -> parse >> returns $p. This allows for method chaining. See the L.

Parses the string of HTML in $html, and builds a tree of nodes.

After calling C<< $p -> parse($html) >>, you must call C<< $p -> traverse($p -> root) >> before calling
C<< $p -> result >>.

Alternately, use C<< $p -> parse_file >>, which calls all these methods for you.

Note: C<< parse() >> may be called directly or via C<< parse_file() >>.

=head2 parse_file($input_file_name, $output_file_name)

Returns the invocant. Thus C<< $p -> parse_file >> returns $p. This allows for method chaining. See the L.

Parses the HTML in the input file, and writes the result to the output file.

C<< parse_file() >> calls L and L, using C<< $p -> root >> for $node.

Note: The parameters passed in to C<< parse_file($input_file_name, $output_file_name) >>, take precedence over the
I and I parameters passed in to C<< new() >>, and over the internal values set with
C<< input_file($in_file_name) >> and C<< output_file($out_file_name) >>.

Lastly, the parameters passed in to C<< parse_file($input_file_name, $output_file_name) >> are used to update
the internal values set with the I and I parameters passed in to C<< new() >>,
or set with calls to C<< input_file($in_file_name) >> and C<< output_file($out_file_name) >>.

=head2 result()

Returns the string which is the result of the parse.

See scripts/parse.html.pl.

=head2 root()

Returns the L object which the parser calls the root of the tree of nodes.

=head2 self_close()

Returns a hashref where the keys are the names of HTML tags of type self close.

The corresponding values in the hashref are just 1.

Typical keys: dd, dt, p, tr.

Note: Some keys, e.g. tr, are also returned by L.

=head2 tagged_attribute()

Returns a string to be used as a regexp, to capture tags and their optional attributes.

It does not return qr/$s/; it just returns $s.

This regexp takes one of two forms, depending on the state of the I option. See L.

The regexp has four (4) sets of capturing parentheses:

=over 4

=item o 1 for the whole tag and attribute and trailing / combination

E.g.: <(....)>

=item o 1 for the tag itself

E.g.: <(img)...>

=item o 1 for the tag's optional attributes

E.g.:

=item o 1 for the tag's optional trailing /

E.g.:

=back

=head2 traverse($node)

Returns the invocant. Thus C<< $p -> traverse >> returns $p. This allows for method chaining. See the L.

Traverses the tree of nodes, starting at $node.

You normally call this as C<< $p -> traverse($p -> root) >>, to ensure all nodes are visited.

See the L for sample code.

Or, see scripts/traverse.file.pl, which uses L, and calls C<< traverse($node) >>
via L.

=head2 verbose($Boolean)

Gets or sets the verbose parameter.

'verbose' is a parameter to L. See L for details.

=head2 xhtml($Boolean)

Gets or sets the xhtml parameter.

If you call this after object creation, the I feature of L is used to call
L so as to correctly set the regexp which recognises xhtml.

'xhtm'> is a parameter to L. See L for details.

=head1 FAQ

=head2 What is the format of the data stored in each node of the tree?

The data of each node is a hash ref. The keys/values of this hash ref are:

=over 4

=item o attributes

This is the string of HTML attributes associated with the HTML tag.

Attributes are stored in lower-case.

So,  will have an attributes string of 




1073




 





 





 





 





 





 



 " align = 'center' summary = 'body'". 




1074




 





 





 





 





 





 



   




1075




 





 





 





 





 





 



 Note the leading space. 




1076




 





 





 





 





 





 



   




1077




 





 





 





 





 





 



 =item o content 




1078




 





 





 





 





 





 



   




1079




 





 





 





 





 





 



 This is an arrayref of bits and pieces of content. 




1080




 





 





 





 





 





 



   




1081




 





 





 





 





 





 



 Consider this fragment of HTML: 




1082




 





 





 





 





 





 



   




1083




 





 





 





 





 





 



 I did not say I liked debugging. 




1084




 





 





 





 





 





 



   




1085




 





 





 





 





 





 



 When parsing 'I did ', the number of child nodes (of ) is 0, since  has not yet been detected. 




1086




 





 





 





 





 





 



   




1087




 





 





 





 





 





 



 So, 'I did ' is stored in the 0th element of the arrayref belonging to . 




1088




 





 





 





 





 





 



   




1089




 





 





 





 





 





 



 Likewise, 'not' is stored in the 0th element of the arrayref belonging to the node . 




1090




 





 





 





 





 





 



   




1091




 





 





 





 





 





 



 Next, ' say I ' is stored in the 1st element of the arrayref belonging to , 




1092




 





 





 





 





 





 



 because it follows the 1st child node (). 




1093




 





 





 





 





 





 



   




1094




 





 





 





 





 





 



 Likewise, ' debugging' is stored in the 2nd element of the arrayref belonging to . 




1095




 





 





 





 





 





 



   




1096




 





 





 





 





 





 



 This way, the input string can be reproduced by successively outputting the elements of the arrayref of content 




1097




 





 





 





 





 





 



 interspersed with the contents of the child nodes (processed recusively). 




1098




 





 





 





 





 





 



   




1099




 





 





 





 





 





 



 Note: If you are processing this tree, never forget that there can be content after the last child node has been closed, 




1100




 





 





 





 





 





 



 but before the current node is closed. 




1101




 





 





 





 





 





 



   




1102




 





 





 





 





 





 



 Note: The DOCTYPE declaration is stored as the 0th element of the content of the root node. 




1103




 





 





 





 





 





 



   




1104




 





 





 





 





 





 



 =item o depth 




1105




 





 





 





 





 





 



   




1106




 





 





 





 





 





 



 The nesting depth of the tag within the document. 




1107




 





 





 





 





 





 



   




1108




 





 





 





 





 





 



 The root is at depth 0, '' is at depth 1, '' and '' are a depth 2, and so on. 




1109




 





 





 





 





 





 



   




1110




 





 





 





 





 





 



 It's just there in case you need it. 




1111




 





 





 





 





 





 



   




1112




 





 





 





 





 





 



 =item o name 




1113




 





 





 





 





 





 



   




1114




 





 





 





 





 





 



 So, the tag '' will mean the name is 'html'. 




1115




 





 





 





 





 





 



   




1116




 





 





 





 





 





 



 Tag names are stored in lower-case. 




1117




 





 





 





 





 





 



   




1118




 





 





 





 





 





 



 The root of the tree is called 'root', and holds the DOCTYPE, if any, as content. 




1119




 





 





 





 





 





 



   




1120




 





 





 





 





 





 



 The root has the node 'html' as the only child, of course. 




1121




 





 





 





 





 





 



   




1122




 





 





 





 





 





 



 =item o node_type 




1123




 





 





 





 





 





 



   




1124




 





 





 





 





 





 



 This holds 'global' before '' and between '' and '', and after ''. 




1125




 





 





 





 





 





 



   




1126




 





 





 





 





 





 



 It holds 'head' for all nodes from '' to '', and holds 'body' from '' to ''. 




1127




 





 





 





 





 





 



   




1128




 





 





 





 





 





 



 It's just there in case you need it. 




1129




 





 





 





 





 





 



   




1130




 





 





 





 





 





 



 =back 




1131




 





 





 





 





 





 



   




1132




 





 





 





 





 





 



 =head2 How are tags and attributes handled? 




1133




 





 





 





 





 





 



   




1134




 





 





 





 





 





 



 Tags are stored in lower-case, in a tree managed by L. 




1135




 





 





 





 





 





 



   




1136




 





 





 





 





 





 



 Attributes are stored in the same case as in the original HTML. 




1137




 





 





 





 





 





 



   




1138




 





 





 





 





 





 



 The root of the tree is returned be L. 




1139




 





 





 





 





 





 



   




1140




 





 





 





 





 





 



 =head2 How are HTML comments handled? 




1141




 





 





 





 





 





 



   




1142




 





 





 





 





 





 



 They are treated as content. This includes the prefix ''. 




1143




 





 





 





 





 





 



   




1144




 





 





 





 





 





 



 =head2 How is DOCTYPE handled? 




1145




 





 





 





 





 





 



   




1146




 





 





 





 





 





 



 It is treated as content belonging to the root of the tree. 




1147




 





 





 





 





 





 



   




1148




 





 





 





 





 





 



 =head2 How is the XML declaration handled? 




1149




 





 





 





 





 





 



   




1150




 





 





 





 





 





 



 It is treated as content belonging to the root of the tree. 




1151




 





 





 





 





 





 



   




1152




 





 





 





 





 





 



 =head2 Does this module handle all HTML pages? 




1153




 





 





 





 





 





 



   




1154




 





 





 





 





 





 



 No, never. 




1155




 





 





 





 





 





 



   




1156




 





 





 





 





 





 



 =head2 Which versions of HTML does this module handle? 




1157




 





 





 





 





 





 



   




1158




 





 





 





 





 





 



 Up to V 4. 




1159




 





 





 





 





 





 



   




1160




 





 





 





 





 





 



 =head2 What do I do if this module does not handle my HTML page? 




1161




 





 





 





 





 





 



   




1162




 





 





 





 





 





 



 Make yourself a nice cup of tea, and then fix your page. 




1163




 





 





 





 





 





 



   




1164




 





 





 





 





 





 



 =head2 Does this validate the HTML input? 




1165




 





 





 





 





 





 



   




1166




 





 





 





 





 





 



 No. 




1167




 





 





 





 





 





 



   




1168




 





 





 





 





 





 



 For example, if you feed in a HTML page without the title tag, this module does not care. 




1169




 





 





 





 





 





 



   




1170




 





 





 





 





 





 



 =head2 How do I view the output HTML? 




1171




 





 





 





 





 





 



   




1172




 





 





 





 





 





 



 There are various ways. 




1173




 





 





 





 





 





 



   




1174




 





 





 





 





 





 



 =over 4 




1175




 





 





 





 





 





 



   




1176




 





 





 





 





 





 



 =item o See scripts/parse.html.pl 




1177




 





 





 





 





 





 



   




1178




 





 





 





 





 





 



 =item o By installing HTML::Revelation, of course! 




1179




 





 





 





 





 





 



   




1180




 





 





 





 





 





 



 Sample output: 




1181




 





 





 





 





 





 



   




1182




 





 





 





 





 





 



 L. 




1183




 





 





 





 





 





 



   




1184




 





 





 





 





 





 



 =back 




1185




 





 





 





 





 





 



   




1186




 





 





 





 





 





 



 =head2 How do I test this module (or my file)? 




1187




 





 





 





 





 





 



   




1188




 





 





 





 





 





 



 Preferably, see the previous question, or... 




1189




 





 





 





 





 





 



   




1190




 





 





 





 





 





 



 Suggested steps: 




1191




 





 





 





 





 





 



   




1192




 





 





 





 





 





 



 Note: There are quite a few files involved. Proceed with caution. 




1193




 





 





 





 





 





 



   




1194




 





 





 





 





 





 



 =over 4 




1195




 





 





 





 





 





 



   




1196




 





 





 





 





 





 



 =item o Select a HTML file to test 




1197




 





 





 





 





 





 



   




1198




 





 





 





 





 





 



 Call this input.html. 




1199




 





 





 





 





 





 



   




1200




 





 





 





 





 





 



 =item o Run input.html thru reveal.pl 




1201




 





 





 





 





 





 



   




1202




 





 





 





 





 





 



 Reveal.pl ships with HTML::Revelation. 




1203




 





 





 





 





 





 



   




1204




 





 





 





 





 





 



 Call the output file output.1.html. 




1205




 





 





 





 





 





 



   




1206




 





 





 





 





 





 



 =item o Run input.html thru parse.html.pl 




1207




 





 





 





 





 





 



   




1208




 





 





 





 





 





 



 parse.html.pl ships with HTML::Parser::Simple. 




1209




 





 





 





 





 





 



   




1210




 





 





 





 





 





 



 Call the output file parsed.html. 




1211




 





 





 





 





 





 



   




1212




 





 





 





 





 





 



 =item o Run parsed.html thru reveal.pl 




1213




 





 





 





 





 





 



   




1214




 





 





 





 





 





 



 Call the output file output.2.html. 




1215




 





 





 





 





 





 



   




1216




 





 





 





 





 





 



 =item o Compare output.1.html and output.2.html 




1217




 





 





 





 





 





 



   




1218




 





 





 





 





 





 



 If they match, or even if they don't match, you're finished. 




1219




 





 





 





 





 





 



   




1220




 





 





 





 





 





 



 =back 




1221




 





 





 





 





 





 



   




1222




 





 





 





 





 





 



 =head2 Will you implement a 'quirks' mode to handle my special HTML file? 




1223




 





 





 





 





 





 



   




1224




 





 





 





 





 





 



 No, never. 




1225




 





 





 





 





 





 



   




1226




 





 





 





 





 





 



 Help with quirks: L. 




1227




 





 





 





 





 





 



   




1228




 





 





 





 





 





 



 =head2 Is there anything I should be aware of? 




1229




 





 





 





 





 





 



   




1230




 





 





 





 





 





 



 Yes. If your HTML file is not nice, the interpretation of tag nesting will not match 




1231




 





 





 





 





 





 



 your preconceptions. 




1232




 





 





 





 





 





 



   




1233




 





 





 





 





 





 



 In such cases, do not seek to fix the code. Instead, fix your (faulty) preconceptions, and fix your HTML file. 




1234




 





 





 





 





 





 



   




1235




 





 





 





 





 





 



 The 'a' tag, for example, is defined to be an inline tag, but the 'div' tag is a block-level tag. 




1236




 





 





 





 





 





 



   




1237




 





 





 





 





 





 



 I don't define 'a' to be inline, others do, e.g. L and hence HTML::Tagset. 




1238




 





 





 





 





 





 



   




1239




 





 





 





 





 





 



 Inline means: 




1240




 





 





 





 





 





 



   




1241




 





 





 





 





 





 



 	NAME 




1242




 





 





 





 





 





 



   




1243




 





 





 





 





 





 



 will I be parsed as an 'a' containing a 'div'. 




1244




 





 





 





 





 





 



   




1245




 





 





 





 





 





 



 The 'a' tag will be closed before the 'div' is opened. So, the result will look like: 




1246




 





 





 





 





 





 



   




1247




 





 





 





 





 





 



 	NAME 




1248




 





 





 





 





 





 



   




1249




 





 





 





 





 





 



 To achieve what was presumably intended, use 'span': 




1250




 





 





 





 





 





 



   




1251




 





 





 





 





 





 



 	NAME 




1252




 





 





 





 





 





 



   




1253




 





 





 





 





 





 



 Some people (*cough* *cough*) have had to redo their entire websites due to this very problem. 




1254




 





 





 





 





 





 



   




1255




 





 





 





 





 





 



 Of course, this is just one of a vast set of possible problems. 




1256




 





 





 





 





 





 



   




1257




 





 





 





 





 





 



 You have been warned. 




1258




 





 





 





 





 





 



   




1259




 





 





 





 





 





 



 =head2 Why did you use Tree::Simple but not Tree or Tree::Fast or Tree::DAG_Node? 




1260




 





 





 





 





 





 



   




1261




 





 





 





 





 





 



 During testing, Tree::Fast crashed, so I replaced it with Tree and everything worked. Spooky. 




1262




 





 





 





 





 





 



   




1263




 





 





 





 





 





 



 Late news: Tree does not cope with an arrayref stored in the metadata, so I've switched to Tree::DAG_Node. 




1264




 





 





 





 





 





 



   




1265




 





 





 





 





 





 



 Stop press: As an experiment I switched to Tree::Simple. Since it also works I'll just keep using it. 




1266




 





 





 





 





 





 



   




1267




 





 





 





 





 





 



 =head2 Why isn't this module called HTML::Parser::PurePerl? 




1268




 





 





 





 





 





 



   




1269




 





 





 





 





 





 



 =over 4 




1270




 





 





 





 





 





 



   




1271




 





 





 





 





 





 



 =item o The API 




1272




 





 





 





 





 





 



   




1273




 





 





 





 





 





 



 That name sounds like a pure Perl version of the same API as used by HTML::Parser. 




1274




 





 





 





 





 





 



   




1275




 





 





 





 





 





 



 But the API's are not, and are not meant to be, compatible. 




1276




 





 





 





 





 





 



   




1277




 





 





 





 





 





 



 =item o The tie-in 




1278




 





 





 





 





 





 



   




1279




 





 





 





 





 





 



 Some people might falsely assume HTML::Parser can automatically fall back to HTML::Parser::PurePerl in the absence of a compiler. 




1280




 





 





 





 





 





 



   




1281




 





 





 





 





 





 



 =back 




1282




 





 





 





 





 





 



   




1283




 





 





 





 





 





 



 =head2 How do I output my own stuff while traversing the tree? 




1284




 





 





 





 





 





 



   




1285




 





 





 





 





 





 



 =over 4 




1286




 





 





 





 





 





 



   




1287




 





 





 





 





 





 



 =item o The sophisticated way 




1288




 





 





 





 





 





 



   




1289




 





 





 





 





 





 



 As always with OO code, sub-class! In this case, you write a new version of the traverse() method. 




1290




 





 





 





 





 





 



   




1291




 





 





 





 





 





 



 See L, for example. It overrides L. 




1292




 





 





 





 





 





 



   




1293




 





 





 





 





 





 



 =item o The crude way 




1294




 





 





 





 





 





 



   




1295




 





 





 





 





 





 



 Alternately, implement another method in your sub-class, e.g. process(), which recurses like traverse(). 




1296




 





 





 





 





 





 



 Then call parse() and process(). 




1297




 





 





 





 





 





 



   




1298




 





 





 





 





 





 



 =back 




1299




 





 





 





 





 





 



   




1300




 





 





 





 





 





 



 =head2 Is the code on github? 




1301




 





 





 





 





 





 



   




1302




 





 





 





 





 





 



 Yes. See: git://github.com/ronsavage/html--parser--simple.git 




1303




 





 





 





 





 





 



   




1304




 





 





 





 





 





 



 =head2 How is the source formatted? 




1305




 





 





 





 





 





 



   




1306




 





 





 





 





 





 



 I edit with UltraEdit. That means, in general, leading 4-space tabs. 




1307




 





 





 





 





 





 



   




1308




 





 





 





 





 





 



 All vertical alignment within lines is done manually with spaces. 




1309




 





 





 





 





 





 



   




1310




 





 





 





 





 





 



 Perl::Critic is off the agenda. 




1311




 





 





 





 





 





 



   




1312




 





 





 





 





 





 



 =head2 Why did you choose Moos? 




1313




 





 





 





 





 





 



   




1314




 





 





 





 





 





 



 For this year's (2012) Google Code-in, I had a quick look at 122 class-building classes, and decided 




1315




 





 





 





 





 





 



 L was suitable, given it is pure-Perl and has the trigger feature I needed. 




1316




 





 





 





 





 





 



   




1317




 





 





 





 





 





 



 See L. 




1318




 





 





 





 





 





 



   




1319




 





 





 





 





 





 



 =head1 Credits 




1320




 





 





 





 





 





 



   




1321




 





 





 





 





 





 



 This Perl HTML parser has been converted from a JavaScript one written by John Resig. 




1322




 





 





 





 





 





 



   




1323




 





 





 





 





 





 



 L. 




1324




 





 





 





 





 





 



   




1325




 





 





 





 





 





 



 Well done John! 




1326




 





 





 





 





 





 



   




1327




 





 





 





 





 





 



 Note also the comments published here: 




1328




 





 





 





 





 





 



   




1329




 





 





 





 





 





 



 L. 




1330




 





 





 





 





 





 



   




1331




 





 





 





 





 





 



 =head1 Author 




1332




 





 





 





 





 





 



   




1333




 





 





 





 





 





 



 C was written by Ron Savage [e-mail]> in 2009. 




1334




 





 





 





 





 





 



   




1335




 





 





 





 





 





 



 Home page: L. 




1336




 





 





 





 





 





 



   




1337




 





 





 





 





 





 



 =head1 Copyright 




1338




 





 





 





 





 





 



   




1339




 





 





 





 





 





 



 Australian copyright (c) 2009 Ron Savage. 




1340




 





 





 





 





 





 



   




1341




 





 





 





 





 





 



 	All Programs of mine are 'OSI Certified Open Source Software'; 




1342




 





 





 





 





 





 



 	you can redistribute them and/or modify them under the terms of 




1343




 





 





 





 





 





 



 	The Artistic License, a copy of which is available at: 




1344




 





 





 





 





 





 



 	http://www.opensource.org/licenses/index.html 




1345




 





 





 





 





 





 



   




1346




 





 





 





 





 





 



 =cut
" align = 'center' summary = 'body'".

Note the leading space.

=item o content

This is an arrayref of bits and pieces of content.

Consider this fragment of HTML:

I did not say I liked debugging.

When parsing 'I did ', the number of child nodes (of ) is 0, since  has not yet been detected.

So, 'I did ' is stored in the 0th element of the arrayref belonging to .

Likewise, 'not' is stored in the 0th element of the arrayref belonging to the node .

Next, ' say I ' is stored in the 1st element of the arrayref belonging to ,
because it follows the 1st child node ().

Likewise, ' debugging' is stored in the 2nd element of the arrayref belonging to .

This way, the input string can be reproduced by successively outputting the elements of the arrayref of content
interspersed with the contents of the child nodes (processed recusively).

Note: If you are processing this tree, never forget that there can be content after the last child node has been closed,
but before the current node is closed.

Note: The DOCTYPE declaration is stored as the 0th element of the content of the root node.

=item o depth

The nesting depth of the tag within the document.

The root is at depth 0, '' is at depth 1, '' and '' are a depth 2, and so on.

It's just there in case you need it.

=item o name

So, the tag '' will mean the name is 'html'.

Tag names are stored in lower-case.

The root of the tree is called 'root', and holds the DOCTYPE, if any, as content.

The root has the node 'html' as the only child, of course.

=item o node_type

This holds 'global' before '' and between '' and '', and after ''.

It holds 'head' for all nodes from '' to '', and holds 'body' from '' to ''.

It's just there in case you need it.

=back

=head2 How are tags and attributes handled?

Tags are stored in lower-case, in a tree managed by L.

Attributes are stored in the same case as in the original HTML.

The root of the tree is returned be L.

=head2 How are HTML comments handled?

They are treated as content. This includes the prefix ''.

=head2 How is DOCTYPE handled?

It is treated as content belonging to the root of the tree.

=head2 How is the XML declaration handled?

It is treated as content belonging to the root of the tree.

=head2 Does this module handle all HTML pages?

No, never.

=head2 Which versions of HTML does this module handle?

Up to V 4.

=head2 What do I do if this module does not handle my HTML page?

Make yourself a nice cup of tea, and then fix your page.

=head2 Does this validate the HTML input?

No.

For example, if you feed in a HTML page without the title tag, this module does not care.

=head2 How do I view the output HTML?

There are various ways.

=over 4

=item o See scripts/parse.html.pl

=item o By installing HTML::Revelation, of course!

Sample output:

L.

=back

=head2 How do I test this module (or my file)?

Preferably, see the previous question, or...

Suggested steps:

Note: There are quite a few files involved. Proceed with caution.

=over 4

=item o Select a HTML file to test

Call this input.html.

=item o Run input.html thru reveal.pl

Reveal.pl ships with HTML::Revelation.

Call the output file output.1.html.

=item o Run input.html thru parse.html.pl

parse.html.pl ships with HTML::Parser::Simple.

Call the output file parsed.html.

=item o Run parsed.html thru reveal.pl

Call the output file output.2.html.

=item o Compare output.1.html and output.2.html

If they match, or even if they don't match, you're finished.

=back

=head2 Will you implement a 'quirks' mode to handle my special HTML file?

No, never.

Help with quirks: L.

=head2 Is there anything I should be aware of?

Yes. If your HTML file is not nice, the interpretation of tag nesting will not match
your preconceptions.

In such cases, do not seek to fix the code. Instead, fix your (faulty) preconceptions, and fix your HTML file.

The 'a' tag, for example, is defined to be an inline tag, but the 'div' tag is a block-level tag.

I don't define 'a' to be inline, others do, e.g. L and hence HTML::Tagset.

Inline means:

	NAME

will I be parsed as an 'a' containing a 'div'.

The 'a' tag will be closed before the 'div' is opened. So, the result will look like:

	NAME

To achieve what was presumably intended, use 'span':

	NAME

Some people (*cough* *cough*) have had to redo their entire websites due to this very problem.

Of course, this is just one of a vast set of possible problems.

You have been warned.

=head2 Why did you use Tree::Simple but not Tree or Tree::Fast or Tree::DAG_Node?

During testing, Tree::Fast crashed, so I replaced it with Tree and everything worked. Spooky.

Late news: Tree does not cope with an arrayref stored in the metadata, so I've switched to Tree::DAG_Node.

Stop press: As an experiment I switched to Tree::Simple. Since it also works I'll just keep using it.

=head2 Why isn't this module called HTML::Parser::PurePerl?

=over 4

=item o The API

That name sounds like a pure Perl version of the same API as used by HTML::Parser.

But the API's are not, and are not meant to be, compatible.

=item o The tie-in

Some people might falsely assume HTML::Parser can automatically fall back to HTML::Parser::PurePerl in the absence of a compiler.

=back

=head2 How do I output my own stuff while traversing the tree?

=over 4

=item o The sophisticated way

As always with OO code, sub-class! In this case, you write a new version of the traverse() method.

See L, for example. It overrides L.

=item o The crude way

Alternately, implement another method in your sub-class, e.g. process(), which recurses like traverse().
Then call parse() and process().

=back

=head2 Is the code on github?

Yes. See: git://github.com/ronsavage/html--parser--simple.git

=head2 How is the source formatted?

I edit with UltraEdit. That means, in general, leading 4-space tabs.

All vertical alignment within lines is done manually with spaces.

Perl::Critic is off the agenda.

=head2 Why did you choose Moos?

For this year's (2012) Google Code-in, I had a quick look at 122 class-building classes, and decided
L was suitable, given it is pure-Perl and has the trigger feature I needed.

See L.

=head1 Credits

This Perl HTML parser has been converted from a JavaScript one written by John Resig.

L.

Well done John!

Note also the comments published here:

L.

=head1 Author

C was written by Ron Savage [e-mail]> in 2009.

Home page: L.

=head1 Copyright

Australian copyright (c) 2009 Ron Savage.

	All Programs of mine are 'OSI Certified Open Source Software';
	you can redistribute them and/or modify them under the terms of
	The Artistic License, a copy of which is available at:
	http://www.opensource.org/licenses/index.html

=cut