File Coverage

blib/lib/HTML/Clean.pm

Criterion	Covered	Total	%
statement	183	202	90.5
branch	52	94	55.3
condition	8	12	66.6
subroutine	20	22	90.9
pod	7	7	100.0
total	270	337	80.1

package HTML::Clean;

use Carp;
use IO::File;
use Fcntl;
use strict;
require 5.004;

use vars qw($VERSION @ISA @EXPORT @EXPORT_OK);

require Exporter;
require AutoLoader;

# Items to export to callers namespace
@EXPORT = qw();

$VERSION = '1.2';

=pod

=head1 NAME

HTML::Clean - Cleans up HTML code for web browsers, not humans

=head1 SYNOPSIS

  use HTML::Clean;
  $h = HTML::Clean->new($filename); # or..
  $h = HTML::Clean->new($htmlcode);

  $h->compat();
  $h->strip();
  $data = $h->data();
  print $$data;

=head1 DESCRIPTION

The HTML::Clean module encapsulates a number of common techniques for
minimizing the size of HTML files.  You can typically save between
10% and 50% of the size of a HTML file using these methods.
It provides the following features:

=over 8

=item Remove unneeded whitespace (begining of line, etc)

=item Remove unneeded META elements.

=item Remove HTML comments (except for styles, javascript and SSI)

=item Replace tags with equivilant shorter tags ( --> )

=item etc.

=back

The entire proces is configurable, so you can pick and choose what you want
to clean.

=cut

=head1 THE HTML::Clean CLASS

=head2 $h = HTML::Clean->new($dataorfile, [$level]);

This creates a new HTML::Clean object.  A Prerequisite for all other
functions in this module.

The $dataorfile parameter supplies the input HTML, either a filename,
or a reference to a scalar value holding the HTML, for example:

  $h = HTML::Clean->new("/htdocs/index.html");
  $html = "Hello!";
  $h = HTML::Clean->new(\$html);

An optional 'level' parameter controls the level of optimization
performed.  Levels range from 1 to 9.  Level 1 includes only simple
fast optimizations.  Level 9 includes all optimizations.

=cut

sub new {
  my $this = shift;
  my $class = ref($this) || $this;
  my $self = {};
  bless $self, $class;

  my $data = shift;
  my $level = shift;

  if ($self->initialize($data)) {
    # set the default level
    $level = 9 if (!$level);
    $self->level($level);
    return $self;
  } else {
    undef $self;
    return undef;
  }
}


#
# Set up the data in the self hash..
#

=head2 $h->initialize($dataorfile)

This function allows you to reinitialize the HTML data used by the
current object.  This is useful if you are processing many files.

$dataorfile has the same usage as the new method.

Return 0 for an error, 1 for success.

=cut

sub initialize {
  my($self, $data) = @_;
  $self->{'DATA'} = undef;

  # Not defined?  Just return true.
  return(1) if (!$data);

  # Check if it's a ref
  if (ref($data)) {
    $self->{DATA} = $data;
    return(1);
  }

  # Newline char, really an error, but just go with it..
  if ($data =~ /\n/) {
    $self->{'DATA'} = \$data;
  }

  # No newline?  Must be a filename
  if (-f $data) {
    my $storage;

    sysopen(IN, "$data", O_RDONLY) || return(0);
    while () {
      $storage .= $_;
    }
    close(IN);
    $self->{'DATA'} = \$storage;
    return(1);
  }

  return(0);  # file not found?
}


=head2 $h->level([$level])

Get/set the optimization level.  $level is a number from 1 to 9.

=cut

sub level {
  my($self, $level) = @_;

  if (defined($level) && ($level > 0) && ($level < 10)) {
    $self->{'LEVEL'} = $level
  }
  return($self->{'LEVEL'});
}

=head2 $myref = $h->data()

Returns the current HTML data as a scalar reference.

=cut

sub data {
  my($self) = @_;

  return $self->{'DATA'};
}


# Junk HTML comments (INTERNAL)

sub _commentcheck($) {
  my($comment) = @_;

  $_ = $comment;

  # Server side include
  return($comment) if (m,^$,si);
  return($comment) if (m,navigator\.app(name|version),si);

  # Stylesheet
  return($comment) if (m,[A-z0-9]+\:[A-z0-9]+\s*\{.*\},si);
  return('');
}


# Remove javascript comments (INTERNAL)

sub _jscomments {
  my($js) = @_;

  $js =~ s,\n\s*//.*?\n,\n,sig;
  $js =~ s,\s+//.*?\n,\n,sig;

  # insure javascript is hidden

  if ($js =~ m,\n,si;
  }
  return($js);
}

# Clean up other javascript stuff..

sub _javascript {
  my($js) = @_;

  # remove excess whitespace at the beginning and end of lines
  $js =~ s,\s*\n+\s*,\n,sig;

  # braces/semicolon at end of line, join next line
  $js =~ s,([;{}])\n,$1,sig;

  # What else is safe to do?

  return($js);
}

# replace #000000 -> black, etc..
# Does the browser render faster with RGB?  You would think so..

sub _defcolorcheck ($) {
  my($c) = @_;

  $c =~ s/\#000000/black/;
  $c =~ s/\#c0c0c0/silver/i;
  $c =~ s/\#808080/gray/;
  $c =~ s/\#ffffff/white/i;
  $c =~ s/\#800000/maroon/;
  $c =~ s/\#ff0000/red/i;
  $c =~ s/\#800080/purple/;
  $c =~ s/\#ff00ff/fuchsia/i;
  $c =~ s/\#ff00ff/fuchsia/i;
  $c =~ s/\#008000/green/;
  $c =~ s/\#00ff00/lime/i;
  $c =~ s/\#808000/olive/;
  $c =~ s/\#ffff00/yellow/i;
  $c =~ s/\#000080/navy/;
  $c =~ s/\#0000ff/blue/i;
  $c =~ s/\#008080/teal/i;
  $c =~ s/\#00ffff/aqua/i;
  return($c);
}

# For replacing entities with numerics
use vars qw/ %_ENTITIES/;
%_ENTITIES =  (
   'Agrave' => 192,
   'Aacute' => 193,
   'Acirc' => 194,
   'Atilde' => 195,
   'Auml' => 196,
   'Aring' => 197,
   'AElig' => 198,
   'Ccedil' => 199,
   'Egrave' => 200,
   'Eacute' => 201,
   'Ecirc' => 202,
   'Euml' => 203,
   'Igrave' => 204,
   'Iacute' => 205,
   'Icirc' => 206,
   'Iuml' => 207,
   'ETH' => 208,
   'Ntilde' => 209,
   'Ograve' => 210,
   'Oacute' => 211,
   'Ocirc' => 212,
   'Otilde' => 213,
   'Ouml' => 214,
   'Oslash' => 216,
   'Ugrave' => 217,
   'Uacute' => 218,
   'Ucirc' => 219,
   'Uuml' => 220,
   'Yacute' => 221,
   'THORN' => 222,
   'szlig' => 223,
   'agrave' => 224,
   'aacute' => 225,
   'acirc' => 226,
   'atilde' => 227,
   'auml' => 228,
   'aring' => 229,
   'aelig' => 230,
   'ccedil' => 231,
   'egrave' => 232,
   'eacute' => 233,
   'ecirc' => 234,
   'euml' => 235,
   'igrave' => 236,
   'iacute' => 237,
   'icirc' => 238,
   'iuml' => 239,
   'eth' => 240,
   'ntilde' => 241,
   'ograve' => 242,
   'oacute' => 243,
   'ocirc' => 244,
   'otilde' => 245,
   'ouml' => 246,
   'oslash' => 248,
   'ugrave' => 249,
   'uacute' => 250,
   'ucirc' => 251,
   'uuml' => 252,
   'yacute' => 253,
   'thorn' => 254,
   'yuml' => 255
);

=head2 strip(\%options);

Removes excess space from HTML

You can control the optimizations used by specifying them in the
%options hash reference.

The following options are recognized:

=over 8

=item boolean values (0 or 1 values)

  whitespace    Remove excess whitespace
  shortertags    -> , etc..
  blink         No blink tags.
  contenttype   Remove default contenttype.
  comments      Remove excess comments.
  entities      " -> ", etc.
  dequote       remove quotes from tag parameters where possible.
  defcolor      recode colors in shorter form. (#ffffff -> white, etc.)
  javascript    remove excess spaces and newlines in javascript code.
  htmldefaults  remove default values for some html tags
  lowercasetags translate all HTML tags to lowercase

=item parameterized values

  meta        Takes a space separated list of meta tags to remove,
              default "GENERATOR FORMATTER"

  emptytags   Takes a space separated list of tags to remove when there is no
              content between the start and end tag, like this: .
              The default is 'b i font center'

=back

=cut

use vars qw/
	  $do_whitespace
	  $do_shortertags
	  $do_meta
	  $do_blink
	  $do_contenttype
	  $do_comments
	  $do_entities
	  $do_dequote
          $do_defcolor
          $do_emptytags
          $do_javascript
          $do_htmldefaults
          $do_lowercasetags
          $do_defbaseurl
  /;

$do_whitespace  = 1;
$do_shortertags = 1;
$do_meta        = "generator formatter";
$do_blink       = 1;
$do_contenttype = 1;
$do_comments    = 1;
$do_entities    = 1;
$do_dequote     = 1;
$do_defcolor    = 1;
$do_emptytags   = 'b i font center';
$do_javascript  = 1;
$do_htmldefaults  = 1;
$do_lowercasetags = 1;
$do_defbaseurl  = '';

sub strip {
  my($self, $options) = @_;

  my $h = $self->{'DATA'};
  my $level = $self->{'LEVEL'};

  # Select a set of options based on $level, and then modify based on
  # user supplied options.

  _level_defaults($level);

  if(defined($options)) {
    no strict 'refs';
    for (keys(%$options)) {
      ${"do_" . lc($_)} = $options->{$_} if defined ${"do_" . lc($_)};
    }
  }

  if ($do_shortertags) {
    $$h =~ s,,,sgi;
    $$h =~ s,,,sgi;
    $$h =~ s,,,sgi;
    $$h =~ s,,,sgi;
  }

  if ($do_whitespace) {
    $$h =~ s,[\r\n]+,\n,sg; # Carriage/LF -> LF
    $$h =~ s,\s+\n,\n,sg;   # empty line
    $$h =~ s,\n\s+<,\n<,sg; # space before tag
    $$h =~ s,\n\s+,\n ,sg;  # other spaces

    $$h =~ s,>\n\s*<,><,sg; # LF/spaces between tags..

    # Remove excess spaces within tags.. note, we could parse out the elements
    # and rewrite for excess spaces between elements.  perhaps next version.
    # removed due to problems with > and < in tag elements..
    #$$h =~ s,\s+>,>,sg;
    #$$h =~ s,<\s+,<,sg;
    # do this again later..
  }

  if ($do_entities) {
    $$h =~ s,",\",sg;
    # Simplify long entity names if using default charset...
    $$h =~ m,charset=([^\"]+)\",;
    if (!defined($1) || ($1 eq 'iso-8859-1')) {
      $$h =~ s,&([A-z]+);,($_ENTITIES{$1}) ? chr($_ENTITIES{$1}) : $&,sige;
    }
  }

  if ($do_meta) {
    foreach my $m (split(/\s+/, $do_meta)) {
      $$h =~ s,]*?>,,sig;
    }
  }
  if ($do_contenttype) {
    # Don't need this, since it is the default for most web servers
    # Also gets rid of 'blinking pages' in older versions of netscape.
    $$h =~ s,,,sig;
  }

  if ($do_defcolor) {
    $$h =~ s,(<[^<]+?color=['"]?\#[0-9A-Fa-f]+["']?),_defcolorcheck($&),sige;
  }
  if ($do_comments) {
    # don't strip server side includes..
    # try not to get javascript, or styles...
    $$h =~ s,,_commentcheck($&),sige;

    # Remove javascript comments
    $$h =~ s,]*(java|ecma)script[^>]*>.*?,_jscomments($&),sige;
  }

  if ($do_javascript) {
    #
    $$h =~ s,]*(java|ecma)script[^>]*>.*?,_javascript($&),sige;
  }

  if ($do_blink) {
    $$h =~ s,,,sgi;
    $$h =~ s,,,sgi;
  }

  if ($do_dequote) {
    while ($$h =~ s,<([A-z]+ [A-z]+=)(['"])([A-z0-9]+)\2(\s*?[^>]*?>),<$1$3$4,sig)
      {
	# Remove alphanumeric quotes.  Note, breaks DTD..
	;
      }
  }
  # remove , etc..
  if ($do_emptytags) {
     my $pat = $do_emptytags;
     $pat =~ s/\s+/|/g;

     while ($$h =~ s,<($pat)(\s+[^>]*?)?>\s*,,siog){}

  }
  if ($do_htmldefaults) {
     # Tables
     # seems to break things..
     #$$h =~ s,(]*)\s+border=0([^>]*>),$1$2,sig;
     $$h =~ s,(]*)\s+rowspan=1([^>]*>),$1$2,sig;
     $$h =~ s,(]*)\s+colspan=1([^>]*>),$1$2,sig;

     #

     # P, TABLE tags are default left aligned..
     # lynx is inconsistent in this manner though..

     $$h =~ s,<(P|table|td)( [^>]*)align=\"?left\"?([^>]*)>,<$1$2$3>,sig;

     # OL start=1
     $$h =~ s,(]*)start=\"?1\"?([^>]*>),$1$2,sig;

     # FORM
     $$h =~ s,(]*)method=\"?get\"?([^>]*>),$1$2,sig;
     $$h =~ s,(]*)enctype=\"application/x-www-form-urlencoded\"([^>]*>),$1$2,sig;

     # hr
     $$h =~ s,(]*)align=\"?center\"?([^>]*>),$1$2,sig;
     $$h =~ s,(]*)width=\"?100%\"?([^>]*>),$1$2,sig;

     # URLs
     $$h =~ s,(href|src)(=\"?http://[^/:]+):80/,$1$2/,sig;
  }

  if ($do_whitespace) {
    # remove space within tags  becomes
    $$h =~ s,\s+>,>,sg;
    $$h =~ s,<\s+,<,sg;
    # join lines with a space at the beginning/end of the line
    # and a line that begins with a tag
    $$h =~ s,>\n ,> ,sig;
    $$h =~ s, \n<, <,sig;
  }

  if ($do_lowercasetags) {
    # translate tags to lowercase to (hopefully) improve compressability..

    # simple tags ,  etc.
    $$h =~ s,(<[/]?[a-zA-Z][a-zA-Z0-9_-]*\s*>),\L$1\E,sg;

    # the rest..
    $$h =~ s/(<[a-zA-Z][a-zA-Z0-9_-]*)(\s+.*?>)/_lowercasetag($1,$2)/sge;
  }
}

sub _lowercasetag {
  my($prefix, $body) = @_;
  $prefix =~ s/^(.+)$/\L$1\E/;
  $body =~ s/(\s+[a-zA-Z][a-zA-Z0-9_-]*)(\s*=\s*[^"\s]+|\s*=\s*"[^"]*"|>|\s)/\L$1\E$2/sg;
  return $prefix.$body;
}

# set options based on the level provided.. INTERNAL

sub _level_defaults($) {
  my ($level) = @_;

  $do_whitespace  = 1; # always do this...

  # level 2
  $do_shortertags = ($level > 1) ? 1 : 0;
  $do_meta        = ($level > 1) ? "generator formatter" : "";
  $do_contenttype = ($level > 1) ? 1 : 0;

  # level 3
  $do_entities    = ($level > 2) ? 1 : 0;
  $do_blink       = ($level > 2) ? 1 : 0;

  # level 4
  $do_comments    = ($level > 3) ? 1 : 0;
  $do_dequote     = ($level > 3) ? 1 : 0;
  $do_defcolor    = ($level > 3) ? 1 : 0;
  $do_emptytags   = ($level > 3) ? 'b i font center' : 0;
  $do_javascript  = ($level > 3) ? 1 : 0;
  $do_htmldefaults = ($level > 3) ? 1 : 0;
  $do_lowercasetags = ($level > 3) ? 1 : 0;

  # higher levels reserved for more intensive optimizations.
}

######################################################################

=head2 compat()

This function improves the cross-platform compatibility of your HTML.
Currently checks for the following problems:

=over 8

=item Insuring all IMG tags have ALT elements.

=item Use of Arial, Futura, or Verdana as a font face.

=item Positioning the  tag immediately after the <head> tag. </td>
</tr>

<tr>
<td  class="h" >
<a name="597">597</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="598">598</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s"> =back </td>
</tr>

<tr>
<td  class="h" >
<a name="599">599</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="600">600</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s"> =cut </td>
</tr>

<tr>
<td  class="h" >
<a name="601">601</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="602">602</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s"> sub compat { </td>
</tr>

<tr>
<td  class="h" >
<a name="603">603</a>
</td>

<td  class="c3" >

3

</td>

<td >

 

</td>

<td >

 

</td>

<td  class="c3" >
 <a href="blib-lib-HTML-Clean-pm--subroutine.html#603-1"> 
3
 </a> 
</td>

<td  class="c3" >
 <a href="blib-lib-HTML-Clean-pm--subroutine.html#603-1"> 
1
 </a> 
</td>

<td >

13

</td>

<td class="s">   my($self, $level, $options) = @_; </td>
</tr>

<tr>
<td  class="h" >
<a name="604">604</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="605">605</a>
</td>

<td  class="c3" >

3

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

5

</td>

<td class="s">   my $h = $self->{'DATA'}; </td>
</tr>

<tr>
<td  class="h" >
<a name="606">606</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="607">607</a>
</td>

<td  class="c3" >

3

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

8

</td>

<td class="s">   $$h =~ s/face="arial"/face="arial,helvetica,sansserif"/sgi; </td>
</tr>

<tr>
<td  class="h" >
<a name="608">608</a>
</td>

<td  class="c3" >

3

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

8

</td>

<td class="s">   $$h =~ s/face="(verdana|futura)"/face="$1,arial,helvetica,sansserif"/sgi; </td>
</tr>

<tr>
<td  class="h" >
<a name="609">609</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="610">610</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   # insure that <title> tag is directly after the <head> tag </td>
</tr>

<tr>
<td  class="h" >
<a name="611">611</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   # Some search engines only search the first N chars. (PLweb for instance..) </td>
</tr>

<tr>
<td  class="h" >
<a name="612">612</a>
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td class="s">   </td>
</tr>

<tr>
<td  class="h" >
<a name="613">613</a>
</td>

<td  class="c3" >

3

</td>

<td  class="c0" >
 <a href="blib-lib-HTML-Clean-pm--branch.html#613-1"> 
 50
 </a> 
</td>

<td >

 

</td>

<td >

 

</td>

<td >

 

</td>

<td >

9

</td>

<td class="s">   if ($$h =~ s,<title>(.*),,si) {
    my $title = $1;
    $$h =~ s,,$title,si;
  }

  # Look for IMG without ALT tags.
  $$h =~ s/(]+>)/_imgalt($1)/segi;
}

sub _imgalt {
  my($tag) = @_;

  $tag =~ s/>/ alt="">/ if ($tag !~ /alt=/i);
  return($tag);
}

=head2 defrontpage();

This function converts pages created with Microsoft Frontpage to
something a Unix server will understand a bit better.  This function
currently does the following:

=over 8

=item Converts Frontpage 'hit counters' into a unix specific format.

=item Removes some frontpage specific html comments

=back

=cut


sub defrontpage {
  my($self) = @_;

  my $h = $self->{'DATA'};

  while ($$h =~ s,,,xis) {
      print "Converted a Hitcounter.. $1, $2, $3\n";
  }
  $$h =~ s,,,sgx;
}


=head1 SEE ALSO

=head2 Modules

FrontPage::Web, FrontPage::File

=head2 Web Sites

=over 6

=item Distribution Site - http://people.itu.int/~lindner/

=back

=head1 AUTHORS and CO-AUTHORS

Paul Lindner for the International Telecommunication Union (ITU)

Pavel Kuptsov

=head1 COPYRIGHT

The HTML::Strip module is Copyright (c) 1998,99 by the ITU, Geneva Switzerland.
All rights reserved.

You may distribute under the terms of either the GNU General Public
License or the Artistic License, as specified in the Perl README file.

=cut

1;
__END__