File Coverage

blib/lib/WWW/Leech/Walker.pm

Criterion	Covered	Total	%
statement	23	82	28.0
branch	0	20	0.0
condition	0	2	0.0
subroutine	7	12	58.3
pod	4	4	100.0
total	34	120	28.3

package WWW::Leech::Walker;
use strict;
use LWP;
use URI::URL;
use WWW::Leech::Parser;

BEGIN {
    use Exporter ();
    use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
    $VERSION     = '0.01';
    @ISA         = qw(Exporter);
    @EXPORT      = qw();
    @EXPORT_OK   = qw();
    %EXPORT_TAGS = ();
}

sub new{
	my $class = shift;
	my $this = shift;

	$this->{'url'} = url $this->{'url'};
	$this->{'state'} ||= {};

	$this->{'_stop_flag'} = undef;

	$this = bless $this, __PACKAGE__;

	$this->{'parser'} = new WWW::Leech::Parser($this->{'parser'});

	return $this;
}

sub leech{
	my $this = shift;
	$this->{'_stop_flag'} = undef;

	my $current_url = $this->{'url'};

	while(1){

		$this->log("Getting links list: $current_url ");

		# get and parse link list
		my $data = $this->_get($current_url);

		my $parse_result = $this->{'parser'}->parseList($data);

		my $links = [map { url($_, $this->{'url'})->abs } @{$parse_result->{'links'}}];

		$this->log("Links list length: ".scalar(@$links));

		if(scalar(@$links) == 0){
			$this->log("No items links found, check your XPath: ".$this->{'parser'}->{'item:link'});

			$this->stop();
			return;
		}

		# filtering if required
		if( $this->{'filter'} ){
			$this->log("Filtering links");
			$links = $this->{'filter'}->($links,$this);
			$this->log("Filtered links list length: ".scalar(@$links));

			return if($this->{'_stop_flag'});
		}

		# get and parse individual items
		foreach(@$links){
			$this->log("\tGetting item: $_ ");

			my $item = $this->{'parser'}->parse($this->_get($_));
			$item->{'_url'} = $_->as_string;

			$this->{'processor'}->($item,$this);

			return if($this->{'_stop_flag'});
		}

		if($parse_result->{'next_page'}){

			my $pnurl;

			if($this->{'next_page_link_post_process'}){
				$pnurl = $this->{'next_page_link_post_process'}->($parse_result->{'next_page'}, $this, $current_url);
			}

			if($pnurl){
				$current_url = url($pnurl)->abs;
			} else {
				$current_url = url($parse_result->{'next_page'}, $this->{'url'})->abs;
			}

			return if($this->{'_stop_flag'});

		} else {
			$this->stop("No next page link found");
			return;
		}
	}


}

sub log{
	my $this = shift;
	my $message = shift;

	if($this->{'logger'}){
		$this->{'logger'}->($message,$this);
	}
}

sub stop{
	my $this = shift;
	my $reason = shift;

	$this->{'_stop_flag'} = 1;
	$this->log("\tWalker stopped.");
	if($reason){
		$this->log("\tReason: $reason.");
	}
}

sub _get{
	my $this = shift;
	my $url = shift;

	my $req = new HTTP::Request('GET',$url);
	my $res = $this->{'ua'}->request($req);

	return $res->decoded_content();
}


1;


__END__