File Coverage

blib/lib/WWW/Crawl4AI/DeepCrawlIterator.pm

Criterion	Covered	Total	%
statement	46	51	90.2
branch	19	20	95.0
condition	15	24	62.5
subroutine	5	9	55.5
pod	3	3	100.0
total	88	107	82.2

package WWW::Crawl4AI::DeepCrawlIterator;
# ABSTRACT: breadth-first iterator for deep_crawl, separating frontier management from crawl logic
use Moo;

our $VERSION = '0.001';


has crawler => (
  is       => 'ro',
  required => 1,
);


has start_url => (
  is       => 'ro',
  required => 1,
);


has max_pages => (
  is      => 'ro',
  default => sub { 25 },
);


has max_depth => (
  is      => 'ro',
  default => sub { 2 },
);


has same_host => (
  is      => 'ro',
  default => sub { 1 },
);


has url_filter => (
  is      => 'ro',
  builder => 1,
);


sub _build_url_filter { undef }

has on_page => (
  is      => 'ro',
  builder => 1,
);


sub _build_on_page { undef }

# Internal state
has _seen        => ( is => 'rwp', default => sub { {} } );
has _queue       => ( is => 'rwp', default => sub { [] } );
has _results     => ( is => 'rwp', default => sub { [] } );
has _start_host  => ( is => 'rw' );
has _crawled => ( is => 'rw', default => sub { 0 } );

sub _canon_url {
  my ( $self, $url ) = @_;
  require URI;
  my $u = eval { URI->new($url) } or return $url;
  $u->fragment(undef);
  return $u->as_string;
}

sub _host_eq {
  my ( $self, $url ) = @_;
  require URI;
  my $host = lc( eval { URI->new($url)->host } // '' );
  return $host eq ( $self->_start_host // '' );
}


sub next {
  my ( $self ) = @_;

  # Seed the queue on first call
  if ( $self->_crawled == 0 && !@{ $self->_queue } ) {
    $self->_push_url( $self->start_url, 0 );
  }

  # Exhausted?
  return undef unless @{ $self->_queue };
  return undef if @{ $self->_results } >= $self->max_pages;

  my $node = shift @{ $self->_queue };
  my $result = $self->crawler->crawl( $node->{url} );
  push @{ $self->_results }, $result;
  $self->_crawled( $self->_crawled + 1 );

  my $depth = $node->{depth};
  $self->on_page->( $result, $depth ) if $self->on_page;

  # Lock start_host from the first real crawl (handles redirects)
  if ( !defined $self->_start_host && $result->final_url ) {
    require URI;
    $self->_start_host(
      lc( eval { URI->new( $result->final_url )->host } // '' ) );
  }

  # Schedule links if not at max depth and result is good
  if ( $depth < $self->max_depth && $result->ok ) {
    for my $url ( @{ $result->urls } ) {
      next if $self->same_host && !$self->_host_eq($url);
      next if $self->url_filter && !$self->url_filter->($url);
      $self->_push_url( $url, $depth + 1 );
    }
  }

  return [ $result, $depth ];
}

sub _push_url {
  my ( $self, $url, $depth ) = @_;
  my $canon = $self->_canon_url($url);
  return if $self->_seen->{$canon}++;
  push @{ $self->_queue }, { url => $url, depth => $depth };
}


sub results { $_[0]->_results }


sub is_exhausted {
  my ( $self ) = @_;
  return !@{ $self->_queue } || @{ $self->_results } >= $self->max_pages;
}

1;

__END__