Crawl4AI/DeepCrawlIterator.pm

filter: Statementstmt 86.4% 64 / 74 Branchbran 95.0% 19 / 20 Conditioncond 62.5% 15 / 24 MC/DCmcdc 46.6% 7 / 15 Subroutinesub 56.2% 9 / 16 Podpod 100% 3 / 3 totaltotal 76.9% 117 / 152 CC 21 SCAR 21.3
CC
21
Cov
83%
next
16.1
is_exhausted
6.0
_host_eq
3.1
CRAP 8.4
« Crawl4AI/Detect.pm ↑ Summary Crawl4AI/Attempt.pm »
1 package WWW::Crawl4AI::DeepCrawlIterator;
2 # ABSTRACT: breadth-first iterator for deep_crawl, separating frontier management from crawl logic
3 5 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 5 our $VERSION = '0.001';
6  
7  
8 5 has crawler => (
9 is => 'ro',
10 required => 1,
11 );
12  
13  
14 5 has start_url => (
15 is => 'ro',
16 required => 1,
17 );
18  
19  
20 has max_pages => (
21 is => 'ro',
22 0 ▶ default => sub { 25 },
Subroutine __ANON__: not called CC=1 SCAR=6.9
23 5 );
24  
25  
26 has max_depth => (
27 is => 'ro',
28 0 ▶ default => sub { 2 },
Subroutine __ANON__: not called CC=1 SCAR=6.9
29 5 );
30  
31  
32 has same_host => (
33 is => 'ro',
34 0 ▶ default => sub { 1 },
Subroutine __ANON__: not called CC=1 SCAR=6.9
35 5 );
36  
37  
38 5 has url_filter => (
39 is => 'ro',
40 builder => 1,
41 );
42  
43  
44 0 ▶sub _build_url_filter { undef }
Subroutine _build_url_filter: not called CC=1 SCAR=6.9
45  
46 5 has on_page => (
47 is => 'ro',
48 builder => 1,
49 );
50  
51  
52 0 ▶sub _build_on_page { undef }
Subroutine _build_on_page: not called CC=1 SCAR=6.9
53  
54 # Internal state
55 7 ▶has _seen => ( is => 'rwp', default => sub { {} } );
Subroutine __ANON__: called CC=1 SCAR=0.0
56 7 ▶has _queue => ( is => 'rwp', default => sub { [] } );
Subroutine __ANON__: called CC=1 SCAR=0.0
57 7 ▶has _results => ( is => 'rwp', default => sub { [] } );
Subroutine __ANON__: called CC=1 SCAR=0.0
58 5 has _start_host => ( is => 'rw' );
59 7 ▶has _crawled => ( is => 'rw', default => sub { 0 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
60  
61 20 ▶sub _canon_url {
Subroutine _canon_url: called CC=2 SCAR=7.0
62 20 my ( $self, $url ) = @_;
63 20 require URI;
64 20 ▶ my $u = eval { URI->new($url) } or return $url;
Branch: 1 branch
BranchTrueFalseTotal
my $u = eval { do { "URI"->new($url) } } or return $url 20 0 20
65 20 $u->fragment(undef);
66 20 return $u->as_string;
67 }
68  
69 14 ▶sub _host_eq {
Subroutine _host_eq: called CC=3 SCAR=11.3
70 14 my ( $self, $url ) = @_;
71 14 require URI;
72 14 ▶ my $host = lc( eval { URI->new($url)->host } // '' );
Condition50%1/2
eval { do { "URI"->new($url)->host } } // ""
l!l
140
MC/DC0%0/1
eval { do { "URI"->new($url)->host } } // ""
eval { do { "URI"->new($url)->host } }
Truth table50%1/2
A
A= eval { do { "URI"->new($url)->host } }
Aresult
11
00
73 14 ▶ return $host eq ( $self->_start_host // '' );
Condition50%1/2
$self->_start_host // ""
l!l
140
MC/DC0%0/1
$self->_start_host // ""
$self->_start_host
Truth table50%1/2
A
A= $self->_start_host
Aresult
11
00
74 }
75  
76  
77 24 ▶sub next {
Subroutine next: called CC=16 SCAR=27.8
Pod documented
78 24 my ( $self ) = @_;
79  
80 # Seed the queue on first call
81 24 ▶ if ( $self->_crawled == 0 && !@{ $self->_queue } ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->_crawled == 0 and not @{$self->_queue;}) 7 17 24
Condition66%2/3
$self->_crawled == 0 and not @{$self->_queue;}
!ll&&!rl&&r
1707
MC/DC50%1/2
$self->_crawled == 0 and not @{$self->_queue;}
$self->_crawled == 0 not @{$self->_queue;}
Truth table66%2/3
A and not(B)
A= $self->_crawled == 0B= not @{$self->_queue;}
ABresult
0X0
100
111
82 7 $self->_push_url( $self->start_url, 0 );
83 }
84  
85 # Exhausted?
86 24 ▶ return undef unless @{ $self->_queue };
Branch: 1 branch
BranchTrueFalseTotal
unless @{$self->_queue;} 6 18 24
87 18 ▶ return undef if @{ $self->_results } >= $self->max_pages;
Branch: 1 branch
BranchTrueFalseTotal
if @{$self->_results;} >= $self->max_pages 1 17 18
88  
89 17 my $node = shift @{ $self->_queue };
90 17 my $result = $self->crawler->crawl( $node->{url} );
91 17 push @{ $self->_results }, $result;
92 17 $self->_crawled( $self->_crawled + 1 );
93  
94 17 my $depth = $node->{depth};
95 17 ▶ $self->on_page->( $result, $depth ) if $self->on_page;
Branch: 1 branch
BranchTrueFalseTotal
if $self->on_page 4 13 17
96  
97 # Lock start_host from the first real crawl (handles redirects)
98 17 ▶ if ( !defined $self->_start_host && $result->final_url ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not defined $self->_start_host and $result->final_url) 7 10 17
Condition66%2/3
not defined $self->_start_host and $result->final_url
!ll&&!rl&&r
1007
MC/DC50%1/2
not defined $self->_start_host and $result->final_url
not defined $self->_start_host $result->final_url
Truth table66%2/3
not(A) and B
A= not defined $self->_start_hostB= $result->final_url
ABresult
0X0
100
111
99 7 require URI;
100 $self->_start_host(
101 7 ▶ lc( eval { URI->new( $result->final_url )->host } // '' ) );
Condition50%1/2
eval { do { "URI"->new($result->final_url)->host } } // ""
l!l
70
MC/DC0%0/1
eval { do { "URI"->new($result->final_url)->host } } // ""
eval { do { "URI"->new($result->final_url)->host } }
Truth table50%1/2
A
A= eval { do { "URI"->new($result->final_url)->host } }
Aresult
11
00
102 }
103  
104 # Schedule links if not at max depth and result is good
105 17 ▶ if ( $depth < $self->max_depth && $result->ok ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($depth < $self->max_depth and $result->ok) 10 7 17
Condition66%2/3
$depth < $self->max_depth and $result->ok
!ll&&!rl&&r
7010
MC/DC50%1/2
$depth < $self->max_depth and $result->ok
$depth < $self->max_depth $result->ok
Truth table66%2/3
A and B
A= $depth < $self->max_depthB= $result->ok
ABresult
0X0
100
111
106 10 for my $url ( @{ $result->urls } ) {
107 15 ▶ next if $self->same_host && !$self->_host_eq($url);
Branch: 1 branch
BranchTrueFalseTotal
if $self->same_host and not $self->_host_eq($url) 1 14 15
Condition100%3/3
$self->same_host and not $self->_host_eq($url)
!ll&&!rl&&r
1131
MC/DC100%2/2
$self->same_host and not $self->_host_eq($url)
$self->same_host not $self->_host_eq($url)
Truth table100%3/3
A and not(B)
A= $self->same_hostB= not $self->_host_eq($url)
ABresult
0X0
100
111
108 14 ▶ next if $self->url_filter && !$self->url_filter->($url);
Branch: 1 branch
BranchTrueFalseTotal
if $self->url_filter and not $self->url_filter->($url) 1 13 14
Condition100%3/3
$self->url_filter and not $self->url_filter->($url)
!ll&&!rl&&r
1211
MC/DC100%2/2
$self->url_filter and not $self->url_filter->($url)
$self->url_filter not $self->url_filter->($url)
Truth table100%3/3
A and not(B)
A= $self->url_filterB= not $self->url_filter->($url)
ABresult
0X0
100
111
109 13 $self->_push_url( $url, $depth + 1 );
110 }
111 }
112  
113 17 return [ $result, $depth ];
114 }
115  
116 20 ▶sub _push_url {
Subroutine _push_url: called CC=2 SCAR=6.9
117 20 my ( $self, $url, $depth ) = @_;
118 20 my $canon = $self->_canon_url($url);
119 20 ▶ return if $self->_seen->{$canon}++;
Branch: 1 branch
BranchTrueFalseTotal
if $self->_seen->{$canon}++ 1 19 20
120 19 push @{ $self->_queue }, { url => $url, depth => $depth };
121 }
122  
123  
124 0 ▶sub results { $_[0]->_results }
Subroutine results: not called CC=1 SCAR=6.9
Pod documented
125  
126  
127 0 ▶sub is_exhausted {
Subroutine is_exhausted: not called CC=2 SCAR=17.9
Pod documented
128 0 my ( $self ) = @_;
129 0 ▶ return !@{ $self->_queue } || @{ $self->_results } >= $self->max_pages;
Condition0%0/3
!@{$self->_queue;} || @{$self->_results;} >= $self->max_pages
l!l&&r!l&&!r
000
MC/DC0%0/2
!@{$self->_queue;} || @{$self->_results;} >= $self->max_pages
!@{$self->_queue;} @{$self->_results;} >= $self->max_pages
Truth table0%0/3
not(A) || B
A= !@{$self->_queue;}B= @{$self->_results;} >= $self->max_pages
ABresult
1X1
011
000
130 }
131  
132 5 1;
133  
134 __END__
« Crawl4AI/Detect.pm ↑ Summary Crawl4AI/Attempt.pm »