Crawl4AI.pm

filter: Statementstmt 81.0% 124 / 153 Branchbran 62.0% 31 / 50 Conditioncond 45.1% 14 / 31 MC/DCmcdc 10.5% 2 / 19 Subroutinesub 75.6% 31 / 41 Podpod 100% 15 / 15 totaltotal 70.2% 217 / 309 CC 38 SCAR 16.5
CC
38
Cov
72%
detect
20.0
_normalize_args
11.1
_failed_result
8.3
CRAP 5.2
« Crawl4AI/Attempt.pm ↑ Summary Crawl4AI/Result.pm »
1 package WWW::Crawl4AI;
2 # ABSTRACT: Perl client and fallback orchestrator for Crawl4AI
3 5 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 5 ▶use Carp qw( croak );
Subroutine BEGIN: called CC=1 SCAR=0.0
5 5 ▶use Time::HiRes ();
Subroutine BEGIN: called CC=1 SCAR=0.0
6 5 ▶use URI ();
Subroutine BEGIN: called CC=1 SCAR=0.0
7 5 ▶use WWW::Crawl4AI::Client ();
Subroutine BEGIN: called CC=1 SCAR=0.0
8 5 ▶use WWW::Crawl4AI::Result ();
Subroutine BEGIN: called CC=1 SCAR=0.0
9 5 ▶use WWW::Crawl4AI::Attempt ();
Subroutine BEGIN: called CC=1 SCAR=0.0
10 5 ▶use WWW::Crawl4AI::Error ();
Subroutine BEGIN: called CC=1 SCAR=0.0
11 5 ▶use WWW::Crawl4AI::Detect ();
Subroutine BEGIN: called CC=1 SCAR=0.0
12 5 ▶use WWW::Crawl4AI::StrategyChain ();
Subroutine BEGIN: called CC=1 SCAR=0.0
13 5 ▶use WWW::Crawl4AI::DeepCrawlIterator ();
Subroutine BEGIN: called CC=1 SCAR=0.0
14  
15 5 our $VERSION = '0.007';
16  
17  
18 5 has strategy_chain => (
19 is => 'lazy',
20 );
21  
22  
23 18 ▶sub _build_strategy_chain {
Subroutine _build_strategy_chain: called CC=1 SCAR=0.0
24 18 my ( $self ) = @_;
25 18 return WWW::Crawl4AI::StrategyChain->new;
26 }
27  
28 # Deprecated: kept for backward compat. Returns the applicable strategy list
29 # the same way the old _build_strategies did.
30 18 ▶sub _strategies_for {
Subroutine _strategies_for: called CC=6 SCAR=18.1
31 18 my ( $self ) = @_;
32 18 my $fb = $self->fallback;
33 18 my @all = @{ $self->strategy_chain->strategies };
34  
35 18 ▶ if ( ref $fb eq 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $fb eq "ARRAY") 1 17 18
36 1 my %by_name = map { $_->name => $_ } @all;
37 1 ▶ return [ grep { $_ && $_->applicable($self) } map { $by_name{$_} } @$fb ];
Condition66%2/3
$_ && $_->applicable($self)
!ll&&!rl&&r
012
MC/DC50%1/2
$_ && $_->applicable($self)
$_ $_->applicable($self)
Truth table66%2/3
A && B
A= $_B= $_->applicable($self)
ABresult
0X0
100
111
38 }
39 17 ▶ if ( !$fb || $fb eq 'none' || $fb eq 'plain' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $fb or $fb eq "none" or $fb eq "plain") 9 8 17
Condition50%3/6
not $fb or $fb eq "none" or $fb eq "plain"
l!l&&r!l&&!r
098
not $fb or $fb eq "none"
l!l&&r!l&&!r
0017
MC/DC33%1/3
not $fb or $fb eq "none" or $fb eq "plain"
not $fb $fb eq "none" $fb eq "plain"
Truth table50%2/4
not(A) or B or C
A= not $fbB= $fb eq "none"C= $fb eq "plain"
ABCresult
1XX1
01X1
0011
0000
40 9 return [ grep { $_->name eq 'crawl4ai_plain' } @all ];
41 }
42 8 return $self->strategy_chain->applicable($self);
43 }
44  
45 5 has strategies => ( is => 'lazy' );
46  
47  
48 18 ▶sub _build_strategies { $_[0]->_strategies_for }
Subroutine _build_strategies: called CC=1 SCAR=0.0
49  
50 has base_url => (
51 is => 'ro',
52 19 ▶ default => sub { $ENV{CRAWL4AI_URL} || $ENV{CRAWL4AI_BASE_URL} || 'http://localhost:11235' },
Subroutine __ANON__: called CC=3 SCAR=14.2
Branch: 1 branch
BranchTrueFalseTotal
$ENV{"CRAWL4AI_URL"} or $ENV{"CRAWL4AI_BASE_URL"} or "http://localhost:11235" 0 19 19
Condition33%1/3
$ENV{"CRAWL4AI_URL"} or $ENV{"CRAWL4AI_BASE_URL"}
l!l&&r!l&&!r
0019
MC/DC0%0/2
$ENV{"CRAWL4AI_URL"} or $ENV{"CRAWL4AI_BASE_URL"}
$ENV{"CRAWL4AI_URL"} $ENV{"CRAWL4AI_BASE_URL"}
Truth table33%1/3
A or B
A= $ENV{"CRAWL4AI_URL"}B= $ENV{"CRAWL4AI_BASE_URL"}
ABresult
1X1
011
000
53 5 );
54  
55  
56 has api_token => (
57 is => 'ro',
58 19 ▶ default => sub { $ENV{CRAWL4AI_API_TOKEN} },
Subroutine __ANON__: called CC=1 SCAR=0.0
59 5 );
60  
61  
62 has cloakbrowser_url => (
63 is => 'ro',
64 18 ▶ default => sub { $ENV{CLOAKBROWSER_CDP_URL} },
Subroutine __ANON__: called CC=1 SCAR=0.0
65 5 );
66  
67  
68 has proxy_url => (
69 is => 'ro',
70 18 ▶ default => sub { $ENV{CRAWL4AI_PROXY_URL} },
Subroutine __ANON__: called CC=1 SCAR=0.0
71 5 );
72  
73  
74 5 has callback => ( is => 'ro' );
75  
76  
77 # 'auto' (full applicable chain), 'plain'/'none' (Plain only), or an arrayref
78 # of backend names to run in that explicit order.
79 9 ▶has fallback => ( is => 'ro', default => sub { 'auto' } );
Subroutine __ANON__: called CC=1 SCAR=0.0
80  
81  
82 19 ▶has timeout => ( is => 'ro', default => sub { 120 } );
Subroutine __ANON__: called CC=1 SCAR=0.0
83  
84  
85 5 has min_markdown => ( is => 'ro' );
86  
87  
88 5 has client => ( is => 'lazy' );
89  
90  
91 0 ▶sub _build_client {
Subroutine _build_client: not called CC=1 SCAR=6.9
92 0 my ( $self ) = @_;
93 0 return WWW::Crawl4AI::Client->new(
94 base_url => $self->base_url,
95 api_token => $self->api_token,
96 timeout => $self->timeout,
97 );
98 }
99  
100 34 ▶sub _normalize_args {
Subroutine _normalize_args: called CC=6 SCAR=24.0
101 34 my ( $self, @args ) = @_;
102 34 ▶ return () unless @args;
Branch: 1 branch
BranchTrueFalseTotal
unless @args 0 34 34
103 # a single hashref: { url => ..., %opts }
104 34 ▶ if ( @args == 1 && ref $args[0] eq 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@args == 1 and ref $args[0] eq "HASH") 0 34 34
Condition66%2/3
@args == 1 and ref $args[0] eq "HASH"
!ll&&!rl&&r
8260
MC/DC0%0/2
@args == 1 and ref $args[0] eq "HASH"
@args == 1 ref $args[0] eq "HASH"
Truth table66%2/3
A and B
A= @args == 1B= ref $args[0] eq "HASH"
ABresult
0X0
100
111
105 0 my %a = %{ $args[0] };
106 0 my $url = delete $a{url};
107 0 return ( $url, %a );
108 }
109 # a leading positional URL, optionally followed by named options
110 34 ▶ if ( @args % 2 == 1 && !ref $args[0] ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@args % 2 == 1 and not ref $args[0]) 34 0 34
Condition33%1/3
@args % 2 == 1 and not ref $args[0]
!ll&&!rl&&r
0034
MC/DC0%0/2
@args % 2 == 1 and not ref $args[0]
@args % 2 == 1 not ref $args[0]
Truth table33%1/3
A and not(B)
A= @args % 2 == 1B= not ref $args[0]
ABresult
0X0
100
111
111 34 my ( $url, %a ) = @args;
112 34 return ( $url, %a );
113 }
114 # all named: ( url => ..., %opts )
115 0 my %a = @args;
116 0 my $url = delete $a{url};
117 0 return ( $url, %a );
118 }
119  
120 # Build the detect option hash (just min_markdown for now) from a per-call
121 # %opts plus the instance default. Shared with Net::Async::Crawl4AI.
122 27 ▶sub _detect_opts {
Subroutine _detect_opts: called CC=3 SCAR=11.0
123 27 my ( $self, %opts ) = @_;
124 27 ▶ my $min = defined $opts{min_markdown} ? $opts{min_markdown} : $self->min_markdown;
Branch: 1 branch
BranchTrueFalseTotal
defined $opts{'min_markdown'} ? : 1 26 27
125 27 ▶ return defined $min ? { min_markdown => $min } : {};
Branch: 1 branch
BranchTrueFalseTotal
defined $min ? : 1 26 27
126 }
127  
128 # Turn one strategy run — its page, error, and timing — into an Attempt,
129 # classifying the page via Detect. Shared with Net::Async::Crawl4AI so the sync
130 # and async chains build identical attempt history.
131 #
132 # Classification is performed by overridable methods. To swap in a different
133 # classifier (e.g. Crawl4AI's own quality score), subclass and override
134 # classify_is_good (the win/continue decision), classify_signals (the reported
135 # signals) and classify_why_failed (the failure token).
136 41 ▶sub _attempt_for {
Subroutine _attempt_for: called CC=5 SCAR=16.2
137 41 my ( $self, $strategy, $page, $err, $elapsed, $detect ) = @_;
138 41 ▶ $detect ||= {};
Condition50%1/2
$detect ||= {}
l!l
410
MC/DC0%0/1
$detect ||= {}
$detect
Truth table50%1/2
A
A= $detect
Aresult
11
00
139 41 ▶ return WWW::Crawl4AI::Attempt->new(
Branch: 1 branch
BranchTrueFalseTotal
if $err 2 39 41
140 backend => $strategy->name,
141 cost_class => $strategy->cost_class,
142 ok => 0,
143 error => $err,
144 why_failed => 'error',
145 elapsed => $elapsed,
146 ) if $err;
147 39 ▶ return WWW::Crawl4AI::Attempt->new(
Branch: 1 branch
BranchTrueFalseTotal
unless defined $page 0 39 39
148 backend => $strategy->name,
149 cost_class => $strategy->cost_class,
150 ok => 0,
151 why_failed => 'empty',
152 elapsed => $elapsed,
153 ) unless defined $page;
154  
155 39 my $signals = $self->classify_signals( $page, %$detect );
156 39 my $good = $self->classify_is_good( $page, %$detect );
157 39 ▶ return WWW::Crawl4AI::Attempt->new(
Branch: 1 branch
BranchTrueFalseTotal
$good ? : 25 14 39
158 backend => $strategy->name,
159 cost_class => $strategy->cost_class,
160 ok => $good,
161 page => $page,
162 signals => $signals,
163 why_failed => ( $good ? undef : $self->classify_why_failed( $page, %$detect ) ),
164 elapsed => $elapsed,
165 );
166 }
167  
168  
169 38 ▶sub classify_is_good {
Subroutine classify_is_good: called CC=1 SCAR=0.0
Pod documented
170 38 my ( $self, $page, %opts ) = @_;
171 38 return WWW::Crawl4AI::Detect::is_good( $page, %opts );
172 }
173  
174  
175 39 ▶sub classify_signals {
Subroutine classify_signals: called CC=1 SCAR=0.0
Pod documented
176 39 my ( $self, $page, %opts ) = @_;
177 39 return WWW::Crawl4AI::Detect::signals( $page, %opts );
178 }
179  
180  
181 14 ▶sub classify_why_failed {
Subroutine classify_why_failed: called CC=1 SCAR=0.0
Pod documented
182 14 my ( $self, $page, %opts ) = @_;
183 14 return WWW::Crawl4AI::Detect::why_failed( $page, %opts );
184 }
185  
186 27 ▶sub crawl {
Subroutine crawl: called CC=5 SCAR=16.2
Pod documented
187 27 my ( $self, @args ) = @_;
188 27 my ( $url, %opts ) = $self->_normalize_args(@args);
189 27 ▶ croak "crawl needs a url" unless defined $url && length $url;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $url and length $url 0 27 27
Condition33%1/3
defined $url and length $url
!ll&&!rl&&r
0027
MC/DC0%0/2
defined $url and length $url
defined $url length $url
Truth table33%1/3
A and B
A= defined $urlB= length $url
ABresult
0X0
100
111
190 27 my $detect = $self->_detect_opts(%opts);
191  
192 27 my @attempts;
193 27 for my $strategy ( @{ $self->strategies } ) {
194 41 my $t0 = Time::HiRes::time();
195 41 my $page = eval { $strategy->crawl( $self, $url, %opts ) };
196 41 my $err = $@;
197 41 my $elapsed = sprintf( '%.3f', Time::HiRes::time() - $t0 ) + 0;
198  
199 41 my $attempt = $self->_attempt_for( $strategy, $page, $err, $elapsed, $detect );
200 41 push @attempts, $attempt;
201 41 ▶ return WWW::Crawl4AI::Result->from_attempt( $attempt, attempts => \@attempts ) if $attempt->ok;
Branch: 1 branch
BranchTrueFalseTotal
if $attempt->ok 25 16 41
202 }
203  
204 2 return $self->_failed_result( $url, \@attempts );
205 }
206  
207  
208 10 ▶sub markdown { my ( $self, @args ) = @_; return $self->crawl(@args) }
Subroutine markdown: called CC=1 SCAR=0.0
Pod documented
209  
210 # Drop the fragment so map apps (#5/lat/lon/...) and trailing-anchor links don't
211 # look like distinct pages during dedup.
212 0 ▶sub _canon_url {
Subroutine _canon_url: not called CC=2 SCAR=17.9
213 0 my ( $self, $url ) = @_;
214 0 ▶ my $u = eval { URI->new($url) } or return $url;
Branch: 1 branch
BranchTrueFalseTotal
my $u = eval { do { "URI"->new($url) } } or return $url 0 0 0
215 0 $u->fragment(undef);
216 0 return $u->as_string;
217 }
218  
219 7 ▶sub deep_crawl {
Subroutine deep_crawl: called CC=7 SCAR=19.9
Pod documented
220 7 my ( $self, @args ) = @_;
221 7 my ( $start, %opts ) = $self->_normalize_args(@args);
222 7 ▶ croak "deep_crawl needs a url" unless defined $start && length $start;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $start and length $start 0 7 7
Condition33%1/3
defined $start and length $start
!ll&&!rl&&r
007
MC/DC0%0/2
defined $start and length $start
defined $start length $start
Truth table33%1/3
A and B
A= defined $startB= length $start
ABresult
0X0
100
111
223  
224 my $iter = WWW::Crawl4AI::DeepCrawlIterator->new(
225 crawler => $self,
226 start_url => $start,
227 max_pages => ( exists $opts{max_pages} ? delete $opts{max_pages} : 25 ),
228 max_depth => ( exists $opts{max_depth} ? delete $opts{max_depth} : 2 ),
229 same_host => ( exists $opts{same_host} ? delete $opts{same_host} : 1 ),
230 url_filter => ( delete $opts{url_filter} ),
231 7 ▶ on_page => ( delete $opts{on_page} ),
Branch: 3 branches
BranchTrueFalseTotal
exists $opts{'max_pages'} ? : 1 6 7
exists $opts{'max_depth'} ? : 7 0 7
exists $opts{'same_host'} ? : 1 6 7
232 );
233  
234 7 my @results;
235 7 ▶ while ( my $page = $iter->next ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $page = $iter->next) 17 7 24
236 17 push @results, $page->[0];
237 }
238 7 return \@results;
239 }
240  
241  
242 2 ▶sub _failed_result {
Subroutine _failed_result: called CC=6 SCAR=21.2
243 2 my ( $self, $url, $attempts ) = @_;
244 2 my $last = $attempts->[-1];
245 2 ▶ return WWW::Crawl4AI::Result->new(
Branch: 1 branch
BranchTrueFalseTotal
unless $last 0 2 2
246 ok => 0,
247 url => $url,
248 attempts => $attempts,
249 error => WWW::Crawl4AI::Error->new(
250 type => 'content',
251 message => 'all crawl strategies failed',
252 url => $url,
253 ),
254 why_failed => 'no_strategies',
255 ) unless $last;
256  
257 return WWW::Crawl4AI::Result->new(
258 ok => 0,
259 url => $url,
260 final_url => ( $last->page ? $last->page->{final_url} : undef ),
261 2 ▶ status => ( $last->page ? $last->page->{status_code} : undef ),
Branch: 2 branches
BranchTrueFalseTotal
$last->page ? : 2 0 2
$last->page ? : 2 0 2
Condition40%2/5
$last->error || 'WWW::Crawl4AI::Error'->new('type', 'content', 'message', 'all crawl strategies failed (last: ' . ($last->why_failed // 'unknown') . ')', 'url', $url, 'backend', $last->backend)
l!l&&r!l&&!r
020
$last->why_failed // "unknown"
l!l
20
MC/DC0%0/2
$last->error || 'WWW::Crawl4AI::Error'->new('type', 'content', 'message', 'all crawl strategies failed (last: ' . ($last->why_failed // 'unknown') . ')', 'url', $url, 'backend', $last->backend)
$last->error 'WWW::Crawl4AI::Error'->new('type', 'content', 'message', 'all crawl strategies failed (last: ' . ($last->why_failed // 'unknown') . ')', 'url', $url, 'backend', $last->backend)
MC/DC0%0/1
$last->why_failed // "unknown"
$last->why_failed
Truth table33%1/3
A || B
A= $last->errorB= 'WWW::Crawl4AI::Error'->new('type', 'content', 'message', 'all crawl strategies failed (last: ' . ($last->why_failed // 'unknown') . ')', 'url', $url, 'backend', $last->backend)
ABresult
1X1
011
000
Truth table50%1/2
A
A= $last->why_failed
Aresult
11
00
262 backend => $last->backend,
263 cost_class => $last->cost_class,
264 signals => $last->signals,
265 why_failed => $last->why_failed,
266 attempts => $attempts,
267 error => (
268 $last->error || WWW::Crawl4AI::Error->new(
269 type => 'content',
270 message => 'all crawl strategies failed (last: ' . ( $last->why_failed // 'unknown' ) . ')',
271 url => $url,
272 backend => $last->backend,
273 )
274 ),
275 );
276 }
277  
278 0 ▶sub health { $_[0]->client->health }
Subroutine health: not called CC=1 SCAR=6.9
Pod documented
279  
280  
281 0 ▶sub screenshot { my $self = shift; $self->client->screenshot(@_) }
Subroutine screenshot: not called CC=1 SCAR=6.9
Pod documented
282 0 ▶sub pdf { my $self = shift; $self->client->pdf(@_) }
Subroutine pdf: not called CC=1 SCAR=6.9
Pod documented
283 0 ▶sub html { my $self = shift; $self->client->html(@_) }
Subroutine html: not called CC=1 SCAR=6.9
Pod documented
284 0 ▶sub execute_js { my $self = shift; $self->client->execute_js(@_) }
Subroutine execute_js: not called CC=1 SCAR=6.9
Pod documented
285 0 ▶sub llm { my $self = shift; $self->client->llm(@_) }
Subroutine llm: not called CC=1 SCAR=6.9
Pod documented
286 0 ▶sub token { my $self = shift; $self->client->token(@_) }
Subroutine token: not called CC=1 SCAR=6.9
Pod documented
287  
288  
289 4 ▶sub available_backends { [ map { $_->name } @{ $_[0]->strategies } ] }
Subroutine available_backends: called CC=1 SCAR=0.0
Pod documented
290  
291  
292 0 ▶sub detect {
Subroutine detect: not called CC=4 SCAR=30.0
Pod documented
293 0 my ( $self ) = @_;
294 return {
295 0 ▶ crawl4ai => $self->health,
Branch: 3 branches
BranchTrueFalseTotal
$self->cloakbrowser_url ? : 0 0 0
$self->proxy_url ? : 0 0 0
$self->callback ? : 0 0 0
296 crawl4ai_url => $self->base_url,
297 cloakbrowser => ( $self->cloakbrowser_url ? WWW::Crawl4AI::Detect::probe_cloakbrowser( $self->cloakbrowser_url ) : 0 ),
298 cloakbrowser_url => $self->cloakbrowser_url,
299 proxy => ( $self->proxy_url ? 1 : 0 ),
300 proxy_url => $self->proxy_url,
301 callback => ( $self->callback ? 1 : 0 ),
302 backends => $self->available_backends,
303 };
304 }
305  
306  
307  
308 5 1;
309  
310 __END__
« Crawl4AI/Attempt.pm ↑ Summary Crawl4AI/Result.pm »