Crawl4AI/Detect.pm

filter: Statementstmt 85.1% 46 / 54 Branchbran 79.4% 27 / 34 Conditioncond 69.5% 32 / 46 MC/DCmcdc 45.8% 11 / 24 Subroutinesub 88.8% 8 / 9 Podpod 100% 5 / 5 totaltotal 75.0% 129 / 172 CC 36 SCAR 22.9
CC
36
Cov
78%
signals
12.2
_probe_ua
12.0
is_good
11.2
CRAP 9.9
« Crawl4AI/Client.pm ↑ Summary Crawl4AI/DeepCrawlIterator.pm »
1 package WWW::Crawl4AI::Detect;
2 # ABSTRACT: service detection and content-quality classification for Crawl4AI
3 6 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 6 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 6 ▶use LWP::UserAgent ();
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 6 our $VERSION = '0.001';
8  
9  
10 # Default: a result needs at least this many markdown characters to count.
11 6 our $MIN_MARKDOWN = 500;
12  
13 # HTTP status codes that mean "the target pushed back", not "transport broke".
14 6 my %SOFT_FAIL = map { $_ => 1 } ( 401, 403, 429 );
15  
16 # A WAF / bot-management gate (Cloudflare, DataDome, PerimeterX, Akamai) often
17 # does not embed a widget into the requested page -- it REDIRECTS to a dedicated
18 # challenge URL. reCAPTCHA / hCaptcha redirects land on the provider's own
19 # verification endpoint. We key purely on the final (post-redirect) URL's
20 # host+path matching a known challenge endpoint: a real content page's
21 # final_url never contains /cdn-cgi/challenge etc., so URL equality with the
22 # requested URL is irrelevant (and checking it would false-positive on cosmetic
23 # http->https / www<->apex / trailing-slash redirects).
24 6 my $RE_CHALLENGE_CAPTCHA = qr{
25 (?:www\.)?google\.com/recaptcha # reCAPTCHA verification endpoint
26 | /recaptcha/api # reCAPTCHA api2/anchor frame
27 | \bhcaptcha\.com\b # hCaptcha challenge host
28 }ix;
29 6 my $RE_CHALLENGE_WALL = qr{
30 /cdn-cgi/challenge # Cloudflare managed challenge
31 | __cf_chl # Cloudflare challenge query/path token
32 | /challenge-platform/ # Cloudflare challenge-platform asset
33 | datadome # DataDome (host or path)
34 | geo\.captcha-delivery\.com # DataDome captcha delivery host
35 | /px/captcha # PerimeterX captcha path
36 | perimeterx # PerimeterX (host or path)
37 }ix;
38  
39 #----------------------------------------------------------------------
40 # Content classification
41 #----------------------------------------------------------------------
42  
43 140 ▶sub signals {
Subroutine signals: called CC=12 SCAR=25.0
Pod documented
44 140 my ( $page, %opt ) = @_;
45 140 ▶ my $min = defined $opt{min_markdown} ? $opt{min_markdown} : $MIN_MARKDOWN;
Branch: 1 branch
BranchTrueFalseTotal
defined $opt{'min_markdown'} ? : 3 137 140
46 140 ▶ $page ||= {};
Condition50%1/2
$page ||= {}
l!l
1400
MC/DC0%0/1
$page ||= {}
$page
Truth table50%1/2
A
A= $page
Aresult
11
00
47 140 ▶ my $md = $page->{markdown} // '';
Condition50%1/2
$page->{"markdown"} // ""
l!l
1400
MC/DC0%0/1
$page->{"markdown"} // ""
$page->{"markdown"}
Truth table50%1/2
A
A= $page->{"markdown"}
Aresult
11
00
48 140 ▶ my $code = $page->{status_code} // 0;
Condition50%1/2
$page->{"status_code"} // 0
l!l
1400
MC/DC0%0/1
$page->{"status_code"} // 0
$page->{"status_code"}
Truth table50%1/2
A
A= $page->{"status_code"}
Aresult
11
00
49 # The post-redirect URL, falling back to the requested URL when the normalized
50 # page omits it. Either may be absent (signals() is also called on bare test
51 # hashes) -- when so, the challenge-URL checks below simply find no match and
52 # no signal is raised. Never warns/dies on missing keys.
53 140 ▶ my $final = $page->{final_url} // $page->{url} // '';
Condition80%4/5
($page->{"final_url"} // $page->{"url"}) // ""
l!l
6476
$page->{"final_url"} // $page->{"url"}
l!l&&r!l&&!r
64076
MC/DC50%1/2
($page->{"final_url"} // $page->{"url"}) // ""
$page->{"final_url"} $page->{"url"}
Truth table66%2/3
A // B
A= $page->{"final_url"}B= $page->{"url"}
ABresult
1X1
011
000
54  
55 # Content volume is the master signal. A bot-wall / JS-shell / captcha gate
56 # REPLACES the page content -- it is, by definition, thin. So a content-rich
57 # page (>= $min markdown chars) that came back 200 IS the scrape: nothing in
58 # its body text or <title> may discard it. The body/title phrase heuristics
59 # ($RE_BLOCK, $RE_JS, $RE_CAPTCHA body arms, $RE_WALL HTML-token, $RE_TITLE)
60 # were removed in 0.005: on a thin page they were redundant (thin_html already
61 # fails the page), and on a full page they were pure false-positives -- a
62 # 386 KB article carrying Cloudflare's passive __cf_ beacon, or a legit
63 # "Access Denied" <title>, was wrongly thrown away. The ONLY size-independent
64 # block signals kept are the fingerprints a real content page can never carry:
65 # an HTTP push-back status, or a redirect whose final_url is a known WAF /
66 # captcha challenge endpoint (the page physically left the origin).
67 140 my $thin = length($md) < $min;
68  
69 # 'blocked' / 'captcha' fire only when the post-redirect final_url is a known
70 # WAF / captcha challenge endpoint. Not HTTP status -- that lives on the
71 # http_error axis. A site that soft-blocks us by serving one identical
72 # interstitial for every URL (200, no redirect) is caught one level up, by the
73 # caller comparing markdown across the fetched pages -- not here, per-page.
74 140 ▶ my $blocked = ( $final =~ $RE_CHALLENGE_WALL ) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$final =~ /$RE_CHALLENGE_WALL/ ? : 15 125 140
75 140 ▶ my $captcha = ( $final =~ $RE_CHALLENGE_CAPTCHA ) ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$final =~ /$RE_CHALLENGE_CAPTCHA/ ? : 3 137 140
76  
77 return {
78 blocked => $blocked,
79 captcha => $captcha,
80 thin_html => $thin ? 1 : 0,
81 140 ▶ http_error => ( $code >= 500 || $SOFT_FAIL{$code} ) ? 1 : 0,
Branch: 2 branches
BranchTrueFalseTotal
$thin ? : 46 94 140
$code >= 500 || $SOFT_FAIL{$code} ? : 2 138 140
Condition100%3/3
$code >= 500 or $SOFT_FAIL{$code}
l!l&&r!l&&!r
11138
MC/DC100%2/2
$code >= 500 or $SOFT_FAIL{$code}
$code >= 500 $SOFT_FAIL{$code}
Truth table100%3/3
A or B
A= $code >= 500B= $SOFT_FAIL{$code}
ABresult
1X1
011
000
82 };
83 }
84  
85  
86 61 ▶sub is_good {
Subroutine is_good: called CC=11 SCAR=24.2
Pod documented
87 61 my ( $page, %opt ) = @_;
88 61 ▶ return 0 unless ref $page eq 'HASH';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $page eq "HASH" 0 61 61
89 61 ▶ return 0 if defined $page->{success} && !$page->{success};
Branch: 1 branch
BranchTrueFalseTotal
if defined $page->{"success"} and not $page->{"success"} 1 60 61
Condition100%3/3
defined $page->{"success"} and not $page->{"success"}
!ll&&!rl&&r
47131
MC/DC100%2/2
defined $page->{"success"} and not $page->{"success"}
defined $page->{"success"} not $page->{"success"}
Truth table100%3/3
A and not(B)
A= defined $page->{"success"}B= not $page->{"success"}
ABresult
0X0
100
111
90 60 ▶ my $code = $page->{status_code} // 0;
Condition50%1/2
$page->{"status_code"} // 0
l!l
600
MC/DC0%0/1
$page->{"status_code"} // 0
$page->{"status_code"}
Truth table50%1/2
A
A= $page->{"status_code"}
Aresult
11
00
91 60 ▶ return 0 if $code && ( $code >= 500 || $SOFT_FAIL{$code} );
Branch: 1 branch
BranchTrueFalseTotal
if $code and $code >= 500 || $SOFT_FAIL{$code} 1 59 60
Condition66%4/6
$code and $code >= 500 || $SOFT_FAIL{$code}
!ll&&!rl&&r
0591
$code >= 500 or $SOFT_FAIL{$code}
l!l&&r!l&&!r
0159
MC/DC33%1/3
$code and $code >= 500 || $SOFT_FAIL{$code}
$code $code >= 500 $SOFT_FAIL{$code}
Truth table50%2/4
A and B or C
A= $codeB= $code >= 500C= $SOFT_FAIL{$code}
ABCresult
0XX0
1000
11X1
1011
92 59 my $sig = signals( $page, %opt );
93 59 ▶ return 0 if $sig->{blocked} || $sig->{captcha} || $sig->{thin_html};
Branch: 1 branch
BranchTrueFalseTotal
if $sig->{"blocked"} or $sig->{"captcha"} or $sig->{"thin_html"} 23 36 59
Condition100%6/6
$sig->{"blocked"} or $sig->{"captcha"} or $sig->{"thin_html"}
l!l&&r!l&&!r
61736
$sig->{"blocked"} or $sig->{"captcha"}
l!l&&r!l&&!r
5153
MC/DC100%3/3
$sig->{"blocked"} or $sig->{"captcha"} or $sig->{"thin_html"}
$sig->{"blocked"} $sig->{"captcha"} $sig->{"thin_html"}
Truth table100%4/4
A or B or C
A= $sig->{"blocked"}B= $sig->{"captcha"}C= $sig->{"thin_html"}
ABCresult
1XX1
01X1
0011
0000
94 36 return 1;
95 }
96  
97  
98 # Most specific reason first.
99 27 ▶sub why_failed {
Subroutine why_failed: called CC=9 SCAR=22.1
Pod documented
100 27 my ( $page, %opt ) = @_;
101 27 ▶ return 'empty' unless ref $page eq 'HASH';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $page eq "HASH" 0 27 27
102 27 my $sig = signals( $page, %opt );
103 27 ▶ return 'captcha' if $sig->{captcha};
Branch: 1 branch
BranchTrueFalseTotal
if $sig->{"captcha"} 1 26 27
104 26 ▶ return 'bot_wall_detected' if $sig->{blocked};
Branch: 1 branch
BranchTrueFalseTotal
if $sig->{"blocked"} 5 21 26
105 21 ▶ my $code = $page->{status_code} // 0;
Condition50%1/2
$page->{"status_code"} // 0
l!l
210
MC/DC0%0/1
$page->{"status_code"} // 0
$page->{"status_code"}
Truth table50%1/2
A
A= $page->{"status_code"}
Aresult
11
00
106 21 ▶ return "http_$code" if $code && ( $code >= 500 || $SOFT_FAIL{$code} );
Branch: 1 branch
BranchTrueFalseTotal
if $code and $code >= 500 || $SOFT_FAIL{$code} 2 19 21
Condition83%5/6
$code and $code >= 500 || $SOFT_FAIL{$code}
!ll&&!rl&&r
0192
$code >= 500 or $SOFT_FAIL{$code}
l!l&&r!l&&!r
1119
MC/DC66%2/3
$code and $code >= 500 || $SOFT_FAIL{$code}
$code $code >= 500 $SOFT_FAIL{$code}
Truth table75%3/4
A and B or C
A= $codeB= $code >= 500C= $SOFT_FAIL{$code}
ABCresult
0XX0
1000
11X1
1011
107 19 ▶ return 'thin_content' if $sig->{thin_html};
Branch: 1 branch
BranchTrueFalseTotal
if $sig->{"thin_html"} 14 5 19
108 5 return undef;
109 }
110  
111  
112 #----------------------------------------------------------------------
113 # Service detection
114 #----------------------------------------------------------------------
115  
116 0 ▶sub _probe_ua {
Subroutine _probe_ua: not called CC=3 SCAR=24.8
117 0 my ( $ua, $timeout ) = @_;
118 0 ▶ return $ua if $ua;
Branch: 1 branch
BranchTrueFalseTotal
if $ua 0 0 0
119 0 ▶ return LWP::UserAgent->new( agent => "WWW-Crawl4AI/$VERSION", timeout => ( $timeout // 5 ) );
Condition0%0/2
$timeout // 5
l!l
00
MC/DC0%0/1
$timeout // 5
$timeout
Truth table0%0/2
A
A= $timeout
Aresult
11
00
120 }
121  
122 1 ▶sub probe_cloakbrowser {
Subroutine probe_cloakbrowser: called CC=4 SCAR=22.9
Pod documented
123 1 my ( $cdp_url, %opt ) = @_;
124 1 ▶ return 0 unless defined $cdp_url && length $cdp_url;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $cdp_url and length $cdp_url 1 0 1
Condition33%1/3
defined $cdp_url and length $cdp_url
!ll&&!rl&&r
100
MC/DC0%0/2
defined $cdp_url and length $cdp_url
defined $cdp_url length $cdp_url
Truth table33%1/3
A and B
A= defined $cdp_urlB= length $cdp_url
ABresult
0X0
100
111
125 0 ( my $base = $cdp_url ) =~ s{/+$}{};
126 0 $base =~ s{\?.*$}{}; # strip CloakBrowser query params (fingerprint=...)
127 0 my $ua = _probe_ua( $opt{ua}, $opt{timeout} );
128 0 my $res = $ua->get( $base . '/json/version' );
129 0 ▶ return $res->is_success ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$res->is_success ? : 0 0 0
130 }
131  
132  
133 1 ▶sub detect_proxy_env {
Subroutine detect_proxy_env: called CC=2 SCAR=7.6
Pod documented
134 1 ▶ return $ENV{CRAWL4AI_PROXY_URL} || undef;
Condition50%1/2
$ENV{'CRAWL4AI_PROXY_URL'} || undef
l!l
10
MC/DC0%0/1
$ENV{'CRAWL4AI_PROXY_URL'} || undef
$ENV{'CRAWL4AI_PROXY_URL'}
Truth table50%1/2
A
A= $ENV{'CRAWL4AI_PROXY_URL'}
Aresult
11
00
135 }
136  
137  
138 6 1;
139  
140 __END__
« Crawl4AI/Client.pm ↑ Summary Crawl4AI/DeepCrawlIterator.pm »