File Coverage

blib/lib/WWW/Crawl4AI/Detect.pm

Criterion	Covered	Total	%
statement	38	47	80.8
branch	31	38	81.5
condition	38	57	66.6
subroutine	7	8	87.5
pod	5	5	100.0
total	119	155	76.7

package WWW::Crawl4AI::Detect;
# ABSTRACT: service detection and content-quality classification for Crawl4AI
use strict;
use warnings;

our $VERSION = '0.001';


# Default: a result needs at least this many markdown characters to count.
our $MIN_MARKDOWN = 500;

# HTTP status codes that mean "the target pushed back", not "transport broke".
my %SOFT_FAIL = map { $_ => 1 } ( 401, 403, 429 );

my $RE_JS      = qr/enable\s+javascript|please\s+enable\s+js|requires?\s+javascript/i;
my $RE_BLOCK   = qr/access\s+denied|checking\s+your\s+browser|are\s+you\s+(?:a\s+)?human|verify\s+you\s+are\s+human|unusual\s+traffic/i;
my $RE_CAPTCHA = qr/\b(?:re)?captcha\b|hcaptcha|g-recaptcha|cf-turnstile/i;
my $RE_WALL    = qr/cf-chl|cf_chl|__cf_|datadome|perimeterx|px-captcha|akamai|incapsula|imperva/i;
my $RE_TITLE   = qr/^\s*just\s+a\s+moment|^\s*attention\s+required|^\s*access\s+denied/i;

#----------------------------------------------------------------------
# Content classification
#----------------------------------------------------------------------

sub signals {
  my ( $page, %opt ) = @_;
  my $min = defined $opt{min_markdown} ? $opt{min_markdown} : $MIN_MARKDOWN;
  $page ||= {};
  my $md    = $page->{markdown} // '';
  my $html  = ( $page->{raw_html} // $page->{html} // '' );
  my $title = $page->{title} // '';
  my $code  = $page->{status_code} // 0;

  # 'blocked' is about content fingerprints (bot walls in the body), not HTTP
  # status — status lives on its own axis (http_error) so why_failed can report
  # a bare 403 as http_403 while a Cloudflare body still reads bot_wall_detected.
  my $blocked =
       ( $md   =~ $RE_BLOCK )
    || ( $html =~ $RE_WALL )
    || ( $title =~ $RE_TITLE );

  # A captcha marker only walls the page when its prompt shows up in the
  # *rendered* text (markdown). An embedded widget -- a comment-form reCAPTCHA,
  # a Cloudflare Turnstile login box -- leaves markers only in the HTML/script
  # markup (class names, script src), not the visible content, so an HTML-only
  # match on an otherwise content-rich page is NOT a wall. Treat an HTML-only
  # match as blocking only when the page is also thin (a JS-rendered gate).
  my $thin    = length($md) < $min;
  my $captcha = ( $md =~ $RE_CAPTCHA ) || ( $html =~ $RE_CAPTCHA && $thin );

  return {
    js_required => ( $md =~ $RE_JS ) ? 1 : 0,
    blocked     => $blocked ? 1 : 0,
    captcha     => $captcha ? 1 : 0,
    thin_html   => $thin ? 1 : 0,
    http_error  => ( $code >= 500 || $SOFT_FAIL{$code} ) ? 1 : 0,
  };
}


sub is_good {
  my ( $page, %opt ) = @_;
  return 0 unless ref $page eq 'HASH';
  return 0 if defined $page->{success} && !$page->{success};
  my $code = $page->{status_code} // 0;
  return 0 if $code && ( $code >= 500 || $SOFT_FAIL{$code} );
  my $sig = signals( $page, %opt );
  return 0 if $sig->{js_required} || $sig->{blocked} || $sig->{captcha} || $sig->{thin_html};
  return 1;
}


# Most specific reason first.
sub why_failed {
  my ( $page, %opt ) = @_;
  return 'empty' unless ref $page eq 'HASH';
  my $sig = signals( $page, %opt );
  return 'captcha'           if $sig->{captcha};
  return 'bot_wall_detected' if $sig->{blocked};
  return 'js_required'       if $sig->{js_required};
  my $code = $page->{status_code} // 0;
  return "http_$code"        if $code && ( $code >= 500 || $SOFT_FAIL{$code} );
  return 'thin_content'      if $sig->{thin_html};
  return undef;
}


#----------------------------------------------------------------------
# Service detection
#----------------------------------------------------------------------

sub _probe_ua {
  my ( $ua, $timeout ) = @_;
  return $ua if $ua;
  require LWP::UserAgent;
  return LWP::UserAgent->new( agent => "WWW-Crawl4AI/$VERSION", timeout => ( $timeout // 5 ) );
}

sub probe_cloakbrowser {
  my ( $cdp_url, %opt ) = @_;
  return 0 unless defined $cdp_url && length $cdp_url;
  ( my $base = $cdp_url ) =~ s{/+$}{};
  $base =~ s{\?.*$}{};    # strip CloakBrowser query params (fingerprint=...)
  my $ua  = _probe_ua( $opt{ua}, $opt{timeout} );
  my $res = $ua->get( $base . '/json/version' );
  return $res->is_success ? 1 : 0;
}


sub detect_proxy_env {
  return $ENV{CRAWL4AI_PROXY_URL} || undef;
}


1;

__END__