File Coverage

blib/lib/WWW/Crawl4AI.pm

Criterion	Covered	Total	%
statement	103	132	78.0
branch	29	48	60.4
condition	10	23	43.4
subroutine	24	34	70.5
pod	14	14	100.0
total	180	251	71.7

package WWW::Crawl4AI;
# ABSTRACT: Perl client and fallback orchestrator for Crawl4AI
use Moo;
use Carp qw( croak );
use Time::HiRes ();
use URI ();
use WWW::Crawl4AI::Client ();
use WWW::Crawl4AI::Result ();
use WWW::Crawl4AI::Attempt ();
use WWW::Crawl4AI::Error ();
use WWW::Crawl4AI::Detect ();
use WWW::Crawl4AI::StrategyChain ();
use WWW::Crawl4AI::DeepCrawlIterator ();

our $VERSION = '0.002';


has strategy_chain => (
  is      => 'lazy',
);


sub _build_strategy_chain {
  my ( $self ) = @_;
  return WWW::Crawl4AI::StrategyChain->new;
}

# Deprecated: kept for backward compat. Returns the applicable strategy list
# the same way the old _build_strategies did.
sub _strategies_for {
  my ( $self ) = @_;
  my $fb = $self->fallback;
  my @all = @{ $self->strategy_chain->strategies };

  if ( ref $fb eq 'ARRAY' ) {
    my %by_name = map { $_->name => $_ } @all;
    return [ grep { $_ && $_->applicable($self) } map { $by_name{$_} } @$fb ];
  }
  if ( !$fb || $fb eq 'none' || $fb eq 'plain' ) {
    return [ grep { $_->name eq 'crawl4ai_plain' } @all ];
  }
  return $self->strategy_chain->applicable($self);
}

has strategies => ( is => 'lazy' );


sub _build_strategies { $_[0]->_strategies_for }

has base_url => (
  is      => 'ro',
  default => sub { $ENV{CRAWL4AI_URL} || $ENV{CRAWL4AI_BASE_URL} || 'http://localhost:11235' },
);


has api_token => (
  is      => 'ro',
  default => sub { $ENV{CRAWL4AI_API_TOKEN} },
);


has cloakbrowser_url => (
  is      => 'ro',
  default => sub { $ENV{CLOAKBROWSER_CDP_URL} },
);


has proxy_url => (
  is      => 'ro',
  default => sub { $ENV{CRAWL4AI_PROXY_URL} },
);


has callback => ( is => 'ro' );


# 'auto' (full applicable chain), 'plain'/'none' (Plain only), or an arrayref
# of backend names to run in that explicit order.
has fallback => ( is => 'ro', default => sub { 'auto' } );


has timeout      => ( is => 'ro', default => sub { 120 } );


has min_markdown => ( is => 'ro' );


has client => ( is => 'lazy' );


sub _build_client {
  my ( $self ) = @_;
  return WWW::Crawl4AI::Client->new(
    base_url  => $self->base_url,
    api_token => $self->api_token,
    timeout   => $self->timeout,
  );
}

sub _normalize_args {
  my ( $self, @args ) = @_;
  return () unless @args;
  # a single hashref: { url => ..., %opts }
  if ( @args == 1 && ref $args[0] eq 'HASH' ) {
    my %a   = %{ $args[0] };
    my $url = delete $a{url};
    return ( $url, %a );
  }
  # a leading positional URL, optionally followed by named options
  if ( @args % 2 == 1 && !ref $args[0] ) {
    my ( $url, %a ) = @args;
    return ( $url, %a );
  }
  # all named: ( url => ..., %opts )
  my %a   = @args;
  my $url = delete $a{url};
  return ( $url, %a );
}

# Build the detect option hash (just min_markdown for now) from a per-call
# %opts plus the instance default. Shared with Net::Async::Crawl4AI.
sub _detect_opts {
  my ( $self, %opts ) = @_;
  my $min = defined $opts{min_markdown} ? $opts{min_markdown} : $self->min_markdown;
  return defined $min ? { min_markdown => $min } : {};
}

# Turn one strategy run — its page, error, and timing — into an Attempt,
# classifying the page via Detect. Shared with Net::Async::Crawl4AI so the sync
# and async chains build identical attempt history.
#
# Classification is performed by overridable methods. To swap in a different
# classifier (e.g. Crawl4AI's own quality score), subclass and override
# classify_signals and classify_why_failed.
sub _attempt_for {
  my ( $self, $strategy, $page, $err, $elapsed, $detect ) = @_;
  $detect ||= {};
  return WWW::Crawl4AI::Attempt->new(
    backend    => $strategy->name,
    cost_class => $strategy->cost_class,
    ok         => 0,
    error      => $err,
    why_failed => 'error',
    elapsed    => $elapsed,
  ) if $err;
  return WWW::Crawl4AI::Attempt->new(
    backend    => $strategy->name,
    cost_class => $strategy->cost_class,
    ok         => 0,
    why_failed => 'empty',
    elapsed    => $elapsed,
  ) unless defined $page;

  my $signals = $self->classify_signals( $page, %$detect );
  my $good    = WWW::Crawl4AI::Detect::is_good( $page, %$detect );
  return WWW::Crawl4AI::Attempt->new(
    backend    => $strategy->name,
    cost_class => $strategy->cost_class,
    ok         => $good,
    page       => $page,
    signals    => $signals,
    why_failed => ( $good ? undef : $self->classify_why_failed( $page, %$detect ) ),
    elapsed    => $elapsed,
  );
}


sub classify_signals {
  my ( $self, $page, %opts ) = @_;
  return WWW::Crawl4AI::Detect::signals( $page, %opts );
}


sub classify_why_failed {
  my ( $self, $page, %opts ) = @_;
  return WWW::Crawl4AI::Detect::why_failed( $page, %opts );
}

sub crawl {
  my ( $self, @args ) = @_;
  my ( $url, %opts ) = $self->_normalize_args(@args);
  croak "crawl needs a url" unless defined $url && length $url;
  my $detect = $self->_detect_opts(%opts);

  my @attempts;
  for my $strategy ( @{ $self->strategies } ) {
    my $t0   = Time::HiRes::time();
    my $page = eval { $strategy->crawl( $self, $url, %opts ) };
    my $err  = $@;
    my $elapsed = sprintf( '%.3f', Time::HiRes::time() - $t0 ) + 0;

    my $attempt = $self->_attempt_for( $strategy, $page, $err, $elapsed, $detect );
    push @attempts, $attempt;
    return WWW::Crawl4AI::Result->from_attempt( $attempt, attempts => \@attempts ) if $attempt->ok;
  }

  return $self->_failed_result( $url, \@attempts );
}


sub markdown { my ( $self, @args ) = @_; return $self->crawl(@args) }

# Drop the fragment so map apps (#5/lat/lon/...) and trailing-anchor links don't
# look like distinct pages during dedup.
sub _canon_url {
  my ( $self, $url ) = @_;
  my $u = eval { URI->new($url) } or return $url;
  $u->fragment(undef);
  return $u->as_string;
}

sub deep_crawl {
  my ( $self, @args ) = @_;
  my ( $start, %opts ) = $self->_normalize_args(@args);
  croak "deep_crawl needs a url" unless defined $start && length $start;

  my $iter = WWW::Crawl4AI::DeepCrawlIterator->new(
    crawler    => $self,
    start_url  => $start,
    max_pages  => ( exists $opts{max_pages}  ? delete $opts{max_pages}  : 25 ),
    max_depth  => ( exists $opts{max_depth}  ? delete $opts{max_depth}  : 2 ),
    same_host  => ( exists $opts{same_host}  ? delete $opts{same_host}  : 1 ),
    url_filter => ( delete $opts{url_filter} ),
    on_page    => ( delete $opts{on_page} ),
  );

  my @results;
  while ( my $page = $iter->next ) {
    push @results, $page->[0];
  }
  return \@results;
}


sub _failed_result {
  my ( $self, $url, $attempts ) = @_;
  my $last = $attempts->[-1];
  return WWW::Crawl4AI::Result->new(
    ok       => 0,
    url      => $url,
    attempts => $attempts,
    error    => WWW::Crawl4AI::Error->new(
      type    => 'content',
      message => 'all crawl strategies failed',
      url     => $url,
    ),
    why_failed => 'no_strategies',
  ) unless $last;

  return WWW::Crawl4AI::Result->new(
    ok         => 0,
    url        => $url,
    final_url  => ( $last->page ? $last->page->{final_url} : undef ),
    status     => ( $last->page ? $last->page->{status_code} : undef ),
    backend    => $last->backend,
    cost_class => $last->cost_class,
    signals    => $last->signals,
    why_failed => $last->why_failed,
    attempts   => $attempts,
    error      => (
      $last->error || WWW::Crawl4AI::Error->new(
        type    => 'content',
        message => 'all crawl strategies failed (last: ' . ( $last->why_failed // 'unknown' ) . ')',
        url     => $url,
        backend => $last->backend,
      )
    ),
  );
}

sub health { $_[0]->client->health }


sub screenshot { my $self = shift; $self->client->screenshot(@_) }
sub pdf        { my $self = shift; $self->client->pdf(@_) }
sub html       { my $self = shift; $self->client->html(@_) }
sub execute_js { my $self = shift; $self->client->execute_js(@_) }
sub llm        { my $self = shift; $self->client->llm(@_) }
sub token      { my $self = shift; $self->client->token(@_) }


sub available_backends { [ map { $_->name } @{ $_[0]->strategies } ] }


sub detect {
  my ( $self ) = @_;
  return {
    crawl4ai         => $self->health,
    crawl4ai_url     => $self->base_url,
    cloakbrowser     => ( $self->cloakbrowser_url ? WWW::Crawl4AI::Detect::probe_cloakbrowser( $self->cloakbrowser_url ) : 0 ),
    cloakbrowser_url => $self->cloakbrowser_url,
    proxy            => ( $self->proxy_url ? 1 : 0 ),
    proxy_url        => $self->proxy_url,
    callback         => ( $self->callback ? 1 : 0 ),
    backends         => $self->available_backends,
  };
}



1;

__END__