File Coverage

blib/lib/WWW/Crawl/Chromium.pm

Criterion	Covered	Total	%
statement	15	64	23.4
branch	0	22	0.0
condition	0	15	0.0
subroutine	5	7	71.4
pod			n/a
total	20	108	18.5

package WWW::Crawl::Chromium;

use strict;
use warnings;

use parent 'WWW::Crawl';

use Carp qw(croak);
use IPC::Run qw(start timeout);

our $VERSION = '0.5';
# $VERSION = eval $VERSION;

sub _fetch_page {
    my ($self, $url) = @_;

    my $chromium_path = $self->{'chromium_path'}
        || $self->{'chrome_path'}
        || $self->{'chromium'}
        || 'chromium';
    my $timeout = $self->{'chromium_timeout'} // 120;
    my $virtual_time_budget = $self->{'chromium_time_budget'} // 20000;
    my $proxy = $self->{'proxy'} // 'direct://';

    my $chrome_ua = $self->{'agent'} // "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36";
    my $retry_count = $self->{'retry_count'} // 0;

    my @base_command = (
        $chromium_path,
        '--headless=new',
        qq{--proxy-server=$proxy},
        qq{--user-agent=$chrome_ua},
        '--window-size=1920,1080',
        '--no-sandbox',
        '--disable-gpu',
        '--disable-dev-shm-usage',
        '--disable-software-rasterizer',
        '--disable-features=VaapiVideoDecoder',
        '--disable-site-isolation-trials',
        '--password-store=basic',
        '--use-mock-keychain',
        '--log-level=3',
        '--no-first-run',
        '--no-default-browser-check',
        '--run-all-compositor-stages-before-draw',
        "--virtual-time-budget=$virtual_time_budget",
        '--dump-dom',
        $url,
    );

    my $result;
    for my $attempt (0 .. $retry_count) {
        if ($attempt > 0) {
            print STDERR "DEBUG: Retry attempt $attempt for $url\n" if $self->{'debug'};
            sleep 1;
        }

        $result = $self->_execute_chromium(\@base_command, $timeout);
        last if $result->{'success'};
    }

    return $result;
}

sub _execute_chromium {
    my ($self, $command, $timeout) = @_;
    my @command = @$command;

    my $content = '';
    my $error_output = '';
    my $timed_out = 0;
    my $launch_error = '';

    my $exit_code = 0;
    my $harness = eval {
        start \@command, '<', \undef, '>', \$content, '2>', \$error_output, timeout($timeout);
    };
    if ($@) {
        $launch_error = $@;
    } elsif (!$harness) {
        $launch_error = 'Unknown failure starting Chromium';
    } else {
        eval { $harness->finish; 1 };
        if ($@) {
            if ($@ =~ /timeout/i) {
                $timed_out = 1;
                $harness->kill_kill;
            } else {
                $launch_error = $@;
                $harness->kill_kill;
            }
        } else {
            $exit_code = $? >> 8;
            if ($exit_code != 0) {
                $launch_error = "Chromium exited with status $exit_code";
            }
        }
    }

    if ($timed_out) {
        return {
            'success' => 0,
            'status'  => 599,
            'reason'  => "Chromium timeout after ${timeout}s",
            'content' => '',
        };
    }

    if ($launch_error ne '') {
        $launch_error =~ s/\s+$//;
        my $reason = $error_output || $launch_error;
        return {
            'success' => 0,
            'status'  => 599,
            'reason'  => $reason,
            'content' => $content,
        };
    }

    if ($content eq '') {
        return {
            'success' => 0,
            'status'  => 599,
            'reason'  => 'Chromium returned empty content',
            'content' => '',
        };
    }

    return {
        'success' => 1,
        'status'  => 200,
        'reason'  => 'OK',
        'content' => $content,
    };
}

1;

__END__