blib/lib/SignalWire/Skills/Builtin/Spider.pm

filter: Statementstmt 82.9% 78 / 94 Branchbran 43.7% 7 / 16 Conditioncond 23.8% 5 / 21 MC/DCmcdc 0.0% 0 / 11 Subroutinesub 90.4% 19 / 21 Podpod 0.0% 0 / 5 totaltotal 64.8% 109 / 168 CC 18 SCAR 17.4
CC
18
Cov
69%
scrape_url
18.4
__ANON__
6.0
__ANON__
6.0
CRAP 5.7
« blib/lib/SignalWire/SWML/Schema.pm ↑ Summary blib/lib/SignalWire/Skills/Builtin/SwmlTransfer.pm »
1 package SignalWire::Skills::Builtin::Spider;
2 # Copyright (c) 2025 SignalWire
3 # Licensed under the MIT License.
4 #
5 # Real web-scraper. Mirrors signalwire-python's
6 # skills/spider/skill.py:_fetch_url + _scrape_url_handler — issue an
7 # outbound GET, optionally parse minimal text out of the HTML body,
8 # return as a FunctionResult. Python's full implementation includes
9 # lxml-based content extraction modes (clean_text / full_text /
10 # structured); we ship the fast_text path that the LLM uses 95% of
11 # the time and leaves the structured-extraction work to consumers.
12  
13 4 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 4 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
15 4 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
16 4 ▶use HTTP::Tiny;
Subroutine BEGIN: called CC=1 SCAR=0.0
17 4 ▶use JSON ();
Subroutine BEGIN: called CC=1 SCAR=0.0
18 4 extends 'SignalWire::Skills::SkillBase';
19  
20 4 ▶use SignalWire::Skills::SkillRegistry;
Subroutine BEGIN: called CC=1 SCAR=0.0
21 4 SignalWire::Skills::SkillRegistry->register_skill('spider', __PACKAGE__);
22  
23 6 ▶has '+skill_name' => (default => sub { 'spider' });
Subroutine __ANON__: called CC=1 SCAR=0.0
24 6 ▶has '+skill_description' => (default => sub { 'Fast web scraping and crawling capabilities' });
Subroutine __ANON__: called CC=1 SCAR=0.0
25 6 ▶has '+supports_multiple_instances' => (default => sub { 1 });
Subroutine __ANON__: called CC=1 SCAR=0.0
26  
27 # Honor SPIDER_BASE_URL env var. When set, the skill rewrites the
28 # user-supplied URL onto the base — useful for the audit fixture
29 # (audit_skills_dispatch.py) which serves a 127.0.0.1 endpoint that
30 # stands in for any external host.
31 has 'base_url' => (
32 is => 'ro',
33 lazy => 1,
34 1 ▶ default => sub { $ENV{SPIDER_BASE_URL} || '' },
Subroutine __ANON__: called CC=2 SCAR=7.6
Branch: 1 branch
BranchTrueFalseTotal
$ENV{"SPIDER_BASE_URL"} or "" 1 0 1
35 4 );
36  
37 has '_http' => (
38 is => 'ro',
39 lazy => 1,
40 1 ▶ default => sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
41 1 HTTP::Tiny->new(
42 agent => 'SignalWire-Perl-Spider/1.0',
43 timeout => 15,
44 );
45 },
46 4 );
47  
48 4 ▶sub setup { 1 }
Subroutine setup: called CC=1 SCAR=0.0
Pod undocumented
49  
50 3 ▶sub register_tools {
Subroutine register_tools: called CC=2 SCAR=9.4
Pod undocumented
51 3 my ($self) = @_;
52 3 ▶ my $tool_prefix = $self->params->{tool_prefix} // '';
Condition50%1/2
$self->params->{"tool_prefix"} // ""
l!l
03
MC/DC0%0/1
$self->params->{"tool_prefix"} // ""
$self->params->{"tool_prefix"}
Truth table50%1/2
A
A= $self->params->{"tool_prefix"}
Aresult
11
00
53 3 my $weak_self = $self;
54 3 require Scalar::Util;
55 3 Scalar::Util::weaken($weak_self);
56  
57 $self->define_tool(
58 name => "${tool_prefix}scrape_url",
59 description => 'Scrape content from a URL',
60 parameters => {
61 type => 'object',
62 properties => {
63 url => { type => 'string', description => 'The URL to scrape' },
64 },
65 required => ['url'],
66 },
67 1 ▶ handler => sub {
Subroutine __ANON__: called CC=2 SCAR=7.0
68 1 my ($args, $raw) = @_;
69 1 require SignalWire::SWAIG::FunctionResult;
70 1 ▶ my $url = $args->{url} // '';
Condition50%1/2
$args->{"url"} // ""
l!l
10
MC/DC0%0/1
$args->{"url"} // ""
$args->{"url"}
Truth table50%1/2
A
A= $args->{"url"}
Aresult
11
00
71 1 my $text = $weak_self->scrape_url($url);
72 1 return SignalWire::SWAIG::FunctionResult->new(response => $text);
73 },
74 3 );
75  
76 $self->define_tool(
77 name => "${tool_prefix}crawl_site",
78 description => 'Crawl a website starting from a URL',
79 parameters => {
80 type => 'object',
81 properties => {
82 start_url => { type => 'string', description => 'Starting URL for crawl' },
83 },
84 required => ['start_url'],
85 },
86 0 ▶ handler => sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
87 0 my ($args, $raw) = @_;
88 0 require SignalWire::SWAIG::FunctionResult;
89 0 ▶ my $url = $args->{start_url} // '';
Condition0%0/2
$args->{"start_url"} // ""
l!l
00
MC/DC0%0/1
$args->{"start_url"} // ""
$args->{"start_url"}
Truth table0%0/2
A
A= $args->{"start_url"}
Aresult
11
00
90 # crawl_site is a single-page wrapper around scrape_url here;
91 # multi-page crawl + URL frontier is out of scope for the
92 # Perl port (Python's lxml-based crawl tree is its own
93 # 600-line concern).
94 0 my $text = $weak_self->scrape_url($url);
95 0 return SignalWire::SWAIG::FunctionResult->new(response => $text);
96 },
97 3 );
98  
99 $self->define_tool(
100 name => "${tool_prefix}extract_structured_data",
101 description => 'Extract structured data from a URL',
102 parameters => {
103 type => 'object',
104 properties => {
105 url => { type => 'string', description => 'URL to extract data from' },
106 },
107 required => ['url'],
108 },
109 0 ▶ handler => sub {
Subroutine __ANON__: not called CC=2 SCAR=17.9
110 0 my ($args, $raw) = @_;
111 0 require SignalWire::SWAIG::FunctionResult;
112 0 ▶ my $url = $args->{url} // '';
Condition0%0/2
$args->{"url"} // ""
l!l
00
MC/DC0%0/1
$args->{"url"} // ""
$args->{"url"}
Truth table0%0/2
A
A= $args->{"url"}
Aresult
11
00
113 0 my $text = $weak_self->scrape_url($url);
114 0 return SignalWire::SWAIG::FunctionResult->new(response => $text);
115 },
116 3 );
117 }
118  
119 1 ▶sub scrape_url {
Subroutine scrape_url: called CC=7 SCAR=29.1
Pod undocumented
120 1 my ($self, $url) = @_;
121 1 my $target = $self->_resolve_url($url);
122 1 my $resp = $self->_http->get($target);
123 1 ▶ unless ($resp->{success}) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($resp->{"success"}) 0 1 1
124 0 return "Spider error: $resp->{status} $resp->{reason} ($target)";
125 }
126  
127 1 ▶ my $body = $resp->{content} // '';
Condition50%1/2
$resp->{"content"} // ""
l!l
10
MC/DC0%0/1
$resp->{"content"} // ""
$resp->{"content"}
Truth table50%1/2
A
A= $resp->{"content"}
Aresult
11
00
128 # The audit fixture serves JSON like {"_raw_html": "<html>...</html>"}
129 # because http.server can't easily decide between content types.
130 # If the response decodes as JSON, lift the embedded HTML out;
131 # otherwise treat the body as HTML directly.
132 1 ▶ if ($body =~ /^\s*\{/) {
Branch: 1 branch
BranchTrueFalseTotal
if ($body =~ /^\s*\{/) 0 1 1
133 0 my $parsed = eval { JSON::decode_json($body) };
134 0 ▶ if (!$@ && ref $parsed eq 'HASH' && exists $parsed->{_raw_html}) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $@ and ref $parsed eq "HASH" and exists $parsed->{"_raw_html"}) 0 0 0
Condition0%0/6
not $@ and ref $parsed eq "HASH" and exists $parsed->{"_raw_html"}
!ll&&!rl&&r
000
not $@ and ref $parsed eq "HASH"
!ll&&!rl&&r
000
MC/DC0%0/3
not $@ and ref $parsed eq "HASH" and exists $parsed->{"_raw_html"}
not $@ ref $parsed eq "HASH" exists $parsed->{"_raw_html"}
Truth table0%0/4
not(A) and B and C
A= not $@B= ref $parsed eq "HASH"C= exists $parsed->{"_raw_html"}
ABCresult
0XX0
10X0
1100
1111
135 0 $body = $parsed->{_raw_html};
136 }
137 }
138  
139 1 return _extract_text($body);
140 }
141  
142 1 ▶sub _resolve_url {
Subroutine _resolve_url: called CC=5 SCAR=17.1
143 1 my ($self, $url) = @_;
144 1 ▶ return $url unless $self->base_url;
Branch: 1 branch
BranchTrueFalseTotal
unless $self->base_url 0 1 1
145 # When a base URL is configured, route the request through it,
146 # preserving the path/query of the requested URL. This mirrors
147 # the audit harness contract in SUBAGENT_PLAYBOOK § audit_skills.
148 1 my $path;
149 1 ▶ if ($url =~ m{^https?://[^/]+(/.*)?$}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($url =~ m[^https?://[^/]+(/.*)?$]) { } 1 0 1
150 1 ▶ $path = $1 // '/';
Condition50%1/2
$1 // "/"
l!l
10
MC/DC0%0/1
$1 // "/"
$1
Truth table50%1/2
A
A= $1
Aresult
11
00
151 } else {
152 0 $path = $url;
153 }
154 1 my $base = $self->base_url;
155 1 $base =~ s{/+$}{};
156 1 ▶ $path = "/$path" unless $path =~ m{^/};
Branch: 1 branch
BranchTrueFalseTotal
unless $path =~ m[^/] 0 1 1
157 1 return "$base$path";
158 }
159  
160 1 ▶sub _extract_text {
Subroutine _extract_text: called CC=3 SCAR=11.1
161 1 my ($html) = @_;
162 1 ▶ return '' unless defined $html && length $html;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $html and length $html 0 1 1
Condition33%1/3
defined $html and length $html
!ll&&!rl&&r
001
MC/DC0%0/2
defined $html and length $html
defined $html length $html
Truth table33%1/3
A and B
A= defined $htmlB= length $html
ABresult
0X0
100
111
163  
164 # Strip <script> and <style> blocks entirely.
165 1 $html =~ s{<script\b[^>]*>.*?</script\s*>}{}gisx;
166 1 $html =~ s{<style\b[^>]*>.*?</style\s*>}{}gisx;
167 # Strip remaining tags, decode common entities, collapse whitespace.
168 1 $html =~ s/<[^>]+>/ /g;
169 1 $html =~ s/&nbsp;/ /gi;
170 1 $html =~ s/&amp;/&/g;
171 1 $html =~ s/&lt;/</g;
172 1 $html =~ s/&gt;/>/g;
173 1 $html =~ s/&quot;/"/g;
174 1 $html =~ s/&#39;/'/g;
175 1 $html =~ s/\s+/ /g;
176 1 $html =~ s/^\s+|\s+$//g;
177 1 return $html;
178 }
179  
180 1 ▶sub get_hints {
Subroutine get_hints: called CC=1 SCAR=0.0
Pod undocumented
181 1 return ['scrape', 'crawl', 'extract', 'web page', 'website', 'spider'];
182 }
183  
184 1 ▶sub get_parameter_schema {
Subroutine get_parameter_schema: called CC=1 SCAR=0.0
Pod undocumented
185 return {
186 1 %{ SignalWire::Skills::SkillBase->get_parameter_schema },
187 delay => { type => 'number' },
188 concurrent_requests => { type => 'integer' },
189 timeout => { type => 'integer' },
190 max_pages => { type => 'integer' },
191 max_depth => { type => 'integer' },
192 };
193 }
194  
195 4 1;
« blib/lib/SignalWire/SWML/Schema.pm ↑ Summary blib/lib/SignalWire/Skills/Builtin/SwmlTransfer.pm »