blib/lib/Langertha/Holmgang/Target.pm

filter: Statementstmt 96.8% 184 / 190 Branchbran 82.5% 99 / 120 Conditioncond 58.4% 69 / 118 MC/DCmcdc 39.3% 26 / 66 Subroutinesub 93.5% 29 / 31 Podpod 83.3% 10 / 12 totaltotal 77.6% 417 / 537 CC 172 SCAR 32.9
CC
172
Cov
82%
check_variant
51.8
check_variant
51.8
BUILD
25.8
CRAP 26.8
« blib/lib/Langertha/Holmgang/Sim/Linux/Accounts.pm ↑ Summary blib/lib/Langertha/Holmgang/Sim/Linux/Calendar.pm »
1 package Langertha::Holmgang::Target;
2 # ABSTRACT: A Langertha engine as a benchmark target: variant settings, one model turn, failed turns classified
3  
4 42 ▶use Moose;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 42 ▶use namespace::autoclean;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 42 ▶use Carp qw( croak );
Subroutine BEGIN: called CC=1 SCAR=0.0
7 42 ▶use Compress::Zlib ();
Subroutine BEGIN: called CC=1 SCAR=0.0
8 42 ▶use Future;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 42 ▶use Future::AsyncAwait;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 42 ▶use JSON::MaybeXS ();
Subroutine BEGIN: called CC=1 SCAR=0.0
11 42 ▶use Langertha::Content::Image;
Subroutine BEGIN: called CC=1 SCAR=0.0
12 42 ▶use Langertha::Holmgang::Error;
Subroutine BEGIN: called CC=1 SCAR=0.0
13 42 ▶use Langertha::Role::HTTP ();
Subroutine BEGIN: called CC=1 SCAR=0.0
14 42 ▶use Scalar::Util qw( blessed );
Subroutine BEGIN: called CC=1 SCAR=0.0
15  
16 42 our $VERSION = '0.001';
17  
18  
19 42 has engine => (
20 is => 'ro',
21 required => 1,
22 );
23  
24  
25 has variant => (
26 is => 'ro',
27 1022 ▶ default => sub { {} },
Subroutine __ANON__: called CC=1 SCAR=0.0
28 42 );
29  
30  
31 42 has deadline => (
32 is => 'ro',
33 predicate => 'has_deadline',
34 );
35  
36  
37 1199 ▶sub BUILD {
Subroutine BUILD: called CC=8 SCAR=32.5
Pod undocumented
38 0 ▶ my ( $self ) = @_;
Subroutine BUILD: not called CC=8 SCAR=32.5
Pod undocumented
39 0 ▶ croak __PACKAGE__.'->new: engine must be a Langertha engine with chat_f'
Branch: 2 branches
BranchTrueFalseTotal
unless builtin::blessed($self->engine) and $self->engine->can("chat_f") 1 1198 1199
unless builtin::blessed($self->engine) and $self->engine->can("chat_f") 0 0 0
Condition33%2/6
builtin::blessed($self->engine) and $self->engine->can("chat_f")
!ll&&!rl&&r
101198
builtin::blessed($self->engine) and $self->engine->can("chat_f")
!ll&&!rl&&r
000
MC/DC50%1/2
builtin::blessed($self->engine) and $self->engine->can("chat_f")
builtin::blessed($self->engine) $self->engine->can("chat_f")
MC/DC0%0/2
builtin::blessed($self->engine) and $self->engine->can("chat_f")
builtin::blessed($self->engine) $self->engine->can("chat_f")
Truth table66%2/3
A and B
A= builtin::blessed($self->engine)B= $self->engine->can("chat_f")
ABresult
0X0
100
111
Truth table0%0/3
A and B
A= builtin::blessed($self->engine)B= $self->engine->can("chat_f")
ABresult
0X0
100
111
40 unless blessed $self->engine && $self->engine->can('chat_f');
41 0 ▶ croak __PACKAGE__.'->new: variant must be a HashRef'
Branch: 2 branches
BranchTrueFalseTotal
unless ref $self->variant eq "HASH" 1 1197 1198
unless ref $self->variant eq "HASH" 0 0 0
42 unless ref $self->variant eq 'HASH';
43 0 ▶ croak __PACKAGE__.'->new: deadline must be a number of seconds above 0'
Branch: 2 branches
BranchTrueFalseTotal
if $self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0 0 1197 1197
if $self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0 0 0 0
Condition22%4/18
$self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
!ll&&!rl&&r
119610
defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
!ll&&!rl&&r
001
defined $self->deadline and $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/
!ll&&!rl&&r
001
$self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
!ll&&!rl&&r
000
defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
!ll&&!rl&&r
000
defined $self->deadline and $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/
!ll&&!rl&&r
000
MC/DC0%0/4
$self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
$self->has_deadline defined $self->deadline $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ $self->deadline > 0
MC/DC0%0/4
$self->has_deadline and not defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0
$self->has_deadline defined $self->deadline $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ $self->deadline > 0
Truth table40%2/5
A and not(B and C && D)
A= $self->has_deadlineB= defined $self->deadlineC= $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/D= $self->deadline > 0
ABCDresult
0XXX0
11110
10XX1
110X1
11101
Truth table0%0/5
A and not(B and C && D)
A= $self->has_deadlineB= defined $self->deadlineC= $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/D= $self->deadline > 0
ABCDresult
0XXX0
11110
10XX1
110X1
11101
44 if $self->has_deadline && !( defined $self->deadline && $self->deadline =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $self->deadline > 0 );
45 0 return;
46 }
47  
48 #### Variants
49  
50 # The settings a variant may name, in the order a message lists them.
51 42 my @VARIANT_KEYS = qw( enable_thinking reasoning_effort max_tokens temperature deadline );
52 42 my %VARIANT_KEY = map { $_ => 1 } @VARIANT_KEYS;
53  
54 # Langertha's reasoning_effort values (Langertha::Role::Chat)
55 42 my %EFFORT = map { $_ => 1 } qw( none minimal low medium high xhigh max );
56  
57 0 ▶sub variant_keys { @VARIANT_KEYS }
Subroutine variant_keys: not called CC=1 SCAR=6.9
Pod documented
58  
59  
60 340 ▶sub check_variant {
Subroutine check_variant: called CC=42 SCAR=39.5
Pod documented
61 340 my ( $self, $settings, $engine ) = @_;
62 340 ▶ my $no = sub { Langertha::Holmgang::Error->throw(@_) };
Subroutine __ANON__: called CC=1 SCAR=0.0
63 340 ▶ $no->( variant => 'not a hash of settings' ) unless ref $settings eq 'HASH';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $settings eq "HASH" 0 340 340
64 340 my %v;
65 340 for my $k ( sort keys %$settings ) {
66 71 my $x = $settings->{$k};
67 71 ▶ $no->( 'variant.'.$k => 'unknown setting (known: '.join( ', ', @VARIANT_KEYS ).')' ) unless $VARIANT_KEY{$k};
Branch: 1 branch
BranchTrueFalseTotal
unless $VARIANT_KEY{$k} 2 69 71
68 # an empty value names nothing (cai-platform check_variant)
69 69 ▶ next if !defined $x || ( !ref $x && !length $x );
Branch: 1 branch
BranchTrueFalseTotal
unless defined $x and ref $x || length $x 0 69 69
Condition50%3/6
defined $x and ref $x || length $x
!ll&&!rl&&r
0069
ref $x or length $x
l!l&&r!l&&!r
16530
MC/DC0%0/3
defined $x and ref $x || length $x
defined $x ref $x length $x
Truth table50%2/4
A and B or C
A= defined $xB= ref $xC= length $x
ABCresult
0XX0
1000
11X1
1011
70 69 ▶ if ( $k eq 'enable_thinking' ) {
Branch: 4 branches
BranchTrueFalseTotal
if ($k eq 'enable_thinking') { } 22 47 69
elsif ($k eq 'reasoning_effort') { } 7 40 47
elsif ($k eq 'max_tokens') { } 24 16 40
elsif ($k eq 'temperature') { } 10 6 16
71 22 ▶ my $on = JSON::MaybeXS::is_bool($x) ? ( $x ? 1 : 0 )
Branch: 5 branches
BranchTrueFalseTotal
JSON::MaybeXS::is_bool($x) ? : 16 6 22
$x ? : 10 6 16
ref $x ? : 0 6 6
$x =~ /\A(?:on|true|1)\z/ ? : 3 3 6
$x =~ /\A(?:off|false|0)\z/ ? : 2 1 3
72 : ref $x ? undef
73 : $x =~ /\A(?:on|true|1)\z/ ? 1
74 : $x =~ /\A(?:off|false|0)\z/ ? 0
75 : undef;
76 22 ▶ $no->( 'variant.enable_thinking' => 'on or off (true/false, 1/0)' ) unless defined $on;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $on 1 21 22
77 21 ▶ $v{$k} = $on ? JSON::MaybeXS::true() : JSON::MaybeXS::false();
Branch: 1 branch
BranchTrueFalseTotal
$on ? : 13 8 21
78 }
79 elsif ( $k eq 'reasoning_effort' ) {
80 $no->( 'variant.reasoning_effort' => 'one of '.join( ', ', qw( none minimal low medium high xhigh max ) ) )
81 7 ▶ unless !ref $x && $EFFORT{$x};
Branch: 1 branch
BranchTrueFalseTotal
unless not ref $x and $EFFORT{$x} 1 6 7
Condition66%2/3
not ref $x and $EFFORT{$x}
!ll&&!rl&&r
016
MC/DC50%1/2
not ref $x and $EFFORT{$x}
not ref $x $EFFORT{$x}
Truth table66%2/3
not(A) and B
A= not ref $xB= $EFFORT{$x}
ABresult
0X0
100
111
82 6 $v{$k} = ''.$x;
83 }
84 elsif ( $k eq 'max_tokens' ) {
85 24 ▶ $no->( 'variant.max_tokens' => 'a whole number from 1 to 9999999' ) unless !ref $x && $x =~ /\A[1-9][0-9]{0,6}\z/;
Branch: 1 branch
BranchTrueFalseTotal
unless not ref $x and $x =~ /\A[1-9][0-9]{0,6}\z/ 3 21 24
Condition66%2/3
not ref $x and $x =~ /\A[1-9][0-9]{0,6}\z/
!ll&&!rl&&r
0321
MC/DC50%1/2
not ref $x and $x =~ /\A[1-9][0-9]{0,6}\z/
not ref $x $x =~ /\A[1-9][0-9]{0,6}\z/
Truth table66%2/3
not(A) and B
A= not ref $xB= $x =~ /\A[1-9][0-9]{0,6}\z/
ABresult
0X0
100
111
86 21 $v{$k} = 0 + $x;
87 }
88 elsif ( $k eq 'temperature' ) {
89 10 ▶ $no->( 'variant.temperature' => 'a number from 0 to 2' )
Branch: 1 branch
BranchTrueFalseTotal
unless not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x <= 2 1 9 10
Condition50%3/6
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x <= 2
!ll&&!rl&&r
019
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/
!ll&&!rl&&r
0010
MC/DC33%1/3
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x <= 2
not ref $x $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ $x <= 2
Truth table50%2/4
not(A) and B and C
A= not ref $xB= $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/C= $x <= 2
ABCresult
0XX0
10X0
1100
1111
90 unless !ref $x && $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $x <= 2;
91 9 $v{$k} = 0 + $x;
92 }
93 else { # deadline
94 6 ▶ $no->( 'variant.deadline' => 'seconds, a number above 0 and at most 86400' )
Branch: 1 branch
BranchTrueFalseTotal
unless not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x > 0 and $x <= 86400 1 5 6
Condition55%5/9
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x > 0 and $x <= 86400
!ll&&!rl&&r
105
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x > 0
!ll&&!rl&&r
015
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/
!ll&&!rl&&r
006
MC/DC25%1/4
not ref $x and $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ and $x > 0 and $x <= 86400
not ref $x $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ $x > 0 $x <= 86400
Truth table40%2/5
not(A) and B and C and D
A= not ref $xB= $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/C= $x > 0D= $x <= 86400
ABCDresult
0XXX0
10XX0
110X0
11100
11111
95 unless !ref $x && $x =~ /\A[0-9]+(?:\.[0-9]+)?\z/ && $x > 0 && $x <= 86400;
96 5 $v{$k} = 0 + $x;
97 }
98 }
99 331 ▶ return \%v unless defined $engine;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $engine 107 224 331
100 224 my $name = ref $engine;
101 224 ▶ my $model = $engine->can('chat_model') ? eval { $engine->chat_model } : undef;
Branch: 1 branch
BranchTrueFalseTotal
$engine->can('chat_model') ? : 224 0 224
102 224 ▶ my $of = 'the engine '.$name.( defined $model ? ' (model '.$model.')' : '' );
Branch: 1 branch
BranchTrueFalseTotal
defined $model ? : 224 0 224
103 224 ▶ my $takes = sub { !$engine->can('supports') || $engine->supports( $_[0] ) };
Subroutine __ANON__: called CC=2 SCAR=7.2
Branch: 1 branch
BranchTrueFalseTotal
not $engine->can("supports") or $engine->supports($_[0]) 0 9 9
104 $no->( 'variant.temperature' => $of.' takes no temperature; Langertha would drop it' )
105 224 ▶ if exists $v{temperature} && !$takes->('temperature');
Branch: 1 branch
BranchTrueFalseTotal
if exists $v{"temperature"} and not &$takes("temperature") 2 222 224
Condition100%3/3
exists $v{"temperature"} and not &$takes("temperature")
!ll&&!rl&&r
21932
MC/DC100%2/2
exists $v{"temperature"} and not &$takes("temperature")
exists $v{"temperature"} not &$takes("temperature")
Truth table100%3/3
A and not(B)
A= exists $v{"temperature"}B= not &$takes("temperature")
ABresult
0X0
100
111
106 $no->( 'variant.reasoning_effort' => $of.' takes no reasoning_effort; Langertha would drop it' )
107 222 ▶ if exists $v{reasoning_effort} && !$takes->('reasoning_effort');
Branch: 1 branch
BranchTrueFalseTotal
if exists $v{"reasoning_effort"} and not &$takes("reasoning_effort") 1 221 222
Condition100%3/3
exists $v{"reasoning_effort"} and not &$takes("reasoning_effort")
!ll&&!rl&&r
21831
MC/DC100%2/2
exists $v{"reasoning_effort"} and not &$takes("reasoning_effort")
exists $v{"reasoning_effort"} not &$takes("reasoning_effort")
Truth table100%3/3
A and not(B)
A= exists $v{"reasoning_effort"}B= not &$takes("reasoning_effort")
ABresult
0X0
100
111
108 $no->( 'variant.enable_thinking' => $of.' speaks no OpenAI-compatible wire, so chat_template_kwargs has no place' )
109 221 ▶ if exists $v{enable_thinking} && !( $engine->can('does') && $engine->does('Langertha::Role::OpenAICompatible') );
Branch: 1 branch
BranchTrueFalseTotal
if exists $v{"enable_thinking"} and not $engine->can("does") && $engine->does("Langertha::Role::OpenAICompatible") 0 221 221
Condition50%3/6
exists $v{"enable_thinking"} and not $engine->can("does") && $engine->does("Langertha::Role::OpenAICompatible")
!ll&&!rl&&r
207140
$engine->can('does') && $engine->does('Langertha::Role::OpenAICompatible')
!ll&&!rl&&r
0014
MC/DC0%0/3
exists $v{"enable_thinking"} and not $engine->can("does") && $engine->does("Langertha::Role::OpenAICompatible")
exists $v{"enable_thinking"} $engine->can('does') $engine->does('Langertha::Role::OpenAICompatible')
Truth table50%2/4
A and not(B && C)
A= exists $v{"enable_thinking"}B= $engine->can('does')C= $engine->does('Langertha::Role::OpenAICompatible')
ABCresult
0XX0
1110
10X1
1101
110 $no->( 'variant.deadline' => $of.' runs on no event loop (IO::Async and Net::Async::HTTP), so a deadline cannot cut a request' )
111 221 ▶ if exists $v{deadline} && !( $engine->can('async_loop') && $engine->async_loop );
Branch: 1 branch
BranchTrueFalseTotal
if exists $v{"deadline"} and not $engine->can("async_loop") && $engine->async_loop 1 220 221
Condition83%5/6
exists $v{"deadline"} and not $engine->can("async_loop") && $engine->async_loop
!ll&&!rl&&r
21821
$engine->can('async_loop') && $engine->async_loop
!ll&&!rl&&r
012
MC/DC66%2/3
exists $v{"deadline"} and not $engine->can("async_loop") && $engine->async_loop
exists $v{"deadline"} $engine->can('async_loop') $engine->async_loop
Truth table75%3/4
A and not(B && C)
A= exists $v{"deadline"}B= $engine->can('async_loop')C= $engine->async_loop
ABCresult
0XX0
1110
10X1
1101
112 220 return \%v;
113 }
114  
115  
116 162 ▶sub for_variant {
Subroutine for_variant: called CC=8 SCAR=22.0
Pod documented
117 162 my ( $self, %arg ) = @_;
118 162 my $engine = $arg{engine};
119 162 ▶ croak __PACKAGE__.'->for_variant: engine must be a Langertha engine with chat_f'
Branch: 1 branch
BranchTrueFalseTotal
unless builtin::blessed($engine) and $engine->can("chat_f") 0 162 162
Condition33%1/3
builtin::blessed($engine) and $engine->can("chat_f")
!ll&&!rl&&r
00162
MC/DC0%0/2
builtin::blessed($engine) and $engine->can("chat_f")
builtin::blessed($engine) $engine->can("chat_f")
Truth table33%1/3
A and B
A= builtin::blessed($engine)B= $engine->can("chat_f")
ABresult
0X0
100
111
120 unless blessed $engine && $engine->can('chat_f');
121 162 ▶ my $s = $self->check_variant( $arg{settings} // {}, $engine );
Condition50%1/2
$arg{"settings"} // {}
l!l
1620
MC/DC0%0/1
$arg{"settings"} // {}
$arg{"settings"}
Truth table50%1/2
A
A= $arg{"settings"}
Aresult
11
00
122 162 ▶ my %opts = map { exists $s->{$_} ? ( $_ => $s->{$_} ) : () } qw( reasoning_effort max_tokens temperature );
Branch: 1 branch
BranchTrueFalseTotal
exists $s->{$_} ? : 13 473 486
123 162 ▶ $opts{chat_template_kwargs} = { enable_thinking => $s->{enable_thinking} } if exists $s->{enable_thinking};
Branch: 1 branch
BranchTrueFalseTotal
if exists $s->{"enable_thinking"} 9 153 162
124 return ( ref $self || $self )->new(
125 engine => $engine,
126 variant => \%opts,
127 162 ▶ ( exists $s->{deadline} ? ( deadline => $s->{deadline} ) : () ),
Branch: 1 branch
BranchTrueFalseTotal
exists $s->{'deadline'} ? : 1 161 162
Condition33%1/3
ref $self || $self
l!l&&r!l&&!r
01620
MC/DC0%0/2
ref $self || $self
ref $self $self
Truth table33%1/3
A || B
A= ref $selfB= $self
ABresult
1X1
011
000
128 );
129 }
130  
131  
132 # How much of an error body Langertha keeps in its croak while a turn runs.
133 # Its default ($Langertha::Role::HTTP::error_body_max_length, 500 characters)
134 # can cut a signature off a long body (a provider error with a long preamble).
135 2041 ▶sub error_body_max_length { 65536 }
Subroutine error_body_max_length: called CC=1 SCAR=0.0
Pod documented
136  
137  
138 #### Signatures
139  
140 # A target that cannot take an image (k27): llama.cpp without an mmproj
141 # ("image input is not supported - hint: ... provide the mmproj"), vLLM
142 # ("... is not a multimodal model").
143 42 my $NO_VISION_RE = qr{
144 image \s+ input \s+ is \s+ not \s+ supported
145 | not \s+ a \s+ multimodal \s+ model
146 }xi;
147  
148 # The request exceeds the context window -- cai-platform's $CONTEXT_RE
149 # (Eval::Runner, af7561a5), unchanged: OpenAI, vLLM, llama.cpp, SGLang, TGI,
150 # LiteLLM, Ollama, Anthropic, and the generic "context window" phrasing.
151 42 my $CONTEXT_RE = qr{
152 context_length_exceeded | exceed_context_size | ContextWindowExceeded
153 | maximum \s+ context \s+ length | maximum \s+ model \s+ length | max_model_len
154 | exceeds? \s+ the \s+ (?:available \s+)? context \s+ (?:size|length|window)
155 | longer \s+ than \s+ the \s+ model'?s? \s+ context \s+ length
156 | tokens \s* \+ \s* `?max_new_tokens`? \s+ must \s+ be \s+ <=
157 | context \s+ window \s+ (?:exceeded|is \s+ too \s+ small)
158 | prompt \s+ is \s+ too \s+ long
159 }xi;
160  
161 53 ▶sub classify_error {
Subroutine classify_error: called CC=5 SCAR=16.1
Pod documented
162 53 my ( $self, $text ) = @_;
163 53 ▶ return 'error' unless defined $text && length $text;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $text and length $text 2 51 53
Condition100%3/3
defined $text and length $text
!ll&&!rl&&r
1151
MC/DC100%2/2
defined $text and length $text
defined $text length $text
Truth table100%3/3
A and B
A= defined $textB= length $text
ABresult
0X0
100
111
164 51 ▶ return $text =~ $NO_VISION_RE ? 'no_vision'
Branch: 2 branches
BranchTrueFalseTotal
$text =~ /$NO_VISION_RE/ ? : 6 45 51
$text =~ /$CONTEXT_RE/ ? : 19 26 45
165 : $text =~ $CONTEXT_RE ? 'context_exceeded'
166 : 'error';
167 }
168  
169  
170 #### Scrubbing
171  
172 # Query parameters and header/JSON fields whose value is a credential.
173 42 my $SECRET_PARAM = qr{ (?: api[_-]?key | key | access[_-]?token | token | auth | secret | password | passwd | sig | signature ) }xi;
174 42 my $AUTH_SCHEME = qr{ (?: Bearer | Basic | Token | Digest ) \s+ }xi;
175 # What ends a value in a header, a query or a JSON string.
176 42 my $VALUE = qr{ [^\s"',;&#]+ }x;
177 # A path segment: anything but whitespace, a slash and the usual delimiters.
178 42 my $SEG = qr{ [^\s/\\"'<>:,;()\[\]{}]+ }x;
179  
180 # A die or warn location: " at FILE line N", with the line of the last read
181 # handle (", <FH> line N" or "chunk N") when Perl adds one; FILE is $1. The one
182 # rule of the dist (k42): World builds its Perl-error line from it, World cuts a
183 # trailing one with it (k44), the CLI
184 # cuts relayed errors and warnings with _strip_location (k41).
185 42 our $LOCATION = qr/ at (\S+) line \d+(?:, <[^>]*> (?:line|chunk) \d+)?/;
186  
187 # die and Carp locations anywhere, the closing dot optional ("called at" lines have none)
188 89 ▶sub _strip_location { my ( $class, $text ) = @_; return $text =~ s/$LOCATION\.?//gr }
Subroutine _strip_location: called CC=1 SCAR=0.0
189  
190 69 ▶sub scrub {
Subroutine scrub: called CC=7 SCAR=19.5
Pod documented
191 69 my ( $self, $text ) = @_;
192 69 ▶ return $text unless defined $text;
Branch: 1 branch
BranchTrueFalseTotal
unless defined $text 1 68 69
193 # the engine's own key, of whatever shape (cai-platform ModelTester::scrub)
194 68 ▶ if ( ref $self ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $self) 56 12 68
195 56 my $engine = $self->engine;
196 56 ▶ my $key = $engine->can('api_key') ? eval { $engine->api_key } : undef;
Branch: 1 branch
BranchTrueFalseTotal
$engine->can('api_key') ? : 54 2 56
197 56 ▶ $text =~ s/\Q$key\E/***/g if defined $key && !ref $key && length $key >= 4;
Branch: 1 branch
BranchTrueFalseTotal
if defined $key and not ref $key and length $key >= 4 9 47 56
Condition83%5/6
defined $key and not ref $key and length $key >= 4
!ll&&!rl&&r
4619
defined $key and not ref $key
!ll&&!rl&&r
46010
MC/DC66%2/3
defined $key and not ref $key and length $key >= 4
defined $key not ref $key length $key >= 4
Truth table75%3/4
A and not(B) and C
A= defined $keyB= not ref $keyC= length $key >= 4
ABCresult
0XX0
10X0
1100
1111
198 }
199 68 $text = $self->_strip_location($text);
200 # credentials in URLs: userinfo and secret query parameters
201 68 $text =~ s{ ( \b [a-z][a-z0-9+.-]* :// ) [^/\s\@"']+ \@ }{$1***\@}gxi;
202 68 $text =~ s{ ( [?&;] $SECRET_PARAM = ) $VALUE }{$1***}gx;
203 # credentials in headers and JSON fields
204 68 $text =~ s{ \b ( Authorization ["']? \s* [:=] \s* ["']? $AUTH_SCHEME? ) $VALUE }{$1***}gxi;
205 68 $text =~ s{ \b ( Bearer \s+ ) (?!\*\*\*) $VALUE }{$1***}gxi;
206 68 $text =~ s{ \b ( (?: x- (?: goog- )? )? api[_-]?key ["']? \s* [:=] \s* ["']? ) $VALUE }{$1***}gxi;
207 # key shapes of the common providers
208 68 $text =~ s{ \b (?: sk | pplx | gsk | xai | hf ) [-_] [A-Za-z0-9_-]{16,} }{***}gx;
209 68 $text =~ s{ \b AIza [0-9A-Za-z_-]{30,} }{***}gx;
210 # absolute host paths: Unix (two segments at least, not inside a URL), home, Windows
211 68 $text =~ s{ (?<! [\w.:/~\\\@-] ) (?: ~ | [A-Za-z]: )? [/\\] (?: $SEG [/\\] )+ $SEG }{<path>}gx;
212 68 return $text;
213 }
214  
215  
216 #### Vision (k27)
217  
218 # The input tokens an image must add at least before the sight test counts it
219 # as seen (k27: "Mindestkosten des Bildes, ~200").
220 8 ▶sub sight_test_min_tokens { 200 }
Subroutine sight_test_min_tokens: called CC=1 SCAR=0.0
Pod documented
221  
222  
223 42 my $SIGHT_PNG;
224  
225 16 ▶sub sight_test_image {
Subroutine sight_test_image: called CC=3 SCAR=11.0
Pod documented
226 16 ▶ return $SIGHT_PNG //= do {
Condition66%2/3
$SIGHT_PNG //= do { () = (512, 512); my $raw = join("", map({my $y = $_; "\000" . join("", map({(int($_ / 64) + int($y / 64)) % 2 ? "\377" : "\@";} 0 .. $w - 1));} 0 .. $h - 1)); my $chunk = sub { my($type, $data) = @_; return pack("N", length $data) . $type . $data . pack("N", Compress::Zlib::crc32($type . $data)); } ; "\211PNG\r\n\cZ\n" . &$chunk("IHDR", pack("NNCCCCC", $w, $h, 8, 0, 0, 0, 0)) . &$chunk("IDAT", &Compress::Zlib::compress($raw, 9)) . &$chunk("IEND", "") }
l!l&&r!l&&!r
1330
MC/DC0%0/2
$SIGHT_PNG //= do { () = (512, 512); my $raw = join("", map({my $y = $_; "\000" . join("", map({(int($_ / 64) + int($y / 64)) % 2 ? "\377" : "\@";} 0 .. $w - 1));} 0 .. $h - 1)); my $chunk = sub { my($type, $data) = @_; return pack("N", length $data) . $type . $data . pack("N", Compress::Zlib::crc32($type . $data)); } ; "\211PNG\r\n\cZ\n" . &$chunk("IHDR", pack("NNCCCCC", $w, $h, 8, 0, 0, 0, 0)) . &$chunk("IDAT", &Compress::Zlib::compress($raw, 9)) . &$chunk("IEND", "") }
$SIGHT_PNG do { () = (512, 512); my $raw = join("", map({my $y = $_; "\000" . join("", map({(int($_ / 64) + int($y / 64)) % 2 ? "\377" : "\@";} 0 .. $w - 1));} 0 .. $h - 1)); my $chunk = sub { my($type, $data) = @_; return pack("N", length $data) . $type . $data . pack("N", Compress::Zlib::crc32($type . $data)); } ; "\211PNG\r\n\cZ\n" . &$chunk("IHDR", pack("NNCCCCC", $w, $h, 8, 0, 0, 0, 0)) . &$chunk("IDAT", &Compress::Zlib::compress($raw, 9)) . &$chunk("IEND", "") }
Truth table66%2/3
A //= B
A= $SIGHT_PNGB= do { () = (512, 512); my $raw = join("", map({my $y = $_; "\000" . join("", map({(int($_ / 64) + int($y / 64)) % 2 ? "\377" : "\@";} 0 .. $w - 1));} 0 .. $h - 1)); my $chunk = sub { my($type, $data) = @_; return pack("N", length $data) . $type . $data . pack("N", Compress::Zlib::crc32($type . $data)); } ; "\211PNG\r\n\cZ\n" . &$chunk("IHDR", pack("NNCCCCC", $w, $h, 8, 0, 0, 0, 0)) . &$chunk("IDAT", &Compress::Zlib::compress($raw, 9)) . &$chunk("IEND", "") }
ABresult
1X1
011
000
227 3 my ( $w, $h ) = ( 512, 512 );
228 # 8-bit grey, a checkerboard of 64-pixel squares; filter byte 0 per row
229 my $raw = join '', map {
230 3 my $y = $_;
231 1536 ▶ "\0".join '', map { ( int( $_ / 64 ) + int( $y / 64 ) ) % 2 ? "\xFF" : "\x40" } 0 .. $w - 1;
Branch: 1 branch
BranchTrueFalseTotal
(int($_ / 64) + int($y / 64)) % 2 ? : 393216 393216 786432
232 } 0 .. $h - 1;
233 9 ▶ my $chunk = sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
234 9 my ( $type, $data ) = @_;
235 9 return pack( 'N', length $data ).$type.$data.pack( 'N', Compress::Zlib::crc32( $type.$data ) );
236 3 };
237 3 "\x89PNG\r\n\x1a\n"
238 .$chunk->( 'IHDR', pack( 'NNCCCCC', $w, $h, 8, 0, 0, 0, 0 ) )
239 .$chunk->( 'IDAT', Compress::Zlib::compress( $raw, 9 ) )
240 .$chunk->( 'IEND', '' );
241 };
242 }
243  
244  
245 42 my $SIGHT_TEXT = 'This is a connection check. Reply with the single word OK.';
246  
247 17 ▶async sub vision_check_f {
Subroutine vision_check_f: called CC=19 SCAR=29.7
Pod documented
248 17 my ( $self ) = @_;
249 17 my $engine = $self->engine;
250 17 my $name = ref $engine;
251 # 1. the wire: an engine without the role has no image part Langertha vouches for
252 17 ▶ return { no_vision => 'the engine '.$name.' takes no image (no Langertha::Role::ImageInput)', check => 'wire' }
Branch: 1 branch
BranchTrueFalseTotal
unless $engine->can("does") and $engine->does("Langertha::Role::ImageInput") 2 15 17
Condition66%2/3
$engine->can("does") and $engine->does("Langertha::Role::ImageInput")
!ll&&!rl&&r
0215
MC/DC50%1/2
$engine->can("does") and $engine->does("Langertha::Role::ImageInput")
$engine->can("does") $engine->does("Langertha::Role::ImageInput")
Truth table66%2/3
A and B
A= $engine->can("does")B= $engine->does("Langertha::Role::ImageInput")
ABresult
0X0
100
111
253 unless $engine->can('does') && $engine->does('Langertha::Role::ImageInput');
254  
255 # 2. what the server says about its model, where Langertha can ask (llama.cpp
256 # /props, Ollama, LM Studio, OpenRouter ...); elsewhere this asks nothing
257 my $learned = $engine->can('probe_model_capabilities_f')
258 15 ▶ ? await( $engine->probe_model_capabilities_f->else( sub { Future->done( {} ) } ) )
Subroutine __ANON__: called CC=1 SCAR=0.2
Branch: 1 branch
BranchTrueFalseTotal
$engine->can('probe_model_capabilities_f') ? : 15 0 15
259 : {};
260 15 ▶ my $model = $engine->can('chat_model') ? eval { $engine->chat_model } : undef;
Branch: 1 branch
BranchTrueFalseTotal
$engine->can('chat_model') ? : 15 0 15
261 my $fact = defined $model && ref $learned eq 'HASH' && ref $learned->{$model} eq 'HASH'
262 15 ▶ ? $learned->{$model}{image_input} : undef;
Branch: 1 branch
BranchTrueFalseTotal
defined $model && ref $learned eq 'HASH' && ref $learned->{$model} eq 'HASH' ? : 2 13 15
Condition50%3/6
defined $model and ref $learned eq "HASH" and ref $learned->{$model} eq "HASH"
!ll&&!rl&&r
0132
defined $model and ref $learned eq "HASH"
!ll&&!rl&&r
0015
MC/DC33%1/3
defined $model and ref $learned eq "HASH" and ref $learned->{$model} eq "HASH"
defined $model ref $learned eq "HASH" ref $learned->{$model} eq "HASH"
Truth table50%2/4
A and B and C
A= defined $modelB= ref $learned eq "HASH"C= ref $learned->{$model} eq "HASH"
ABCresult
0XX0
10X0
1100
1111
263 15 ▶ return { no_vision => 'the server\'s model metadata says the model '.$model.' takes no image', check => 'probe' }
Branch: 1 branch
BranchTrueFalseTotal
if defined $fact and not $fact 1 14 15
Condition100%3/3
defined $fact and not $fact
!ll&&!rl&&r
1311
MC/DC100%2/2
defined $fact and not $fact
defined $fact not $fact
Truth table100%3/3
A and not(B)
A= defined $factB= not $fact
ABresult
0X0
100
111
264 if defined $fact && !$fact;
265  
266 # 3. the sight test: the same short request without and with an image; a
267 # template that drops the image leaves the input tokens where they were
268 14 my %controls = ( max_tokens => 16 );
269 14 my $plain = await $self->chat_turn_f( messages => [ { role => 'user', content => $SIGHT_TEXT } ], %controls );
270 14 my $image = Langertha::Content::Image->from_data( $self->sight_test_image, media_type => 'image/png' );
271 14 my $seen = await $self->chat_turn_f( messages => [ { role => 'user', content => [ { type => 'text', text => $SIGHT_TEXT }, $image ] } ], %controls );
272 14 ▶ if ( defined $seen->{error_class} && $seen->{error_class} eq 'no_vision' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $seen->{"error_class"} and $seen->{"error_class"} eq "no_vision") 1 13 14
Condition100%3/3
defined $seen->{"error_class"} and $seen->{"error_class"} eq "no_vision"
!ll&&!rl&&r
1211
MC/DC100%2/2
defined $seen->{"error_class"} and $seen->{"error_class"} eq "no_vision"
defined $seen->{"error_class"} $seen->{"error_class"} eq "no_vision"
Truth table100%3/3
A and B
A= defined $seen->{"error_class"}B= $seen->{"error_class"} eq "no_vision"
ABresult
0X0
100
111
273 1 return { no_vision => 'the target refused the image: '.$seen->{error}, check => 'sight_test' };
274 }
275 13 for my $t ( $plain, $seen ) {
276 return { no_vision => undef, check => 'sight_test', note => 'sight test failed: '.$t->{error} }
277 25 ▶ if defined $t->{error_class};
Branch: 1 branch
BranchTrueFalseTotal
if defined $t->{"error_class"} 1 24 25
278 }
279 my ( $without, $with ) = map {
280 12 my $r = $_->{response};
281 24 ▶ $r && $r->has_usage ? $r->usage->input_tokens : undef;
Branch: 1 branch
BranchTrueFalseTotal
$r && $r->has_usage ? : 14 10 24
Condition66%2/3
$r and $r->has_usage
!ll&&!rl&&r
01014
MC/DC50%1/2
$r and $r->has_usage
$r $r->has_usage
Truth table66%2/3
A and B
A= $rB= $r->has_usage
ABresult
0X0
100
111
282 } $plain, $seen;
283 12 ▶ return { no_vision => undef, check => 'sight_test', note => 'sight test unmeasured: the target reports no input tokens' }
Branch: 1 branch
BranchTrueFalseTotal
unless defined $without and defined $with 5 7 12
Condition66%2/3
defined $without and defined $with
!ll&&!rl&&r
507
MC/DC50%1/2
defined $without and defined $with
defined $without defined $with
Truth table66%2/3
A and B
A= defined $withoutB= defined $with
ABresult
0X0
100
111
284 unless defined $without && defined $with;
285 7 my $added = $with - $without;
286 7 my $min = $self->sight_test_min_tokens;
287 7 ▶ return { no_vision => 'sight test: the image added '.$added.' input tokens, at least '.$min
Branch: 1 branch
BranchTrueFalseTotal
if $added < $min 2 5 7
288 .' expected -- the target drops the image', check => 'sight_test', added_tokens => $added }
289 if $added < $min;
290 5 return { no_vision => undef, check => 'sight_test', added_tokens => $added };
291 42 }
292  
293  
294 #### The turn
295  
296 2041 ▶sub chat_turn_f {
Subroutine chat_turn_f: called CC=9 SCAR=22.2
Pod documented
297 2041 my ( $self, %opts ) = @_;
298 2041 my $f;
299 {
300 2041 my $max = $self->error_body_max_length;
301 2041 ▶ local $Langertha::Role::HTTP::error_body_max_length =
Branch: 1 branch
BranchTrueFalseTotal
$Langertha::Role::HTTP::error_body_max_length > $max ? : 0 2041 2041
302 $Langertha::Role::HTTP::error_body_max_length > $max ? $Langertha::Role::HTTP::error_body_max_length : $max;
303 2041 $f = eval { $self->engine->chat_f( %opts, %{ $self->variant } ) };
304 2041 ▶ $f = Future->fail( $@ || ref( $self->engine ).'->chat_f returned no Future' )
Branch: 1 branch
BranchTrueFalseTotal
unless builtin::blessed($f) and $f->isa("Future") 6 2035 2041
Condition80%4/5
builtin::blessed($f) and $f->isa("Future")
!ll&&!rl&&r
602035
$@ || ref($self->engine) . '->chat_f returned no Future'
l!l
51
MC/DC50%1/2
builtin::blessed($f) and $f->isa("Future")
builtin::blessed($f) $f->isa("Future")
MC/DC100%1/1
$@ || ref($self->engine) . '->chat_f returned no Future'
$@
Truth table66%2/3
A and B
A= builtin::blessed($f)B= $f->isa("Future")
ABresult
0X0
100
111
Truth table100%2/2
A
A= $@
Aresult
11
00
305 unless blessed $f && $f->isa('Future');
306 }
307 # the variant's deadline: the request races a timer on the engine's loop
308 2041 ▶ if ( $self->has_deadline && !$f->is_ready ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->has_deadline and not $f->is_ready) 1 2040 2041
Condition66%2/3
$self->has_deadline and not $f->is_ready
!ll&&!rl&&r
204001
MC/DC50%1/2
$self->has_deadline and not $f->is_ready
$self->has_deadline not $f->is_ready
Truth table66%2/3
A and not(B)
A= $self->has_deadlineB= not $f->is_ready
ABresult
0X0
100
111
309 1 ▶ my $loop = $self->engine->can('async_loop') ? $self->engine->async_loop : undef;
Branch: 1 branch
BranchTrueFalseTotal
$self->engine->can('async_loop') ? : 1 0 1
310 1 ▶ if ( $loop ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($loop) 1 0 1
311 1 my $d = $self->deadline;
312 1 $f = Future->wait_any( $f, $loop->delay_future( after => $d )->then_fail( 'timeout: deadline ('.$d.' s)' ) );
313 }
314 }
315 return $f->then(
316 1991 ▶ sub {
Subroutine __ANON__: called CC=1 SCAR=0.0
317 1991 my ( $response ) = @_;
318 1991 return Future->done( { response => $response, error => undef, error_class => undef } );
319 },
320 50 ▶ sub {
Subroutine __ANON__: called CC=2 SCAR=7.0
321 50 my ( $failure ) = @_;
322 50 ▶ my $text = defined $failure ? "$failure" : '';
Branch: 1 branch
BranchTrueFalseTotal
defined $failure ? : 50 0 50
323 50 $text =~ s/\s+\z//;
324 # classified on the raw text, scrubbed before it leaves (k38)
325 50 my $class = $self->classify_error($text);
326 50 return Future->done( { response => undef, error => $self->scrub($text), error_class => $class } );
327 },
328 2041 );
329 }
330  
331  
332 42 __PACKAGE__->meta->make_immutable;
333  
334 42 1;
335  
336 __END__
« blib/lib/Langertha/Holmgang/Sim/Linux/Accounts.pm ↑ Summary blib/lib/Langertha/Holmgang/Sim/Linux/Calendar.pm »