| 1 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 2 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | |||
| 4 | package App::HTTP_Proxy_IMP::IMP; | ||
| 5 | |||
| 6 | 1 | ▶ | use Net::Inspect::Debug qw(:DEFAULT $DEBUG); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 7 | 1 | ▶ | use Net::IMP::Debug var => \$DEBUG, sub => \&debug; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 8 | 1 | ▶ | use Net::IMP qw(:DEFAULT :log); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 1 | ▶ | use Net::IMP::HTTP; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 10 | use Scalar::Util 'weaken'; | ||
| 11 | use Hash::Util 'lock_ref_keys'; | ||
| 12 | use Compress::Raw::Zlib; | ||
| 13 | use Carp; | ||
| 14 | |||
| 15 | use base 'Exporter'; | ||
| 16 | our @EXPORT_OK = qw($IMP_MAX_IN_ANALYZER); | ||
| 17 | |||
| 18 | # if more than IMP_MAX_IN_ANALYZER are in the ibuf we will stop receiving | ||
| 19 | # data and wait until callbacks decrease size of ibuf again | ||
| 20 | our $IMP_MAX_IN_ANALYZER = 2**20; | ||
| 21 | |||
| 22 | my %METHODS_RFC2616 = map { ($_,1) } qw( GET HEAD POST PUT DELETE OPTIONS CONNECT TRACE ); | ||
| 23 | my %METHODS_WITHOUT_RQBODY = map { ($_,1) } qw( GET HEAD DELETE CONNECT ); | ||
| 24 | my %METHODS_WITH_RQBODY = map { ($_,1) } qw( POST PUT ); | ||
| 25 | my %CODE_WITHOUT_RPBODY = map { ($_,1) } qw(204 205 304); | ||
| 26 | my %METHODS_WITHOUT_RPBODY = map { ($_,1) } qw(HEAD); | ||
| 27 | |||
| 28 | # we want plugins to suppport the HTTP Request innterface | ||
| 29 | my $interface = [ | ||
| 30 | IMP_DATA_HTTPRQ, | ||
| 31 | [ | ||
| 32 | IMP_PASS, | ||
| 33 | IMP_PREPASS, | ||
| 34 | IMP_REPLACE, | ||
| 35 | IMP_TOSENDER, | ||
| 36 | IMP_DENY, | ||
| 37 | IMP_LOG, | ||
| 38 | IMP_ACCTFIELD, | ||
| 39 | ] | ||
| 40 | ]; | ||
| 41 | |||
| 42 | sub can_modify { | ||
| 43 | return shift->{can_modify}; | ||
| 44 | } | ||
| 45 | |||
| 46 | # create a new factory object | ||
| 47 | sub new_factory { | ||
| 48 | my ($class,@mod) = @_; | ||
| 49 | my @factory; | ||
| 50 | for my $module (@mod) { | ||
| 51 | if ( ref($module)) { | ||
| 52 | # assume it is already an IMP factory object | ||
| 53 | push @factory, $module; | ||
| 54 | next; | ||
| 55 | } | ||
| 56 | |||
| 57 | # --filter mod=args | ||
| 58 | my ($mod,$args) = $module =~m{^([a-z][\w:]*)(?:=(.*))?$}i | ||
| 59 | or die "invalid module $module"; | ||
| 60 | eval "require $mod" or die "cannot load $mod args=$args: $@"; | ||
| 61 | my %args = $mod->str2cfg($args//''); | ||
| 62 | my $factory = $mod->new_factory(%args) | ||
| 63 | or croak("cannot create Net::IMP factory for $mod"); | ||
| 64 | $factory = $factory->set_interface( $interface ) | ||
| 65 | or croak("$mod does not implement the interface supported by us"); | ||
| 66 | push @factory,$factory; | ||
| 67 | } | ||
| 68 | |||
| 69 | @factory or return; | ||
| 70 | if (@factory>1) { | ||
| 71 | # for cascading filters we need Net::IMP::Cascade | ||
| 72 | require Net::IMP::Cascade; | ||
| 73 | my $cascade = Net::IMP::Cascade->new_factory( parts => [ @factory ]) | ||
| 74 | or croak("cannot create Net::IMP::Cascade factory"); | ||
| 75 | @factory = $cascade; | ||
| 76 | } | ||
| 77 | |||
| 78 | my $factory = $factory[0]->set_interface( $interface ) or | ||
| 79 | croak("factory[0] does not implement the interface supported by us"); | ||
| 80 | |||
| 81 | my $self = bless { | ||
| 82 | imp => $factory, # IMP factory object | ||
| 83 | can_modify => 0, # does interface support IMP_REPLACE, IMP_TOSENDER | ||
| 84 | }, $class; | ||
| 85 | lock_ref_keys($self); | ||
| 86 | |||
| 87 | # update can_modify | ||
| 88 | CHKIF: for my $if ( $self->{imp}->get_interface ) { | ||
| 89 | my ($dt,$rt) = @$if; | ||
| 90 | for (@$rt) { | ||
| 91 | $_ ~~ [ IMP_REPLACE, IMP_TOSENDER ] or next; | ||
| 92 | $self->{can_modify} =1; | ||
| 93 | last CHKIF; | ||
| 94 | } | ||
| 95 | } | ||
| 96 | |||
| 97 | return $self; | ||
| 98 | } | ||
| 99 | |||
| 100 | # create a new analyzer based on the factory | ||
| 101 | sub new_analyzer { | ||
| 102 | my ($factory,$request,$meta) = @_; | ||
| 103 | |||
| 104 | my %meta = %$meta; | ||
| 105 | # IMP uses different *addr than Net::Inspect, translate | ||
| 106 | # [s]ource -> [c]lient, [d]estination -> [s]erver | ||
| 107 | $meta{caddr} = delete $meta{saddr}; | ||
| 108 | $meta{cport} = delete $meta{sport}; | ||
| 109 | $meta{saddr} = delete $meta{daddr}; | ||
| 110 | $meta{sport} = delete $meta{dport}; | ||
| 111 | |||
| 112 | my $analyzer = $factory->{imp}->new_analyzer( meta => \%meta ); | ||
| 113 | |||
| 114 | my $self = bless { | ||
| 115 | request => $request, # App::HTTP_Proxy_IMP::Request object | ||
| 116 | imp => $analyzer, | ||
| 117 | # incoming data, put into analyzer | ||
| 118 | # \@list of [ buf_base,buf,type,callback,$cb_arg ] per dir | ||
| 119 | ibuf => [ | ||
| 120 | [ [0,''] ], | ||
| 121 | [ [0,''] ], | ||
| 122 | ], | ||
| 123 | pass => [0,0], # pass allowed up to given offset (per dir) | ||
| 124 | prepass => [0,0], # prepass allowed up to given offset (per dir) | ||
| 125 | fixup_header => [], # sub to fixup content-length in header once known | ||
| 126 | stalled => [0,0], # too much data in ibuf? | ||
| 127 | eof => [0,0], # got eof in dir ? | ||
| 128 | decode => undef, # decoder for content-encoding decode{type}[dir] | ||
| 129 | pass_encoded => undef, # pass body encoded (analyzer will not change body) | ||
| 130 | method => undef, # request method | ||
| 131 | }, ref($factory); | ||
| 132 | lock_ref_keys($self); | ||
| 133 | weaken($self->{request}); | ||
| 134 | |||
| 135 | # set callback, this might trigger callback immediately if there are | ||
| 136 | # results pending | ||
| 137 | weaken( my $wself = $self ); | ||
| 138 | $analyzer->set_callback( sub { _imp_callback($wself,@_) } ); | ||
| 139 | return $self; | ||
| 140 | } | ||
| 141 | |||
| 142 | |||
| 143 | sub request_header { | ||
| 144 | my ($self,$hdr,$xhdr,@callback) = @_; | ||
| 145 | my $clen = $xhdr->{content_length}; | ||
| 146 | if ( ! defined $clen and $xhdr->{method} ne 'CONNECT') { | ||
| 147 | # length not known -> chunking | ||
| 148 | die "FIXME: chunking request body not yet supported"; | ||
| 149 | } | ||
| 150 | |||
| 151 | # new body might change content-length info in request header | ||
| 152 | # need to defer sending header until body length is known | ||
| 153 | if ( ! $METHODS_WITHOUT_RQBODY{$xhdr->{method}} ) { | ||
| 154 | my $hlen = length($hdr); | ||
| 155 | $self->{fixup_header}[0] = sub { | ||
| 156 | my ($self,$hdr,%args) = @_; | ||
| 157 | my $size = $args{content}; | ||
| 158 | goto fix_clen if defined $size; | ||
| 159 | |||
| 160 | if ( my $pass = $self->{pass}[0] ) { | ||
| 161 | if ( $pass == IMP_MAXOFFSET or $pass >= $hlen + $clen ) { | ||
| 162 | # will not change body | ||
| 163 | goto fix_clen; | ||
| 164 | } | ||
| 165 | } | ||
| 166 | if ( my $prepass = $self->{prepass}[0] ) { | ||
| 167 | if ( $prepass == IMP_MAXOFFSET or $prepass >= $hlen + $clen ) { | ||
| 168 | # will not change body | ||
| 169 | goto fix_clen; | ||
| 170 | } | ||
| 171 | } | ||
| 172 | if ($self->{ibuf}[0][0][0] >= $hlen + $clen) { # ibuf[client].base | ||
| 173 | # everything passed thru ibuf | ||
| 174 | goto fix_clen; | ||
| 175 | } | ||
| 176 | |||
| 177 | # need to defer header until all of the body is passed | ||
| 178 | # or replaced, then we know the size | ||
| 179 | return; | ||
| 180 | |||
| 181 | fix_clen: | ||
| 182 | |||
| 183 | if (!defined $size) { | ||
| 184 | # bytes in ibuf and outstanding bytes will not be changed, so: | ||
| 185 | # new_content_length = | ||
| 186 | # ( orig_clen + orig_hlen - received ) # not yet received | ||
| 187 | # + ( received - ibuf.base ) # still in ibuf | ||
| 188 | # + defered_body.length # ready to forward | ||
| 189 | # ---> | ||
| 190 | # orig_clen + orig_hlen - ibuf.base + defered_body.length | ||
| 191 | $size = $clen + $hlen # orig_clen + orig_hlen | ||
| 192 | - $self->{ibuf}[0][0][0] # ibuf.base | ||
| 193 | + $args{defered}; # defered_body.length | ||
| 194 | } | ||
| 195 | |||
| 196 | $DEBUG && $self->{request}->xdebug("fixup header with clen=%d",$size); | ||
| 197 | # replace or add content-length header | ||
| 198 | $$hdr =~s{^(Content-length:[ \t]*)(\d+)}{$1$size}mi | ||
| 199 | || $$hdr =~s{(\n)}{$1Content-length: $size\r\n}; | ||
| 200 | return 1; | ||
| 201 | }; | ||
| 202 | } | ||
| 203 | |||
| 204 | |||
| 205 | # send data to analyzer. | ||
| 206 | # will call back into request on processed data | ||
| 207 | _imp_data($self,0,$hdr,0,IMP_DATA_HTTPRQ_HEADER, | ||
| 208 | \&_request_header_imp,[ $xhdr,@callback ]); | ||
| 209 | } | ||
| 210 | |||
| 211 | |||
| 212 | ############################################################################ | ||
| 213 | # callback from IMP after passing/replacing the HTTP request header | ||
| 214 | # will reparse the header if changed and continue in @callback from request | ||
| 215 | ############################################################################ | ||
| 216 | sub _request_header_imp { | ||
| 217 | my ($self,$hdr,$changed,$args) = @_; | ||
| 218 | my ($xhdr,$callback,@cb_args) = @$args; | ||
| 219 | |||
| 220 | if ( $changed ) { | ||
| 221 | # we need to parse the header again and update xhdr | ||
| 222 | my ($met,$url,$version,$fields) = $hdr =~m{ \A | ||
| 223 | (\S+)[\040\t]+ | ||
| 224 | (\S+)[\040\t]+ | ||
| 225 | HTTP/(1\.[01])[\040\t]* | ||
| 226 | \r?\n | ||
| 227 | (.*?\n) | ||
| 228 | \r?\n\Z | ||
| 229 | }isx; | ||
| 230 | |||
| 231 | # internal URL are not accepted by the client itself, only from | ||
| 232 | # plugin. Set xhdr.internal_url if we see, that IMP plugin rewrote | ||
| 233 | # url to internal one and strip internal:// again so that original | ||
| 234 | # URL could be logged | ||
| 235 | my $internal = $met ne 'CONNECT' | ||
| 236 | && $xhdr->{url} !~m{^internal://}i | ||
| 237 | && $url =~s{^internal://}{}i; | ||
| 238 | |||
| 239 | my %kv; | ||
| 240 | my $bad = _parse_hdrfields($fields,\%kv); | ||
| 241 | $xhdr = { | ||
| 242 | method => uc($met), | ||
| 243 | version => $version, | ||
| 244 | url => $url, | ||
| 245 | fields => \%kv, | ||
| 246 | $bad ? ( junk => $bad ) :(), | ||
| 247 | $internal ? ( internal_url => 1 ):(), | ||
| 248 | }; | ||
| 249 | } | ||
| 250 | |||
| 251 | # we don't know the content length yet, unless it can be determined by the | ||
| 252 | # request method. If we got a (pre)pass until the end of the request body | ||
| 253 | # fixup_header will know it and adjust the header | ||
| 254 | $xhdr->{content_length} = | ||
| 255 | $METHODS_WITHOUT_RQBODY{$xhdr->{method}} ? 0:undef; | ||
| 256 | |||
| 257 | $self->{method} = $xhdr->{method}; | ||
| 258 | return $callback->(@cb_args,$hdr,$xhdr); | ||
| 259 | } | ||
| 260 | |||
| 261 | ############################################################################ | ||
| 262 | # fix request header by setting correct content-length | ||
| 263 | # returns true if header could be fixed | ||
| 264 | ############################################################################ | ||
| 265 | sub fixup_request_header { | ||
| 266 | my ($self,$hdr_ref,%args) = @_; | ||
| 267 | my $sub = $self->{fixup_header}[0] or return 1; | ||
| 268 | my $ok = $sub->($self,$hdr_ref,%args); | ||
| 269 | $self->{fixup_header}[0] = undef if $ok; | ||
| 270 | return $ok; | ||
| 271 | } | ||
| 272 | |||
| 273 | |||
| 274 | ############################################################################ | ||
| 275 | # process request body data | ||
| 276 | # just feed to analyzer and call back into request once done | ||
| 277 | ############################################################################ | ||
| 278 | sub request_body { | ||
| 279 | my ($self,$data,@callback) = @_; | ||
| 280 | |||
| 281 | # feed data into IMP | ||
| 282 | $self->{eof}[0] = 1 if $data eq ''; | ||
| 283 | _imp_data($self,0,$data,0,IMP_DATA_HTTPRQ_CONTENT, | ||
| 284 | \&_request_body_imp,\@callback ); | ||
| 285 | } | ||
| 286 | |||
| 287 | sub _request_body_imp { | ||
| 288 | my ($self,$data,$changed,$args) = @_; | ||
| 289 | my ($callback,@cb_args) = @$args; | ||
| 290 | my $eof = _check_eof($self,0); | ||
| 291 | $callback->(@cb_args,$data,$eof) if $data ne '' || $eof; | ||
| 292 | } | ||
| 293 | |||
| 294 | ############################################################################ | ||
| 295 | # process response header | ||
| 296 | ############################################################################ | ||
| 297 | sub response_header { | ||
| 298 | my ($self,$hdr,$xhdr,@callback) = @_; | ||
| 299 | |||
| 300 | # if content is encoded we need to decode it in order to analyze | ||
| 301 | # it. For now only set decode to the encoding method, this will | ||
| 302 | # be changed to a decoding function once we need it in the body | ||
| 303 | if ( my $ce = $xhdr->{fields}{'content-encoding'} ) { | ||
| 304 | # the right way would be to extract all encodings and then complain, if | ||
| 305 | # there is an encoding we don't support. Instead we just look for the | ||
| 306 | # encodings we support | ||
| 307 | my %ce = map { lc($_) => 1 } map { m{\b(?:x-)?(gzip|deflate)\b}ig } @$ce; | ||
| 308 | $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] = join(", ",keys %ce) | ||
| 309 | if %ce; | ||
| 310 | } | ||
| 311 | |||
| 312 | # header length is needed in callback | ||
| 313 | $xhdr->{header_length} = length($hdr); | ||
| 314 | _imp_data($self,1,$hdr,0,IMP_DATA_HTTPRQ_HEADER, | ||
| 315 | \&_response_header_imp,[$xhdr,@callback] ); | ||
| 316 | } | ||
| 317 | |||
| 318 | |||
| 319 | ############################################################################ | ||
| 320 | # callback after passing/replacing the HTTP response header | ||
| 321 | # will reparse the header if changed and continue in the request via | ||
| 322 | # callback | ||
| 323 | ############################################################################ | ||
| 324 | sub _response_header_imp { | ||
| 325 | my ($self,$hdr,$changed,$args) = @_; | ||
| 326 | my ($xhdr,$callback,@cb_args) = @$args; | ||
| 327 | |||
| 328 | my $orig_clen = $xhdr->{content_length}; | ||
| 329 | my $orig_hlen = $xhdr->{header_length}; | ||
| 330 | |||
| 331 | if ( $changed ) { | ||
| 332 | # we need to parse the header again and update xhdr | ||
| 333 | my ($version,$code,$reason,$fields) = $hdr =~m{ \A | ||
| 334 | HTTP/(1\.[01])[\040\t]+ | ||
| 335 | (\d\d\d) | ||
| 336 | (?:[\040\t]+([^\r\n]*))? | ||
| 337 | \r?\n | ||
| 338 | (.*?\n) | ||
| 339 | \r?\n\Z | ||
| 340 | }isx; | ||
| 341 | |||
| 342 | my %kv; | ||
| 343 | my $bad = _parse_hdrfields($fields,\%kv); | ||
| 344 | $xhdr = { | ||
| 345 | code => $code, | ||
| 346 | version => $version, | ||
| 347 | reason => $reason, | ||
| 348 | fields => \%kv, | ||
| 349 | $bad ? ( junk => $bad ) :(), | ||
| 350 | }; | ||
| 351 | } | ||
| 352 | |||
| 353 | # except for some codes or request methods we don't know the | ||
| 354 | # content-length of the body yet | ||
| 355 | # in these cases we try in this order | ||
| 356 | # - check if we got a (pre)pass for the whole body already | ||
| 357 | # - use chunked encoding if client speaks HTTP/1.1 | ||
| 358 | # - don't specify content-length and close request with connection close | ||
| 359 | |||
| 360 | # we don't change $hdr here because it will be rebuild from fields anyway | ||
| 361 | if ( $CODE_WITHOUT_RPBODY{$xhdr->{code}} ) { | ||
| 362 | $xhdr->{content_length} = 0; | ||
| 363 | # better remove them | ||
| 364 | delete @{ $xhdr->{fields} }{ qw/ content-length transfer-encoding / }; | ||
| 365 | goto callback; | ||
| 366 | } | ||
| 367 | |||
| 368 | if ( $METHODS_WITHOUT_RPBODY{ $self->{method} } ) { | ||
| 369 | $xhdr->{content_length} = 0; | ||
| 370 | # keep content-length etc, client might want to peek into it using HEAD | ||
| 371 | goto callback; | ||
| 372 | } | ||
| 373 | |||
| 374 | # reset infos about content-length | ||
| 375 | $xhdr->{content_length} = $xhdr->{chunked} = undef; | ||
| 376 | delete @{ $xhdr->{fields} }{ qw/ content-length transfer-encoding / }; | ||
| 377 | |||
| 378 | # if we have read the whole body already or at least know, that we will | ||
| 379 | # not change anymore data, we could compute the new content-length | ||
| 380 | my $clen; | ||
| 381 | my $nochange; | ||
| 382 | while ( defined $orig_clen ) { | ||
| 383 | my $rpsize = $orig_hlen + $orig_clen; | ||
| 384 | |||
| 385 | if ( my $pass = $self->{pass}[1] ) { | ||
| 386 | if ( $pass == IMP_MAXOFFSET or $pass >= $rpsize ) { | ||
| 387 | # will not look at and not change body | ||
| 388 | $nochange = 1; | ||
| 389 | goto compute_clen; | ||
| 390 | } | ||
| 391 | } | ||
| 392 | if ( my $prepass = $self->{prepass}[1] ) { | ||
| 393 | if ( $prepass == IMP_MAXOFFSET or $prepass >= $rpsize ) { | ||
| 394 | # will not change body | ||
| 395 | $nochange = 1; | ||
| 396 | goto compute_clen; | ||
| 397 | } | ||
| 398 | } | ||
| 399 | if ($self->{ibuf}[1][0][0] >= $rpsize) { # ibuf[server].base | ||
| 400 | # everything passed thru ibuf | ||
| 401 | goto compute_clen; | ||
| 402 | } | ||
| 403 | |||
| 404 | # we still don't know final size | ||
| 405 | last; | ||
| 406 | |||
| 407 | compute_clen: | ||
| 408 | # bytes in ibuf and outstanding bytes will not be changed, so: | ||
| 409 | # new_content_length = | ||
| 410 | # ( total_size - received ) # not yet received | ||
| 411 | # + ( received - ibuf.base ) # still in ibuf | ||
| 412 | # ---> | ||
| 413 | # total_size - ibuf.base | ||
| 414 | $clen = $rpsize - $self->{ibuf}[1][0][0]; | ||
| 415 | |||
| 416 | last; | ||
| 417 | } | ||
| 418 | |||
| 419 | if ( $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] ) { | ||
| 420 | if ( $nochange ) { | ||
| 421 | # we will pass encoded stuff, either no decoding needs to | ||
| 422 | # be done (pass) or we will decode only for the analyzer (prepass) | ||
| 423 | # which will only watch at the content, but not change it | ||
| 424 | $self->{pass_encoded}[1] = 1; | ||
| 425 | |||
| 426 | my $pass = $self->{pass}[1]; | ||
| 427 | if ( $pass and defined $orig_clen and ( | ||
| 428 | $pass == IMP_MAXOFFSET or | ||
| 429 | $pass >= $orig_clen + $orig_hlen )) { | ||
| 430 | # no need to decode body | ||
| 431 | $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] = undef; | ||
| 432 | } | ||
| 433 | } else { | ||
| 434 | # content is encoded and inspection wants to see decoded stuff, | ||
| 435 | # which we then will forward too | ||
| 436 | # but decoding might change length | ||
| 437 | $clen = undef; | ||
| 438 | # the content will be delivered decoded | ||
| 439 | delete $xhdr->{fields}{'content-encoding'} | ||
| 440 | } | ||
| 441 | } | ||
| 442 | if ( defined $clen ) { | ||
| 443 | $xhdr->{fields}{'content-length'} = [ $clen ]; | ||
| 444 | $xhdr->{content_length} = $clen; | ||
| 445 | } | ||
| 446 | |||
| 447 | callback: | ||
| 448 | $callback->(@cb_args,$hdr,$xhdr); | ||
| 449 | } | ||
| 450 | |||
| 451 | |||
| 452 | |||
| 453 | ############################################################################ | ||
| 454 | # handle response body data | ||
| 455 | ############################################################################ | ||
| 456 | sub response_body { | ||
| 457 | my ($self,$data,@callback) = @_; | ||
| 458 | |||
| 459 | # forward to IMP analyzer | ||
| 460 | $self->{eof}[1] = 1 if $data eq ''; | ||
| 461 | _imp_data($self,1,$data,0,IMP_DATA_HTTPRQ_CONTENT, | ||
| 462 | \&_response_body_imp,\@callback); | ||
| 463 | } | ||
| 464 | |||
| 465 | sub _response_body_imp { | ||
| 466 | my ($self,$data,$changed,$args) = @_; | ||
| 467 | my ($callback,@cb_args) = @$args; | ||
| 468 | my $eof = _check_eof($self,1); | ||
| 469 | $callback->(@cb_args,$data,$eof) if $data ne '' || $eof; | ||
| 470 | } | ||
| 471 | |||
| 472 | |||
| 473 | sub _check_eof { | ||
| 474 | my ($self,$dir) = @_; | ||
| 475 | $DEBUG && $self->{request}->xdebug( | ||
| 476 | "check eof[%d] - eof=%d - %s - (pre)pass=%d/%d", | ||
| 477 | $dir,$self->{eof}[$dir], _show_buf($self,$dir), | ||
| 478 | $self->{prepass}[$dir], | ||
| 479 | $self->{pass}[$dir] | ||
| 480 | ); | ||
| 481 | return $self->{eof}[$dir] # received eof | ||
| 482 | && ! defined $self->{ibuf}[$dir][0][2] # no more data in buf | ||
| 483 | && ( # (pre)pass til end ok | ||
| 484 | $self->{prepass}[$dir] == IMP_MAXOFFSET | ||
| 485 | || $self->{pass}[$dir] == IMP_MAXOFFSET | ||
| 486 | ); | ||
| 487 | } | ||
| 488 | |||
| 489 | sub _show_buf { | ||
| 490 | my ($self,$dir) = @_; | ||
| 491 | return join('|', | ||
| 492 | map { ($_->[2]||'none')."($_->[0],+".length($_->[1]).")" } | ||
| 493 | @{ $self->{ibuf}[$dir] } | ||
| 494 | ); | ||
| 495 | } | ||
| 496 | |||
| 497 | |||
| 498 | |||
| 499 | ############################################################################ | ||
| 500 | # Websockets, TLS upgrades etc | ||
| 501 | # if not IMP the forwarding will be done inside this function, otherwise it | ||
| 502 | # will be done in _in_data_imp, which gets called by IMP callback | ||
| 503 | ############################################################################ | ||
| 504 | sub data { | ||
| 505 | my ($self,$dir,$data,@callback) = @_; | ||
| 506 | |||
| 507 | # forward to IMP analyzer | ||
| 508 | $self->{eof}[$dir] = 1 if $data eq ''; | ||
| 509 | _imp_data($self,$dir,$data,0,IMP_DATA_HTTPRQ_CONTENT, | ||
| 510 | \&_data_imp,[$dir,@callback]); | ||
| 511 | } | ||
| 512 | |||
| 513 | sub _data_imp { | ||
| 514 | my ($self,$data,$changed,$args) = @_; | ||
| 515 | my ($dir,$callback,@cb_args) = @$args; | ||
| 516 | my $eof = $self->{eof}[$dir] && # got eof from server | ||
| 517 | ! defined $self->{ibuf}[$dir][0][2]; # no more data in ibuf[server] | ||
| 518 | $callback->(@cb_args,$dir,$data,$eof) if $data ne '' || $eof; | ||
| 519 | } | ||
| 520 | |||
| 521 | |||
| 522 | |||
| 523 | ############################################################################ | ||
| 524 | # callback from IMP | ||
| 525 | # process return types and trigger type specific callbacks on (pre)pass/replace | ||
| 526 | ############################################################################ | ||
| 527 | sub _imp_callback { | ||
| 528 | my $self = shift; | ||
| 529 | |||
| 530 | my %fwd; # forwarded data, per dir | ||
| 531 | for my $rv (@_) { | ||
| 532 | |||
| 533 | # if the request got closed in between just return | ||
| 534 | my $request = $self->{request} or return; | ||
| 535 | |||
| 536 | my $rtype = shift(@$rv); | ||
| 537 | |||
| 538 | # deny further data | ||
| 539 | if ( $rtype == IMP_DENY ) { | ||
| 540 | my ($impdir,$msg) = @$rv; | ||
| 541 | $DEBUG && $request->xdebug("got deny($impdir) $msg"); | ||
| 542 | return $request->deny($msg // 'closed by imp'); | ||
| 543 | } | ||
| 544 | |||
| 545 | # log some data | ||
| 546 | if ( $rtype == IMP_LOG ) { | ||
| 547 | my ($impdir,$offset,$len,$level,$msg) = @$rv; | ||
| 548 | $DEBUG && $request->xdebug("got log($impdir,$level) $msg"); | ||
| 549 | my %args = ( msg => $msg ); | ||
| 550 | $args{offset} = $offset if defined $offset; | ||
| 551 | $args{len} = $len if defined $len; | ||
| 552 | $msg = join(' ',"log[$level]", map { "$_=$args{$_}" } keys %args); | ||
| 553 | $request->xdebug($msg); | ||
| 554 | next; | ||
| 555 | } | ||
| 556 | |||
| 557 | # set accounting field | ||
| 558 | if ( $rtype == IMP_ACCTFIELD ) { | ||
| 559 | my ($key,$value) = @$rv; | ||
| 560 | $DEBUG && $request->xdebug("got acct $key => $value"); | ||
| 561 | $request->{acct}{$key} = $value; | ||
| 562 | next; | ||
| 563 | } | ||
| 564 | |||
| 565 | # (pre)pass data up to offset | ||
| 566 | if ( $rtype ~~ [ IMP_PASS, IMP_PREPASS ]) { | ||
| 567 | my ($dir,$offset) = @$rv; | ||
| 568 | $DEBUG && $request->xdebug("got $rtype($dir) off=$offset "._show_buf($self,$dir)); | ||
| 569 | |||
| 570 | if ( $rtype == IMP_PASS ) { | ||
| 571 | # ignore pass if it's not better than a previous pass | ||
| 572 | if ( $self->{pass}[$dir] == IMP_MAXOFFSET ) { | ||
| 573 | # there is no better thing than IMP_MAXOFFSET | ||
| 574 | next; | ||
| 575 | } elsif ( $offset == IMP_MAXOFFSET | ||
| 576 | or $offset > $self->{ibuf}[$dir][0][0] ) { | ||
| 577 | # we can pass new data | ||
| 578 | $self->{pass}[$dir] = $offset; | ||
| 579 | } else { | ||
| 580 | # offset is no better than previous pass | ||
| 581 | next; | ||
| 582 | } | ||
| 583 | |||
| 584 | } else { # IMP_PREPASS | ||
| 585 | # ignore prepass if it's not better than a previous pass | ||
| 586 | # and a previous prepaself->{ibuf}[1][0] | ||
| 587 | if ( $self->{pass}[$dir] == IMP_MAXOFFSET | ||
| 588 | or $self->{prepass}[$dir] == IMP_MAXOFFSET ) { | ||
| 589 | # there is no better thing than IMP_MAXOFFSET | ||
| 590 | $DEBUG && debug("new off $offset no better than existing (pre)pass=max"); | ||
| 591 | next; | ||
| 592 | } elsif ( $offset == IMP_MAXOFFSET | ||
| 593 | or $offset > $self->{ibuf}[$dir][0][0] ) { | ||
| 594 | # we can prepass new data | ||
| 595 | $self->{prepass}[$dir] = $offset; | ||
| 596 | $DEBUG && debug("update prepass with new off $offset"); | ||
| 597 | } else { | ||
| 598 | # offset is no better than previous pass | ||
| 599 | $DEBUG && debug( | ||
| 600 | "new off $offset no better than existing $self->{ibuf}[$dir][0][0]"); | ||
| 601 | next; | ||
| 602 | } | ||
| 603 | } | ||
| 604 | |||
| 605 | # collect data up to offset for forwarding | ||
| 606 | # list of [ changed,data,callback,cbarg ] | ||
| 607 | my $fwd = $fwd{$dir} ||= []; | ||
| 608 | |||
| 609 | my $ibuf = $self->{ibuf}[$dir]; | ||
| 610 | my $ib0; # top of ibuf, e.g. ibuf[0] | ||
| 611 | |||
| 612 | while ( @$ibuf ) { | ||
| 613 | $ib0 = shift(@$ibuf); | ||
| 614 | defined $ib0->[2] or last; # dummy entry with no type | ||
| 615 | |||
| 616 | if ( $offset == IMP_MAXOFFSET ) { | ||
| 617 | # forward this buf and maybe more | ||
| 618 | push @$fwd, [ 0, @{$ib0}[1,3,4] ]; | ||
| 619 | } else { | ||
| 620 | my $pass = $offset - $ib0->[0]; | ||
| 621 | my $len0 = length($ib0->[1]); | ||
| 622 | if ( $pass > $len0 ) { | ||
| 623 | # forward this buf and maybe more | ||
| 624 | push @$fwd, [ 0, @{$ib0}[1,3,4] ]; | ||
| 625 | } elsif ( $pass == $len0 ) { | ||
| 626 | # forward this buf, but not more | ||
| 627 | push @$fwd, [ 0, @{$ib0}[1,3,4] ]; | ||
| 628 | |||
| 629 | # add empty buf if this was the last, this will also | ||
| 630 | # trigger resetting pass,prepass below | ||
| 631 | if ( @$ibuf ) { # still data in buffer | ||
| 632 | } elsif ( $ib0->[2] < 0 ) { | ||
| 633 | # no eof yet and no further data in ibuf | ||
| 634 | # we might get a replacement at the end of the | ||
| 635 | # buffer so put emptied buffer back | ||
| 636 | $ib0->[1] = ''; | ||
| 637 | push @$ibuf, $ib0; | ||
| 638 | } else { | ||
| 639 | push @$ibuf, [ $offset,'' ]; | ||
| 640 | } | ||
| 641 | last; | ||
| 642 | } elsif ( $ib0->[2] < 0 ) { | ||
| 643 | # streaming type: | ||
| 644 | # forward part of buf | ||
| 645 | push @$fwd, [ | ||
| 646 | 0, # not changed | ||
| 647 | substr($ib0->[1],0,$pass,''), # data | ||
| 648 | $ib0->[3], # callback | ||
| 649 | $ib0->[4], # args | ||
| 650 | ]; | ||
| 651 | # keep rest in ibuf | ||
| 652 | unshift @$ibuf,$ib0; | ||
| 653 | $ib0->[0] += $pass; | ||
| 654 | last; # nothing more to forward | ||
| 655 | } else { | ||
| 656 | # packet type: they need to be processed in total | ||
| 657 | return $request->fatal("partial $rtype for $ib0->[2]"); | ||
| 658 | } | ||
| 659 | } | ||
| 660 | } | ||
| 661 | |||
| 662 | if ( @$ibuf ) { | ||
| 663 | # there are still data in ibuf which cannot get passed, | ||
| 664 | # so reset pass, prepass | ||
| 665 | $self->{pass}[$dir] = $self->{prepass}[$dir] = 0; | ||
| 666 | } else { | ||
| 667 | # add empty buffer containing only current offset based on | ||
| 668 | # what we last removed from ibuf | ||
| 669 | push @$ibuf, [ $ib0->[0] + length($ib0->[1]),'' ]; | ||
| 670 | } | ||
| 671 | |||
| 672 | next; | ||
| 673 | } | ||
| 674 | |||
| 675 | |||
| 676 | # replace data up to offset | ||
| 677 | if ( $rtype == IMP_REPLACE ) { | ||
| 678 | my ($dir,$offset,$newdata) = @$rv; | ||
| 679 | $DEBUG && $request->xdebug("got replace($dir) off=$offset data.len=". | ||
| 680 | length($newdata)); | ||
| 681 | my $ibuf = $self->{ibuf}[$dir]; | ||
| 682 | @$ibuf or die "no ibuf"; | ||
| 683 | |||
| 684 | # if there is an active pass|prepass (e.g. pointing to future data) | ||
| 685 | # the data cannot be replaced | ||
| 686 | return $request->fatal( | ||
| 687 | "cannot replace data which are said to be passed") | ||
| 688 | if $self->{pass}[$dir] or $self->{prepass}[$dir]; | ||
| 689 | |||
| 690 | # we cannot replace future data | ||
| 691 | return $request->fatal('IMP', "cannot use replace with maxoffset") | ||
| 692 | if $offset == IMP_MAXOFFSET; | ||
| 693 | |||
| 694 | # data to replace cannot span different types, so they must be in | ||
| 695 | # the first ibuf | ||
| 696 | my $ib0 = $ibuf->[0]; | ||
| 697 | my $rlen = $offset - $ib0->[0]; | ||
| 698 | my $len0 = length($ib0->[1]); | ||
| 699 | |||
| 700 | # some sanity checks | ||
| 701 | if ( $rlen < 0 ) { | ||
| 702 | return $request->fatal("cannot replace already passed data"); | ||
| 703 | } elsif ( $rlen > $len0 ) { | ||
| 704 | return $request->fatal( | ||
| 705 | "replacement cannot span multiple data types") | ||
| 706 | if @$ibuf>1 or $ib0->[2]>0; | ||
| 707 | return $request->fatal("cannot replace future data"); | ||
| 708 | } elsif ( $rlen < $len0 ) { | ||
| 709 | # replace part of buffer | ||
| 710 | return $request->fatal("cannot replace part of packet type") | ||
| 711 | if $ib0->[2]>0; | ||
| 712 | |||
| 713 | # keep rest and update position | ||
| 714 | substr( $ib0->[1],0,$rlen,'' ) if $rlen; | ||
| 715 | $ib0->[0] += $rlen; | ||
| 716 | } else { | ||
| 717 | # remove complete buffer | ||
| 718 | if ( @$ibuf>1 ) { # still data in buffer | ||
| 719 | } elsif ( $ib0->[2] < 0 ) { | ||
| 720 | # no eof yet and no further data in ibuf | ||
| 721 | # we might get a replacement at the end of the | ||
| 722 | # buffer so put emptied buffer back | ||
| 723 | $ib0->[1] = ''; | ||
| 724 | } else { | ||
| 725 | # replace with empty | ||
| 726 | @$ibuf = [ $offset,'' ]; | ||
| 727 | } | ||
| 728 | } | ||
| 729 | |||
| 730 | push @{$fwd{$dir}}, [ | ||
| 731 | 1, # changed | ||
| 732 | $newdata, # new data | ||
| 733 | $ib0->[3], # callback | ||
| 734 | $ib0->[4], # cbargs | ||
| 735 | ]; | ||
| 736 | |||
| 737 | next; | ||
| 738 | } | ||
| 739 | |||
| 740 | return $request->fatal("unsupported IMP return type: $rtype"); | ||
| 741 | } | ||
| 742 | |||
| 743 | %fwd or return; # no passes/replacements... | ||
| 744 | |||
| 745 | while ( my ($dir,$fwd) = each %fwd ) { | ||
| 746 | while ( my $fw = shift(@$fwd)) { | ||
| 747 | #warn Dumper($fw); use Data::Dumper; | ||
| 748 | my ($changed,$data,$callback,$args) = @$fw; | ||
| 749 | $callback->($self,$data,$changed,$args); | ||
| 750 | } | ||
| 751 | # unstall ? | ||
| 752 | _check_stalled($self,$dir) if $self->{stalled}[$dir]; | ||
| 753 | } | ||
| 754 | } | ||
| 755 | |||
| 756 | ############################################################################ | ||
| 757 | # IMP specific stalled handling | ||
| 758 | # if more than IMP_MAX_IN_ANALYZER are in the ibuf we will stop receiving | ||
| 759 | # data and wait until callbacks decrease size of ibuf again | ||
| 760 | ############################################################################ | ||
| 761 | sub _check_stalled { | ||
| 762 | my ($self,$dir) = @_; | ||
| 763 | my $ibuf = $self->{ibuf}[$dir]; | ||
| 764 | my $size = $ibuf->[-1][0] + length($ibuf->[-1][1]) - $ibuf->[0][0]; | ||
| 765 | |||
| 766 | if ( $IMP_MAX_IN_ANALYZER ) { | ||
| 767 | if ( $size > $IMP_MAX_IN_ANALYZER ) { | ||
| 768 | return if $self->{stalled}[$dir]; # no change - still stalled | ||
| 769 | # set stalled and disable fd | ||
| 770 | $self->{stalled}[$dir] = 1; | ||
| 771 | $DEBUG && $self->{request}->xdebug( | ||
| 772 | "max in analyzer reached - set $dir to stalled"); | ||
| 773 | my $relay = $self->{request}{conn}{relay} or return; | ||
| 774 | if ( my $fo = $relay->fd($dir)) { | ||
| 775 | $fo->mask( r => 0 ); | ||
| 776 | } | ||
| 777 | } else { | ||
| 778 | return if ! $self->{stalled}[$dir]; # no change - still not stalled | ||
| 779 | # set unstalled and enable fd | ||
| 780 | $DEBUG && $self->{request}->xdebug( | ||
| 781 | "below max in analyzer - unstall $dir"); | ||
| 782 | $self->{stalled}[$dir] = 0; | ||
| 783 | my $relay = $self->{request}{conn}{relay} or return; | ||
| 784 | if ( my $fo = $relay->fd($dir)) { | ||
| 785 | $fo->mask( r => 1 ); | ||
| 786 | } | ||
| 787 | } | ||
| 788 | } | ||
| 789 | } | ||
| 790 | |||
| 791 | ############################################################################ | ||
| 792 | # send data to IMP analyzer | ||
| 793 | # if we had a previous (pre)pass some data can be forwarded immediatly, for | ||
| 794 | # others we have to wait for the analyzer callback | ||
| 795 | # returns how many bytes of data are waiting for callback, e.g. 0 if we | ||
| 796 | # we can pass everything immediately | ||
| 797 | ############################################################################ | ||
| 798 | sub _imp_data { | ||
| 799 | my ($self,$dir,$data,$offset,$type,$callback,$args) = @_; | ||
| 800 | my $ibuf = $self->{ibuf}[$dir]; | ||
| 801 | my $eobuf = $ibuf->[-1][0] + length($ibuf->[-1][1]); | ||
| 802 | |||
| 803 | my $encoded_data; | ||
| 804 | if ( my $decode = $self->{decode}{$type+0}[$dir] ) { | ||
| 805 | # set up decoder if not set up yet | ||
| 806 | if ( ! ref($decode)) { | ||
| 807 | # create function to decode content | ||
| 808 | $self->{decode}{$type+0}[$dir] = $decode = _create_decoder($decode) | ||
| 809 | || return $self->{request}->fatal( | ||
| 810 | "cannot decode content-encoding $decode"); | ||
| 811 | } | ||
| 812 | |||
| 813 | # offsets relates to original stream, but we put the decoded stream | ||
| 814 | # into ibuf. And offset>0 means, that we have a gap in the input, | ||
| 815 | # which is not allowed, when decoding a stream. | ||
| 816 | die "cannot use content decoder with gap in data" if $offset; | ||
| 817 | |||
| 818 | $encoded_data = $data if $self->{pass_encoded}[$dir]; | ||
| 819 | defined( $data = $decode->($data) ) | ||
| 820 | or return $self->{request}->fatal("decoding content failed"); | ||
| 821 | } | ||
| 822 | |||
| 823 | if ( $offset ) { | ||
| 824 | die "offset($offset)<eobuf($eobuf)" if $offset < $eobuf; | ||
| 825 | $offset = 0 if $offset == $eobuf; | ||
| 826 | } | ||
| 827 | |||
| 828 | my $fwd; # what gets send to analyzer | ||
| 829 | |||
| 830 | my $dlen = length($data); | ||
| 831 | my $pass = $self->{pass}[$dir]; | ||
| 832 | if ( $pass ) { | ||
| 833 | # if pass is set there should be no data in ibuf, e.g. everything | ||
| 834 | # before should have been passed | ||
| 835 | ! $ibuf->[0][2] or die "unexpected data in ibuf"; | ||
| 836 | |||
| 837 | if ( $pass == IMP_MAXOFFSET ) { | ||
| 838 | # pass thru w/o analyzing | ||
| 839 | $ibuf->[0][0] += $dlen; | ||
| 840 | $DEBUG && $self->{request}->xdebug("can pass($dir) infinite"); | ||
| 841 | return $callback->($self,$encoded_data // $data,0,$args); | ||
| 842 | } | ||
| 843 | |||
| 844 | my $canpass = $pass - ( $offset||$eobuf ); | ||
| 845 | if ( $canpass <= 0 ) { | ||
| 846 | # cannot pass anything, pass should have been reset already | ||
| 847 | die "pass($dir,$pass) must be point into future ($canpass)"; | ||
| 848 | } elsif ( $canpass >= $dlen) { | ||
| 849 | # can pass everything | ||
| 850 | $ibuf->[0][0] += $dlen; | ||
| 851 | if ( $data eq '' ) { | ||
| 852 | # forward eof to analyzer | ||
| 853 | $fwd = $data; | ||
| 854 | $DEBUG && $self->{request}->xdebug("pass($dir) eof"); | ||
| 855 | goto SEND2IMP; | ||
| 856 | } | ||
| 857 | $DEBUG && $self->{request}->xdebug( | ||
| 858 | "can pass($dir) all: pass($canpass)>=data.len($dlen)"); | ||
| 859 | return $callback->($self,$encoded_data // $data,0,$args); | ||
| 860 | } elsif ( $type < 0 ) { | ||
| 861 | # can pass part of data, only for streaming types | ||
| 862 | # remove from data what can be passed | ||
| 863 | die "body might change" if $self->{pass_encoded}[$dir]; | ||
| 864 | $ibuf->[0][0] += $canpass; | ||
| 865 | my $passed_data = substr($data,0,$canpass,''); | ||
| 866 | $eobuf += $canpass; | ||
| 867 | $dlen = length($data); | ||
| 868 | $DEBUG && $self->{request}->xdebug( | ||
| 869 | "can pass($dir) part: pass($canpass)<data.len($dlen)"); | ||
| 870 | $callback->($self,$passed_data,0,$args); # callback but continue | ||
| 871 | } | ||
| 872 | } | ||
| 873 | |||
| 874 | $fwd = $data; # this must be forwarded to analyzer | ||
| 875 | |||
| 876 | my $prepass = $self->{prepass}[$dir]; | ||
| 877 | if ( $prepass ) { | ||
| 878 | # if prepass is set there should be no data in ibuf, e.g. everything | ||
| 879 | # before should have been passed | ||
| 880 | ! $ibuf->[0][2] or die "unexpected data in ibuf"; | ||
| 881 | if ( $prepass == IMP_MAXOFFSET ) { | ||
| 882 | # prepass everything | ||
| 883 | $ibuf->[0][0] += $dlen; | ||
| 884 | $DEBUG && $self->{request}->xdebug("can prepass($dir) infinite"); | ||
| 885 | $callback->($self,$encoded_data // $data,0,$args); # callback but continue | ||
| 886 | goto SEND2IMP; | ||
| 887 | } | ||
| 888 | |||
| 889 | my $canprepass = $prepass - ( $offset||$eobuf ); | ||
| 890 | if ( $canprepass <= 0 ) { | ||
| 891 | # cannot prepass anything, prepass should have been reset already | ||
| 892 | die "prepass must be point into future"; | ||
| 893 | } elsif ( $canprepass >= $dlen) { | ||
| 894 | # can prepass everything | ||
| 895 | $ibuf->[0][0] += $dlen; | ||
| 896 | $callback->($self,$encoded_data // $data,0,$args); # callback but continue | ||
| 897 | $DEBUG && $self->{request}->xdebug( | ||
| 898 | "can prepass($dir) all: pass($canprepass)>=data.len($dlen)"); | ||
| 899 | goto SEND2IMP; | ||
| 900 | } elsif ( $type < 0 ) { | ||
| 901 | # can prepass part of data, only for streaming types | ||
| 902 | # remove from data what can be prepassed | ||
| 903 | die "body might change" if $self->{pass_encoded}[$dir]; | ||
| 904 | $ibuf->[0][0] += $canprepass; | ||
| 905 | my $passed_data = substr($data,0,$canprepass,''); | ||
| 906 | $eobuf += $canprepass; | ||
| 907 | $dlen = length($data); | ||
| 908 | $DEBUG && $self->{request}->xdebug( | ||
| 909 | "can prepass($dir) part: prepass($canprepass)<data.len($dlen)"); | ||
| 910 | $callback->($self,$passed_data,0,$args); # callback but continue | ||
| 911 | } | ||
| 912 | } | ||
| 913 | |||
| 914 | # everything else in $data must be added to buffer | ||
| 915 | |||
| 916 | # there can be no gaps inside ibuf because caller is only allowed to | ||
| 917 | # pass data which we explicitly allowed | ||
| 918 | if ( $offset && $offset > $eobuf ) { | ||
| 919 | defined $ibuf->[0][2] and # we have still data in ibuf! | ||
| 920 | die "there can be no gaps in ibuf"; | ||
| 921 | } | ||
| 922 | if ( ! defined $ibuf->[-1][2] ) { | ||
| 923 | # replace buf, because it was empty | ||
| 924 | $ibuf->[-1] = [ $offset||$eobuf,$data,$type,$callback,$args ]; | ||
| 925 | } elsif ( $type < 0 | ||
| 926 | and $type == $ibuf->[-1][2] | ||
| 927 | and $callback == $ibuf->[-1][3] | ||
| 928 | ) { | ||
| 929 | # streaming data, concatinate to existing buf of same type | ||
| 930 | $ibuf->[-1][1] .= $data; | ||
| 931 | } else { | ||
| 932 | # different type or non-streaming data, add new buf | ||
| 933 | push @$ibuf,[ $offset||$eobuf,$data,$type,$callback,$args ]; | ||
| 934 | } | ||
| 935 | $DEBUG && $self->{request}->xdebug( "ibuf.length=%d", | ||
| 936 | $ibuf->[-1][0] + length($ibuf->[-1][1]) - $ibuf->[0][0]); | ||
| 937 | _check_stalled($self,$dir) if ! $self->{stalled}[$dir]; | ||
| 938 | |||
| 939 | SEND2IMP: | ||
| 940 | $DEBUG && $self->{request}->xdebug("forward(%d) %d bytes type=%s off=%d to analyzer", | ||
| 941 | $dir,length($fwd),$type,$offset); | ||
| 942 | $self->{imp}->data($dir,$fwd,$offset,$type); | ||
| 943 | return length($fwd); | ||
| 944 | } | ||
| 945 | |||
| 946 | ##################################################################### | ||
| 947 | # parse header fields | ||
| 948 | # taken from Net::Inspect::L7::HTTP (where it got put in by myself) | ||
| 949 | ##################################################################### | ||
| 950 | my $token = qr{[^()<>@,;:\\"/\[\]?={}\x00-\x20\x7f-\xff]+}; | ||
| 951 | my $token_value_cont = qr{ | ||
| 952 | ($token): # key: | ||
| 953 | [\040\t]*([^\r\n]*?)[\040\t]* # <space>value<space> | ||
| 954 | ((?:\r?\n[\040\t][^\r\n]*)*) # continuation lines | ||
| 955 | \r?\n # (CR)LF | ||
| 956 | }x; | ||
| 957 | sub _parse_hdrfields { | ||
| 958 | my ($hdr,$fields) = @_; | ||
| 959 | my $bad = ''; | ||
| 960 | parse: | ||
| 961 | while ( $hdr =~m{\G$token_value_cont}gc ) { | ||
| 962 | if ($3 eq '') { | ||
| 963 | # no continuation line | ||
| 964 | push @{$fields->{ lc($1) }},$2; | ||
| 965 | } else { | ||
| 966 | # with continuation line | ||
| 967 | my ($k,$v) = ($1,$2.$3); | ||
| 968 | # <space>value-part<space> -> ' ' + value-part | ||
| 969 | $v =~s{[\r\n]+[ \t](.*?)[ \t]*}{ $1}g; | ||
| 970 | push @{$fields->{ lc($k) }},$v; | ||
| 971 | } | ||
| 972 | } | ||
| 973 | if (pos($hdr)//0 != length($hdr)) { | ||
| 974 | # bad line inside | ||
| 975 | substr($hdr,0,pos($hdr),''); | ||
| 976 | $bad .= $1 if $hdr =~s{\A([^\n]*)\n}{}; | ||
| 977 | goto parse; | ||
| 978 | } | ||
| 979 | return $bad; | ||
| 980 | } | ||
| 981 | |||
| 982 | ##################################################################### | ||
| 983 | # create decoder function for gzip|deflate content-encoding | ||
| 984 | ##################################################################### | ||
| 985 | sub _create_decoder { | ||
| 986 | my $typ = shift; | ||
| 987 | $typ ~~ [ 'gzip','deflate' ] or return; # not supported | ||
| 988 | |||
| 989 | my $gzip_csum; | ||
| 990 | my $buf = ''; | ||
| 991 | my $inflate; | ||
| 992 | |||
| 993 | return sub { | ||
| 994 | my $data = shift; | ||
| 995 | $buf .= $data; | ||
| 996 | |||
| 997 | goto inflate if defined $inflate; | ||
| 998 | |||
| 999 | # read gzip|deflate header | ||
| 1000 | my $wb; | ||
| 1001 | my $more = $data eq '' ? undef:''; # need more data if possible | ||
| 1002 | if ( $typ eq 'gzip' ) { | ||
| 1003 | my $hdr_len = 10; # minimum gzip header | ||
| 1004 | |||
| 1005 | return $more if length($buf) < $hdr_len; | ||
| 1006 | my ($magic,$method,$flags) = unpack('vCC',$buf); | ||
| 1007 | if ( $magic != 0x8b1f or $method != Z_DEFLATED or $flags & 0xe0 ) { | ||
| 1008 | $DEBUG && debug("no valid gzip header. assuming plain text"); | ||
| 1009 | $inflate = ''; # defined but false | ||
| 1010 | goto inflate; | ||
| 1011 | } | ||
| 1012 | if ( $flags & 4 ) { | ||
| 1013 | # skip extra section | ||
| 1014 | return $more if length($buf) < ($hdr_len+=2); | ||
| 1015 | $hdr_len += unpack('x10v',$buf); | ||
| 1016 | return $more if length($buf) < $hdr_len; | ||
| 1017 | } | ||
| 1018 | if ( $flags & 8 ) { | ||
| 1019 | # skip filename | ||
| 1020 | my $o = index($buf,"\0",$hdr_len); | ||
| 1021 | return $more if $o == -1; # end of filename not found | ||
| 1022 | $hdr_len = $o+1; | ||
| 1023 | } | ||
| 1024 | if ( $flags & 16 ) { | ||
| 1025 | # skip comment | ||
| 1026 | my $o = index($buf,"\0",$hdr_len); | ||
| 1027 | return $more if $o == -1; # end of comment not found | ||
| 1028 | $hdr_len = $o+1; | ||
| 1029 | } | ||
| 1030 | if ( $flags & 2 ) { | ||
| 1031 | # skip CRC | ||
| 1032 | return $more if length($buf) < ($hdr_len+=2); | ||
| 1033 | } | ||
| 1034 | |||
| 1035 | # remove header | ||
| 1036 | substr($buf,0,$hdr_len,''); | ||
| 1037 | $gzip_csum = 8; # 8 byte Adler CRC at end | ||
| 1038 | $wb = -MAX_WBITS(); # see Compress::Raw::Zlib | ||
| 1039 | |||
| 1040 | } else { | ||
| 1041 | # deflate | ||
| 1042 | # according to RFC it should be zlib, but due to the encoding name | ||
| 1043 | # often real deflate is used instead | ||
| 1044 | # check magic bytes to decide | ||
| 1045 | |||
| 1046 | # lets see if it looks like a zlib header | ||
| 1047 | # check for CM=8, CMID<=7 in first byte and valid FCHECK in | ||
| 1048 | # seconds byte | ||
| 1049 | return $more if length($buf)<2; | ||
| 1050 | my $magic = unpack('C',substr($buf,0,1)); | ||
| 1051 | if ( | ||
| 1052 | ( $magic & 0b1111 ) == 8 # CM = 8 | ||
| 1053 | and $magic >> 4 <= 7 # CMID <= 7 | ||
| 1054 | and unpack('n',substr($buf,0,2)) % 31 == 0 # valid FCHECK | ||
| 1055 | ) { | ||
| 1056 | # looks like zlib header | ||
| 1057 | $wb = +MAX_WBITS(); # see Compress::Raw::Zlib | ||
| 1058 | } else { | ||
| 1059 | # assume deflate | ||
| 1060 | $wb = -MAX_WBITS(); # see Compress::Raw::Zlib | ||
| 1061 | } | ||
| 1062 | } | ||
| 1063 | |||
| 1064 | $inflate = Compress::Raw::Zlib::Inflate->new( | ||
| 1065 | -WindowBits => $wb, | ||
| 1066 | -AppendOutput => 1, | ||
| 1067 | -ConsumeInput => 1 | ||
| 1068 | ) or die "cannot create inflation stream"; | ||
| 1069 | |||
| 1070 | inflate: | ||
| 1071 | |||
| 1072 | return '' if $buf eq ''; | ||
| 1073 | |||
| 1074 | if ( ! $inflate ) { | ||
| 1075 | # wrong gzip header: sometimes servers claim to use gzip | ||
| 1076 | # if confronted with "Accept-Encoding: identity" but in reality | ||
| 1077 | # they send plain text | ||
| 1078 | # so consider it plain text and don't decode | ||
| 1079 | my $out = $buf; | ||
| 1080 | $buf = ''; | ||
| 1081 | return $out | ||
| 1082 | } | ||
| 1083 | |||
| 1084 | my $out = ''; | ||
| 1085 | my $stat = $inflate->inflate(\$buf,\$out); | ||
| 1086 | if ( $stat == Z_STREAM_END ) { | ||
| 1087 | if ( $gzip_csum and length($buf) >= $gzip_csum ) { | ||
| 1088 | # TODO - check checksum - but what would it help? | ||
| 1089 | substr($buf,0,$gzip_csum,''); | ||
| 1090 | $gzip_csum = 0; | ||
| 1091 | } | ||
| 1092 | } elsif ( $stat != Z_OK ) { | ||
| 1093 | $DEBUG && debug("decode failed: $stat"); | ||
| 1094 | return; # error | ||
| 1095 | } | ||
| 1096 | return $out | ||
| 1097 | }; | ||
| 1098 | } | ||
| 1099 | |||
| 1100 | 1; | ||