HTTP_Proxy_IMP/IMP.pm

filter: Statementstmt 88.8% 16 / 18 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 6 / 6 Podpod n/a totaltotal 91.6% 22 / 24 CC 1 SCAR 0.5
CC
1
Cov
89%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« HTTP_Proxy_IMP.pm ↑ Summary HTTP_Proxy_IMP/IMP/CSRFprotect.pm »
1 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
2 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
3  
4 package App::HTTP_Proxy_IMP::IMP;
5  
6 1 ▶use Net::Inspect::Debug qw(:DEFAULT $DEBUG);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 ▶use Net::IMP::Debug var => \$DEBUG, sub => \&debug;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use Net::IMP qw(:DEFAULT :log);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 1 ▶use Net::IMP::HTTP;
Subroutine BEGIN: called CC=1 SCAR=2.6
10 use Scalar::Util 'weaken';
11 use Hash::Util 'lock_ref_keys';
12 use Compress::Raw::Zlib;
13 use Carp;
14  
15 use base 'Exporter';
16 our @EXPORT_OK = qw($IMP_MAX_IN_ANALYZER);
17  
18 # if more than IMP_MAX_IN_ANALYZER are in the ibuf we will stop receiving
19 # data and wait until callbacks decrease size of ibuf again
20 our $IMP_MAX_IN_ANALYZER = 2**20;
21  
22 my %METHODS_RFC2616 = map { ($_,1) } qw( GET HEAD POST PUT DELETE OPTIONS CONNECT TRACE );
23 my %METHODS_WITHOUT_RQBODY = map { ($_,1) } qw( GET HEAD DELETE CONNECT );
24 my %METHODS_WITH_RQBODY = map { ($_,1) } qw( POST PUT );
25 my %CODE_WITHOUT_RPBODY = map { ($_,1) } qw(204 205 304);
26 my %METHODS_WITHOUT_RPBODY = map { ($_,1) } qw(HEAD);
27  
28 # we want plugins to suppport the HTTP Request innterface
29 my $interface = [
30 IMP_DATA_HTTPRQ,
31 [
32 IMP_PASS,
33 IMP_PREPASS,
34 IMP_REPLACE,
35 IMP_TOSENDER,
36 IMP_DENY,
37 IMP_LOG,
38 IMP_ACCTFIELD,
39 ]
40 ];
41  
42 sub can_modify {
43 return shift->{can_modify};
44 }
45  
46 # create a new factory object
47 sub new_factory {
48 my ($class,@mod) = @_;
49 my @factory;
50 for my $module (@mod) {
51 if ( ref($module)) {
52 # assume it is already an IMP factory object
53 push @factory, $module;
54 next;
55 }
56  
57 # --filter mod=args
58 my ($mod,$args) = $module =~m{^([a-z][\w:]*)(?:=(.*))?$}i
59 or die "invalid module $module";
60 eval "require $mod" or die "cannot load $mod args=$args: $@";
61 my %args = $mod->str2cfg($args//'');
62 my $factory = $mod->new_factory(%args)
63 or croak("cannot create Net::IMP factory for $mod");
64 $factory = $factory->set_interface( $interface )
65 or croak("$mod does not implement the interface supported by us");
66 push @factory,$factory;
67 }
68  
69 @factory or return;
70 if (@factory>1) {
71 # for cascading filters we need Net::IMP::Cascade
72 require Net::IMP::Cascade;
73 my $cascade = Net::IMP::Cascade->new_factory( parts => [ @factory ])
74 or croak("cannot create Net::IMP::Cascade factory");
75 @factory = $cascade;
76 }
77  
78 my $factory = $factory[0]->set_interface( $interface ) or
79 croak("factory[0] does not implement the interface supported by us");
80  
81 my $self = bless {
82 imp => $factory, # IMP factory object
83 can_modify => 0, # does interface support IMP_REPLACE, IMP_TOSENDER
84 }, $class;
85 lock_ref_keys($self);
86  
87 # update can_modify
88 CHKIF: for my $if ( $self->{imp}->get_interface ) {
89 my ($dt,$rt) = @$if;
90 for (@$rt) {
91 $_ ~~ [ IMP_REPLACE, IMP_TOSENDER ] or next;
92 $self->{can_modify} =1;
93 last CHKIF;
94 }
95 }
96
97 return $self;
98 }
99  
100 # create a new analyzer based on the factory
101 sub new_analyzer {
102 my ($factory,$request,$meta) = @_;
103  
104 my %meta = %$meta;
105 # IMP uses different *addr than Net::Inspect, translate
106 # [s]ource -> [c]lient, [d]estination -> [s]erver
107 $meta{caddr} = delete $meta{saddr};
108 $meta{cport} = delete $meta{sport};
109 $meta{saddr} = delete $meta{daddr};
110 $meta{sport} = delete $meta{dport};
111  
112 my $analyzer = $factory->{imp}->new_analyzer( meta => \%meta );
113  
114 my $self = bless {
115 request => $request, # App::HTTP_Proxy_IMP::Request object
116 imp => $analyzer,
117 # incoming data, put into analyzer
118 # \@list of [ buf_base,buf,type,callback,$cb_arg ] per dir
119 ibuf => [
120 [ [0,''] ],
121 [ [0,''] ],
122 ],
123 pass => [0,0], # pass allowed up to given offset (per dir)
124 prepass => [0,0], # prepass allowed up to given offset (per dir)
125 fixup_header => [], # sub to fixup content-length in header once known
126 stalled => [0,0], # too much data in ibuf?
127 eof => [0,0], # got eof in dir ?
128 decode => undef, # decoder for content-encoding decode{type}[dir]
129 pass_encoded => undef, # pass body encoded (analyzer will not change body)
130 method => undef, # request method
131 }, ref($factory);
132 lock_ref_keys($self);
133 weaken($self->{request});
134  
135 # set callback, this might trigger callback immediately if there are
136 # results pending
137 weaken( my $wself = $self );
138 $analyzer->set_callback( sub { _imp_callback($wself,@_) } );
139 return $self;
140 }
141  
142  
143 sub request_header {
144 my ($self,$hdr,$xhdr,@callback) = @_;
145 my $clen = $xhdr->{content_length};
146 if ( ! defined $clen and $xhdr->{method} ne 'CONNECT') {
147 # length not known -> chunking
148 die "FIXME: chunking request body not yet supported";
149 }
150  
151 # new body might change content-length info in request header
152 # need to defer sending header until body length is known
153 if ( ! $METHODS_WITHOUT_RQBODY{$xhdr->{method}} ) {
154 my $hlen = length($hdr);
155 $self->{fixup_header}[0] = sub {
156 my ($self,$hdr,%args) = @_;
157 my $size = $args{content};
158 goto fix_clen if defined $size;
159  
160 if ( my $pass = $self->{pass}[0] ) {
161 if ( $pass == IMP_MAXOFFSET or $pass >= $hlen + $clen ) {
162 # will not change body
163 goto fix_clen;
164 }
165 }
166 if ( my $prepass = $self->{prepass}[0] ) {
167 if ( $prepass == IMP_MAXOFFSET or $prepass >= $hlen + $clen ) {
168 # will not change body
169 goto fix_clen;
170 }
171 }
172 if ($self->{ibuf}[0][0][0] >= $hlen + $clen) { # ibuf[client].base
173 # everything passed thru ibuf
174 goto fix_clen;
175 }
176  
177 # need to defer header until all of the body is passed
178 # or replaced, then we know the size
179 return;
180  
181 fix_clen:
182  
183 if (!defined $size) {
184 # bytes in ibuf and outstanding bytes will not be changed, so:
185 # new_content_length =
186 # ( orig_clen + orig_hlen - received ) # not yet received
187 # + ( received - ibuf.base ) # still in ibuf
188 # + defered_body.length # ready to forward
189 # --->
190 # orig_clen + orig_hlen - ibuf.base + defered_body.length
191 $size = $clen + $hlen # orig_clen + orig_hlen
192 - $self->{ibuf}[0][0][0] # ibuf.base
193 + $args{defered}; # defered_body.length
194 }
195  
196 $DEBUG && $self->{request}->xdebug("fixup header with clen=%d",$size);
197 # replace or add content-length header
198 $$hdr =~s{^(Content-length:[ \t]*)(\d+)}{$1$size}mi
199 || $$hdr =~s{(\n)}{$1Content-length: $size\r\n};
200 return 1;
201 };
202 }
203  
204  
205 # send data to analyzer.
206 # will call back into request on processed data
207 _imp_data($self,0,$hdr,0,IMP_DATA_HTTPRQ_HEADER,
208 \&_request_header_imp,[ $xhdr,@callback ]);
209 }
210  
211  
212 ############################################################################
213 # callback from IMP after passing/replacing the HTTP request header
214 # will reparse the header if changed and continue in @callback from request
215 ############################################################################
216 sub _request_header_imp {
217 my ($self,$hdr,$changed,$args) = @_;
218 my ($xhdr,$callback,@cb_args) = @$args;
219  
220 if ( $changed ) {
221 # we need to parse the header again and update xhdr
222 my ($met,$url,$version,$fields) = $hdr =~m{ \A
223 (\S+)[\040\t]+
224 (\S+)[\040\t]+
225 HTTP/(1\.[01])[\040\t]*
226 \r?\n
227 (.*?\n)
228 \r?\n\Z
229 }isx;
230  
231 # internal URL are not accepted by the client itself, only from
232 # plugin. Set xhdr.internal_url if we see, that IMP plugin rewrote
233 # url to internal one and strip internal:// again so that original
234 # URL could be logged
235 my $internal = $met ne 'CONNECT'
236 && $xhdr->{url} !~m{^internal://}i
237 && $url =~s{^internal://}{}i;
238  
239 my %kv;
240 my $bad = _parse_hdrfields($fields,\%kv);
241 $xhdr = {
242 method => uc($met),
243 version => $version,
244 url => $url,
245 fields => \%kv,
246 $bad ? ( junk => $bad ) :(),
247 $internal ? ( internal_url => 1 ):(),
248 };
249 }
250  
251 # we don't know the content length yet, unless it can be determined by the
252 # request method. If we got a (pre)pass until the end of the request body
253 # fixup_header will know it and adjust the header
254 $xhdr->{content_length} =
255 $METHODS_WITHOUT_RQBODY{$xhdr->{method}} ? 0:undef;
256  
257 $self->{method} = $xhdr->{method};
258 return $callback->(@cb_args,$hdr,$xhdr);
259 }
260  
261 ############################################################################
262 # fix request header by setting correct content-length
263 # returns true if header could be fixed
264 ############################################################################
265 sub fixup_request_header {
266 my ($self,$hdr_ref,%args) = @_;
267 my $sub = $self->{fixup_header}[0] or return 1;
268 my $ok = $sub->($self,$hdr_ref,%args);
269 $self->{fixup_header}[0] = undef if $ok;
270 return $ok;
271 }
272  
273  
274 ############################################################################
275 # process request body data
276 # just feed to analyzer and call back into request once done
277 ############################################################################
278 sub request_body {
279 my ($self,$data,@callback) = @_;
280  
281 # feed data into IMP
282 $self->{eof}[0] = 1 if $data eq '';
283 _imp_data($self,0,$data,0,IMP_DATA_HTTPRQ_CONTENT,
284 \&_request_body_imp,\@callback );
285 }
286  
287 sub _request_body_imp {
288 my ($self,$data,$changed,$args) = @_;
289 my ($callback,@cb_args) = @$args;
290 my $eof = _check_eof($self,0);
291 $callback->(@cb_args,$data,$eof) if $data ne '' || $eof;
292 }
293  
294 ############################################################################
295 # process response header
296 ############################################################################
297 sub response_header {
298 my ($self,$hdr,$xhdr,@callback) = @_;
299  
300 # if content is encoded we need to decode it in order to analyze
301 # it. For now only set decode to the encoding method, this will
302 # be changed to a decoding function once we need it in the body
303 if ( my $ce = $xhdr->{fields}{'content-encoding'} ) {
304 # the right way would be to extract all encodings and then complain, if
305 # there is an encoding we don't support. Instead we just look for the
306 # encodings we support
307 my %ce = map { lc($_) => 1 } map { m{\b(?:x-)?(gzip|deflate)\b}ig } @$ce;
308 $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] = join(", ",keys %ce)
309 if %ce;
310 }
311  
312 # header length is needed in callback
313 $xhdr->{header_length} = length($hdr);
314 _imp_data($self,1,$hdr,0,IMP_DATA_HTTPRQ_HEADER,
315 \&_response_header_imp,[$xhdr,@callback] );
316 }
317  
318  
319 ############################################################################
320 # callback after passing/replacing the HTTP response header
321 # will reparse the header if changed and continue in the request via
322 # callback
323 ############################################################################
324 sub _response_header_imp {
325 my ($self,$hdr,$changed,$args) = @_;
326 my ($xhdr,$callback,@cb_args) = @$args;
327  
328 my $orig_clen = $xhdr->{content_length};
329 my $orig_hlen = $xhdr->{header_length};
330  
331 if ( $changed ) {
332 # we need to parse the header again and update xhdr
333 my ($version,$code,$reason,$fields) = $hdr =~m{ \A
334 HTTP/(1\.[01])[\040\t]+
335 (\d\d\d)
336 (?:[\040\t]+([^\r\n]*))?
337 \r?\n
338 (.*?\n)
339 \r?\n\Z
340 }isx;
341  
342 my %kv;
343 my $bad = _parse_hdrfields($fields,\%kv);
344 $xhdr = {
345 code => $code,
346 version => $version,
347 reason => $reason,
348 fields => \%kv,
349 $bad ? ( junk => $bad ) :(),
350 };
351 }
352  
353 # except for some codes or request methods we don't know the
354 # content-length of the body yet
355 # in these cases we try in this order
356 # - check if we got a (pre)pass for the whole body already
357 # - use chunked encoding if client speaks HTTP/1.1
358 # - don't specify content-length and close request with connection close
359  
360 # we don't change $hdr here because it will be rebuild from fields anyway
361 if ( $CODE_WITHOUT_RPBODY{$xhdr->{code}} ) {
362 $xhdr->{content_length} = 0;
363 # better remove them
364 delete @{ $xhdr->{fields} }{ qw/ content-length transfer-encoding / };
365 goto callback;
366 }
367  
368 if ( $METHODS_WITHOUT_RPBODY{ $self->{method} } ) {
369 $xhdr->{content_length} = 0;
370 # keep content-length etc, client might want to peek into it using HEAD
371 goto callback;
372 }
373
374 # reset infos about content-length
375 $xhdr->{content_length} = $xhdr->{chunked} = undef;
376 delete @{ $xhdr->{fields} }{ qw/ content-length transfer-encoding / };
377  
378 # if we have read the whole body already or at least know, that we will
379 # not change anymore data, we could compute the new content-length
380 my $clen;
381 my $nochange;
382 while ( defined $orig_clen ) {
383 my $rpsize = $orig_hlen + $orig_clen;
384  
385 if ( my $pass = $self->{pass}[1] ) {
386 if ( $pass == IMP_MAXOFFSET or $pass >= $rpsize ) {
387 # will not look at and not change body
388 $nochange = 1;
389 goto compute_clen;
390 }
391 }
392 if ( my $prepass = $self->{prepass}[1] ) {
393 if ( $prepass == IMP_MAXOFFSET or $prepass >= $rpsize ) {
394 # will not change body
395 $nochange = 1;
396 goto compute_clen;
397 }
398 }
399 if ($self->{ibuf}[1][0][0] >= $rpsize) { # ibuf[server].base
400 # everything passed thru ibuf
401 goto compute_clen;
402 }
403  
404 # we still don't know final size
405 last;
406  
407 compute_clen:
408 # bytes in ibuf and outstanding bytes will not be changed, so:
409 # new_content_length =
410 # ( total_size - received ) # not yet received
411 # + ( received - ibuf.base ) # still in ibuf
412 # --->
413 # total_size - ibuf.base
414 $clen = $rpsize - $self->{ibuf}[1][0][0];
415  
416 last;
417 }
418  
419 if ( $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] ) {
420 if ( $nochange ) {
421 # we will pass encoded stuff, either no decoding needs to
422 # be done (pass) or we will decode only for the analyzer (prepass)
423 # which will only watch at the content, but not change it
424 $self->{pass_encoded}[1] = 1;
425  
426 my $pass = $self->{pass}[1];
427 if ( $pass and defined $orig_clen and (
428 $pass == IMP_MAXOFFSET or
429 $pass >= $orig_clen + $orig_hlen )) {
430 # no need to decode body
431 $self->{decode}{IMP_DATA_HTTPRQ_CONTENT+0}[1] = undef;
432 }
433 } else {
434 # content is encoded and inspection wants to see decoded stuff,
435 # which we then will forward too
436 # but decoding might change length
437 $clen = undef;
438 # the content will be delivered decoded
439 delete $xhdr->{fields}{'content-encoding'}
440 }
441 }
442 if ( defined $clen ) {
443 $xhdr->{fields}{'content-length'} = [ $clen ];
444 $xhdr->{content_length} = $clen;
445 }
446  
447 callback:
448 $callback->(@cb_args,$hdr,$xhdr);
449 }
450  
451  
452  
453 ############################################################################
454 # handle response body data
455 ############################################################################
456 sub response_body {
457 my ($self,$data,@callback) = @_;
458  
459 # forward to IMP analyzer
460 $self->{eof}[1] = 1 if $data eq '';
461 _imp_data($self,1,$data,0,IMP_DATA_HTTPRQ_CONTENT,
462 \&_response_body_imp,\@callback);
463 }
464  
465 sub _response_body_imp {
466 my ($self,$data,$changed,$args) = @_;
467 my ($callback,@cb_args) = @$args;
468 my $eof = _check_eof($self,1);
469 $callback->(@cb_args,$data,$eof) if $data ne '' || $eof;
470 }
471  
472  
473 sub _check_eof {
474 my ($self,$dir) = @_;
475 $DEBUG && $self->{request}->xdebug(
476 "check eof[%d] - eof=%d - %s - (pre)pass=%d/%d",
477 $dir,$self->{eof}[$dir], _show_buf($self,$dir),
478 $self->{prepass}[$dir],
479 $self->{pass}[$dir]
480 );
481 return $self->{eof}[$dir] # received eof
482 && ! defined $self->{ibuf}[$dir][0][2] # no more data in buf
483 && ( # (pre)pass til end ok
484 $self->{prepass}[$dir] == IMP_MAXOFFSET
485 || $self->{pass}[$dir] == IMP_MAXOFFSET
486 );
487 }
488  
489 sub _show_buf {
490 my ($self,$dir) = @_;
491 return join('|',
492 map { ($_->[2]||'none')."($_->[0],+".length($_->[1]).")" }
493 @{ $self->{ibuf}[$dir] }
494 );
495 }
496  
497  
498  
499 ############################################################################
500 # Websockets, TLS upgrades etc
501 # if not IMP the forwarding will be done inside this function, otherwise it
502 # will be done in _in_data_imp, which gets called by IMP callback
503 ############################################################################
504 sub data {
505 my ($self,$dir,$data,@callback) = @_;
506  
507 # forward to IMP analyzer
508 $self->{eof}[$dir] = 1 if $data eq '';
509 _imp_data($self,$dir,$data,0,IMP_DATA_HTTPRQ_CONTENT,
510 \&_data_imp,[$dir,@callback]);
511 }
512  
513 sub _data_imp {
514 my ($self,$data,$changed,$args) = @_;
515 my ($dir,$callback,@cb_args) = @$args;
516 my $eof = $self->{eof}[$dir] && # got eof from server
517 ! defined $self->{ibuf}[$dir][0][2]; # no more data in ibuf[server]
518 $callback->(@cb_args,$dir,$data,$eof) if $data ne '' || $eof;
519 }
520  
521  
522  
523 ############################################################################
524 # callback from IMP
525 # process return types and trigger type specific callbacks on (pre)pass/replace
526 ############################################################################
527 sub _imp_callback {
528 my $self = shift;
529  
530 my %fwd; # forwarded data, per dir
531 for my $rv (@_) {
532  
533 # if the request got closed in between just return
534 my $request = $self->{request} or return;
535  
536 my $rtype = shift(@$rv);
537  
538 # deny further data
539 if ( $rtype == IMP_DENY ) {
540 my ($impdir,$msg) = @$rv;
541 $DEBUG && $request->xdebug("got deny($impdir) $msg");
542 return $request->deny($msg // 'closed by imp');
543 }
544  
545 # log some data
546 if ( $rtype == IMP_LOG ) {
547 my ($impdir,$offset,$len,$level,$msg) = @$rv;
548 $DEBUG && $request->xdebug("got log($impdir,$level) $msg");
549 my %args = ( msg => $msg );
550 $args{offset} = $offset if defined $offset;
551 $args{len} = $len if defined $len;
552 $msg = join(' ',"log[$level]", map { "$_=$args{$_}" } keys %args);
553 $request->xdebug($msg);
554 next;
555 }
556  
557 # set accounting field
558 if ( $rtype == IMP_ACCTFIELD ) {
559 my ($key,$value) = @$rv;
560 $DEBUG && $request->xdebug("got acct $key => $value");
561 $request->{acct}{$key} = $value;
562 next;
563 }
564  
565 # (pre)pass data up to offset
566 if ( $rtype ~~ [ IMP_PASS, IMP_PREPASS ]) {
567 my ($dir,$offset) = @$rv;
568 $DEBUG && $request->xdebug("got $rtype($dir) off=$offset "._show_buf($self,$dir));
569  
570 if ( $rtype == IMP_PASS ) {
571 # ignore pass if it's not better than a previous pass
572 if ( $self->{pass}[$dir] == IMP_MAXOFFSET ) {
573 # there is no better thing than IMP_MAXOFFSET
574 next;
575 } elsif ( $offset == IMP_MAXOFFSET
576 or $offset > $self->{ibuf}[$dir][0][0] ) {
577 # we can pass new data
578 $self->{pass}[$dir] = $offset;
579 } else {
580 # offset is no better than previous pass
581 next;
582 }
583  
584 } else { # IMP_PREPASS
585 # ignore prepass if it's not better than a previous pass
586 # and a previous prepaself->{ibuf}[1][0]
587 if ( $self->{pass}[$dir] == IMP_MAXOFFSET
588 or $self->{prepass}[$dir] == IMP_MAXOFFSET ) {
589 # there is no better thing than IMP_MAXOFFSET
590 $DEBUG && debug("new off $offset no better than existing (pre)pass=max");
591 next;
592 } elsif ( $offset == IMP_MAXOFFSET
593 or $offset > $self->{ibuf}[$dir][0][0] ) {
594 # we can prepass new data
595 $self->{prepass}[$dir] = $offset;
596 $DEBUG && debug("update prepass with new off $offset");
597 } else {
598 # offset is no better than previous pass
599 $DEBUG && debug(
600 "new off $offset no better than existing $self->{ibuf}[$dir][0][0]");
601 next;
602 }
603 }
604  
605 # collect data up to offset for forwarding
606 # list of [ changed,data,callback,cbarg ]
607 my $fwd = $fwd{$dir} ||= [];
608  
609 my $ibuf = $self->{ibuf}[$dir];
610 my $ib0; # top of ibuf, e.g. ibuf[0]
611  
612 while ( @$ibuf ) {
613 $ib0 = shift(@$ibuf);
614 defined $ib0->[2] or last; # dummy entry with no type
615  
616 if ( $offset == IMP_MAXOFFSET ) {
617 # forward this buf and maybe more
618 push @$fwd, [ 0, @{$ib0}[1,3,4] ];
619 } else {
620 my $pass = $offset - $ib0->[0];
621 my $len0 = length($ib0->[1]);
622 if ( $pass > $len0 ) {
623 # forward this buf and maybe more
624 push @$fwd, [ 0, @{$ib0}[1,3,4] ];
625 } elsif ( $pass == $len0 ) {
626 # forward this buf, but not more
627 push @$fwd, [ 0, @{$ib0}[1,3,4] ];
628  
629 # add empty buf if this was the last, this will also
630 # trigger resetting pass,prepass below
631 if ( @$ibuf ) { # still data in buffer
632 } elsif ( $ib0->[2] < 0 ) {
633 # no eof yet and no further data in ibuf
634 # we might get a replacement at the end of the
635 # buffer so put emptied buffer back
636 $ib0->[1] = '';
637 push @$ibuf, $ib0;
638 } else {
639 push @$ibuf, [ $offset,'' ];
640 }
641 last;
642 } elsif ( $ib0->[2] < 0 ) {
643 # streaming type:
644 # forward part of buf
645 push @$fwd, [
646 0, # not changed
647 substr($ib0->[1],0,$pass,''), # data
648 $ib0->[3], # callback
649 $ib0->[4], # args
650 ];
651 # keep rest in ibuf
652 unshift @$ibuf,$ib0;
653 $ib0->[0] += $pass;
654 last; # nothing more to forward
655 } else {
656 # packet type: they need to be processed in total
657 return $request->fatal("partial $rtype for $ib0->[2]");
658 }
659 }
660 }
661  
662 if ( @$ibuf ) {
663 # there are still data in ibuf which cannot get passed,
664 # so reset pass, prepass
665 $self->{pass}[$dir] = $self->{prepass}[$dir] = 0;
666 } else {
667 # add empty buffer containing only current offset based on
668 # what we last removed from ibuf
669 push @$ibuf, [ $ib0->[0] + length($ib0->[1]),'' ];
670 }
671  
672 next;
673 }
674  
675  
676 # replace data up to offset
677 if ( $rtype == IMP_REPLACE ) {
678 my ($dir,$offset,$newdata) = @$rv;
679 $DEBUG && $request->xdebug("got replace($dir) off=$offset data.len=".
680 length($newdata));
681 my $ibuf = $self->{ibuf}[$dir];
682 @$ibuf or die "no ibuf";
683  
684 # if there is an active pass|prepass (e.g. pointing to future data)
685 # the data cannot be replaced
686 return $request->fatal(
687 "cannot replace data which are said to be passed")
688 if $self->{pass}[$dir] or $self->{prepass}[$dir];
689  
690 # we cannot replace future data
691 return $request->fatal('IMP', "cannot use replace with maxoffset")
692 if $offset == IMP_MAXOFFSET;
693  
694 # data to replace cannot span different types, so they must be in
695 # the first ibuf
696 my $ib0 = $ibuf->[0];
697 my $rlen = $offset - $ib0->[0];
698 my $len0 = length($ib0->[1]);
699  
700 # some sanity checks
701 if ( $rlen < 0 ) {
702 return $request->fatal("cannot replace already passed data");
703 } elsif ( $rlen > $len0 ) {
704 return $request->fatal(
705 "replacement cannot span multiple data types")
706 if @$ibuf>1 or $ib0->[2]>0;
707 return $request->fatal("cannot replace future data");
708 } elsif ( $rlen < $len0 ) {
709 # replace part of buffer
710 return $request->fatal("cannot replace part of packet type")
711 if $ib0->[2]>0;
712  
713 # keep rest and update position
714 substr( $ib0->[1],0,$rlen,'' ) if $rlen;
715 $ib0->[0] += $rlen;
716 } else {
717 # remove complete buffer
718 if ( @$ibuf>1 ) { # still data in buffer
719 } elsif ( $ib0->[2] < 0 ) {
720 # no eof yet and no further data in ibuf
721 # we might get a replacement at the end of the
722 # buffer so put emptied buffer back
723 $ib0->[1] = '';
724 } else {
725 # replace with empty
726 @$ibuf = [ $offset,'' ];
727 }
728 }
729  
730 push @{$fwd{$dir}}, [
731 1, # changed
732 $newdata, # new data
733 $ib0->[3], # callback
734 $ib0->[4], # cbargs
735 ];
736  
737 next;
738 }
739  
740 return $request->fatal("unsupported IMP return type: $rtype");
741 }
742  
743 %fwd or return; # no passes/replacements...
744  
745 while ( my ($dir,$fwd) = each %fwd ) {
746 while ( my $fw = shift(@$fwd)) {
747 #warn Dumper($fw); use Data::Dumper;
748 my ($changed,$data,$callback,$args) = @$fw;
749 $callback->($self,$data,$changed,$args);
750 }
751 # unstall ?
752 _check_stalled($self,$dir) if $self->{stalled}[$dir];
753 }
754 }
755  
756 ############################################################################
757 # IMP specific stalled handling
758 # if more than IMP_MAX_IN_ANALYZER are in the ibuf we will stop receiving
759 # data and wait until callbacks decrease size of ibuf again
760 ############################################################################
761 sub _check_stalled {
762 my ($self,$dir) = @_;
763 my $ibuf = $self->{ibuf}[$dir];
764 my $size = $ibuf->[-1][0] + length($ibuf->[-1][1]) - $ibuf->[0][0];
765  
766 if ( $IMP_MAX_IN_ANALYZER ) {
767 if ( $size > $IMP_MAX_IN_ANALYZER ) {
768 return if $self->{stalled}[$dir]; # no change - still stalled
769 # set stalled and disable fd
770 $self->{stalled}[$dir] = 1;
771 $DEBUG && $self->{request}->xdebug(
772 "max in analyzer reached - set $dir to stalled");
773 my $relay = $self->{request}{conn}{relay} or return;
774 if ( my $fo = $relay->fd($dir)) {
775 $fo->mask( r => 0 );
776 }
777 } else {
778 return if ! $self->{stalled}[$dir]; # no change - still not stalled
779 # set unstalled and enable fd
780 $DEBUG && $self->{request}->xdebug(
781 "below max in analyzer - unstall $dir");
782 $self->{stalled}[$dir] = 0;
783 my $relay = $self->{request}{conn}{relay} or return;
784 if ( my $fo = $relay->fd($dir)) {
785 $fo->mask( r => 1 );
786 }
787 }
788 }
789 }
790  
791 ############################################################################
792 # send data to IMP analyzer
793 # if we had a previous (pre)pass some data can be forwarded immediatly, for
794 # others we have to wait for the analyzer callback
795 # returns how many bytes of data are waiting for callback, e.g. 0 if we
796 # we can pass everything immediately
797 ############################################################################
798 sub _imp_data {
799 my ($self,$dir,$data,$offset,$type,$callback,$args) = @_;
800 my $ibuf = $self->{ibuf}[$dir];
801 my $eobuf = $ibuf->[-1][0] + length($ibuf->[-1][1]);
802  
803 my $encoded_data;
804 if ( my $decode = $self->{decode}{$type+0}[$dir] ) {
805 # set up decoder if not set up yet
806 if ( ! ref($decode)) {
807 # create function to decode content
808 $self->{decode}{$type+0}[$dir] = $decode = _create_decoder($decode)
809 || return $self->{request}->fatal(
810 "cannot decode content-encoding $decode");
811 }
812  
813 # offsets relates to original stream, but we put the decoded stream
814 # into ibuf. And offset>0 means, that we have a gap in the input,
815 # which is not allowed, when decoding a stream.
816 die "cannot use content decoder with gap in data" if $offset;
817  
818 $encoded_data = $data if $self->{pass_encoded}[$dir];
819 defined( $data = $decode->($data) )
820 or return $self->{request}->fatal("decoding content failed");
821 }
822  
823 if ( $offset ) {
824 die "offset($offset)<eobuf($eobuf)" if $offset < $eobuf;
825 $offset = 0 if $offset == $eobuf;
826 }
827  
828 my $fwd; # what gets send to analyzer
829  
830 my $dlen = length($data);
831 my $pass = $self->{pass}[$dir];
832 if ( $pass ) {
833 # if pass is set there should be no data in ibuf, e.g. everything
834 # before should have been passed
835 ! $ibuf->[0][2] or die "unexpected data in ibuf";
836  
837 if ( $pass == IMP_MAXOFFSET ) {
838 # pass thru w/o analyzing
839 $ibuf->[0][0] += $dlen;
840 $DEBUG && $self->{request}->xdebug("can pass($dir) infinite");
841 return $callback->($self,$encoded_data // $data,0,$args);
842 }
843  
844 my $canpass = $pass - ( $offset||$eobuf );
845 if ( $canpass <= 0 ) {
846 # cannot pass anything, pass should have been reset already
847 die "pass($dir,$pass) must be point into future ($canpass)";
848 } elsif ( $canpass >= $dlen) {
849 # can pass everything
850 $ibuf->[0][0] += $dlen;
851 if ( $data eq '' ) {
852 # forward eof to analyzer
853 $fwd = $data;
854 $DEBUG && $self->{request}->xdebug("pass($dir) eof");
855 goto SEND2IMP;
856 }
857 $DEBUG && $self->{request}->xdebug(
858 "can pass($dir) all: pass($canpass)>=data.len($dlen)");
859 return $callback->($self,$encoded_data // $data,0,$args);
860 } elsif ( $type < 0 ) {
861 # can pass part of data, only for streaming types
862 # remove from data what can be passed
863 die "body might change" if $self->{pass_encoded}[$dir];
864 $ibuf->[0][0] += $canpass;
865 my $passed_data = substr($data,0,$canpass,'');
866 $eobuf += $canpass;
867 $dlen = length($data);
868 $DEBUG && $self->{request}->xdebug(
869 "can pass($dir) part: pass($canpass)<data.len($dlen)");
870 $callback->($self,$passed_data,0,$args); # callback but continue
871 }
872 }
873  
874 $fwd = $data; # this must be forwarded to analyzer
875  
876 my $prepass = $self->{prepass}[$dir];
877 if ( $prepass ) {
878 # if prepass is set there should be no data in ibuf, e.g. everything
879 # before should have been passed
880 ! $ibuf->[0][2] or die "unexpected data in ibuf";
881 if ( $prepass == IMP_MAXOFFSET ) {
882 # prepass everything
883 $ibuf->[0][0] += $dlen;
884 $DEBUG && $self->{request}->xdebug("can prepass($dir) infinite");
885 $callback->($self,$encoded_data // $data,0,$args); # callback but continue
886 goto SEND2IMP;
887 }
888  
889 my $canprepass = $prepass - ( $offset||$eobuf );
890 if ( $canprepass <= 0 ) {
891 # cannot prepass anything, prepass should have been reset already
892 die "prepass must be point into future";
893 } elsif ( $canprepass >= $dlen) {
894 # can prepass everything
895 $ibuf->[0][0] += $dlen;
896 $callback->($self,$encoded_data // $data,0,$args); # callback but continue
897 $DEBUG && $self->{request}->xdebug(
898 "can prepass($dir) all: pass($canprepass)>=data.len($dlen)");
899 goto SEND2IMP;
900 } elsif ( $type < 0 ) {
901 # can prepass part of data, only for streaming types
902 # remove from data what can be prepassed
903 die "body might change" if $self->{pass_encoded}[$dir];
904 $ibuf->[0][0] += $canprepass;
905 my $passed_data = substr($data,0,$canprepass,'');
906 $eobuf += $canprepass;
907 $dlen = length($data);
908 $DEBUG && $self->{request}->xdebug(
909 "can prepass($dir) part: prepass($canprepass)<data.len($dlen)");
910 $callback->($self,$passed_data,0,$args); # callback but continue
911 }
912 }
913  
914 # everything else in $data must be added to buffer
915
916 # there can be no gaps inside ibuf because caller is only allowed to
917 # pass data which we explicitly allowed
918 if ( $offset && $offset > $eobuf ) {
919 defined $ibuf->[0][2] and # we have still data in ibuf!
920 die "there can be no gaps in ibuf";
921 }
922 if ( ! defined $ibuf->[-1][2] ) {
923 # replace buf, because it was empty
924 $ibuf->[-1] = [ $offset||$eobuf,$data,$type,$callback,$args ];
925 } elsif ( $type < 0
926 and $type == $ibuf->[-1][2]
927 and $callback == $ibuf->[-1][3]
928 ) {
929 # streaming data, concatinate to existing buf of same type
930 $ibuf->[-1][1] .= $data;
931 } else {
932 # different type or non-streaming data, add new buf
933 push @$ibuf,[ $offset||$eobuf,$data,$type,$callback,$args ];
934 }
935 $DEBUG && $self->{request}->xdebug( "ibuf.length=%d",
936 $ibuf->[-1][0] + length($ibuf->[-1][1]) - $ibuf->[0][0]);
937 _check_stalled($self,$dir) if ! $self->{stalled}[$dir];
938  
939 SEND2IMP:
940 $DEBUG && $self->{request}->xdebug("forward(%d) %d bytes type=%s off=%d to analyzer",
941 $dir,length($fwd),$type,$offset);
942 $self->{imp}->data($dir,$fwd,$offset,$type);
943 return length($fwd);
944 }
945  
946 #####################################################################
947 # parse header fields
948 # taken from Net::Inspect::L7::HTTP (where it got put in by myself)
949 #####################################################################
950 my $token = qr{[^()<>@,;:\\"/\[\]?={}\x00-\x20\x7f-\xff]+};
951 my $token_value_cont = qr{
952 ($token): # key:
953 [\040\t]*([^\r\n]*?)[\040\t]* # <space>value<space>
954 ((?:\r?\n[\040\t][^\r\n]*)*) # continuation lines
955 \r?\n # (CR)LF
956 }x;
957 sub _parse_hdrfields {
958 my ($hdr,$fields) = @_;
959 my $bad = '';
960 parse:
961 while ( $hdr =~m{\G$token_value_cont}gc ) {
962 if ($3 eq '') {
963 # no continuation line
964 push @{$fields->{ lc($1) }},$2;
965 } else {
966 # with continuation line
967 my ($k,$v) = ($1,$2.$3);
968 # <space>value-part<space> -> ' ' + value-part
969 $v =~s{[\r\n]+[ \t](.*?)[ \t]*}{ $1}g;
970 push @{$fields->{ lc($k) }},$v;
971 }
972 }
973 if (pos($hdr)//0 != length($hdr)) {
974 # bad line inside
975 substr($hdr,0,pos($hdr),'');
976 $bad .= $1 if $hdr =~s{\A([^\n]*)\n}{};
977 goto parse;
978 }
979 return $bad;
980 }
981  
982 #####################################################################
983 # create decoder function for gzip|deflate content-encoding
984 #####################################################################
985 sub _create_decoder {
986 my $typ = shift;
987 $typ ~~ [ 'gzip','deflate' ] or return; # not supported
988  
989 my $gzip_csum;
990 my $buf = '';
991 my $inflate;
992  
993 return sub {
994 my $data = shift;
995 $buf .= $data;
996  
997 goto inflate if defined $inflate;
998  
999 # read gzip|deflate header
1000 my $wb;
1001 my $more = $data eq '' ? undef:''; # need more data if possible
1002 if ( $typ eq 'gzip' ) {
1003 my $hdr_len = 10; # minimum gzip header
1004  
1005 return $more if length($buf) < $hdr_len;
1006 my ($magic,$method,$flags) = unpack('vCC',$buf);
1007 if ( $magic != 0x8b1f or $method != Z_DEFLATED or $flags & 0xe0 ) {
1008 $DEBUG && debug("no valid gzip header. assuming plain text");
1009 $inflate = ''; # defined but false
1010 goto inflate;
1011 }
1012 if ( $flags & 4 ) {
1013 # skip extra section
1014 return $more if length($buf) < ($hdr_len+=2);
1015 $hdr_len += unpack('x10v',$buf);
1016 return $more if length($buf) < $hdr_len;
1017 }
1018 if ( $flags & 8 ) {
1019 # skip filename
1020 my $o = index($buf,"\0",$hdr_len);
1021 return $more if $o == -1; # end of filename not found
1022 $hdr_len = $o+1;
1023 }
1024 if ( $flags & 16 ) {
1025 # skip comment
1026 my $o = index($buf,"\0",$hdr_len);
1027 return $more if $o == -1; # end of comment not found
1028 $hdr_len = $o+1;
1029 }
1030 if ( $flags & 2 ) {
1031 # skip CRC
1032 return $more if length($buf) < ($hdr_len+=2);
1033 }
1034  
1035 # remove header
1036 substr($buf,0,$hdr_len,'');
1037 $gzip_csum = 8; # 8 byte Adler CRC at end
1038 $wb = -MAX_WBITS(); # see Compress::Raw::Zlib
1039  
1040 } else {
1041 # deflate
1042 # according to RFC it should be zlib, but due to the encoding name
1043 # often real deflate is used instead
1044 # check magic bytes to decide
1045  
1046 # lets see if it looks like a zlib header
1047 # check for CM=8, CMID<=7 in first byte and valid FCHECK in
1048 # seconds byte
1049 return $more if length($buf)<2;
1050 my $magic = unpack('C',substr($buf,0,1));
1051 if (
1052 ( $magic & 0b1111 ) == 8 # CM = 8
1053 and $magic >> 4 <= 7 # CMID <= 7
1054 and unpack('n',substr($buf,0,2)) % 31 == 0 # valid FCHECK
1055 ) {
1056 # looks like zlib header
1057 $wb = +MAX_WBITS(); # see Compress::Raw::Zlib
1058 } else {
1059 # assume deflate
1060 $wb = -MAX_WBITS(); # see Compress::Raw::Zlib
1061 }
1062 }
1063  
1064 $inflate = Compress::Raw::Zlib::Inflate->new(
1065 -WindowBits => $wb,
1066 -AppendOutput => 1,
1067 -ConsumeInput => 1
1068 ) or die "cannot create inflation stream";
1069  
1070 inflate:
1071  
1072 return '' if $buf eq '';
1073  
1074 if ( ! $inflate ) {
1075 # wrong gzip header: sometimes servers claim to use gzip
1076 # if confronted with "Accept-Encoding: identity" but in reality
1077 # they send plain text
1078 # so consider it plain text and don't decode
1079 my $out = $buf;
1080 $buf = '';
1081 return $out
1082 }
1083  
1084 my $out = '';
1085 my $stat = $inflate->inflate(\$buf,\$out);
1086 if ( $stat == Z_STREAM_END ) {
1087 if ( $gzip_csum and length($buf) >= $gzip_csum ) {
1088 # TODO - check checksum - but what would it help?
1089 substr($buf,0,$gzip_csum,'');
1090 $gzip_csum = 0;
1091 }
1092 } elsif ( $stat != Z_OK ) {
1093 $DEBUG && debug("decode failed: $stat");
1094 return; # error
1095 }
1096 return $out
1097 };
1098 }
1099  
1100 1;
« HTTP_Proxy_IMP.pm ↑ Summary HTTP_Proxy_IMP/IMP/CSRFprotect.pm »