blib/lib/Mail/DMARC/Report/Receive.pm untested

filter: Statementstmt 0.0% 0 / 281 Branchbran 0.0% 0 / 104 Conditioncond 0.0% 0 / 50 MC/DCmcdc 0.0% 0 / 30 Subroutinesub 0.0% 0 / 35 Podpod 0.0% 0 / 14 totaltotal 0.0% 0 / 514 CC 53 SCAR 41.1
CC
53
Cov
0%
from_imap
182.0
from_email_simple
110.0
from_file
90.0
CRAP 61.0
1 package Mail::DMARC::Report::Receive;
2 use strict;
3 use warnings;
4 use feature 'signatures';
5 use feature 'try';
6 no warnings 'experimental::try'; ## no critic (ProhibitNoWarnings)
7  
8 our $VERSION = '2.20260827';
9  
10 use Carp;
11 use Data::Dumper;
12 use Email::MIME;
13 use Email::Simple;
14 use Encode;
15 use File::Basename;
16 use IO::Uncompress::Unzip;
17 use IO::Uncompress::Gunzip;
18 use Module::Load;
19 use XML::LibXML;
20  
21 use parent 'Mail::DMARC::Base';
22 require Mail::DMARC::Policy;
23 require Mail::DMARC::Report;
24 require Mail::DMARC::Report::Aggregate::Record;
25  
26 sub from_imap($self) {
27 try {
28 load "Net::IMAP::Simple";
29 }
30 catch ($error) {
31 croak "Net::IMAP::Simple seems to not work, is it installed? $error";
32 }
33  
34 my $server = $self->config->{imap}{server} or croak "no imap server conf";
35 my $folder = $self->config->{imap}{folder} or croak "no imap folder conf";
36 my $a_done = $self->config->{imap}{a_done};
37 my $f_done = $self->config->{imap}{f_done};
38 my $port = $self->config->{imap}{port} // 993;
39  
40 if ( $port != 143 ) {
41 try {
42 require IO::Socket::SSL;
43 }
44 catch ($error) {
45 croak "Can't load IO::Socket::SSL: $error\n";
46 }
47  
48 if ( defined $self->config->{imap}{SSL_verify_mode} ) {
49 IO::Socket::SSL::set_ctx_defaults(
50 SSL_verifycn_scheme => 'imap',
51 SSL_verify_mode => $self->config->{imap}{SSL_verify_mode},
52 );
53 }
54 }
55  
56 no warnings qw(once); ## no critic (Warn)
57 my $imap = Net::IMAP::Simple->new(
58 $server,
59 port => $port,
60 use_ssl => $port != 143,
61 )
62 or do {
63 ## no critic (PackageVar)
64 my $err
65 = $port == 143
66 ? $Net::IMAP::Simple::errstr
67 : $Net::IMAP::Simple::SSL::errstr;
68 croak "Unable to connect to IMAP: $err\n";
69 };
70  
71 print "connected to IMAP server $server:$port\n" if $self->verbose;
72  
73 $imap->login( $self->config->{imap}{user}, $self->config->{imap}{pass} )
74 or croak "Login failed: " . $imap->errstr . "\n";
75  
76 print "\tlogged in\n" if $self->verbose;
77  
78 my $nm = $imap->select( $self->config->{imap}{folder} );
79 $imap->expunge_mailbox( $self->config->{imap}{folder} );
80 my @mess = $imap->search( 'UNSEEN', 'DATE' );
81 if ( !@mess ) {
82  
83 # imap server might not support SORT extension *Gmail*
84 @mess = $imap->search('UNSEEN');
85 }
86  
87 print "\tfound " . scalar @mess . " messages\n" if $self->verbose;
88  
89 foreach my $i (@mess) {
90 print $imap->seen($i) ? '*' : ' ';
91 printf "[%03d] ", $i;
92 my $message = $imap->get($i) or do {
93 carp "unable to get message $i\n";
94 next;
95 };
96 my $type = $self->from_email_simple( Email::Simple->new("$message") );
97 next if !$type;
98 my $done_box
99 = $type eq 'aggregate' ? $a_done
100 : $type eq 'forensic' ? $f_done
101 : croak "unknown type!";
102  
103 $imap->add_flags( $i, '\Seen' );
104 if ($done_box) {
105 $imap->copy( $i, $done_box ) or do {
106 carp $imap->errstr;
107 next;
108 };
109 $imap->add_flags( $i, '\Deleted' );
110 }
111 }
112  
113 $imap->quit;
114 return 1;
115 }
116  
117 sub from_file( $self, $file = undef ) {
118 croak "missing message" if !$file;
119 croak "No such file $file: $!" if !-f $file;
120  
121 my $contents = $self->slurp($file);
122  
123 # Detect gzip by magic bytes \x1f\x8b
124 if ( substr( $contents, 0, 2 ) eq "\x1f\x8b" ) {
125 my $xml;
126 IO::Uncompress::Gunzip::gunzip( \$contents, \$xml )
127 or croak "gunzip failed: $IO::Uncompress::Gunzip::GunzipError";
128 $self->_init_for_file($file);
129 return $self->handle_body($xml) ? 'aggregate' : undef;
130 }
131  
132 # Detect zip by magic bytes PK\x03\x04
133 if ( substr( $contents, 0, 4 ) eq "PK\x03\x04" ) {
134 my $xml;
135 IO::Uncompress::Unzip::unzip( \$contents, \$xml )
136 or croak "unzip failed: $IO::Uncompress::Unzip::UnzipError";
137 $self->_init_for_file($file);
138 return $self->handle_body($xml) ? 'aggregate' : undef;
139 }
140  
141 # Detect XML by content (starts with optional BOM/whitespace then '<')
142 if ( $contents =~ /\A(?:\xef\xbb\xbf)?\s*</ ) {
143 $self->_init_for_file($file);
144 return $self->handle_body($contents) ? 'aggregate' : undef;
145 }
146  
147 return $self->from_email_simple( Email::Simple->new($contents) );
148 }
149  
150 sub _init_for_file( $self, $file ) {
151 $self->report->init();
152 $self->{_envelope_to} = undef;
153 $self->{_header_from} = undef;
154 $self->get_submitter_from_filename( File::Basename::basename($file) );
155 return;
156 }
157  
158 sub from_mbox( $self, $file_name ) {
159 croak "missing mbox file" if !$file_name;
160  
161 # TODO: replace this module
162 # commented out due to build test failures
163 # load "Mail::Mbox::MessageParser";
164 # croak "is Mail::Mbox::MessageParser installed?" if $@;
165  
166 # my $file_handle = FileHandle->new($file_name);
167  
168 my $folder_reader; # = Mail::Mbox::MessageParser->new(
169 # { 'file_name' => $file_name,
170 # 'file_handle' => $file_handle,
171 # 'enable_cache' => 1,
172 # 'enable_grep' => 1,
173 # }
174 # );
175  
176 croak $folder_reader unless ref $folder_reader;
177  
178 my $prologue = $folder_reader->prologue;
179 print $prologue;
180  
181 while ( !$folder_reader->end_of_file() ) {
182 $self->from_email_simple(
183 Email::Simple->new( $folder_reader->read_next_email() ) );
184 }
185 return 1;
186 }
187  
188 sub from_email_simple( $self, $email ) {
189  
190 $self->report->init();
191 $self->{_envelope_to} = undef;
192 $self->{_header_from} = undef;
193 $self->get_submitter_from_subject( $email->header('Subject') );
194  
195 my $unzipper = {
196 gz => \&IO::Uncompress::Gunzip::gunzip, # 2013 draft
197 zip => \&IO::Uncompress::Unzip::unzip, # legacy format
198 };
199  
200 my $rep_type;
201 foreach my $part ( Email::MIME->new( $email->as_string )->parts ) {
202 my ( $c_type, @ignore ) = split ';', ( $part->content_type || '' );
203 next if $c_type =~ m{^text/(plain|html)$};
204 if ( $c_type eq 'text/rfc822-headers' ) {
205 warn "TODO: handle forensic reports\n"; ## no critic (Carp)
206 $rep_type = 'forensic';
207 next;
208 }
209 if ( $c_type eq 'message/feedback-report' ) {
210 warn "TODO: handle forensic reports\n"; ## no critic (Carp)
211 $rep_type = 'forensic';
212 next;
213 }
214  
215 my $bigger;
216 my $filename = $part->{ct}{attributes}{name} || '';
217  
218 if ( $c_type eq 'application/zip'
219 || $c_type eq 'application/x-zip-compressed' )
220 {
221 try {
222 $self->get_submitter_from_filename($filename);
223 $unzipper->{zip}->( \$part->body, \$bigger );
224 $self->handle_body($bigger);
225 $rep_type = 'aggregate';
226 }
227 catch ($error) {
228 warn "failure processing $filename: $error";
229 }
230 next;
231 }
232 if ( $c_type eq 'application/gzip' ) {
233 try {
234 $self->get_submitter_from_filename($filename);
235 $unzipper->{gz}->( \$part->body, \$bigger );
236 $self->handle_body($bigger);
237 $rep_type = 'aggregate';
238 }
239 catch ($error) {
240 warn "failure processing $filename: $error";
241 }
242 next;
243 }
244 if ( $filename =~ /xml\.gz$/ ) {
245 if ( $c_type eq 'application/octet-stream'
246 || $c_type eq 'multipart/alternative' )
247 {
248 try {
249 $self->get_submitter_from_filename($filename);
250 $unzipper->{gz}->( \$part->body, \$bigger );
251 $self->handle_body($bigger);
252 $rep_type = 'aggregate';
253 }
254 catch ($error) {
255 warn "failure processing $filename: $error";
256 }
257 next;
258 }
259 }
260  
261 if ( $c_type eq 'multipart/alternative' ) {
262 }
263 elsif ( $c_type eq 'multipart/related' ) {
264 }
265 else {
266 warn "Unknown message part $c_type\n"; ## no critic (Carp)
267 }
268 }
269 return $rep_type;
270 }
271  
272 sub get_submitter_from_filename( $self, $filename ) {
273 return if $self->{_envelope_to}; # already parsed from Subject:
274 my ( $submitter_dom, $report_dom, $begin, $end ) = split /!/, $filename;
275 $self->{_header_from} ||= $report_dom;
276 return $self->{_envelope_to} = $submitter_dom;
277 }
278  
279 sub get_submitter_from_subject( $self, $subject ) {
280  
281 # The 2013 DMARC spec section 12.2.1 suggests that the header SHOULD conform
282 # to a supplied ABNF. Rather than "require" such conformance, this method is
283 # more concerned with reliably extracting the submitter domain. Quickly.
284 $subject = lc Encode::decode( 'MIME-Header', $subject );
285 print $subject . "\n";
286 $subject = substr( $subject, 8 ) if 'subject:' eq substr( $subject, 0, 8 );
287 $subject =~ s/(?:report\sdomain|submitter|report-id)//gx; # strip keywords
288 $subject =~ s/\s+//g; # remove white space
289 my ( undef, $report_dom, $sub_dom, $report_id ) = split /:/, $subject;
290 my $meta = $self->report->aggregate->metadata;
291  
292 if ( $report_id && !$meta->uuid ) {
293  
294 # remove <containment brackets> if present
295 $report_id = substr( $report_id, 1 ) if '<' eq substr( $report_id, 0, 1 );
296 chop $report_id if '>' eq substr( $report_id, -1, 1 );
297 $meta->uuid($report_id);
298 }
299 $self->{_header_from} ||= $report_dom;
300 return $self->{_envelope_to} = $sub_dom;
301 }
302  
303 sub handle_body( $self, $body ) {
304  
305 print "handling decompressed body\n" if $self->{verbose};
306 if ( $body =~ /xmlns=/ ) {
307  
308 # RFC 9990 defines xmlns="urn:ietf:params:xml:ns:dmarc-2.0" as the
309 # official namespace. Strip it so un-namespaced XPath queries
310 # (/feedback/...) continue to work with XML::LibXML.
311 print "NOTICE: removing xmlns from XML document\n" if $self->{verbose};
312 $body =~ s/\s+xmlns="[^"]*"//g;
313 }
314  
315 my $dom = XML::LibXML->load_xml( string => $body );
316 $self->do_node_report_metadata( $dom->findnodes("/feedback/report_metadata") );
317 $self->do_node_policy_published(
318 $dom->findnodes("/feedback/policy_published") );
319  
320 foreach my $record ( $dom->findnodes("/feedback/record") ) {
321 $self->do_node_record($record);
322 }
323  
324 return $self->report->save_aggregate();
325 }
326  
327 sub report($self) {
328 return $self->{report} if ref $self->{report};
329 return $self->{report} = Mail::DMARC::Report->new();
330 }
331  
332 sub do_node_report_metadata( $self, $node ) {
333  
334 foreach my $n (qw/ org_name email extra_contact_info /) {
335 $self->report->aggregate->metadata->$n(
336 $node->findnodes("./$n")->string_value );
337 }
338  
339 my $rid = $node->findnodes("./report_id")->string_value;
340 $rid = substr( $rid, 1 ) if '<' eq substr( $rid, 0, 1 ); # remove <
341 chop $rid if '>' eq substr( $rid, -1, 1 ); # remove >
342 $self->report->aggregate->metadata->report_id($rid);
343  
344 if ( !$self->report->aggregate->metadata->uuid ) {
345 $self->report->aggregate->metadata->uuid($rid);
346 }
347  
348 foreach my $n (qw/ begin end /) {
349 $self->report->aggregate->metadata->$n(
350 $node->findnodes("./date_range/$n")->string_value );
351 }
352  
353 foreach my $n ( $node->findnodes("./error") ) {
354 $self->report->aggregate->metadata->error( $n->string_value );
355 }
356 return $self->report->aggregate->metadata;
357 }
358  
359 sub do_node_policy_published( $self, $node ) {
360  
361 my $pol = Mail::DMARC::Policy->new();
362  
363 foreach my $n (qw/ domain adkim aspf p sp np t psd fo discovery_method /) {
364 my $val = $node->findnodes("./$n")->string_value or next;
365 $val =~ s/\s*//g; # remove whitespace
366 try { $pol->$n($val) } catch ($e) { }; # ignore unknown/invalid tag values
367 }
368  
369 $self->report->aggregate->policy_published($pol);
370 return $pol;
371 }
372  
373 sub do_node_record( $self, $node ) {
374  
375 my $rec = Mail::DMARC::Report::Aggregate::Record->new;
376 $self->do_node_record_auth( \$rec, $node );
377  
378 foreach my $r (qw/ source_ip count /) {
379 $rec->row->$r( $node->findnodes("./row/$r")->string_value );
380 }
381  
382 # policy_evaluated
383 foreach my $pe (qw/ disposition dkim spf /) {
384 my $ResultType
385 = $node->findnodes("./row/policy_evaluated/$pe")->string_value;
386 if ( $pe eq 'disposition' ) {
387 if ( $ResultType !~ /^(none|quarantine|reject)$/ ) {
388 $ResultType = 'none'; # invalid DispositionType (Facebook)
389 }
390 }
391 else {
392 if ( $ResultType !~ /^(pass|fail)$/ ) {
393 $ResultType = 'pass'; # invalid ResultType (also FaceBook)
394 }
395 }
396 $rec->row->policy_evaluated->$pe($ResultType);
397 }
398  
399 # reason
400 $self->do_node_record_reason( \$rec, $node );
401  
402 # identifiers
403 foreach my $i (qw/ envelope_to envelope_from header_from /) {
404 $rec->identifiers->$i( $node->findnodes("./identifiers/$i")->string_value );
405 }
406  
407 # for reports from junc.org with mis-labeled identifiers
408 if ( !$rec->identifiers->header_from ) {
409 $rec->identifiers->header_from(
410 $node->findnodes("./identities/header_from")->string_value );
411 }
412  
413 # last resort...
414 if ( !$rec->identifiers->envelope_to ) {
415 $rec->identifiers->envelope_to( $self->{_envelope_to} );
416 }
417 if ( !$rec->identifiers->header_from ) {
418 $rec->identifiers->header_from( $self->{_header_from} );
419 }
420  
421 print Data::Dumper::Dumper($rec) if $self->verbose;
422 $self->report->aggregate->record($rec);
423 return $rec;
424 }
425  
426 sub do_node_record_auth( $self, $row, $node ) {
427  
428 my @spf = qw/ domain scope result /;
429  
430 foreach ( $node->findnodes("./auth_results/spf") ) {
431 my %spf
432 = map { $_ => $node->findnodes("./auth_results/spf/$_")->string_value }
433 @spf;
434  
435 if ( $spf{scope} && !$self->is_valid_spf_scope( $spf{scope} ) ) {
436 carp "invalid scope: $spf{scope}, ignoring";
437 delete $spf{scope};
438 }
439  
440 # this is for reports from ivenue.com with result=unknown
441 if ( $spf{result} && !$self->is_valid_spf_result( $spf{result} ) ) {
442 carp "invalid SPF result: $spf{result}, setting to temperror";
443 $spf{result} = 'temperror';
444 }
445 $$row->auth_results->spf( \%spf );
446 }
447  
448 my @dkim = qw/ domain selector result human_result /;
449 foreach ( $node->findnodes("./auth_results/dkim") ) {
450 my %dkim
451 = map { $_ => $node->findnodes("./auth_results/dkim/$_")->string_value }
452 @dkim;
453 $$row->auth_results->dkim( \%dkim );
454 }
455  
456 return;
457 }
458  
459 sub do_node_record_reason( $self, $row, $node ) {
460  
461 # my @types = qw/ forwarded sampled_out trusted_forwarder mailing_list
462 # local_policy other /;
463  
464 foreach my $r ( $node->findnodes("./row/policy_evaluated/reason") ) {
465 my $type = $r->findnodes('./type')->string_value or next;
466 my $comment = $r->findnodes('./comment')->string_value;
467 $$row->row->policy_evaluated->reason(
468 { type => $type, comment => $comment } );
469 }
470 return;
471 }
472  
473 1;
474  
475 __END__
476  
477 =pod
478  
479 =head1 NAME
480  
481 Mail::DMARC::Report::Receive - process incoming DMARC reports
482  
483 =head1 VERSION
484  
485 version 2.20260827
486  
487 =head1 DESCRIPTION
488  
489 Receive DMARC reports and save them to the report store/database.
490  
491 =head1 METHODS
492  
493 =head2 from_imap, from_file, from_mbox
494  
495 These methods are called by L<dmarc_receive> program, which has its own documentation and usage instructions.
496  
497 C<from_file> accepts a file path and auto-detects the file type:
498  
499 =over 4
500  
501 =item * B<gzip> (magic bytes C<\x1f\x8b>) - decompresses with L<IO::Uncompress::Gunzip> and parses as XML
502  
503 =item * B<zip> (magic bytes C<PK\x03\x04>) - decompresses with L<IO::Uncompress::Unzip> and parses as XML
504  
505 =item * B<XML> (content starting with C<E<lt>>) - parses directly
506  
507 =item * B<email message> - falls back to passing the file to from_email_simple
508  
509 =back
510  
511 C<from_imap> and C<from_mbox> accept a message (or list of messages) and create an Email::Simple object from each, passing that object to from_email_simple.
512  
513 =head2 from_email_simple
514  
515 Accepts an Email::Simple message object. Returns the type of DMARC report detected or undef if no DMARC report was detected.
516  
517 When forensic reports are detected, no further processing is done.
518  
519 When an aggregate report is detected, the report details are extracted from the message body as well as the Subject field/header and attachment metadata.
520  
521 Parsing of the Subject and MIME metadata is necessary because the 2013 draft DMARC specification does not REQUIRE the envelope_to domain name to be included in the XML report. For example, the only way to B<know> that the email which generated this particular report was sent to hotmail.com is to extract the envelope_to domain from the message metadata (Org Name=Microsoft, hotmail.com is not in the XML). So far, every messsage I have seen has had the envelope_to domain in one location or the other.
522  
523 To extract messages from the message body, the MIME attachments are decompressed and passed to L<handle_body>.
524  
525 =head2 handle_body
526  
527 Accepts a XML message, parsing it with XML::LibXML and XPath expressions. The parsed data is stored in a L<Mail::DMARC::Report> object. When the parsing is complete, the report object is saved to the report store.
528  
529 =head1 AUTHORS
530  
531 =over 4
532  
533 =item *
534  
535 Matt Simerson <msimerson@cpan.org>
536  
537 =item *
538  
539 Davide Migliavacca <shari@cpan.org>
540  
541 =item *
542  
543 Marc Bradshaw <marc@marcbradshaw.net>
544  
545 =back
546  
547 =head1 COPYRIGHT AND LICENSE
548  
549 This software is copyright (c) 2026 by Matt Simerson.
550  
551 This is free software; you can redistribute it and/or modify it under
552 the same terms as the Perl 5 programming language system itself.
553  
554 =cut