script/cfi_create_training_data.pl untested

filter: Statementstmt 0.0% 0 / 78 Branchbran 0.0% 0 / 6 Conditioncond 0.0% 0 / 9 MC/DCmcdc 0.0% 0 / 5 Subroutinesub 0.0% 0 / 13 Podpod 0.0% 0 / 1 totaltotal 0.0% 0 / 112 CC 1 SCAR 6.9
CC
1
Cov
0%
print_histogram
2.0
BEGIN
2.0
BEGIN
2.0
CRAP 2.0
1 #!/usr/local/bin/perl
2 # reads all the files given on command line,
3 # calcs histogram and detect mimetype
4 # stores a line in CSV-file
5 # used to execute a learner to train decision-trees
6 # example call:
7 # find ~/git/format-corpus -print0| xargs -0 perl -I lib cfi_create_training_data.pl --csv_file=test.csv
8 #
9 # (c) 2020 by Andreas Romeyke
10 # licensed via GPL v3.0 or later
11 # PODNAME: cfi_create_training_data.pl
12 use strict;
13 use warnings FATAL => 'all';
14 use feature qw(say);
15 use lib '../lib';
16 use Pod::Usage;
17 use IO::Handle;
18 use Path::Tiny;
19 use Time::Progress;
20 use Time::HiRes qw(time);
21 use Getopt::Long;
22 use MIME::Types;
23 use File::FormatIdentification::RandomSampling;
24  
25 STDOUT->autoflush(1);
26 # guarantee, that output will be utf-8
27 binmode(STDOUT, ":encoding(UTF-8)");
28  
29 sub print_histogram {
30 my $fh = shift;
31 my $category = shift;
32 my $histogram = shift;
33 foreach my $sk (qw(onegram bigram)) {
34 foreach my $ngram (@{$histogram->{$sk}}) {
35 print $fh "$ngram,";
36 }
37 }
38 say $fh $category;
39 return;
40 }
41  
42 #### MAIN
43  
44 my $csv_file;
45 my $help;
46 my $sectorsize = 512; # 1 Sector = 512 Bytes
47 GetOptions(
48 "help|?" => \$help,
49 "csv_file=s" => \$csv_file,
50 ) or pod2usage(2);
51 pod2usage(1) if $help;
52 if (!defined $csv_file) {
53 die "you must define csv_file!";
54 }
55  
56 open(my $fhout, ">>", $csv_file) or die "could not open csv_file '$csv_file' for appending, $!";
57 say $fhout "onegram1, onegram2, onegram3, onegram4, onegram5, onegram6, onegram7, onegram8, bigram1, bigram2, bigram3, bigram4, bigram5, bigram6, bigram7, bigram8, mimetype";
58 close($fhout);
59 foreach my $image (@ARGV) {
60 my $types = MIME::Types->new;
61 my $mime = $types->mimeTypeOf($image);
62  
63 my $category = "unknown";
64 if (defined $mime) {
65 $category = $mime->simplified();
66 }
67 my $rs = File::FormatIdentification::RandomSampling->new();
68 open(my $fh, "<", $image) or die "could not open image $image for reading, $!";
69 binmode($fh);
70 my $buffer;
71 while (read $fh, $buffer, $sectorsize) {
72 $rs->update_bytegram($buffer);
73 }
74 close $fh;
75 my $hist = $rs->calc_histogram();
76 open(my $fhout, ">>", $csv_file) or die "could not open csv_file '$csv_file' for appending, $!";
77 print_histogram($fhout, $category, $hist);
78 close($fhout);
79 }
80  
81 1;
82  
83 __END__