| 1 | #!/usr/local/bin/perl | ||
| 2 | # reads all the files given on command line, | ||
| 3 | # calcs histogram and detect mimetype | ||
| 4 | # stores a line in CSV-file | ||
| 5 | # used to execute a learner to train decision-trees | ||
| 6 | # example call: | ||
| 7 | # find ~/git/format-corpus -print0| xargs -0 perl -I lib cfi_create_training_data.pl --csv_file=test.csv | ||
| 8 | # | ||
| 9 | # (c) 2020 by Andreas Romeyke | ||
| 10 | # licensed via GPL v3.0 or later | ||
| 11 | # PODNAME: cfi_create_training_data.pl | ||
| 12 | use strict; | ||
| 13 | use warnings FATAL => 'all'; | ||
| 14 | use feature qw(say); | ||
| 15 | use lib '../lib'; | ||
| 16 | use Pod::Usage; | ||
| 17 | use IO::Handle; | ||
| 18 | use Path::Tiny; | ||
| 19 | use Time::Progress; | ||
| 20 | use Time::HiRes qw(time); | ||
| 21 | use Getopt::Long; | ||
| 22 | use MIME::Types; | ||
| 23 | use File::FormatIdentification::RandomSampling; | ||
| 24 | |||
| 25 | STDOUT->autoflush(1); | ||
| 26 | # guarantee, that output will be utf-8 | ||
| 27 | binmode(STDOUT, ":encoding(UTF-8)"); | ||
| 28 | |||
| 29 | sub print_histogram { | ||
| 30 | my $fh = shift; | ||
| 31 | my $category = shift; | ||
| 32 | my $histogram = shift; | ||
| 33 | foreach my $sk (qw(onegram bigram)) { | ||
| 34 | foreach my $ngram (@{$histogram->{$sk}}) { | ||
| 35 | print $fh "$ngram,"; | ||
| 36 | } | ||
| 37 | } | ||
| 38 | say $fh $category; | ||
| 39 | return; | ||
| 40 | } | ||
| 41 | |||
| 42 | #### MAIN | ||
| 43 | |||
| 44 | my $csv_file; | ||
| 45 | my $help; | ||
| 46 | my $sectorsize = 512; # 1 Sector = 512 Bytes | ||
| 47 | GetOptions( | ||
| 48 | "help|?" => \$help, | ||
| 49 | "csv_file=s" => \$csv_file, | ||
| 50 | ) or pod2usage(2); | ||
| 51 | pod2usage(1) if $help; | ||
| 52 | if (!defined $csv_file) { | ||
| 53 | die "you must define csv_file!"; | ||
| 54 | } | ||
| 55 | |||
| 56 | open(my $fhout, ">>", $csv_file) or die "could not open csv_file '$csv_file' for appending, $!"; | ||
| 57 | say $fhout "onegram1, onegram2, onegram3, onegram4, onegram5, onegram6, onegram7, onegram8, bigram1, bigram2, bigram3, bigram4, bigram5, bigram6, bigram7, bigram8, mimetype"; | ||
| 58 | close($fhout); | ||
| 59 | foreach my $image (@ARGV) { | ||
| 60 | my $types = MIME::Types->new; | ||
| 61 | my $mime = $types->mimeTypeOf($image); | ||
| 62 | |||
| 63 | my $category = "unknown"; | ||
| 64 | if (defined $mime) { | ||
| 65 | $category = $mime->simplified(); | ||
| 66 | } | ||
| 67 | my $rs = File::FormatIdentification::RandomSampling->new(); | ||
| 68 | open(my $fh, "<", $image) or die "could not open image $image for reading, $!"; | ||
| 69 | binmode($fh); | ||
| 70 | my $buffer; | ||
| 71 | while (read $fh, $buffer, $sectorsize) { | ||
| 72 | $rs->update_bytegram($buffer); | ||
| 73 | } | ||
| 74 | close $fh; | ||
| 75 | my $hist = $rs->calc_histogram(); | ||
| 76 | open(my $fhout, ">>", $csv_file) or die "could not open csv_file '$csv_file' for appending, $!"; | ||
| 77 | print_histogram($fhout, $category, $hist); | ||
| 78 | close($fhout); | ||
| 79 | } | ||
| 80 | |||
| 81 | 1; | ||
| 82 | |||
| 83 | __END__ |