| 1 | #!/usr/local/bin/perl | ||
| 2 | # (c) 2020 by Andreas Romeyke | ||
| 3 | # licensed via GPL v3.0 or later | ||
| 4 | # call ./cfi_lear_model.pl CSV-file | ||
| 5 | # outputs perl code for a model of File::FormatIdentification::RandomSampling::Model | ||
| 6 | # PODNAME: cfi_learn_model.pl | ||
| 7 | use strict; | ||
| 8 | use warnings FATAL => 'all'; | ||
| 9 | use feature qw(say); | ||
| 10 | use AI::DecisionTree; | ||
| 11 | my $dtree = AI::DecisionTree->new( noise_mode => 'pick_best', verbose => 1, max_depth => 8); | ||
| 12 | my $training_file = shift @ARGV; | ||
| 13 | open (my $fh, "<", $training_file); | ||
| 14 | my $header_line = <$fh>; | ||
| 15 | #chomp $header_line; | ||
| 16 | #my @header = split(/,/, $header_line); | ||
| 17 | my @header = qw(onegram1 onegram2 onegram3 onegram4 onegram5 onegram6 onegram7 onegram8 bigram1 bigram2 bigram3 bigram4 bigram5 bigram6 bigram7 bigram8 mimetype); | ||
| 18 | foreach my $line (<$fh>) { | ||
| 19 | chomp $line; | ||
| 20 | if ($line =~ m/^onegram/) { next;} | ||
| 21 | my @values = split(/, */, $line); | ||
| 22 | $dtree->add_instance( | ||
| 23 | attributes => { | ||
| 24 | $header[0] => $values[0], | ||
| 25 | $header[1] => $values[1], | ||
| 26 | $header[2] => $values[2], | ||
| 27 | $header[3] => $values[3], | ||
| 28 | $header[4] => $values[4], | ||
| 29 | $header[5] => $values[5], | ||
| 30 | $header[6] => $values[6], | ||
| 31 | $header[7] => $values[7], | ||
| 32 | $header[8] => $values[8], | ||
| 33 | $header[9] => $values[9], | ||
| 34 | $header[10] => $values[10], | ||
| 35 | $header[11] => $values[11], | ||
| 36 | $header[12] => $values[12], | ||
| 37 | $header[13] => $values[13], | ||
| 38 | $header[14] => $values[14], | ||
| 39 | $header[15] => $values[15], | ||
| 40 | }, | ||
| 41 | result => $values[16], | ||
| 42 | name => $header[16], | ||
| 43 | ); | ||
| 44 | #p($dtree); | ||
| 45 | |||
| 46 | } | ||
| 47 | $dtree->train(); | ||
| 48 | #use Data::Printer; | ||
| 49 | #my $ruletree = $dtree->rule_tree(); | ||
| 50 | #p($ruletree); | ||
| 51 | #p( $dtree); | ||
| 52 | my @rule_statements = $dtree->rule_statements(); | ||
| 53 | say <<'HEADER'; | ||
| 54 | package File::FormatIdentification::RandomSampling::Model; | ||
| 55 | # ABSTRACT: methods to identify files using random sampling | ||
| 56 | # VERSION: | ||
| 57 | # (c) 2020 by Andreas Romeyke | ||
| 58 | # licensed via GPL v3.0 or later | ||
| 59 | use strict; | ||
| 60 | use warnings; | ||
| 61 | use feature qw(say); | ||
| 62 | use Moose; | ||
| 63 | use List::Util qw( any ); | ||
| 64 | |||
| 65 | sub calc_mimetype { | ||
| 66 | my $self = shift; | ||
| 67 | my $histogram = shift; | ||
| 68 | my @bigrams = @{$histogram->{bigram}}; | ||
| 69 | my @onegrams = @{$histogram->{onegram}}; | ||
| 70 | HEADER | ||
| 71 | |||
| 72 | |||
| 73 | say join("\n", map { | ||
| 74 | s/if ([^-]*)-> ('[^']*')/\tif ($1) { return $2 ; }/; | ||
| 75 | s/bigram[0-9]*='([^']*)'/(any {\$_ == $1 } \@bigrams)/g; | ||
| 76 | s/onegram[0-9]*='([^']*)'/(any {\$_ == $1 } \@onegrams)/g; | ||
| 77 | $_; | ||
| 78 | } @rule_statements); | ||
| 79 | |||
| 80 | say <<'FOOTER'; | ||
| 81 | return 'unknown'; | ||
| 82 | } | ||
| 83 | |||
| 84 | 1; | ||
| 85 | FOOTER | ||
| 86 | |||
| 87 | 1; | ||
| 88 | |||
| 89 | __END__ |