lib/File/FormatIdentification/RandomSampling.pm

filter: Statementstmt 83.3% 10 / 12 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 4 / 4 Podpod n/a totaltotal 87.5% 14 / 16 CC 1 SCAR 0.7
CC
1
Cov
83%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
« script/cfi_learn_model.pl ↑ Summary
1 package File::FormatIdentification::RandomSampling;
2 # ABSTRACT: methods to identify files using random sampling
3 our $VERSION = '0.002'; # TRIAL VERSION:
4 # (c) 2020 by Andreas Romeyke
5 # licensed via GPL v3.0 or later
6 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
7 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use feature qw(say);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 0 ▶use Moose;
Subroutine BEGIN: called CC=1 SCAR=2.6
10  
11 has 'bytegram' => (
12 is => 'rw',
13 isa => 'ArrayRef',
14 default => sub {[]},
15 );
16  
17 sub init_bytegrams {
18 my $self = shift;
19 my $bytegram_ref = $self->{'bytegram'};
20 $bytegram_ref->[0] = [(0) x 256]; # onegram
21 $bytegram_ref->[1] = [(0) x 65536]; #bigram
22 return 1;
23 }
24  
25 sub BUILD {
26 my $self = shift;
27 $self->init_bytegrams();
28 return 1;
29 }
30  
31 sub update_bytegram {
32 my $self = shift;
33 my $buffer = shift;
34 if (defined $buffer) {
35 my $bytegram_ref = $self->{'bytegram'};
36 my @bytes = unpack "C*", $buffer;
37 my @words = unpack "S*", $buffer;
38 # my @bytes = map{ ord($_)} split //, $buffer;
39 if (scalar @bytes > 0) {
40 my @onegram = @{$bytegram_ref->[0]};
41 my @bigram = @{$bytegram_ref->[1]};
42 foreach my $byte (@bytes) {
43 $onegram[$byte]++;
44 }
45 foreach my $word (@words) {
46 $bigram[$word]++;
47 }
48 $bytegram_ref->[0] = \@onegram;
49 $bytegram_ref->[1] = \@bigram;
50 }
51 }
52 return 1;
53 }
54  
55 sub calc_histogram { # use only the most significant first 8 entries
56 my $self = shift;
57 my $bytegram_ref = $self->{'bytegram'};
58 my @bytes_sorted = sort {$bytegram_ref->[0]->[$b] <=> $bytegram_ref->[0]->[$a]} (0..255);
59 my @words_sorted = sort {$bytegram_ref->[1]->[$b] <=> $bytegram_ref->[1]->[$a]} (0 .. 65535);
60 # show only 8 most onegrame bytes
61 my @bytes_truncated = @bytes_sorted[0..7];
62 my @words_truncated = @words_sorted[0..7];
63 my %histogram;
64 foreach my $byte (@bytes_truncated) {
65 push @{$histogram{onegram}}, $byte; #$bytegram_ref->[0]->[$byte];
66 }
67 foreach my $word (@words_truncated) {
68 push @{$histogram{bigram}}, $word; #$bytegram_ref->[1]->[$word];
69 }
70 return \%histogram;
71 }
72  
73 sub is_uniform {
74 my $self = shift;
75 #say "is_uniform?";
76 my $bytegram_ref = $self->{'bytegram'};
77 my $sum = 0;
78 my $n = 0;
79 my @unigram = @{$bytegram_ref->[0]};
80 foreach my $byte (0 .. 255) {
81 if ($unigram[$byte] > 0) {
82 $n += $unigram[$byte];
83 $sum += ($unigram[$byte] * $byte);
84 }
85 }
86 if ($n == 0) { return;}
87 my $expected = (256)/2;
88 my $mean = ($sum/$n);
89 #say "expected=$expected, sum=$sum, mean=$mean";
90 return (abs($expected - $mean) < 4);
91 }
92  
93 sub is_empty {
94 my $self = shift;
95 #say "is_empty?";
96 my $bytegram_ref = $self->{'bytegram'};
97 my $sum = 0;
98 my $n = 0;
99 my @unigram = @{$bytegram_ref->[0]};
100 foreach my $byte (0 .. 255) {
101 if ($unigram[$byte] > 0) {
102 $n += $unigram[$byte];
103 $sum += ($unigram[$byte] * $byte);
104 }
105 }
106 if ($n == 0) { return;}
107 my $expected = 0;
108 my $mean = ($sum/$n);
109 # say "expected=$expected, mean=$mean";
110 my $criteria = abs($expected - $mean) < 4;
111 return ( $criteria);
112 }
113  
114 sub is_text {
115 my $self = shift;
116 #say "is_text?";
117 my $bytegram_ref = $self->{'bytegram'};
118 # many Bytes in range 32 .. 173
119 my $printable = 0;
120 my $non_printable = 0;
121 my @unigram = @{$bytegram_ref->[0]};
122 foreach my $byte (0 .. 255) {
123 #say "bytegram[$byte] = ". $bytegram_ref->[0]->[$byte];
124 if ($unigram[$byte] > 0) {
125 if (($byte >= 32) && ($byte <= 173)) {
126 $printable += ($unigram[$byte]);
127 }
128 else {
129 $non_printable += ($unigram[$byte]);
130 }
131 }
132 }
133 my $ratio = $printable / ($printable + $non_printable + 1); # +1 to avoid division by zero
134 #say "ratio text = $ratio (print=$printable, nonprint=$non_printable";
135 return ($ratio > 0.9);
136 }
137  
138 sub is_video { # quicktime
139 my $self = shift;
140 #say "is_video?";
141 my $bytegram_ref = $self->{'bytegram'};
142 # many Bytes with 0x6d, ratio > 1/256 per read Byte
143 my $mp_indicator = 0;
144 my $other = 0;
145 my @unigram = @{$bytegram_ref->[0]};
146 # MPEG-TS: Synchrobyte = 0x47 5times with distance of 188bytes
147 # MP4/Quicktime: Atom 'mvhd'
148 # General: 0x6d value
149 foreach my $byte ( 0 .. 255) {
150 if ($unigram[$byte] > 0) {
151 if ($byte != 0x6d) {
152 $other += $unigram[$byte];
153 } else { # $byte = 0x6d
154 $mp_indicator += $unigram[$byte];
155 }
156 }
157 }
158 my $ratio = $mp_indicator / ($mp_indicator + $other + 1); # +1 to avoid division by zero
159 #say "ratio=$ratio ($mp_indicator / ".($mp_indicator + $other + 1).") 47=", chr(0x47);
160 return ($ratio > 2/256);
161 }
162  
163  
164 sub calc_type {
165 my $self = shift;
166 my $buffer = shift;
167  
168 $self->init_bytegrams();
169 $self->update_bytegram($buffer);
170  
171 if ($self->is_empty()) {
172 return "empty";
173 }
174 elsif ($self->is_text()) {
175 return "text";
176 }
177 elsif ($self->is_video()) {
178 return "video/audio";
179 }
180 elsif ($self->is_uniform()) {
181 return "random/encrypted/compressed";
182 }
183 return "undef";
184 }
185  
186 no Moose;
187  
188 __PACKAGE__->meta->make_immutable;
189  
190 1;
191  
192 __END__
« script/cfi_learn_model.pl ↑ Summary