script/crazy_fast_image_scan.pl untested

filter: Statementstmt 0.0% 0 / 146 Branchbran 0.0% 0 / 22 Conditioncond 0.0% 0 / 11 MC/DCmcdc 0.0% 0 / 7 Subroutinesub 0.0% 0 / 16 Podpod 0.0% 0 / 2 totaltotal 0.0% 0 / 204 CC 2 SCAR 9.0
CC
2
Cov
0%
calc_probability_of_not_finding_data
6.0
calc_null_probability
2.0
BEGIN
2.0
CRAP 2.5
1 #!/usr/local/bin/perl
2 # (c) 2020 by Andreas Romeyke
3 # licensed via GPL v3.0 or later
4 use strict;
5 use warnings FATAL => 'all';
6 use Getopt::Long;
7 use feature qw(say);
8 use Pod::Usage;
9 use IO::Handle;
10 use Fcntl qw(:seek);
11 use Path::Tiny;
12 use Math::Random qw(random_uniform_integer);
13 use Time::Progress;
14 use Time::HiRes qw(time);
15 use File::FormatIdentification::RandomSampling;
16 use File::FormatIdentification::RandomSampling::Model;
17 #use File::FormatIdentification::Pronom;
18 use List::Util qw(all);
19  
20 # PODNAME: crazy-fast-image-scan
21 # ABSTRACT tool to detect content of media using random sampling
22  
23  
24  
25  
26 STDOUT->autoflush(1);
27 # guarantee, that output will be utf-8
28 binmode(STDOUT, ":encoding(UTF-8)");
29  
30  
31 sub calc_null_probability {
32 my $filesize = shift;
33 my $sectorsize=shift;
34 my $sectorcount = shift;
35 # totalsector-sectors
36 # ------------------- = p_1
37 # totalsectors
38 my $totalsectors = $filesize/$sectorsize;
39 my $probability= ($totalsectors - $sectorcount) / $totalsectors;
40 return $probability;
41 }
42  
43 sub calc_probability_of_not_finding_data {
44 my $imagesize = shift; # size of image
45 my $sectorsize=shift;
46 my $sectorcount = shift;
47 my $probsize = shift; # chance to find file of size $probsize
48 if ($probsize > $imagesize) { return; }
49 # totalsector-sectors
50 # ------------------- = p_1
51 # totalsectors
52  
53 my $totalsectors = $imagesize/$sectorsize;
54 my $probsectors = int(0.5 + $probsize/$sectorsize);
55 my $probability= ($totalsectors - $sectorcount) / $totalsectors;
56 #say "p_1 = $prob";
57 for (my $i = 1; $i<=$probsectors; $i++) {
58 my $den = $totalsectors - ($i -1);
59 my $nom = $den - $sectorcount;
60 $probability *= $nom / $den ;
61 #say "p_$i = $prob ($nom / $den)";
62 }
63 return $probability;
64 }
65  
66 my $help;
67 my $image;
68 my $filesize;
69 my $percentage = 0.01; # 1%
70 my $sectorsize = 512; # 1 Sector = 512 Bytes
71 my $pronom_droid_file;
72 my $pronom;
73 GetOptions(
74 "help|?" => \$help,
75 "image=s" => \$image,
76 "percentage=f" => \$percentage,
77 "sectorsize=i" => \$sectorsize,
78 "pronomfile=s" => \$pronom_droid_file,
79 ) or pod2usage(2);
80 pod2usage(1) if $help;
81  
82 if (! defined $image or length($image) < 1) {
83 die "image missed!";
84 }
85  
86 if (!defined $filesize) {
87 $filesize = -s $image;
88 if ((!defined $filesize) or ($filesize == 0) ) {
89 my $fh = path($image)->openr();
90 my $seeksize = sysseek $fh, 0, SEEK_END;
91 sysseek $fh, 0, SEEK_SET;
92 $filesize = $seeksize;
93 }
94 }
95  
96 #if (defined $pronom_droid_file) {
97 # $pronom = File::FormatIdentification::Pronom->new(
98 # "droid_signature_filename" => $pronom_droid_file
99 # );
100 #}
101 my $sectors = int($filesize / $sectorsize)+1;
102 if ($sectors > 2147483561) {
103 die "too many sectors ($sectors, restart it with smaller percentage value (current: $percentage)";
104 }
105 if (int($sectors * $percentage) == 0) {
106 die "too few sectors ($sectors, restart it with higher percentage value (current: $percentage)";
107 }
108 # prepare random sample positions
109 my @seek_positions = sort map {$_*$sectorsize} random_uniform_integer(int($sectors * $percentage), 0, $sectors);
110 my $count_seeks = scalar @seek_positions;
111 say "Scanning Image $image with size $filesize, checking $count_seeks sectors";
112  
113 my $buffer;
114 my %stats;
115 $stats{start} = time();
116 my $progressbar=Time::Progress->new(min => 0, max => $count_seeks, smoothing => 1);
117 open( my $fh, "<", $image) or die "could not open image $image for reading, $!";
118 binmode($fh);
119 my $pos = 0;
120 my $ff = File::FormatIdentification::RandomSampling->new();
121 foreach my $seek (@seek_positions) {
122 seek $fh, $seek, SEEK_SET;
123 read $fh, $buffer, $sectorsize;
124 my $type = $ff->calc_type($buffer);
125 my $histogram = $ff->calc_histogram();
126 my $model = File::FormatIdentification::RandomSampling::Model->new();
127 my $mimetype = $model->calc_mimetype( $histogram );
128 $stats{mimetype}->{$mimetype}++;
129 # if (defined $pronom) {
130 # foreach my $internalid ( $pronom->get_all_internal_ids() ) {
131 # my $sig = $pronom->get_signature_id_by_internal_id($internalid);
132 # if (!defined $sig) {next;}
133 # my $puid = $pronom->get_puid_by_signature_id($sig);
134 # my $name = $pronom->get_name_by_signature_id($sig);
135 # my $quality = $pronom->get_qualities_by_internal_id($internalid);
136 # my @regexes = $pronom->get_regular_expressions_by_internal_id($internalid);
137 # if ( all {$buffer =~ m/$_/saa} @regexes ) {
138 # $stats{puid}->{$puid}->{name} = $name;
139 # $stats{puid}->{$puid}->{count}++;
140 # }
141 # }
142 # }
143 $stats{type}->{$type}++;
144 print $progressbar->report("scanning %40b ETA: %E \r", $pos++);
145 }
146 $stats{end} = time();
147 $stats{duration} = $stats{end} - $stats{start};
148 $stats{throughput} = ($count_seeks * $sectorsize) / ($stats{duration});
149 $stats{fullscan_duration} = $filesize / $stats{throughput};
150 $stats{speedup} = 0.5 + $stats{fullscan_duration} / $stats{duration} ;
151 say "";
152 # report
153 say "Estimate, that the image\n\t'$image'\nhas percent of following data types:";
154 foreach my $type (sort {
155 $stats{type}->{$b} <=> $stats{type}->{$a}
156 }keys %{$stats{type}}) {
157 my $value = 100 / $count_seeks * $stats{type}->{$type};
158 # TODO: calc probability or Unsureness
159 printf("\t%4.1f%% %s\n", $value, $type);
160 }
161 say "The next mimetype estimation is experimental and needs further work:";
162 foreach my $type (sort {
163 $stats{mimetype}->{$b} <=> $stats{mimetype}->{$a}
164 } keys %{ $stats{mimetype} }) {
165 my $value = 100 / $count_seeks * $stats{mimetype}->{$type};
166 printf("\t%4.1f%% %s\n", $value, $type);
167 }
168 my $thr_string = $stats{throughput};
169 if ($stats{throughput} > (1024 * 1024 * 1024)) { $thr_string=sprintf("%0.1f GByte/s",$stats{throughput}/(1024*1024*1024));}
170 elsif ($stats{throughput} > (1024 * 1024)) { $thr_string=sprintf("%0.1f MByte/s",$stats{throughput}/(1024*1024));}
171 elsif ($stats{throughput} > (1024)) { $thr_string=sprintf("%0.1f kByte/s",$stats{throughput}/(1024));}
172 else {$thr_string=sprintf("%0.1f Byte/s",$stats{throughput});}
173 printf "Scanned in %f s, real throughput %s, speedup %dx\n", $stats{duration}, $thr_string, $stats{speedup};
174 foreach my $probsize (qw(1000000000 1000000 1000)) {
175 my $notfind_prob = calc_probability_of_not_finding_data($filesize, $sectorsize, $count_seeks, $probsize);
176 if (defined $notfind_prob) {
177 my $prob = 100 * (1 - $notfind_prob);
178 printf "There was a %0.1f%% chance to find data files with size less than %d Bytes\n", $prob, $probsize;
179 }
180 }
181 printf "The probability to pick one empty sector of %d Bytes is: p_1=%0.4f\n", $sectorsize, calc_null_probability($filesize, $sectorsize, $count_seeks);
182 #use Data::Printer; p( $stats{puid});
183 1;
184  
185 __END__