« lib/File/FormatIdentification/RandomSampling/Model.pm
↑ Summary
script/cfi_create_training_data.pl »
| 1 | #!/usr/local/bin/perl | ||
| 2 | # (c) 2020 by Andreas Romeyke | ||
| 3 | # licensed via GPL v3.0 or later | ||
| 4 | use strict; | ||
| 5 | use warnings FATAL => 'all'; | ||
| 6 | use Getopt::Long; | ||
| 7 | use feature qw(say); | ||
| 8 | use Pod::Usage; | ||
| 9 | use IO::Handle; | ||
| 10 | use Fcntl qw(:seek); | ||
| 11 | use Path::Tiny; | ||
| 12 | use Math::Random qw(random_uniform_integer); | ||
| 13 | use Time::Progress; | ||
| 14 | use Time::HiRes qw(time); | ||
| 15 | use File::FormatIdentification::RandomSampling; | ||
| 16 | use File::FormatIdentification::RandomSampling::Model; | ||
| 17 | #use File::FormatIdentification::Pronom; | ||
| 18 | use List::Util qw(all); | ||
| 19 | |||
| 20 | # PODNAME: crazy-fast-image-scan | ||
| 21 | # ABSTRACT tool to detect content of media using random sampling | ||
| 22 | |||
| 23 | |||
| 24 | |||
| 25 | |||
| 26 | STDOUT->autoflush(1); | ||
| 27 | # guarantee, that output will be utf-8 | ||
| 28 | binmode(STDOUT, ":encoding(UTF-8)"); | ||
| 29 | |||
| 30 | |||
| 31 | sub calc_null_probability { | ||
| 32 | my $filesize = shift; | ||
| 33 | my $sectorsize=shift; | ||
| 34 | my $sectorcount = shift; | ||
| 35 | # totalsector-sectors | ||
| 36 | # ------------------- = p_1 | ||
| 37 | # totalsectors | ||
| 38 | my $totalsectors = $filesize/$sectorsize; | ||
| 39 | my $probability= ($totalsectors - $sectorcount) / $totalsectors; | ||
| 40 | return $probability; | ||
| 41 | } | ||
| 42 | |||
| 43 | sub calc_probability_of_not_finding_data { | ||
| 44 | my $imagesize = shift; # size of image | ||
| 45 | my $sectorsize=shift; | ||
| 46 | my $sectorcount = shift; | ||
| 47 | my $probsize = shift; # chance to find file of size $probsize | ||
| 48 | if ($probsize > $imagesize) { return; } | ||
| 49 | # totalsector-sectors | ||
| 50 | # ------------------- = p_1 | ||
| 51 | # totalsectors | ||
| 52 | |||
| 53 | my $totalsectors = $imagesize/$sectorsize; | ||
| 54 | my $probsectors = int(0.5 + $probsize/$sectorsize); | ||
| 55 | my $probability= ($totalsectors - $sectorcount) / $totalsectors; | ||
| 56 | #say "p_1 = $prob"; | ||
| 57 | for (my $i = 1; $i<=$probsectors; $i++) { | ||
| 58 | my $den = $totalsectors - ($i -1); | ||
| 59 | my $nom = $den - $sectorcount; | ||
| 60 | $probability *= $nom / $den ; | ||
| 61 | #say "p_$i = $prob ($nom / $den)"; | ||
| 62 | } | ||
| 63 | return $probability; | ||
| 64 | } | ||
| 65 | |||
| 66 | my $help; | ||
| 67 | my $image; | ||
| 68 | my $filesize; | ||
| 69 | my $percentage = 0.01; # 1% | ||
| 70 | my $sectorsize = 512; # 1 Sector = 512 Bytes | ||
| 71 | my $pronom_droid_file; | ||
| 72 | my $pronom; | ||
| 73 | GetOptions( | ||
| 74 | "help|?" => \$help, | ||
| 75 | "image=s" => \$image, | ||
| 76 | "percentage=f" => \$percentage, | ||
| 77 | "sectorsize=i" => \$sectorsize, | ||
| 78 | "pronomfile=s" => \$pronom_droid_file, | ||
| 79 | ) or pod2usage(2); | ||
| 80 | pod2usage(1) if $help; | ||
| 81 | |||
| 82 | if (! defined $image or length($image) < 1) { | ||
| 83 | die "image missed!"; | ||
| 84 | } | ||
| 85 | |||
| 86 | if (!defined $filesize) { | ||
| 87 | $filesize = -s $image; | ||
| 88 | if ((!defined $filesize) or ($filesize == 0) ) { | ||
| 89 | my $fh = path($image)->openr(); | ||
| 90 | my $seeksize = sysseek $fh, 0, SEEK_END; | ||
| 91 | sysseek $fh, 0, SEEK_SET; | ||
| 92 | $filesize = $seeksize; | ||
| 93 | } | ||
| 94 | } | ||
| 95 | |||
| 96 | #if (defined $pronom_droid_file) { | ||
| 97 | # $pronom = File::FormatIdentification::Pronom->new( | ||
| 98 | # "droid_signature_filename" => $pronom_droid_file | ||
| 99 | # ); | ||
| 100 | #} | ||
| 101 | my $sectors = int($filesize / $sectorsize)+1; | ||
| 102 | if ($sectors > 2147483561) { | ||
| 103 | die "too many sectors ($sectors, restart it with smaller percentage value (current: $percentage)"; | ||
| 104 | } | ||
| 105 | if (int($sectors * $percentage) == 0) { | ||
| 106 | die "too few sectors ($sectors, restart it with higher percentage value (current: $percentage)"; | ||
| 107 | } | ||
| 108 | # prepare random sample positions | ||
| 109 | my @seek_positions = sort map {$_*$sectorsize} random_uniform_integer(int($sectors * $percentage), 0, $sectors); | ||
| 110 | my $count_seeks = scalar @seek_positions; | ||
| 111 | say "Scanning Image $image with size $filesize, checking $count_seeks sectors"; | ||
| 112 | |||
| 113 | my $buffer; | ||
| 114 | my %stats; | ||
| 115 | $stats{start} = time(); | ||
| 116 | my $progressbar=Time::Progress->new(min => 0, max => $count_seeks, smoothing => 1); | ||
| 117 | open( my $fh, "<", $image) or die "could not open image $image for reading, $!"; | ||
| 118 | binmode($fh); | ||
| 119 | my $pos = 0; | ||
| 120 | my $ff = File::FormatIdentification::RandomSampling->new(); | ||
| 121 | foreach my $seek (@seek_positions) { | ||
| 122 | seek $fh, $seek, SEEK_SET; | ||
| 123 | read $fh, $buffer, $sectorsize; | ||
| 124 | my $type = $ff->calc_type($buffer); | ||
| 125 | my $histogram = $ff->calc_histogram(); | ||
| 126 | my $model = File::FormatIdentification::RandomSampling::Model->new(); | ||
| 127 | my $mimetype = $model->calc_mimetype( $histogram ); | ||
| 128 | $stats{mimetype}->{$mimetype}++; | ||
| 129 | # if (defined $pronom) { | ||
| 130 | # foreach my $internalid ( $pronom->get_all_internal_ids() ) { | ||
| 131 | # my $sig = $pronom->get_signature_id_by_internal_id($internalid); | ||
| 132 | # if (!defined $sig) {next;} | ||
| 133 | # my $puid = $pronom->get_puid_by_signature_id($sig); | ||
| 134 | # my $name = $pronom->get_name_by_signature_id($sig); | ||
| 135 | # my $quality = $pronom->get_qualities_by_internal_id($internalid); | ||
| 136 | # my @regexes = $pronom->get_regular_expressions_by_internal_id($internalid); | ||
| 137 | # if ( all {$buffer =~ m/$_/saa} @regexes ) { | ||
| 138 | # $stats{puid}->{$puid}->{name} = $name; | ||
| 139 | # $stats{puid}->{$puid}->{count}++; | ||
| 140 | # } | ||
| 141 | # } | ||
| 142 | # } | ||
| 143 | $stats{type}->{$type}++; | ||
| 144 | print $progressbar->report("scanning %40b ETA: %E \r", $pos++); | ||
| 145 | } | ||
| 146 | $stats{end} = time(); | ||
| 147 | $stats{duration} = $stats{end} - $stats{start}; | ||
| 148 | $stats{throughput} = ($count_seeks * $sectorsize) / ($stats{duration}); | ||
| 149 | $stats{fullscan_duration} = $filesize / $stats{throughput}; | ||
| 150 | $stats{speedup} = 0.5 + $stats{fullscan_duration} / $stats{duration} ; | ||
| 151 | say ""; | ||
| 152 | # report | ||
| 153 | say "Estimate, that the image\n\t'$image'\nhas percent of following data types:"; | ||
| 154 | foreach my $type (sort { | ||
| 155 | $stats{type}->{$b} <=> $stats{type}->{$a} | ||
| 156 | }keys %{$stats{type}}) { | ||
| 157 | my $value = 100 / $count_seeks * $stats{type}->{$type}; | ||
| 158 | # TODO: calc probability or Unsureness | ||
| 159 | printf("\t%4.1f%% %s\n", $value, $type); | ||
| 160 | } | ||
| 161 | say "The next mimetype estimation is experimental and needs further work:"; | ||
| 162 | foreach my $type (sort { | ||
| 163 | $stats{mimetype}->{$b} <=> $stats{mimetype}->{$a} | ||
| 164 | } keys %{ $stats{mimetype} }) { | ||
| 165 | my $value = 100 / $count_seeks * $stats{mimetype}->{$type}; | ||
| 166 | printf("\t%4.1f%% %s\n", $value, $type); | ||
| 167 | } | ||
| 168 | my $thr_string = $stats{throughput}; | ||
| 169 | if ($stats{throughput} > (1024 * 1024 * 1024)) { $thr_string=sprintf("%0.1f GByte/s",$stats{throughput}/(1024*1024*1024));} | ||
| 170 | elsif ($stats{throughput} > (1024 * 1024)) { $thr_string=sprintf("%0.1f MByte/s",$stats{throughput}/(1024*1024));} | ||
| 171 | elsif ($stats{throughput} > (1024)) { $thr_string=sprintf("%0.1f kByte/s",$stats{throughput}/(1024));} | ||
| 172 | else {$thr_string=sprintf("%0.1f Byte/s",$stats{throughput});} | ||
| 173 | printf "Scanned in %f s, real throughput %s, speedup %dx\n", $stats{duration}, $thr_string, $stats{speedup}; | ||
| 174 | foreach my $probsize (qw(1000000000 1000000 1000)) { | ||
| 175 | my $notfind_prob = calc_probability_of_not_finding_data($filesize, $sectorsize, $count_seeks, $probsize); | ||
| 176 | if (defined $notfind_prob) { | ||
| 177 | my $prob = 100 * (1 - $notfind_prob); | ||
| 178 | printf "There was a %0.1f%% chance to find data files with size less than %d Bytes\n", $prob, $probsize; | ||
| 179 | } | ||
| 180 | } | ||
| 181 | printf "The probability to pick one empty sector of %d Bytes is: p_1=%0.4f\n", $sectorsize, calc_null_probability($filesize, $sectorsize, $count_seeks); | ||
| 182 | #use Data::Printer; p( $stats{puid}); | ||
| 183 | 1; | ||
| 184 | |||
| 185 | __END__ |