regexp-table-maker.pl untested

filter: Statementstmt 0.0% 0 / 178 Branchbran 0.0% 0 / 24 Conditioncond 0.0% 0 / 20 MC/DCmcdc n/a Subroutinesub 0.0% 0 / 28 Podpod 0.0% 0 / 16 totaltotal 0.0% 0 / 266 CC 11 SCAR 22.8
CC
11
Cov
0%
make_table
42.0
main
12.0
http_get_and_unzip
6.0
CRAP 9.7
1 #!/usr/bin/env perl
2  
3 use strict;
4 use warnings;
5 use Encode;
6 use FindBin;
7 use LWP::Simple;
8 use Regexp::Assemble::Compressed;
9 use IO::Uncompress::Unzip qw(unzip $UnzipError);
10 use Text::CSV_XS;
11 use File::Basename;
12 use File::Path;
13 use Getopt::Long;
14  
15 our $DEBUG;
16 our $UPDATED;
17 our $VERSION;
18 our $STOREDIR;
19 our $TABLEDIR;
20 our $TESTDIR ;
21  
22 GetOptions(
23 'date=s' => sub { updated_on(\$UPDATED, @_) },
24 verbose => \$DEBUG,
25 ) or die <DATA>;
26  
27 $DEBUG = defined $ENV{DEBUG} ? $ENV{DEBUG} : $DEBUG;
28 $UPDATED = defined $UPDATED ? $UPDATED : today();
29 $VERSION = "0." . do { (my $ymd = $UPDATED) =~ s/-//g; $ymd };
30 $STOREDIR = "$FindBin::Bin/share";
31 $TABLEDIR = "$FindBin::Bin/lib/Number/ZipCode/JP/Table";
32 $TESTDIR = "$FindBin::Bin/t";
33  
34 main();
35  
36 sub main {
37 unless (-d $STOREDIR) {
38 mkpath($STOREDIR) or die $!;
39 }
40 opendir my $dh, $TABLEDIR or die "$TABLEDIR: $!";
41 for my $file (readdir $dh) {
42 next unless $file =~ /^(\w+)\.pm$/;
43 my $class = $1;
44 no strict 'refs';
45 my $func = lc($class);
46 _warn("calling $func()");
47 $func->($class);
48 }
49 }
50  
51 sub make_table {
52 my($class, $zip_uri, $zip_pos, $desc_pos, $exp_pos, $exp_val) = @_;
53 my $lc_class = lc($class);
54 my $filename = "$TABLEDIR/$class.pm";
55 my %table = ();
56 my @ok = ();
57 my @ng = ();
58  
59 unless (ref($desc_pos) eq 'ARRAY') {
60 $desc_pos = [$desc_pos];
61 }
62  
63 my $stored = http_get_and_unzip($zip_uri);
64  
65 open my $fh, '<:encoding(shiftjis)', $stored or die "$stored: $!";
66 my $csv = Text::CSV_XS->new ({ binary => 1 });
67  
68 my $last_pref;
69 my $last_suff;
70  
71 my @all_rows = ();
72 while (my $row = $csv->getline($fh)) {
73 push @all_rows, $row;
74 }
75 @all_rows = sort { $a->[$zip_pos] <=> $b->[$zip_pos] } @all_rows;
76  
77 for my $row (@all_rows) {
78 my $zip = $row->[$zip_pos];
79 if ($row->[$exp_pos] == $exp_val) { # expired
80 push @ng, $zip;
81 next;
82 }
83 if (my($pref, $suff) = split_zipcode($zip)) {
84 $last_pref ||= $pref;
85 if ($pref != $last_pref) {
86 push @ng, $last_pref . sprintf('%04d', $last_suff + 1);
87 $last_pref = $pref;
88 }
89 $last_suff = $suff;
90 my $description = join '', @{$row}[@$desc_pos];
91 $description = convert_value($description);
92 push @ok, [$zip, $description];
93 unless (exists $table{$pref}) {
94 $table{$pref} = Regexp::Assemble::Compressed->new;
95 }
96 $table{$pref}->add($suff);
97 _warn(sprintf "%s: %s", $zip, $description);
98 }
99 else {
100 die "malformed zipcode: $zip";
101 }
102 }
103 $csv->eof;
104 close $fh;
105 open my $pm, '>', $filename or die $!;
106 print $pm table_class_header($class);
107 for my $pref (sort { $a cmp $b } keys %table) {
108 my $re = compress($table{$pref}->re);
109 printf $pm " '%s' => '%s',\n", $pref, $re;
110 }
111 print $pm table_class_footer();
112 close $pm;
113 close $pm;
114 make_test($lc_class, \@ok, \@ng);
115 }
116  
117 sub split_zipcode {
118 my $zipcode = shift;
119 return $zipcode =~ /^(\d{3})(\d{4})$/;
120 }
121  
122 sub area {
123 my $class = shift;
124 make_table($class, 'kogaki/zip/ken_all.zip', 2, [qw(6 7 8)], 14, 6);
125 }
126  
127 sub company {
128 my $class = shift;
129 make_table($class, 'office/zip/jigyosyo.zip', 7, 2, 12, 5);
130 }
131  
132 sub http_get_and_unzip {
133 my $file = shift;
134 my $basename = basename($file);
135 my $stored_zip = "$STOREDIR/$basename";
136 (my $uncompressed = $stored_zip) =~ s/\.zip$/\.csv/i;
137 my $url = sprintf 'https://www.post.japanpost.jp/service/search/zipcode/download/%s', $file;
138 _warn($url);
139 my $res = LWP::Simple::mirror($url, $stored_zip);
140 unless ($res == 200 || $res == 304) {
141 _warn("fail to get new file: $basename ($res)");
142 return;
143 }
144 unzip $stored_zip => $uncompressed or die "unzip failed: $UnzipError";
145 return $uncompressed;
146 }
147  
148 sub convert_value {
149 my $cell_value = shift;
150 $cell_value =~ tr/\x{3000}\x{FF01}-\x{FF5E}/\x20\x21-\x7E/;
151 $cell_value =~ tr/\x{201D}\x{2019}\x{FFE5}\x{2018}\x{301C}/"'\\`~/;
152 $cell_value =~ tr/\x{2010}-\x{2015}\x{2212}/\-\-\-\-\-\-\-/;
153 $cell_value =~ s/'/\\'/g;
154 return encode('utf-8', $cell_value);
155 }
156  
157 sub compress { # makes regexp more compressed
158 my $regexp = shift;
159 $regexp =~ s/^\(\?(?:-xism|\^):/(?:/;
160 $regexp =~ s&\(\?:([^\(\)]+)\)&
161 my $re = $1;
162 my @re = ();
163 my %suffixes = ();
164 my @parts = split /\|/, $re;
165 for my $part (@parts) {
166 unless ($part =~ m{^(\d)(\[[-\d]+\]|\\d)$}) {
167 push @re, $part;
168 next;
169 }
170 my($pref, $suff) = ($1, $2);
171 $suffixes{$suff} ||= [];
172 push @{$suffixes{$suff}}, $pref;
173 }
174 for my $suff (keys %suffixes) {
175 if (@{$suffixes{$suff}} == 1) {
176 push @re, $suffixes{$suff}->[0] . $suff;
177 next;
178 }
179 my $rac = Regexp::Assemble::Compressed->new;
180 for my $pref (sort @{$suffixes{$suff}}) {
181 $rac->add($pref);
182 }
183 (my $pref_class = $rac->re) =~ s/^\(\?(?:-xism|\^):(.*?)\)/$1/;
184 push @re, $pref_class . $suff;
185 }
186 '(?:' . join('|', @re) . ')';
187 &eg;
188 return $regexp;
189 }
190  
191 sub table_class_header {
192 my $name = shift;
193 return sprintf <<'END', $name, $VERSION, $UPDATED;
194 package Number::ZipCode::JP::Table::%s;
195  
196 use strict;
197 use warnings;
198  
199 our $VERSION = '%s';
200  
201 # Table last modified: %s
202 our %%ZIP_TABLE = (
203 END
204 ;
205 }
206  
207 sub table_class_footer {
208 return <<'END';
209 );
210  
211 1;
212 __END__