| 1 | #!/usr/bin/env perl | ||
| 2 | |||
| 3 | use strict; | ||
| 4 | use warnings; | ||
| 5 | use Encode; | ||
| 6 | use FindBin; | ||
| 7 | use LWP::Simple; | ||
| 8 | use Regexp::Assemble::Compressed; | ||
| 9 | use IO::Uncompress::Unzip qw(unzip $UnzipError); | ||
| 10 | use Text::CSV_XS; | ||
| 11 | use File::Basename; | ||
| 12 | use File::Path; | ||
| 13 | use Getopt::Long; | ||
| 14 | |||
| 15 | our $DEBUG; | ||
| 16 | our $UPDATED; | ||
| 17 | our $VERSION; | ||
| 18 | our $STOREDIR; | ||
| 19 | our $TABLEDIR; | ||
| 20 | our $TESTDIR ; | ||
| 21 | |||
| 22 | GetOptions( | ||
| 23 | 'date=s' => sub { updated_on(\$UPDATED, @_) }, | ||
| 24 | verbose => \$DEBUG, | ||
| 25 | ) or die <DATA>; | ||
| 26 | |||
| 27 | $DEBUG = defined $ENV{DEBUG} ? $ENV{DEBUG} : $DEBUG; | ||
| 28 | $UPDATED = defined $UPDATED ? $UPDATED : today(); | ||
| 29 | $VERSION = "0." . do { (my $ymd = $UPDATED) =~ s/-//g; $ymd }; | ||
| 30 | $STOREDIR = "$FindBin::Bin/share"; | ||
| 31 | $TABLEDIR = "$FindBin::Bin/lib/Number/ZipCode/JP/Table"; | ||
| 32 | $TESTDIR = "$FindBin::Bin/t"; | ||
| 33 | |||
| 34 | main(); | ||
| 35 | |||
| 36 | sub main { | ||
| 37 | unless (-d $STOREDIR) { | ||
| 38 | mkpath($STOREDIR) or die $!; | ||
| 39 | } | ||
| 40 | opendir my $dh, $TABLEDIR or die "$TABLEDIR: $!"; | ||
| 41 | for my $file (readdir $dh) { | ||
| 42 | next unless $file =~ /^(\w+)\.pm$/; | ||
| 43 | my $class = $1; | ||
| 44 | no strict 'refs'; | ||
| 45 | my $func = lc($class); | ||
| 46 | _warn("calling $func()"); | ||
| 47 | $func->($class); | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | sub make_table { | ||
| 52 | my($class, $zip_uri, $zip_pos, $desc_pos, $exp_pos, $exp_val) = @_; | ||
| 53 | my $lc_class = lc($class); | ||
| 54 | my $filename = "$TABLEDIR/$class.pm"; | ||
| 55 | my %table = (); | ||
| 56 | my @ok = (); | ||
| 57 | my @ng = (); | ||
| 58 | |||
| 59 | unless (ref($desc_pos) eq 'ARRAY') { | ||
| 60 | $desc_pos = [$desc_pos]; | ||
| 61 | } | ||
| 62 | |||
| 63 | my $stored = http_get_and_unzip($zip_uri); | ||
| 64 | |||
| 65 | open my $fh, '<:encoding(shiftjis)', $stored or die "$stored: $!"; | ||
| 66 | my $csv = Text::CSV_XS->new ({ binary => 1 }); | ||
| 67 | |||
| 68 | my $last_pref; | ||
| 69 | my $last_suff; | ||
| 70 | |||
| 71 | my @all_rows = (); | ||
| 72 | while (my $row = $csv->getline($fh)) { | ||
| 73 | push @all_rows, $row; | ||
| 74 | } | ||
| 75 | @all_rows = sort { $a->[$zip_pos] <=> $b->[$zip_pos] } @all_rows; | ||
| 76 | |||
| 77 | for my $row (@all_rows) { | ||
| 78 | my $zip = $row->[$zip_pos]; | ||
| 79 | if ($row->[$exp_pos] == $exp_val) { # expired | ||
| 80 | push @ng, $zip; | ||
| 81 | next; | ||
| 82 | } | ||
| 83 | if (my($pref, $suff) = split_zipcode($zip)) { | ||
| 84 | $last_pref ||= $pref; | ||
| 85 | if ($pref != $last_pref) { | ||
| 86 | push @ng, $last_pref . sprintf('%04d', $last_suff + 1); | ||
| 87 | $last_pref = $pref; | ||
| 88 | } | ||
| 89 | $last_suff = $suff; | ||
| 90 | my $description = join '', @{$row}[@$desc_pos]; | ||
| 91 | $description = convert_value($description); | ||
| 92 | push @ok, [$zip, $description]; | ||
| 93 | unless (exists $table{$pref}) { | ||
| 94 | $table{$pref} = Regexp::Assemble::Compressed->new; | ||
| 95 | } | ||
| 96 | $table{$pref}->add($suff); | ||
| 97 | _warn(sprintf "%s: %s", $zip, $description); | ||
| 98 | } | ||
| 99 | else { | ||
| 100 | die "malformed zipcode: $zip"; | ||
| 101 | } | ||
| 102 | } | ||
| 103 | $csv->eof; | ||
| 104 | close $fh; | ||
| 105 | open my $pm, '>', $filename or die $!; | ||
| 106 | print $pm table_class_header($class); | ||
| 107 | for my $pref (sort { $a cmp $b } keys %table) { | ||
| 108 | my $re = compress($table{$pref}->re); | ||
| 109 | printf $pm " '%s' => '%s',\n", $pref, $re; | ||
| 110 | } | ||
| 111 | print $pm table_class_footer(); | ||
| 112 | close $pm; | ||
| 113 | close $pm; | ||
| 114 | make_test($lc_class, \@ok, \@ng); | ||
| 115 | } | ||
| 116 | |||
| 117 | sub split_zipcode { | ||
| 118 | my $zipcode = shift; | ||
| 119 | return $zipcode =~ /^(\d{3})(\d{4})$/; | ||
| 120 | } | ||
| 121 | |||
| 122 | sub area { | ||
| 123 | my $class = shift; | ||
| 124 | make_table($class, 'kogaki/zip/ken_all.zip', 2, [qw(6 7 8)], 14, 6); | ||
| 125 | } | ||
| 126 | |||
| 127 | sub company { | ||
| 128 | my $class = shift; | ||
| 129 | make_table($class, 'office/zip/jigyosyo.zip', 7, 2, 12, 5); | ||
| 130 | } | ||
| 131 | |||
| 132 | sub http_get_and_unzip { | ||
| 133 | my $file = shift; | ||
| 134 | my $basename = basename($file); | ||
| 135 | my $stored_zip = "$STOREDIR/$basename"; | ||
| 136 | (my $uncompressed = $stored_zip) =~ s/\.zip$/\.csv/i; | ||
| 137 | my $url = sprintf 'https://www.post.japanpost.jp/service/search/zipcode/download/%s', $file; | ||
| 138 | _warn($url); | ||
| 139 | my $res = LWP::Simple::mirror($url, $stored_zip); | ||
| 140 | unless ($res == 200 || $res == 304) { | ||
| 141 | _warn("fail to get new file: $basename ($res)"); | ||
| 142 | return; | ||
| 143 | } | ||
| 144 | unzip $stored_zip => $uncompressed or die "unzip failed: $UnzipError"; | ||
| 145 | return $uncompressed; | ||
| 146 | } | ||
| 147 | |||
| 148 | sub convert_value { | ||
| 149 | my $cell_value = shift; | ||
| 150 | $cell_value =~ tr/\x{3000}\x{FF01}-\x{FF5E}/\x20\x21-\x7E/; | ||
| 151 | $cell_value =~ tr/\x{201D}\x{2019}\x{FFE5}\x{2018}\x{301C}/"'\\`~/; | ||
| 152 | $cell_value =~ tr/\x{2010}-\x{2015}\x{2212}/\-\-\-\-\-\-\-/; | ||
| 153 | $cell_value =~ s/'/\\'/g; | ||
| 154 | return encode('utf-8', $cell_value); | ||
| 155 | } | ||
| 156 | |||
| 157 | sub compress { # makes regexp more compressed | ||
| 158 | my $regexp = shift; | ||
| 159 | $regexp =~ s/^\(\?(?:-xism|\^):/(?:/; | ||
| 160 | $regexp =~ s&\(\?:([^\(\)]+)\)& | ||
| 161 | my $re = $1; | ||
| 162 | my @re = (); | ||
| 163 | my %suffixes = (); | ||
| 164 | my @parts = split /\|/, $re; | ||
| 165 | for my $part (@parts) { | ||
| 166 | unless ($part =~ m{^(\d)(\[[-\d]+\]|\\d)$}) { | ||
| 167 | push @re, $part; | ||
| 168 | next; | ||
| 169 | } | ||
| 170 | my($pref, $suff) = ($1, $2); | ||
| 171 | $suffixes{$suff} ||= []; | ||
| 172 | push @{$suffixes{$suff}}, $pref; | ||
| 173 | } | ||
| 174 | for my $suff (keys %suffixes) { | ||
| 175 | if (@{$suffixes{$suff}} == 1) { | ||
| 176 | push @re, $suffixes{$suff}->[0] . $suff; | ||
| 177 | next; | ||
| 178 | } | ||
| 179 | my $rac = Regexp::Assemble::Compressed->new; | ||
| 180 | for my $pref (sort @{$suffixes{$suff}}) { | ||
| 181 | $rac->add($pref); | ||
| 182 | } | ||
| 183 | (my $pref_class = $rac->re) =~ s/^\(\?(?:-xism|\^):(.*?)\)/$1/; | ||
| 184 | push @re, $pref_class . $suff; | ||
| 185 | } | ||
| 186 | '(?:' . join('|', @re) . ')'; | ||
| 187 | ⪚ | ||
| 188 | return $regexp; | ||
| 189 | } | ||
| 190 | |||
| 191 | sub table_class_header { | ||
| 192 | my $name = shift; | ||
| 193 | return sprintf <<'END', $name, $VERSION, $UPDATED; | ||
| 194 | package Number::ZipCode::JP::Table::%s; | ||
| 195 | |||
| 196 | use strict; | ||
| 197 | use warnings; | ||
| 198 | |||
| 199 | our $VERSION = '%s'; | ||
| 200 | |||
| 201 | # Table last modified: %s | ||
| 202 | our %%ZIP_TABLE = ( | ||
| 203 | END | ||
| 204 | ; | ||
| 205 | } | ||
| 206 | |||
| 207 | sub table_class_footer { | ||
| 208 | return <<'END'; | ||
| 209 | ); | ||
| 210 | |||
| 211 | 1; | ||
| 212 | __END__ |