| 1 | #-*-perl-*- | ||
| 2 | |||
| 3 | package Uplug::PreProcess::Tokenizer; | ||
| 4 | |||
| 5 | =head1 NAME | ||
| 6 | |||
| 7 | Uplug::PreProcess::Tokenizer | ||
| 8 | |||
| 9 | =head1 SYNOPSIS | ||
| 10 | |||
| 11 | my $tokenizer = new Uplug::PreProcess::Tokenizer( lang => 'en' ); | ||
| 12 | my @tokens = tokenizer->tokenize( 'Mr. Smith says: "What is a text anyway?"' ); | ||
| 13 | my $text = detokenize( '" Big improvement ! " says Mr. Smith .'); | ||
| 14 | |||
| 15 | =head1 IMPLEMENTS | ||
| 16 | |||
| 17 | =head2 C<tokenize> | ||
| 18 | |||
| 19 | Tokenize a given text. Returns a list of tokens. | ||
| 20 | |||
| 21 | =head2 C<detokenize> | ||
| 22 | |||
| 23 | De-tokenize a space-separated text or a list of tokens. Returns plain text. | ||
| 24 | |||
| 25 | =head2 C<load_prefixes> | ||
| 26 | |||
| 27 | Load language specific abbreviations and other non-breaking prefixes. | ||
| 28 | |||
| 29 | =head1 DESCRIPTION | ||
| 30 | |||
| 31 | This module heavily relies on the implementation of the tokenizer and detokenizer used in the Moses toolkit for SMT. All credits go to the original authors (Josh Schroeder and Philipp Koehn). | ||
| 32 | |||
| 33 | =cut | ||
| 34 | |||
| 35 | use strict; | ||
| 36 | |||
| 37 | |||
| 38 | use Uplug::Config; | ||
| 39 | |||
| 40 | # defaults: language = English | ||
| 41 | |||
| 42 | our $DEFAULT_LANG = 'en'; | ||
| 43 | |||
| 44 | our %NONBREAKING_PREFIX; | ||
| 45 | our $NONBREAKING_PREFIX_DIR = &shared_lang() . '/nonbreaking_prefixes'; | ||
| 46 | our $DEFAULT_NONBREAKING_PREFIX = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.en'; | ||
| 47 | |||
| 48 | |||
| 49 | sub new { | ||
| 50 | my $class = shift; | ||
| 51 | my %self = @_; | ||
| 52 | |||
| 53 | if ( $self{lang} ) { | ||
| 54 | load_prefixes( $self{lang}, \%NONBREAKING_PREFIX ); | ||
| 55 | } | ||
| 56 | |||
| 57 | return bless \%self, $class; | ||
| 58 | } | ||
| 59 | |||
| 60 | |||
| 61 | sub tokenize { | ||
| 62 | my $self = shift; | ||
| 63 | |||
| 64 | my ($text) = @_; | ||
| 65 | chomp($text); | ||
| 66 | $text = " $text "; | ||
| 67 | |||
| 68 | # seperate out all "other" special characters | ||
| 69 | $text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g; #` | ||
| 70 | |||
| 71 | #multi-dots stay together | ||
| 72 | $text =~ s/\.([\.]+)/ DOTMULTI$1/g; | ||
| 73 | while ( $text =~ /DOTMULTI\./ ) { | ||
| 74 | $text =~ s/DOTMULTI\.([^\.])/DOTDOTMULTI $1/g; | ||
| 75 | $text =~ s/DOTMULTI\./DOTDOTMULTI/g; | ||
| 76 | } | ||
| 77 | |||
| 78 | # seperate out "," except if within numbers (5,300) | ||
| 79 | $text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; | ||
| 80 | |||
| 81 | # separate , pre and post number | ||
| 82 | $text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g; | ||
| 83 | $text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g; | ||
| 84 | |||
| 85 | # turn `into ' | ||
| 86 | $text =~ s/\`/\'/g; #` | ||
| 87 | |||
| 88 | #turn '' into " | ||
| 89 | $text =~ s/\'\'/ \" /g; | ||
| 90 | |||
| 91 | if ( $$self{lang} eq "en" ) { | ||
| 92 | #split contractions right | ||
| 93 | $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; | ||
| 94 | $text =~ s/([^\p{IsAlpha}\p{IsN}])[']([\p{IsAlpha}])/$1 ' $2/g; | ||
| 95 | $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; | ||
| 96 | $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1 '$2/g; | ||
| 97 | |||
| 98 | #special case for "1990's" | ||
| 99 | $text =~ s/([\p{IsN}])[']([s])/$1 '$2/g; | ||
| 100 | } | ||
| 101 | elsif ( ( $$self{lang} eq "fr" ) or ( $$self{lang} eq "it" ) ) { | ||
| 102 | #split contractions left | ||
| 103 | $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; | ||
| 104 | $text =~ s/([^\p{IsAlpha}])[']([\p{IsAlpha}])/$1 ' $2/g; | ||
| 105 | $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g; | ||
| 106 | $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1' $2/g; | ||
| 107 | } | ||
| 108 | else { | ||
| 109 | $text =~ s/\'/ \' /g; | ||
| 110 | } | ||
| 111 | |||
| 112 | #word token method | ||
| 113 | my @words = split( /\s/, $text ); | ||
| 114 | $text = ""; | ||
| 115 | for ( my $i = 0; $i < ( scalar(@words) ); $i++ ) { | ||
| 116 | my $word = $words[$i]; | ||
| 117 | if ( $word =~ /^(\S+)\.$/ ) { | ||
| 118 | my $pre = $1; | ||
| 119 | if (( $pre =~ /\./ && $pre =~ /\p{IsAlpha}/ ) | ||
| 120 | || ( $NONBREAKING_PREFIX{$pre} | ||
| 121 | && $NONBREAKING_PREFIX{$pre} == 1 ) | ||
| 122 | || ( $i < scalar(@words) - 1 | ||
| 123 | && ( $words[ $i + 1 ] =~ /^[\p{IsLower}]/ ) ) ) | ||
| 124 | { | ||
| 125 | #no change | ||
| 126 | } | ||
| 127 | elsif ( | ||
| 128 | ( $NONBREAKING_PREFIX{$pre} | ||
| 129 | && $NONBREAKING_PREFIX{$pre} == 2 | ||
| 130 | ) | ||
| 131 | && ( $i < scalar(@words) - 1 | ||
| 132 | && ( $words[ $i + 1 ] =~ /^[0-9]+/ ) ) ) | ||
| 133 | { | ||
| 134 | #no change | ||
| 135 | } | ||
| 136 | else { | ||
| 137 | $word = $pre . " ."; | ||
| 138 | } | ||
| 139 | } | ||
| 140 | $text .= $word . " "; | ||
| 141 | } | ||
| 142 | |||
| 143 | # clean up extraneous spaces | ||
| 144 | $text =~ s/ +/ /g; | ||
| 145 | $text =~ s/^ //g; | ||
| 146 | $text =~ s/ $//g; | ||
| 147 | |||
| 148 | #restore multi-dots | ||
| 149 | while ( $text =~ /DOTDOTMULTI/ ) { | ||
| 150 | $text =~ s/DOTDOTMULTI/DOTMULTI./g; | ||
| 151 | } | ||
| 152 | $text =~ s/DOTMULTI/./g; | ||
| 153 | |||
| 154 | #ensure final line break | ||
| 155 | $text .= "\n" unless $text =~ /\n$/; | ||
| 156 | return split( /\s+/, $text ); | ||
| 157 | } | ||
| 158 | |||
| 159 | |||
| 160 | sub load_prefixes { | ||
| 161 | my ( $language, $PREFIX_REF ) = @_; | ||
| 162 | |||
| 163 | my $prefixfile | ||
| 164 | = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.' . $language; | ||
| 165 | |||
| 166 | #default back to English if we don't have a language-specific prefix file | ||
| 167 | if ( !( -e $prefixfile ) ) { | ||
| 168 | $prefixfile = $DEFAULT_NONBREAKING_PREFIX; | ||
| 169 | unless ( -e $prefixfile ) { | ||
| 170 | return 0; | ||
| 171 | } | ||
| 172 | } | ||
| 173 | |||
| 174 | if ( -e "$prefixfile" ) { | ||
| 175 | open( PREFIX, "<:encoding(utf8)", "$prefixfile" ); | ||
| 176 | while (<PREFIX>) { | ||
| 177 | my $item = $_; | ||
| 178 | chomp($item); | ||
| 179 | if ( ($item) && ( substr( $item, 0, 1 ) ne "#" ) ) { | ||
| 180 | if ( $item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/ ) { | ||
| 181 | $PREFIX_REF->{$1} = 2; | ||
| 182 | } | ||
| 183 | else { | ||
| 184 | $PREFIX_REF->{$item} = 1; | ||
| 185 | } | ||
| 186 | } | ||
| 187 | } | ||
| 188 | close(PREFIX); | ||
| 189 | } | ||
| 190 | } | ||
| 191 | |||
| 192 | |||
| 193 | # this is the Moses detokenizer | ||
| 194 | # written by Josh Schroeder, based on code by Philipp Koehn | ||
| 195 | |||
| 196 | sub detokenize { | ||
| 197 | my $self = shift; | ||
| 198 | my $token = shift; | ||
| 199 | |||
| 200 | my $text = ref($token) eq 'ARRAY' ? join( ' ', @$token ) : $token; | ||
| 201 | my $language = $self->{lang} || 'en'; | ||
| 202 | |||
| 203 | # chomp($text); | ||
| 204 | $text =~ s/\n/ /gs; | ||
| 205 | $text =~ s/\s\s+/ /gs; | ||
| 206 | $text = " $text "; | ||
| 207 | |||
| 208 | my $word; | ||
| 209 | my $i; | ||
| 210 | my @words = split( / /, $text ); | ||
| 211 | $text = ""; | ||
| 212 | my %quoteCount = ( "\'" => 0, "\"" => 0 ); | ||
| 213 | my $prependSpace = " "; | ||
| 214 | for ( $i = 0; $i < ( scalar(@words) ); $i++ ) { | ||
| 215 | if ( $words[$i] =~ /^[\p{IsSc}\(\[\{\¿\¡]+$/ ) { | ||
| 216 | # perform right shift on currency and other random punctuation items | ||
| 217 | $text = $text . $prependSpace . $words[$i]; | ||
| 218 | $prependSpace = ""; | ||
| 219 | } | ||
| 220 | elsif ( $words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/ ) { | ||
| 221 | # perform left shift on punctuation items | ||
| 222 | $text = $text . $words[$i]; | ||
| 223 | $prependSpace = " "; | ||
| 224 | } | ||
| 225 | elsif (( $language eq "en" ) | ||
| 226 | && ( $i > 0 ) | ||
| 227 | && ( $words[$i] =~ /^[\'][\p{IsAlpha}]/ ) | ||
| 228 | && ( $words[ $i - 1 ] =~ /[\p{IsAlnum}]$/ ) ) | ||
| 229 | { | ||
| 230 | # left-shift the contraction for English | ||
| 231 | $text = $text . $words[$i]; | ||
| 232 | $prependSpace = " "; | ||
| 233 | } | ||
| 234 | elsif (( $language eq "fr" ) | ||
| 235 | && ( $i < ( scalar(@words) - 2 ) ) | ||
| 236 | && ( $words[$i] =~ /[\p{IsAlpha}][\']$/ ) | ||
| 237 | && ( $words[ $i + 1 ] =~ /^[\p{IsAlpha}]/ ) ) | ||
| 238 | { | ||
| 239 | # right-shift the contraction for French | ||
| 240 | $text = $text . $prependSpace . $words[$i]; | ||
| 241 | $prependSpace = ""; | ||
| 242 | } | ||
| 243 | elsif ( $words[$i] =~ /^[\'\"]+$/ ) { | ||
| 244 | # combine punctuation smartly | ||
| 245 | if ( ( $quoteCount{ $words[$i] } % 2 ) eq 0 ) { | ||
| 246 | if ( ( $language eq "en" ) | ||
| 247 | && ( $words[$i] eq "'" ) | ||
| 248 | && ( $i > 0 ) | ||
| 249 | && ( $words[ $i - 1 ] =~ /[s]$/ ) ) | ||
| 250 | { | ||
| 251 | # single quote for posesssives ending in s... "The Jones' house" | ||
| 252 | # left shift | ||
| 253 | $text = $text . $words[$i]; | ||
| 254 | $prependSpace = " "; | ||
| 255 | } | ||
| 256 | else { | ||
| 257 | # right shift | ||
| 258 | $text = $text . $prependSpace . $words[$i]; | ||
| 259 | $prependSpace = ""; | ||
| 260 | $quoteCount{ $words[$i] } = $quoteCount{ $words[$i] } + 1; | ||
| 261 | } | ||
| 262 | } | ||
| 263 | else { | ||
| 264 | # left shift | ||
| 265 | $text = $text . $words[$i]; | ||
| 266 | $prependSpace = " "; | ||
| 267 | $quoteCount{ $words[$i] } = $quoteCount{ $words[$i] } + 1; | ||
| 268 | } | ||
| 269 | } | ||
| 270 | else { | ||
| 271 | $text = $text . $prependSpace . $words[$i]; | ||
| 272 | $prependSpace = " "; | ||
| 273 | } | ||
| 274 | } | ||
| 275 | |||
| 276 | # clean up spaces at head and tail of each line as well as any double-spacing | ||
| 277 | $text =~ s/ +/ /g; | ||
| 278 | $text =~ s/\n /\n/g; | ||
| 279 | $text =~ s/ \n/\n/g; | ||
| 280 | $text =~ s/^ //g; | ||
| 281 | $text =~ s/ $//g; | ||
| 282 | |||
| 283 | # add trailing break | ||
| 284 | #$text .= "\n" unless $text =~ /\n$/; | ||
| 285 | |||
| 286 | return $text; | ||
| 287 | } | ||
| 288 | |||
| 289 | |||
| 290 | 1; | ||
| 291 |