blib/lib/Uplug/PreProcess/Tokenizer.pm untested

filter: Statementstmt 0.0% 0 / 128 Branchbran 0.0% 0 / 40 Conditioncond 0.0% 0 / 59 MC/DCmcdc 0.0% 0 / 32 Subroutinesub 0.0% 0 / 6 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 269 CC 21 SCAR 40.3
CC
21
Cov
0%
detokenize
90.0
tokenize
56.0
load_prefixes
42.0
CRAP 56.5
1 #-*-perl-*-
2  
3 package Uplug::PreProcess::Tokenizer;
4  
5 =head1 NAME
6  
7 Uplug::PreProcess::Tokenizer
8  
9 =head1 SYNOPSIS
10  
11 my $tokenizer = new Uplug::PreProcess::Tokenizer( lang => 'en' );
12 my @tokens = tokenizer->tokenize( 'Mr. Smith says: "What is a text anyway?"' );
13 my $text = detokenize( '" Big improvement ! " says Mr. Smith .');
14  
15 =head1 IMPLEMENTS
16  
17 =head2 C<tokenize>
18  
19 Tokenize a given text. Returns a list of tokens.
20  
21 =head2 C<detokenize>
22  
23 De-tokenize a space-separated text or a list of tokens. Returns plain text.
24  
25 =head2 C<load_prefixes>
26  
27 Load language specific abbreviations and other non-breaking prefixes.
28  
29 =head1 DESCRIPTION
30  
31 This module heavily relies on the implementation of the tokenizer and detokenizer used in the Moses toolkit for SMT. All credits go to the original authors (Josh Schroeder and Philipp Koehn).
32  
33 =cut
34  
35 use strict;
36  
37  
38 use Uplug::Config;
39  
40 # defaults: language = English
41  
42 our $DEFAULT_LANG = 'en';
43  
44 our %NONBREAKING_PREFIX;
45 our $NONBREAKING_PREFIX_DIR = &shared_lang() . '/nonbreaking_prefixes';
46 our $DEFAULT_NONBREAKING_PREFIX = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.en';
47  
48  
49 sub new {
50 my $class = shift;
51 my %self = @_;
52  
53 if ( $self{lang} ) {
54 load_prefixes( $self{lang}, \%NONBREAKING_PREFIX );
55 }
56  
57 return bless \%self, $class;
58 }
59  
60  
61 sub tokenize {
62 my $self = shift;
63  
64 my ($text) = @_;
65 chomp($text);
66 $text = " $text ";
67  
68 # seperate out all "other" special characters
69 $text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g; #`
70  
71 #multi-dots stay together
72 $text =~ s/\.([\.]+)/ DOTMULTI$1/g;
73 while ( $text =~ /DOTMULTI\./ ) {
74 $text =~ s/DOTMULTI\.([^\.])/DOTDOTMULTI $1/g;
75 $text =~ s/DOTMULTI\./DOTDOTMULTI/g;
76 }
77  
78 # seperate out "," except if within numbers (5,300)
79 $text =~ s/([^\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;
80  
81 # separate , pre and post number
82 $text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;
83 $text =~ s/([^\p{IsN}])[,]([\p{IsN}])/$1 , $2/g;
84  
85 # turn `into '
86 $text =~ s/\`/\'/g; #`
87  
88 #turn '' into "
89 $text =~ s/\'\'/ \" /g;
90  
91 if ( $$self{lang} eq "en" ) {
92 #split contractions right
93 $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g;
94 $text =~ s/([^\p{IsAlpha}\p{IsN}])[']([\p{IsAlpha}])/$1 ' $2/g;
95 $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g;
96 $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1 '$2/g;
97  
98 #special case for "1990's"
99 $text =~ s/([\p{IsN}])[']([s])/$1 '$2/g;
100 }
101 elsif ( ( $$self{lang} eq "fr" ) or ( $$self{lang} eq "it" ) ) {
102 #split contractions left
103 $text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g;
104 $text =~ s/([^\p{IsAlpha}])[']([\p{IsAlpha}])/$1 ' $2/g;
105 $text =~ s/([\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g;
106 $text =~ s/([\p{IsAlpha}])[']([\p{IsAlpha}])/$1' $2/g;
107 }
108 else {
109 $text =~ s/\'/ \' /g;
110 }
111  
112 #word token method
113 my @words = split( /\s/, $text );
114 $text = "";
115 for ( my $i = 0; $i < ( scalar(@words) ); $i++ ) {
116 my $word = $words[$i];
117 if ( $word =~ /^(\S+)\.$/ ) {
118 my $pre = $1;
119 if (( $pre =~ /\./ && $pre =~ /\p{IsAlpha}/ )
120 || ( $NONBREAKING_PREFIX{$pre}
121 && $NONBREAKING_PREFIX{$pre} == 1 )
122 || ( $i < scalar(@words) - 1
123 && ( $words[ $i + 1 ] =~ /^[\p{IsLower}]/ ) ) )
124 {
125 #no change
126 }
127 elsif (
128 ( $NONBREAKING_PREFIX{$pre}
129 && $NONBREAKING_PREFIX{$pre} == 2
130 )
131 && ( $i < scalar(@words) - 1
132 && ( $words[ $i + 1 ] =~ /^[0-9]+/ ) ) )
133 {
134 #no change
135 }
136 else {
137 $word = $pre . " .";
138 }
139 }
140 $text .= $word . " ";
141 }
142  
143 # clean up extraneous spaces
144 $text =~ s/ +/ /g;
145 $text =~ s/^ //g;
146 $text =~ s/ $//g;
147  
148 #restore multi-dots
149 while ( $text =~ /DOTDOTMULTI/ ) {
150 $text =~ s/DOTDOTMULTI/DOTMULTI./g;
151 }
152 $text =~ s/DOTMULTI/./g;
153  
154 #ensure final line break
155 $text .= "\n" unless $text =~ /\n$/;
156 return split( /\s+/, $text );
157 }
158  
159  
160 sub load_prefixes {
161 my ( $language, $PREFIX_REF ) = @_;
162  
163 my $prefixfile
164 = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.' . $language;
165  
166 #default back to English if we don't have a language-specific prefix file
167 if ( !( -e $prefixfile ) ) {
168 $prefixfile = $DEFAULT_NONBREAKING_PREFIX;
169 unless ( -e $prefixfile ) {
170 return 0;
171 }
172 }
173  
174 if ( -e "$prefixfile" ) {
175 open( PREFIX, "<:encoding(utf8)", "$prefixfile" );
176 while (<PREFIX>) {
177 my $item = $_;
178 chomp($item);
179 if ( ($item) && ( substr( $item, 0, 1 ) ne "#" ) ) {
180 if ( $item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/ ) {
181 $PREFIX_REF->{$1} = 2;
182 }
183 else {
184 $PREFIX_REF->{$item} = 1;
185 }
186 }
187 }
188 close(PREFIX);
189 }
190 }
191  
192  
193 # this is the Moses detokenizer
194 # written by Josh Schroeder, based on code by Philipp Koehn
195  
196 sub detokenize {
197 my $self = shift;
198 my $token = shift;
199  
200 my $text = ref($token) eq 'ARRAY' ? join( ' ', @$token ) : $token;
201 my $language = $self->{lang} || 'en';
202  
203 # chomp($text);
204 $text =~ s/\n/ /gs;
205 $text =~ s/\s\s+/ /gs;
206 $text = " $text ";
207  
208 my $word;
209 my $i;
210 my @words = split( / /, $text );
211 $text = "";
212 my %quoteCount = ( "\'" => 0, "\"" => 0 );
213 my $prependSpace = " ";
214 for ( $i = 0; $i < ( scalar(@words) ); $i++ ) {
215 if ( $words[$i] =~ /^[\p{IsSc}\(\[\{\¿\¡]+$/ ) {
216 # perform right shift on currency and other random punctuation items
217 $text = $text . $prependSpace . $words[$i];
218 $prependSpace = "";
219 }
220 elsif ( $words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/ ) {
221 # perform left shift on punctuation items
222 $text = $text . $words[$i];
223 $prependSpace = " ";
224 }
225 elsif (( $language eq "en" )
226 && ( $i > 0 )
227 && ( $words[$i] =~ /^[\'][\p{IsAlpha}]/ )
228 && ( $words[ $i - 1 ] =~ /[\p{IsAlnum}]$/ ) )
229 {
230 # left-shift the contraction for English
231 $text = $text . $words[$i];
232 $prependSpace = " ";
233 }
234 elsif (( $language eq "fr" )
235 && ( $i < ( scalar(@words) - 2 ) )
236 && ( $words[$i] =~ /[\p{IsAlpha}][\']$/ )
237 && ( $words[ $i + 1 ] =~ /^[\p{IsAlpha}]/ ) )
238 {
239 # right-shift the contraction for French
240 $text = $text . $prependSpace . $words[$i];
241 $prependSpace = "";
242 }
243 elsif ( $words[$i] =~ /^[\'\"]+$/ ) {
244 # combine punctuation smartly
245 if ( ( $quoteCount{ $words[$i] } % 2 ) eq 0 ) {
246 if ( ( $language eq "en" )
247 && ( $words[$i] eq "'" )
248 && ( $i > 0 )
249 && ( $words[ $i - 1 ] =~ /[s]$/ ) )
250 {
251 # single quote for posesssives ending in s... "The Jones' house"
252 # left shift
253 $text = $text . $words[$i];
254 $prependSpace = " ";
255 }
256 else {
257 # right shift
258 $text = $text . $prependSpace . $words[$i];
259 $prependSpace = "";
260 $quoteCount{ $words[$i] } = $quoteCount{ $words[$i] } + 1;
261 }
262 }
263 else {
264 # left shift
265 $text = $text . $words[$i];
266 $prependSpace = " ";
267 $quoteCount{ $words[$i] } = $quoteCount{ $words[$i] } + 1;
268 }
269 }
270 else {
271 $text = $text . $prependSpace . $words[$i];
272 $prependSpace = " ";
273 }
274 }
275  
276 # clean up spaces at head and tail of each line as well as any double-spacing
277 $text =~ s/ +/ /g;
278 $text =~ s/\n /\n/g;
279 $text =~ s/ \n/\n/g;
280 $text =~ s/^ //g;
281 $text =~ s/ $//g;
282  
283 # add trailing break
284 #$text .= "\n" unless $text =~ /\n$/;
285  
286 return $text;
287 }
288  
289  
290 1;
291