blib/lib/Uplug/PreProcess/SentDetect.pm untested

filter: Statementstmt 0.0% 0 / 85 Branchbran 0.0% 0 / 32 Conditioncond 0.0% 0 / 24 MC/DCmcdc 0.0% 0 / 11 Subroutinesub 0.0% 0 / 6 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 162 CC 17 SCAR 38.4
CC
17
Cov
0%
init
72.0
_preprocess
56.0
split
12.0
CRAP 46.4
1 #-*-perl-*-
2  
3 package Uplug::PreProcess::SentDetect;
4  
5 # sentence boundary detection based on the Moses/Europarl sentence splitter
6 # (adjusted from Lingua::Sentence to Uplug)
7  
8 =head1 NAME
9  
10 Uplug::PreProcess::SentDetect - Moses/Europarl sentence boundary detector
11  
12 =head1 SYNOPSIS
13  
14 use Uplug::PreProcess::SentDetect;
15 my $splitter = Uplug::PreProcess::SentDetect->new (lang => 'en');
16 my $text = 'This is a paragraph. It contains several sentences. "But why," you ask?';
17 print $splitter->split($text);
18  
19 =head1 DESCRIPTION
20  
21 This module is basically a copy of L<Lingua::Sentence> by Achim Ruopp adapted to Uplug which is based on tools developed for Moses and the Europarl corpus. All credits go to the original authors. This version includes some additional non-breaking prefix files.
22  
23 This module allows splitting of text paragraphs into sentences.
24 It is based on scripts developed by Philipp Koehn and Josh Schroeder
25 for processing the Europarl corpus (L<http://www.statmt.org/europarl/>).
26  
27 The module uses punctuation and capitalization clues to split paragraphs
28 into an newline-separated string with one sentence per line.
29 For example:
30  
31 This is a paragraph. It contains several sentences. "But why," you ask?
32  
33 goes to:
34  
35 This is a paragraph.
36 It contains several sentences.
37 "But why," you ask?
38  
39 Languages currently supported by the module are:
40  
41 =over
42  
43 =item ca (Catalan)
44  
45 =item da (Danish)
46  
47 =item de (German)
48  
49 =item el (Greek)
50  
51 =item en (English)
52  
53 =item es (Spanish)
54  
55 =item fr (French)
56  
57 =item is (Icelandic)
58  
59 =item it (Italian)
60  
61 =item nl (Dutch)
62  
63 -item pl (Polish)
64  
65 =item pt (Portuguese)
66  
67 =item ro (Romanian)
68  
69 =item ru (Russian)
70  
71 =item sk (Slovak)
72  
73 =item sl (Slovene)
74  
75 =item sv (Swedish)
76  
77 =back
78  
79  
80 =head2 Nonbreaking-Prefixes Files
81  
82 Nonbreaking prefixes are loosely defined as any word ending in a period
83 that does NOT indicate an end of sentence marker.
84 A basic example is Mr. and Ms. in English.
85  
86 The sentence splitter module uses the nonbreaking prefix files included in this distribution.
87  
88 To add a file for other languages, follow the naming convention nonbreaking_prefix.??
89 and use the two-letter language code you intend to use when creating a Lingua::Sentence object.
90  
91 The sentence splitter module will first look for a file for the language it is processing,
92 and fall back to English if a file for that language is not found.
93  
94 For the splitter, normally a period followed by an uppercase word results in a sentence split.
95 If the word preceeding the period is a nonbreaking prefix, this line break is not inserted.
96  
97 A special case of prefixes, NUMERIC_ONLY, is included for special cases where the prefix should be handled ONLY when before numbers.
98 For example, "Article No. 24 states this." the No. is a nonbreaking prefix.
99 However, in "No. It is not true." No functions as a word.
100  
101 See the example prefix files included in the distribution for more examples.
102  
103 =cut
104  
105 use strict;
106  
107 use Uplug::Config;
108  
109 # defaults: language = English
110  
111 our $DEFAULT_LANG = 'en';
112  
113 our $NONBREAKING_PREFIX_DIR = &shared_lang() . '/nonbreaking_prefixes';
114 our $DEFAULT_NONBREAKING_PREFIX = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.en';
115  
116  
117 =head1 CONSTRUCTOR
118  
119 The constructor can be called in two ways:
120  
121 Uplug::PreProcess::SentDetect->new (lang => $lang_id)
122  
123 Instantiate an object to split sentences in language C<$lang_id>.
124 If the language is not supported, a splitter object for English will be instantiated.
125  
126 =cut
127  
128 sub new {
129 my $class = shift;
130  
131 my $self = {};
132 %{$self} = @_;
133  
134 $self->{lang} = $DEFAULT_LANG unless ( defined $self->{lang} );
135 bless $self, $class;
136 $self->init( $self->{lang} );
137 return $self;
138 }
139  
140  
141 # Preloaded methods go here.
142  
143 sub init {
144 my $self = shift;
145  
146 my $langid = shift;
147 my $prefixfile = shift;
148  
149 # Try loading nonbreaking prefix file specified in constructor
150 my $dir = $NONBREAKING_PREFIX_DIR;
151 if ( defined($prefixfile) ) {
152 if ( !( -e $prefixfile ) ) {
153 $prefixfile = "$dir/nonbreaking_prefix.$langid";
154 }
155 }
156 else {
157 $prefixfile = "$NONBREAKING_PREFIX_DIR/nonbreaking_prefix.$langid";
158 }
159  
160 my %NONBREAKING_PREFIX;
161  
162 #default back to English if we don't have a language-specific prefix file
163 if ( !( -e $prefixfile ) ) {
164 $prefixfile = $DEFAULT_NONBREAKING_PREFIX;
165 unless ( -e $prefixfile ) {
166 die "ERROR: No abbreviations files found in $dir";
167 }
168 }
169 if ( -e "$prefixfile" ) {
170 open( PREFIX, "<:encoding(utf8)", "$prefixfile" );
171 while (<PREFIX>) {
172 my $item = $_;
173 chomp($item);
174 if ( ($item) && ( substr( $item, 0, 1 ) ne "#" ) ) {
175 if ( $item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/ ) {
176 $NONBREAKING_PREFIX{$1} = 2;
177 }
178 else {
179 $NONBREAKING_PREFIX{$item} = 1;
180 }
181 }
182 }
183 close(PREFIX);
184 }
185  
186 $self->{LangID} = $langid;
187 $self->{Nonbreaking} = \%NONBREAKING_PREFIX;
188 return $self;
189 }
190  
191  
192 sub split {
193 my $self = shift;
194 if ( !ref $self ) {
195 return "Unnamed $self";
196 }
197 my $text = shift;
198 if ( !$text ) {
199 return ();
200 }
201 my $splittext = _preprocess( $self, $text );
202 chomp $splittext;
203 return split( /\n/, $splittext );
204 }
205  
206  
207 sub _preprocess {
208 my ( $self, $text ) = @_;
209  
210 # clean up spaces at head and tail of each line as well as any double-spacing
211 $text =~ s/ +/ /g;
212 $text =~ s/\n /\n/g;
213 $text =~ s/ \n/\n/g;
214 $text =~ s/^ //g;
215 $text =~ s/ $//g;
216  
217 ##### add sentence breaks as needed #####
218  
219 #non-period end of sentence markers (?!) followed by sentence starters.
220 $text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\p{IsUpper}])/$1\n$2/g;
221  
222 #multi-dots followed by sentence starters
223 $text =~ s/(\.[\.]+) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\p{IsUpper}])/$1\n$2/g;
224  
225 # add breaks for sentences that end with some sort of punctuation
226 # inside a quote or parenthetical and are followed by a possible
227 # sentence starter punctuation and upper case
228 $text =~ s/([?!\.][\ ]*[\'\"\)\]\p{IsPf}]+) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\ ]*[\p{IsUpper}])/$1\n$2/g;
229  
230 # add breaks for sentences that end with some sort of punctuation are
231 # followed by a sentence starter punctuation and upper case
232 $text =~ s/([?!\.]) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]+[\ ]*[\p{IsUpper}])/$1\n$2/g;
233  
234 # special punctuation cases are covered. Check all remaining periods.
235 my $word;
236 my $i;
237 my @words = split( / /, $text );
238 $text = "";
239 for ( $i = 0; $i < ( scalar(@words) - 1 ); $i++ ) {
240 if ( $words[$i]
241 =~ /([\p{IsAlnum}\.\-]*)([\'\"\)\]\%\p{IsPf}]*)(\.+)$/ )
242 {
243 #check if $1 is a known honorific and $2 is empty, never break
244 my $prefix = $1;
245 my $starting_punct = $2;
246 if ( $prefix
247 && $self->{Nonbreaking}{$prefix}
248 && $self->{Nonbreaking}{$prefix} == 1
249 && !$starting_punct )
250 {
251 #not breaking;
252 }
253 elsif ( $words[$i] =~ /(\.)[\p{IsUpper}\-]+(\.+)$/ ) {
254 #not breaking - upper case acronym
255 }
256 elsif ( $words[ $i + 1 ] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[ ]*[\p{IsUpper}0-9])/ ) {
257 # the next word has a bunch of initial quotes,
258 # maybe a space, then either upper case or a number
259 $words[$i] = $words[$i] . "\n"
260 unless ( $prefix
261 && $self->{Nonbreaking}{$prefix}
262 && $self->{Nonbreaking}{$prefix} == 2
263 && !$starting_punct
264 && ( $words[ $i + 1 ] =~ /^[0-9]+/ ) );
265 # we always add a return for these unless we have
266 # a numeric non-breaker and a number start
267 }
268  
269 }
270 $text = $text . $words[$i] . " ";
271 }
272  
273 # we stopped one token from the end to allow for easy look-ahead. Append it now.
274 $text = $text . $words[$i];
275  
276 # clean up spaces at head and tail of each line as well as any double-spacing
277 $text =~ s/ +/ /g;
278 $text =~ s/\n /\n/g;
279 $text =~ s/ \n/\n/g;
280 $text =~ s/^ //g;
281 $text =~ s/ $//g;
282  
283 #add trailing break
284 $text .= "\n" unless $text =~ /\n$/;
285  
286 return $text;
287 }
288  
289  
290 1;
291  
292 =head2 CREDITS
293  
294 Thanks for the following individuals for supplying nonbreaking prefix files:
295 Bas Rozema (Dutch), HilE<aacute>rio Leal Fontes (Portuguese), JesE<uacute>s GimE<eacute>nez (Catalan & Spanish)
296  
297 =head1 SUPPORT
298  
299 Bugs should always be submitted via the project hosting bug tracker
300  
301 L<http://code.google.com/p/corpus-tools/issues/list>
302  
303 For other issues, contact the maintainer.
304  
305 =head1 SEE ALSO
306  
307 L<Text::Sentence>,
308 L<Lingua::EN::Sentence>,
309 L<Lingua::DE::Sentence>,
310 L<Lingua::HE::Sentence>
311  
312 =head1 AUTHOR
313  
314 Lingua::Sentence: Achim Ruopp, E<lt>achimru@gmail.comE<gt>
315  
316 Adapted to Uplug: Joerg Tiedemann
317  
318 =head1 COPYRIGHT AND LICENSE
319  
320 Copyright (C) 2010 by Digital Silk Road
321  
322 Portions Copyright (C) 2005 by Philip Koehn and Josh Schroeder (used with permission)
323  
324 This library is free software; you can redistribute it and/or modify
325 it under the same terms as Perl itself, either Perl version 5.8.8 or,
326 at your option, any later version of Perl 5 you may have available.
327