| 1 | #-*-perl-*- | ||
| 2 | |||
| 3 | package Uplug::PreProcess::SentDetect; | ||
| 4 | |||
| 5 | # sentence boundary detection based on the Moses/Europarl sentence splitter | ||
| 6 | # (adjusted from Lingua::Sentence to Uplug) | ||
| 7 | |||
| 8 | =head1 NAME | ||
| 9 | |||
| 10 | Uplug::PreProcess::SentDetect - Moses/Europarl sentence boundary detector | ||
| 11 | |||
| 12 | =head1 SYNOPSIS | ||
| 13 | |||
| 14 | use Uplug::PreProcess::SentDetect; | ||
| 15 | my $splitter = Uplug::PreProcess::SentDetect->new (lang => 'en'); | ||
| 16 | my $text = 'This is a paragraph. It contains several sentences. "But why," you ask?'; | ||
| 17 | print $splitter->split($text); | ||
| 18 | |||
| 19 | =head1 DESCRIPTION | ||
| 20 | |||
| 21 | This module is basically a copy of L<Lingua::Sentence> by Achim Ruopp adapted to Uplug which is based on tools developed for Moses and the Europarl corpus. All credits go to the original authors. This version includes some additional non-breaking prefix files. | ||
| 22 | |||
| 23 | This module allows splitting of text paragraphs into sentences. | ||
| 24 | It is based on scripts developed by Philipp Koehn and Josh Schroeder | ||
| 25 | for processing the Europarl corpus (L<http://www.statmt.org/europarl/>). | ||
| 26 | |||
| 27 | The module uses punctuation and capitalization clues to split paragraphs | ||
| 28 | into an newline-separated string with one sentence per line. | ||
| 29 | For example: | ||
| 30 | |||
| 31 | This is a paragraph. It contains several sentences. "But why," you ask? | ||
| 32 | |||
| 33 | goes to: | ||
| 34 | |||
| 35 | This is a paragraph. | ||
| 36 | It contains several sentences. | ||
| 37 | "But why," you ask? | ||
| 38 | |||
| 39 | Languages currently supported by the module are: | ||
| 40 | |||
| 41 | =over | ||
| 42 | |||
| 43 | =item ca (Catalan) | ||
| 44 | |||
| 45 | =item da (Danish) | ||
| 46 | |||
| 47 | =item de (German) | ||
| 48 | |||
| 49 | =item el (Greek) | ||
| 50 | |||
| 51 | =item en (English) | ||
| 52 | |||
| 53 | =item es (Spanish) | ||
| 54 | |||
| 55 | =item fr (French) | ||
| 56 | |||
| 57 | =item is (Icelandic) | ||
| 58 | |||
| 59 | =item it (Italian) | ||
| 60 | |||
| 61 | =item nl (Dutch) | ||
| 62 | |||
| 63 | -item pl (Polish) | ||
| 64 | |||
| 65 | =item pt (Portuguese) | ||
| 66 | |||
| 67 | =item ro (Romanian) | ||
| 68 | |||
| 69 | =item ru (Russian) | ||
| 70 | |||
| 71 | =item sk (Slovak) | ||
| 72 | |||
| 73 | =item sl (Slovene) | ||
| 74 | |||
| 75 | =item sv (Swedish) | ||
| 76 | |||
| 77 | =back | ||
| 78 | |||
| 79 | |||
| 80 | =head2 Nonbreaking-Prefixes Files | ||
| 81 | |||
| 82 | Nonbreaking prefixes are loosely defined as any word ending in a period | ||
| 83 | that does NOT indicate an end of sentence marker. | ||
| 84 | A basic example is Mr. and Ms. in English. | ||
| 85 | |||
| 86 | The sentence splitter module uses the nonbreaking prefix files included in this distribution. | ||
| 87 | |||
| 88 | To add a file for other languages, follow the naming convention nonbreaking_prefix.?? | ||
| 89 | and use the two-letter language code you intend to use when creating a Lingua::Sentence object. | ||
| 90 | |||
| 91 | The sentence splitter module will first look for a file for the language it is processing, | ||
| 92 | and fall back to English if a file for that language is not found. | ||
| 93 | |||
| 94 | For the splitter, normally a period followed by an uppercase word results in a sentence split. | ||
| 95 | If the word preceeding the period is a nonbreaking prefix, this line break is not inserted. | ||
| 96 | |||
| 97 | A special case of prefixes, NUMERIC_ONLY, is included for special cases where the prefix should be handled ONLY when before numbers. | ||
| 98 | For example, "Article No. 24 states this." the No. is a nonbreaking prefix. | ||
| 99 | However, in "No. It is not true." No functions as a word. | ||
| 100 | |||
| 101 | See the example prefix files included in the distribution for more examples. | ||
| 102 | |||
| 103 | =cut | ||
| 104 | |||
| 105 | use strict; | ||
| 106 | |||
| 107 | use Uplug::Config; | ||
| 108 | |||
| 109 | # defaults: language = English | ||
| 110 | |||
| 111 | our $DEFAULT_LANG = 'en'; | ||
| 112 | |||
| 113 | our $NONBREAKING_PREFIX_DIR = &shared_lang() . '/nonbreaking_prefixes'; | ||
| 114 | our $DEFAULT_NONBREAKING_PREFIX = $NONBREAKING_PREFIX_DIR . '/nonbreaking_prefix.en'; | ||
| 115 | |||
| 116 | |||
| 117 | =head1 CONSTRUCTOR | ||
| 118 | |||
| 119 | The constructor can be called in two ways: | ||
| 120 | |||
| 121 | Uplug::PreProcess::SentDetect->new (lang => $lang_id) | ||
| 122 | |||
| 123 | Instantiate an object to split sentences in language C<$lang_id>. | ||
| 124 | If the language is not supported, a splitter object for English will be instantiated. | ||
| 125 | |||
| 126 | =cut | ||
| 127 | |||
| 128 | sub new { | ||
| 129 | my $class = shift; | ||
| 130 | |||
| 131 | my $self = {}; | ||
| 132 | %{$self} = @_; | ||
| 133 | |||
| 134 | $self->{lang} = $DEFAULT_LANG unless ( defined $self->{lang} ); | ||
| 135 | bless $self, $class; | ||
| 136 | $self->init( $self->{lang} ); | ||
| 137 | return $self; | ||
| 138 | } | ||
| 139 | |||
| 140 | |||
| 141 | # Preloaded methods go here. | ||
| 142 | |||
| 143 | sub init { | ||
| 144 | my $self = shift; | ||
| 145 | |||
| 146 | my $langid = shift; | ||
| 147 | my $prefixfile = shift; | ||
| 148 | |||
| 149 | # Try loading nonbreaking prefix file specified in constructor | ||
| 150 | my $dir = $NONBREAKING_PREFIX_DIR; | ||
| 151 | if ( defined($prefixfile) ) { | ||
| 152 | if ( !( -e $prefixfile ) ) { | ||
| 153 | $prefixfile = "$dir/nonbreaking_prefix.$langid"; | ||
| 154 | } | ||
| 155 | } | ||
| 156 | else { | ||
| 157 | $prefixfile = "$NONBREAKING_PREFIX_DIR/nonbreaking_prefix.$langid"; | ||
| 158 | } | ||
| 159 | |||
| 160 | my %NONBREAKING_PREFIX; | ||
| 161 | |||
| 162 | #default back to English if we don't have a language-specific prefix file | ||
| 163 | if ( !( -e $prefixfile ) ) { | ||
| 164 | $prefixfile = $DEFAULT_NONBREAKING_PREFIX; | ||
| 165 | unless ( -e $prefixfile ) { | ||
| 166 | die "ERROR: No abbreviations files found in $dir"; | ||
| 167 | } | ||
| 168 | } | ||
| 169 | if ( -e "$prefixfile" ) { | ||
| 170 | open( PREFIX, "<:encoding(utf8)", "$prefixfile" ); | ||
| 171 | while (<PREFIX>) { | ||
| 172 | my $item = $_; | ||
| 173 | chomp($item); | ||
| 174 | if ( ($item) && ( substr( $item, 0, 1 ) ne "#" ) ) { | ||
| 175 | if ( $item =~ /(.*)[\s]+(\#NUMERIC_ONLY\#)/ ) { | ||
| 176 | $NONBREAKING_PREFIX{$1} = 2; | ||
| 177 | } | ||
| 178 | else { | ||
| 179 | $NONBREAKING_PREFIX{$item} = 1; | ||
| 180 | } | ||
| 181 | } | ||
| 182 | } | ||
| 183 | close(PREFIX); | ||
| 184 | } | ||
| 185 | |||
| 186 | $self->{LangID} = $langid; | ||
| 187 | $self->{Nonbreaking} = \%NONBREAKING_PREFIX; | ||
| 188 | return $self; | ||
| 189 | } | ||
| 190 | |||
| 191 | |||
| 192 | sub split { | ||
| 193 | my $self = shift; | ||
| 194 | if ( !ref $self ) { | ||
| 195 | return "Unnamed $self"; | ||
| 196 | } | ||
| 197 | my $text = shift; | ||
| 198 | if ( !$text ) { | ||
| 199 | return (); | ||
| 200 | } | ||
| 201 | my $splittext = _preprocess( $self, $text ); | ||
| 202 | chomp $splittext; | ||
| 203 | return split( /\n/, $splittext ); | ||
| 204 | } | ||
| 205 | |||
| 206 | |||
| 207 | sub _preprocess { | ||
| 208 | my ( $self, $text ) = @_; | ||
| 209 | |||
| 210 | # clean up spaces at head and tail of each line as well as any double-spacing | ||
| 211 | $text =~ s/ +/ /g; | ||
| 212 | $text =~ s/\n /\n/g; | ||
| 213 | $text =~ s/ \n/\n/g; | ||
| 214 | $text =~ s/^ //g; | ||
| 215 | $text =~ s/ $//g; | ||
| 216 | |||
| 217 | ##### add sentence breaks as needed ##### | ||
| 218 | |||
| 219 | #non-period end of sentence markers (?!) followed by sentence starters. | ||
| 220 | $text =~ s/([?!]) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\p{IsUpper}])/$1\n$2/g; | ||
| 221 | |||
| 222 | #multi-dots followed by sentence starters | ||
| 223 | $text =~ s/(\.[\.]+) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\p{IsUpper}])/$1\n$2/g; | ||
| 224 | |||
| 225 | # add breaks for sentences that end with some sort of punctuation | ||
| 226 | # inside a quote or parenthetical and are followed by a possible | ||
| 227 | # sentence starter punctuation and upper case | ||
| 228 | $text =~ s/([?!\.][\ ]*[\'\"\)\]\p{IsPf}]+) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[\ ]*[\p{IsUpper}])/$1\n$2/g; | ||
| 229 | |||
| 230 | # add breaks for sentences that end with some sort of punctuation are | ||
| 231 | # followed by a sentence starter punctuation and upper case | ||
| 232 | $text =~ s/([?!\.]) +([\'\"\(\[\¿\¡\p{IsPi}\x{201E}]+[\ ]*[\p{IsUpper}])/$1\n$2/g; | ||
| 233 | |||
| 234 | # special punctuation cases are covered. Check all remaining periods. | ||
| 235 | my $word; | ||
| 236 | my $i; | ||
| 237 | my @words = split( / /, $text ); | ||
| 238 | $text = ""; | ||
| 239 | for ( $i = 0; $i < ( scalar(@words) - 1 ); $i++ ) { | ||
| 240 | if ( $words[$i] | ||
| 241 | =~ /([\p{IsAlnum}\.\-]*)([\'\"\)\]\%\p{IsPf}]*)(\.+)$/ ) | ||
| 242 | { | ||
| 243 | #check if $1 is a known honorific and $2 is empty, never break | ||
| 244 | my $prefix = $1; | ||
| 245 | my $starting_punct = $2; | ||
| 246 | if ( $prefix | ||
| 247 | && $self->{Nonbreaking}{$prefix} | ||
| 248 | && $self->{Nonbreaking}{$prefix} == 1 | ||
| 249 | && !$starting_punct ) | ||
| 250 | { | ||
| 251 | #not breaking; | ||
| 252 | } | ||
| 253 | elsif ( $words[$i] =~ /(\.)[\p{IsUpper}\-]+(\.+)$/ ) { | ||
| 254 | #not breaking - upper case acronym | ||
| 255 | } | ||
| 256 | elsif ( $words[ $i + 1 ] =~ /^([ ]*[\'\"\(\[\¿\¡\p{IsPi}\x{201E}]*[ ]*[\p{IsUpper}0-9])/ ) { | ||
| 257 | # the next word has a bunch of initial quotes, | ||
| 258 | # maybe a space, then either upper case or a number | ||
| 259 | $words[$i] = $words[$i] . "\n" | ||
| 260 | unless ( $prefix | ||
| 261 | && $self->{Nonbreaking}{$prefix} | ||
| 262 | && $self->{Nonbreaking}{$prefix} == 2 | ||
| 263 | && !$starting_punct | ||
| 264 | && ( $words[ $i + 1 ] =~ /^[0-9]+/ ) ); | ||
| 265 | # we always add a return for these unless we have | ||
| 266 | # a numeric non-breaker and a number start | ||
| 267 | } | ||
| 268 | |||
| 269 | } | ||
| 270 | $text = $text . $words[$i] . " "; | ||
| 271 | } | ||
| 272 | |||
| 273 | # we stopped one token from the end to allow for easy look-ahead. Append it now. | ||
| 274 | $text = $text . $words[$i]; | ||
| 275 | |||
| 276 | # clean up spaces at head and tail of each line as well as any double-spacing | ||
| 277 | $text =~ s/ +/ /g; | ||
| 278 | $text =~ s/\n /\n/g; | ||
| 279 | $text =~ s/ \n/\n/g; | ||
| 280 | $text =~ s/^ //g; | ||
| 281 | $text =~ s/ $//g; | ||
| 282 | |||
| 283 | #add trailing break | ||
| 284 | $text .= "\n" unless $text =~ /\n$/; | ||
| 285 | |||
| 286 | return $text; | ||
| 287 | } | ||
| 288 | |||
| 289 | |||
| 290 | 1; | ||
| 291 | |||
| 292 | =head2 CREDITS | ||
| 293 | |||
| 294 | Thanks for the following individuals for supplying nonbreaking prefix files: | ||
| 295 | Bas Rozema (Dutch), HilE<aacute>rio Leal Fontes (Portuguese), JesE<uacute>s GimE<eacute>nez (Catalan & Spanish) | ||
| 296 | |||
| 297 | =head1 SUPPORT | ||
| 298 | |||
| 299 | Bugs should always be submitted via the project hosting bug tracker | ||
| 300 | |||
| 301 | L<http://code.google.com/p/corpus-tools/issues/list> | ||
| 302 | |||
| 303 | For other issues, contact the maintainer. | ||
| 304 | |||
| 305 | =head1 SEE ALSO | ||
| 306 | |||
| 307 | L<Text::Sentence>, | ||
| 308 | L<Lingua::EN::Sentence>, | ||
| 309 | L<Lingua::DE::Sentence>, | ||
| 310 | L<Lingua::HE::Sentence> | ||
| 311 | |||
| 312 | =head1 AUTHOR | ||
| 313 | |||
| 314 | Lingua::Sentence: Achim Ruopp, E<lt>achimru@gmail.comE<gt> | ||
| 315 | |||
| 316 | Adapted to Uplug: Joerg Tiedemann | ||
| 317 | |||
| 318 | =head1 COPYRIGHT AND LICENSE | ||
| 319 | |||
| 320 | Copyright (C) 2010 by Digital Silk Road | ||
| 321 | |||
| 322 | Portions Copyright (C) 2005 by Philip Koehn and Josh Schroeder (used with permission) | ||
| 323 | |||
| 324 | This library is free software; you can redistribute it and/or modify | ||
| 325 | it under the same terms as Perl itself, either Perl version 5.8.8 or, | ||
| 326 | at your option, any later version of Perl 5 you may have available. | ||
| 327 |