blib/lib/Lingua/EN/Bigram.pm

filter: Statementstmt 100% 91 / 91 Branchbran 100% 24 / 24 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 17 / 17 Podpod 100% 13 / 13 totaltotal 100% 145 / 145 CC 13 SCAR 8.4
CC
13
Cov
100%
ngram
5.0
ngram_count
3.0
text
2.0
CRAP 2.3
↑ Summary
1 package Lingua::EN::Bigram;
2  
3 # Bigram.pm - Extract n-grams from a text and list them according to frequency and/or T-Score
4  
5 # Eric Lease Morgan <eric_morgan@infomotions.com>
6 # June 18, 2009 - first investigations
7 # June 19, 2009 - "finished" POD
8 # August 22, 2010 - added trigrams and quadgrams; can I say "n-grams"?
9 # August 23, 2010 - yes, I can say n-gram
10  
11  
12 # include
13 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 1 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
15  
16 # define
17 1 our $VERSION = '0.03';
18  
19  
20 1 ▶sub new {
Subroutine new: called CC=1 SCAR=0.0
Pod documented
21  
22 # get input
23 1 my ( $class ) = @_;
24
25 # initialize
26 1 my $self = {};
27
28 # return
29 1 return bless $self, $class;
30
31 }
32  
33  
34 20 ▶sub text {
Subroutine text: called CC=2 SCAR=6.9
Pod documented
35  
36 # get input
37 20 my ( $self, $text ) = @_;
38
39 # set
40 20 ▶ if ( $text ) { $self->{ text } = $text }
Branch: 1 branch
BranchTrueFalseTotal
if ($text) 1 19 20
41
42 # return
43 20 return $self->{ text };
44
45 }
46  
47  
48 18 ▶sub words {
Subroutine words: called CC=1 SCAR=0.0
Pod documented
49  
50 # get input
51 18 my ( $self ) = shift;
52  
53 # parse
54 18 my $text = $self->text;
55 18 $text =~ tr/a-zA-Z'()\-,.?!;:/\n/cs;
56 18 $text =~ s/([,.?!:;()\-])/\n$1\n/g;
57 18 $text =~ s/\n+/\n/g;
58
59 # done
60 18 return split /\n/, lc( $text );
61  
62 }
63  
64  
65 2 ▶sub word_count {
Subroutine word_count: called CC=2 SCAR=6.9
Pod documented
66  
67 # get input
68 2 my ( $self ) = shift;
69  
70 # initialize
71 2 my @words = $self->words;
72 2 my %word_count = ();
73
74 # do the work
75 2 ▶ for ( my $i = 0; $i <= $#words; $i++ ) { $word_count{ $words[ $i ] }++ }
Branch: 1 branch
BranchTrueFalseTotal
if ($i <= $#words) 21774 2 21776
76  
77 # done
78 2 return \%word_count;
79
80 }
81  
82  
83 4 ▶sub bigrams {
Subroutine bigrams: called CC=1 SCAR=0.0
Pod documented
84  
85 # get input
86 4 my ( $self ) = shift;
87
88 # done
89 4 return $self->ngram( 2 );
90  
91 }
92  
93  
94 2 ▶sub bigram_count {
Subroutine bigram_count: called CC=2 SCAR=6.9
Pod documented
95  
96 # get input
97 2 my ( $self ) = shift;
98  
99 # initialize
100 2 my @words = $self->words;
101 2 my @bigrams = $self->bigrams;
102 2 my %bigram_count = ();
103
104 # do the work
105 2 ▶ for ( my $i = 0; $i < $#words; $i++ ) { $bigram_count{ $bigrams[ $i ] }++ }
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 21772 2 21774
106
107 # done
108 2 return \%bigram_count;
109  
110 }
111  
112  
113 1 ▶sub tscore {
Subroutine tscore: called CC=2 SCAR=6.9
Pod documented
114  
115 # get input
116 1 my ( $self ) = shift;
117
118 # initialize
119 1 my @words = $self->words;
120 1 my $word_count = $self->word_count;
121 1 my @bigrams = $self->bigrams;
122 1 my $bigram_count = $self->bigram_count;
123  
124 # calculate t-score
125 1 my %tscore = ();
126 1 ▶ for ( my $i = 0; $i < $#words; $i++ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 10886 1 10887
127  
128 $tscore{ $bigrams[ $i ] } = ( $$bigram_count{ $bigrams[ $i ] } -
129 $$word_count{ $words[ $i ] } *
130 $$word_count{ $words[ $i + 1 ] } /
131 ( $#words + 1 ) ) /
132 10886 sqrt( $$bigram_count{ $bigrams[ $i ] }
133 );
134  
135 }
136
137 # done
138 1 return \%tscore;
139
140 }
141  
142  
143 2 ▶sub trigrams {
Subroutine trigrams: called CC=1 SCAR=0.0
Pod documented
144
145 # get input
146 2 my ( $self ) = shift;
147  
148 # done
149 2 return $self->ngram( 3 );
150  
151 }
152  
153  
154 1 ▶sub trigram_count {
Subroutine trigram_count: called CC=2 SCAR=6.9
Pod documented
155  
156 # get input
157 1 my ( $self ) = shift;
158  
159 # initialize
160 1 my @words = $self->words;
161 1 my @trigrams = $self->trigrams;
162 1 my %trigram_count = ();
163
164 # do the work
165 1 ▶ for ( my $i = 0; $i < $#words; $i++ ) { $trigram_count{ $trigrams[ $i ] }++ }
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 10886 1 10887
166
167 # done
168 1 return \%trigram_count;
169  
170 }
171  
172  
173 2 ▶sub quadgrams {
Subroutine quadgrams: called CC=1 SCAR=0.0
Pod documented
174
175 # get input
176 2 my ( $self ) = shift;
177  
178 # done
179 2 return $self->ngram( 4 );
180  
181 }
182  
183  
184 1 ▶sub quadgram_count {
Subroutine quadgram_count: called CC=2 SCAR=6.9
Pod documented
185  
186 # get input
187 1 my ( $self ) = shift;
188  
189 # initialize
190 1 my @words = $self->words;
191 1 my @quadgrams = $self->quadgrams;
192 1 my %quadgram_count = ();
193
194 # do the work
195 1 ▶ for ( my $i = 0; $i < $#words; $i++ ) { $quadgram_count{ $quadgrams[ $i ] }++ }
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 10886 1 10887
196
197 # done
198 1 return \%quadgram_count;
199  
200 }
201  
202  
203 11 ▶sub ngram {
Subroutine ngram: called CC=5 SCAR=16.1
Pod documented
204  
205 # get input
206 11 my ( $self, $n ) = @_;
207  
208 # sanity check
209 11 ▶ if ( ! $n ) { die "This method -- ngram -- requires an integer as an argument." }
Branch: 1 branch
BranchTrueFalseTotal
unless ($n) 1 10 11
210 10 ▶ if ( $n =~ /\D/ ) { die "This method -- ngram -- requires an integer as an argument." }
Branch: 1 branch
BranchTrueFalseTotal
if ($n =~ /\D/) 1 9 10
211
212 # initialize
213 1 ▶ no warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
214 9 my @words = $self->words;
215 9 my @ngrams = ();
216
217 # process each word
218 9 ▶ for ( my $i = 0; $i < $#words; $i++ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 97974 9 97983
219
220 # repeat n number of times
221 97974 my $tokens = '';
222 97974 ▶ for ( my $j = $i; $j < $i + $n; $j++ ) { $tokens .= $words[ $j ] . ' ' }
Branch: 1 branch
BranchTrueFalseTotal
if ($j < $i + $n) 283036 97974 381010
223
224 # remove the trailing space
225 97974 chop $tokens;
226
227 # build the ngram
228 97974 $ngrams[ $i ] = $tokens;
229
230 }
231
232 # done
233 9 return @ngrams;
234  
235 }
236  
237  
238 2 ▶sub ngram_count {
Subroutine ngram_count: called CC=3 SCAR=11.0
Pod documented
239  
240 # get input
241 2 my ( $self, $ngrams ) = @_;
242  
243 # sanity check
244 2 ▶ if ( ref( $ngrams ) ne 'ARRAY' ) { die "This method -- ngram_count -- requires you pass it a reference to an array." }
Branch: 1 branch
BranchTrueFalseTotal
if (ref $ngrams ne "ARRAY") 1 1 2
245
246 # initialize
247 1 ▶ no warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
248 1 my @words = $self->words;
249 1 my %ngram_count = ();
250
251 # do the work
252 1 ▶ for ( my $i = 0; $i < $#words; $i++ ) { $ngram_count{ $$ngrams[ $i ] }++ }
Branch: 1 branch
BranchTrueFalseTotal
if ($i < $#words) 10886 1 10887
253
254 # done
255 1 return \%ngram_count;
256  
257 }
258  
259  
260 =head1 NAME
261  
262 Lingua::EN::Bigram - Extract n-grams from a text and list them according to frequency and/or T-Score
263  
264  
265 =head1 SYNOPSIS
266  
267 # initalize
268 use Lingua::EN::Bigram;
269 $ngrams = Lingua::EN::Bigram->new;
270 $ngrams->text( 'All men by nature desire to know. An indication of this...' );
271  
272 # calculate t-score for bigrams; t-score is only available for bigrams
273 $tscore = $ngrams->tscore;
274 foreach ( sort { $$tscore{ $b } <=> $$tscore{ $a } } keys %$tscore ) {
275  
276 print "$$tscore{ $_ }\t" . "$_\n";
277  
278 }
279  
280 # list trigrams according to frequency
281 @trigrams = $ngrams->ngram( 3 );
282 $count = $ngrams->ngram_count( \@trigrams );
283 foreach my $trigram ( sort { $$count{ $b } <=> $$count{ $a } } keys %$count ) {
284  
285 print $$count{ $trigram }, "\t$trigram\n";
286  
287 }
288  
289  
290 =head1 DESCRIPTION
291  
292 This module is designed to: 1) pull out all of the ngrams (multi-word phrases) in a given text, and 2) list these phrases according to their frequency. Using this module is it possible to create lists of the most common phrases in a text as well as order them by their statistical occurance, thus implying significance. This process is useful for the purposes of textual analysis and "distant reading".
293  
294  
295 =head1 METHODS
296  
297  
298 =head2 new
299  
300 Create a new, empty Lingua::EN::Bigram object:
301  
302 # initalize
303 $ngrams = Lingua::EN::Bigram->new;
304  
305  
306 =head2 text
307  
308 Set or get the text to be analyzed:
309  
310 # fill Lingua::EN::Bigram object with content
311 $ngrams->text( 'All good things must come to an end...' );
312  
313 # get the Lingua::EN::Bigram object's content
314 $text = $ngrams->text;
315  
316  
317 =head2 words
318  
319 Return a list of all the tokens in a text. Each token will be a word or puncutation mark:
320  
321 # get words
322 @words = $ngrams->words;
323  
324  
325 =head2 word_count
326  
327 Return a reference to a hash whose keys are a token and whose values are the number of times the token occurs in the text:
328  
329 # get word count
330 $word_count = $ngrams->word_count;
331  
332 # list the words according to frequency
333 foreach ( sort { $$word_count{ $b } <=> $$word_count{ $a } } keys %$word_count ) {
334  
335 print $$word_count{ $_ }, "\t$_\n";
336  
337 }
338  
339  
340 =head2 bigrams
341  
342 Return a list of all bigrams in the text. Each item will be a pair of tokens and the tokens may consist of words or puncutation marks:
343  
344 # get bigrams
345 @bigrams = $ngrams->bigrams;
346  
347 This is a convienience method for the ngram method, described below. It is identical to $ngrams->ngram( 2 ). In fact, that is exactly what is called within the module itself.
348  
349  
350 =head2 bigram_count
351  
352 Return a reference to a hash whose keys are a bigram and whose values are the frequency of the bigram in the text:
353  
354 # get bigram count
355 $count = $ngrams->bigram_count;
356  
357 # list the bigrams according to frequency
358 foreach ( sort { $$count{ $b } <=> $$count{ $a } } keys %$count ) {
359  
360 print $$count{ $_ }, "\t$_\n";
361  
362 }
363  
364  
365 =head2 tscore
366  
367 Return a reference to a hash whose keys are a bigram and whose values are a T-Score -- a probabalistic calculation determining the significance of the bigram occuring in the text:
368  
369 # get t-score
370 $tscore = $ngrams->tscore;
371  
372 # list bigrams according to t-score
373 foreach ( sort { $$tscore{ $b } <=> $$tscore{ $a } } keys %$tscore ) {
374  
375 print "$$tscore{ $_ }\t" . "$_\n";
376  
377 }
378  
379 T-Score can only be computed against bigrams.
380  
381  
382 =head2 trigrams
383  
384 Return a list of all trigrams (three-word phrases) in the text. Each item will include three tokens and the tokens may consist of words or puncutation marks:
385  
386 # get trigrams
387 @trigrams = $ngrams->trigrams;
388  
389 This is a convienience method for the ngram method, described below. It is identical to $ngrams->ngram( 3 ). In fact, that is exactly what is called within the module itself.
390  
391  
392 =head2 trigram_count
393  
394 Return a reference to a hash whose keys are a trigram and whose values are the frequency of the trigram in the text:
395  
396 # get trigram count
397 $count = $ngrams->trigram_count;
398  
399 # list the trigrams according to frequency
400 foreach ( sort { $$count{ $b } <=> $$count{ $a } } keys %$count ) {
401  
402 print $$count{ $_ }, "\t$_\n";
403  
404 }
405  
406  
407 =head2 quadgrams
408  
409 Return a list of all quadgrams (four-word phrases) in the text. Each item will include four tokens and the tokens may consist of words or puncutation marks:
410  
411 # get quadgrams
412 @quadgrams = $ngrams->quadgrams;
413  
414 This is a convienience method for the ngram method, described below. It is identical to $ngrams->ngram( 4 ). In fact, that is exactly what is called within the module itself.
415  
416  
417 =head2 quadgram_count
418  
419 Return a reference to a hash whose keys are a quadgram and whose values are the frequency of the quadgram in the text:
420  
421 # get quadgram count
422 $count = $ngrams->quadgram_count;
423  
424 # list the trigrams according to frequency
425 foreach ( sort { $$count{ $b } <=> $$count{ $a } } keys %$count ) {
426  
427 print $$count{ $_ }, "\t$_\n";
428  
429 }
430  
431  
432 =head2 ngram
433  
434 Return a list of ngrams where the length of each ngram is denoted by the method's parameter:
435  
436 # create a list of trigrams
437 @trigrams = $ngrams->ngram( 3 );
438
439 This method requires a single parameter and that parameter must be an integer.
440  
441  
442 =head2 ngram_count
443  
444 Given a reference to an array, return a reference to a hash whose keys are an ngram and whose values are the frequency of the ngram in the text:
445  
446 # count ngram frequency
447 $counts = $ngrams->ngram_count( \@trigrams );
448 foreach ( sort { $$counts{ $b } <=> $$counts{ $a } } keys %$counts ) {
449  
450 print $$counts{ $_ }, "\t$_\n";
451
452 }
453  
454  
455 =head1 DISCUSSION
456  
457 Given the increasing availability of full text materials, this module is intended to help "digital humanists" apply mathematical methods to the analysis of texts. For example, the developer can extract the high-frequency words using the word_count method and allow the user to search for those words in a concordance. The bigram_count method simply returns the frequency of a given bigram, but the tscore method can order them in a more finely tuned manner.
458  
459 Consider using T-Score-weighted bigrams as classification terms to supplement the "aboutness" of texts. Concatonate many texts together and look for common phrases written by the author. Compare these commonly used phrases to the commonly used phrases of other authors.
460  
461 Each bigram, trigram, quadgram, or ngram includes punctuation. This is intentional. Developers may need want to remove bigrams, trigrams, quadgrams, or ngrams containing such values from the output. Similarly, no effort has been made to remove commonly used words -- stop words -- from the methods. Consider the use of Lingua::StopWords, Lingua::EN::StopWords, or the creation of your own stop word list to make output more meaningful. The distribution came with a script (bin/ngrams.pl) demonstrating how to remove puncutation and stop words from the displayed output.
462  
463 Finally, this is not the only module supporting bigram extraction. See also Text::NSP which supports n-gram extraction.
464  
465  
466 =head1 TODO
467  
468 There are probably a number of ways the module can be improved:
469  
470 =over
471  
472 * the constructor method could take a scalar as input, thus reducing the need for the text method
473  
474 * the distribution's license should probably be changed to the Perl Aristic License
475  
476 * the addition of alternative T-Score calculations would be nice
477  
478 * make sure the module works with character sets beyond ASCII
479  
480 =back
481  
482  
483 =head1 CHANGES
484  
485 =over
486  
487 * August 23, 2010 (version 0.03) - added ngram and ngram_counts methods
488  
489 * August 22, 2010 (version 0.02) - added trigrams and quadgrams; tweaked documentation; removed bigrams.pl from the distribution and substituted it wih n-grams.pl
490  
491 * June 19, 2009 (version 0.01) - initial release
492  
493 =back
494  
495 =head1 ACKNOWLEDGEMENTS
496  
497 T-Score, as well as a number of the module's methods, is calculated as per Nugues, P. M. (2006). An introduction to language processing with Perl and Prolog: An outline of theories, implementation, and application with special consideration of English, French, and German. Cognitive technologies. Berlin: Springer.
498  
499  
500 =head1 AUTHOR
501  
502 Eric Lease Morgan <eric_morgan@infomotions.com>
503  
504 =cut
505  
506 # return true or die
507 1 1;
↑ Summary