Ngram.pm

filter: Statementstmt 69.2% 264 / 381 Branchbran 44.9% 71 / 158 Conditioncond 25.0% 3 / 12 MC/DCmcdc 0.0% 0 / 8 Subroutinesub 60.6% 20 / 33 Podpod 48.2% 14 / 29 totaltotal 59.9% 372 / 621 CC 100 SCAR 35.6
CC
100
Cov
61%
valid
150.6
set_nontoken_regex
72.0
create_stop_list
56.0
CRAP 35.3
↑ Summary text-positional-ngram-driver.pl »
1 #########################################################################
2 # PACKAGE: Text::Positional::Ngram
3 #
4 # Copyright (C), 2004-2007
5 # Bridget Thomson McInnes, bthomson@d.umn.edu
6 #
7 # University of Minnesota, Duluth
8 #
9 # USAGE:
10 # use Text::Positional::Ngram
11 #
12 # DESCRIPTION:
13 #
14 # The Text::Positional::Ngram module determines contiguous and
15 # noncontiguous n-grams and their frequency from a given corpus.
16 # See perldoc Text::Positional::Ngram
17 #
18 #########################################################################
19 package Text::Positional::Ngram;
20  
21 1 ▶use 5.008;
Subroutine BEGIN: called CC=1 SCAR=0.0
22 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 1 ▶use bytes;
Subroutine BEGIN: called CC=1 SCAR=0.0
24  
25 1 require Exporter;
26 1 ▶use AutoLoader qw(AUTOLOAD);
Subroutine BEGIN: called CC=1 SCAR=0.0
27  
28 1 our @ISA = qw(Exporter);
29  
30 # Items to export into callers namespace by default. Note: do not export
31 # names by default without a very good reason. Use EXPORT_OK instead.
32 # Do not simply export all your public functions/methods/constants.
33  
34 # This allows declaration use Text::Positional::Ngram ':all';
35 # If you do not need this, moving things directly into @EXPORT or @EXPORT_OK
36 # will save memory.
37 1 our %EXPORT_TAGS = ( 'all' => [ qw() ] );
38  
39 1 our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
40  
41 1 our @EXPORT = qw();
42  
43 1 our $VERSION = '0.5';
44  
45 #########################
46 # File Name Variables #
47 #########################
48 1 my $CORPUS_FILE = "";
49 1 my $VOCAB_FILE = "";
50 1 my $SNT_FILE = "";
51 1 my $SNTNGRAM_FILE = "";
52 1 my $NGRAM_FILE = "";
53 1 my $STOPLIST = "";
54 1 my $TOKEN_FILE = "";
55 1 my $NONTOKEN_FILE = "";
56  
57 ###########################
58 # User defined Variables #
59 ###########################
60 1 my $max_ngram_size = 2; #default is 2
61 1 my $min_ngram_size = 2; #default is 2
62 1 my $ngram_size = 2; #default is 2
63 1 my $frequency = 0; #default is 0
64 1 my $window_size = 0; #default is 0
65  
66 #########################
67 # Stop List Variables #
68 #########################
69 1 my $stop_mode = "AND"; #AND/OR default is AND
70 1 my $stop_regex = ""; #regex to store stop list
71
72 ############################
73 # Token Obtion Variables #
74 ############################
75 1 my $tokenizerRegex = "";
76 1 my $nontokenizerRegex = "";
77  
78 ####################
79 # Flag Variables #
80 ####################
81 1 my $stop_flag = 0; #default is false
82 1 my $marginals = 0; #default is false
83 1 my $remove = 0; #default is false;
84 1 my $new_line = 0; #default is false
85  
86 #####################
87 # Cache Variables #
88 #####################
89 1 my $unigrams = "";
90 1 my %remove_hash = ();
91  
92 #####################
93 # Array Variables #
94 #####################
95 1 my @vocab_array = ();
96 1 my @window_array = ();
97  
98 #########################
99 # Main Mask Variables #
100 #########################
101 # VEC VARIABLES
102 1 my $corpus = ""; # corpus vec
103  
104 # MISC VARIABLES
105 1 my $N = 0; # the length of the corpus
106 1 my $bit = 32; # the bit size for the vec array
107 1 my $ngram_count = 0; # the number of ngrams
108 1 my $win_bit = 1; # the bit size for the windowing
109 1 my $timestamp = ""; # the time stamp for the files
110  
111 ###############
112 # new method #
113 ###############
114 1 my $location;
115 sub new
116 1 ▶{
Subroutine new: called CC=5 SCAR=16.2
Pod undocumented
117 # First argument is class
118 1 my $class = shift;
119  
120 1 my $self = {};
121  
122 1 bless $self, $class;
123  
124 1 ▶ $self->{dir} = shift if (defined(@_ > 0));
Branch: 1 branch
BranchTrueFalseTotal
if defined(@_ > 0) 1 0 1
125 1 ▶ $self->{verbose} = @_ ? shift : 0;
Branch: 1 branch
BranchTrueFalseTotal
@_ ? : 0 1 1
126  
127 1 ▶ warn "Dir = ", $self->{dir}, "\n" if ($self->{verbose});
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 1 1
128 1 ▶ warn "Verbose = ", $self->{verbose}, "\n" if ($self->{verbose});
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 1 1
129  
130 # Initialize some variables at new
131 1 $CORPUS_FILE = ""; $VOCAB_FILE = "";
132 1 $SNT_FILE = ""; $SNTNGRAM_FILE = "";
133 1 $NGRAM_FILE = ""; $STOPLIST = "";
134 1 $TOKEN_FILE = ""; $NONTOKEN_FILE = "";
135  
136 1 $stop_flag = 0; $marginals = 0;
137 1 $remove = 0; $new_line = 0;
138 1 $ngram_size = 2; $frequency = 0;
139 1 $window_size = 0;
140  
141 1 $unigrams = ""; %remove_hash = ();
142 1 $nontokenizerRegex = ""; $tokenizerRegex = "";
143  
144 1 return $self;
145 }
146  
147  
148 #######################################
149 # Create the vocabulary and snt file #
150 #######################################
151 sub create_files
152 1 ▶{
Subroutine create_files: called CC=10 SCAR=23.4
Pod documented
153 1 my $self = shift; my @files = @_;
154  
155 # Open the corpus, vocab and snt files
156 1 ▶ open(VOCAB , ">$VOCAB_FILE") || die "Could not open the vocabfile: $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open VOCAB, ">$VOCAB_FILE" or die "Could not open the vocabfile: $!\n" 1 0 1
157 1 ▶ open(SNT, ">$SNT_FILE") || die "Could not open the sntfile : $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open SNT, ">$SNT_FILE" or die "Could not open the sntfile : $!\n" 1 0 1
158
159 # Create the token and nontoken regular expression
160 0 ▶ if($NONTOKEN_FILE ne "") { set_nontoken_regex(); } set_token_regex();
Branch: 1 branch
BranchTrueFalseTotal
if ($NONTOKEN_FILE ne "") 0 1 1
161
162 ################################################
163 # Index always starts at 2 because 1 is #
164 # considered a new line parameter if defined #
165 ################################################
166  
167 1 my $index = 2; my %vocab_hash = ();
168  
169 1 foreach (@files) {
170 1 ▶ open(CORPUS, $_) || die "Could not find the corpus file: $_\n";
Branch: 1 branch
BranchTrueFalseTotal
open CORPUS, $_ or die "Could not find the corpus file: $_\n" 1 0 1
171 1 ▶ while(<CORPUS>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline CORPUS)) 3 1 4
172 3 chomp;
173
174 3 s/$nontokenizerRegex//g;
175
176 3 ▶ while( /$tokenizerRegex/g ) {
Branch: 1 branch
BranchTrueFalseTotal
if (/$tokenizerRegex/g) 22 3 25
177 22 my $token = $&;
178
179 22 ▶ if (! exists $vocab_hash{$token} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not exists $vocab_hash{$token}) { } 18 4 22
180 18 print SNT "$index ";
181 18 print VOCAB "$index\n"; print VOCAB "$token\n";
182 18 $vocab_hash{$token} = $index++;
183 }
184 else {
185 4 print SNT "$vocab_hash{$token} ";
186 }
187 }
188 3 ▶ print SNT "1" if $new_line;
Branch: 1 branch
BranchTrueFalseTotal
if $new_line 0 3 3
189 3 print SNT "\n";
190 }
191 }
192 }
193  
194 ######################
195 # Remove the files #
196 ######################
197 sub remove_files
198 1 ▶{
Subroutine remove_files: called CC=1 SCAR=0.0
Pod documented
199 1 my $self = shift;
200  
201 1 system("rm -rf $VOCAB_FILE");
202 1 system("rm -rf $SNT_FILE");
203 1 system("rm -rf $SNTNGRAM_FILE");
204 }
205  
206 ###########################
207 # Remove the ngram file #
208 ###########################
209 sub remove_ngram_file
210 0 ▶{
Subroutine remove_ngram_file: not called CC=1 SCAR=6.9
Pod undocumented
211 0 system("rm -rf $NGRAM_FILE");
212 }
213  
214  
215 ############################
216 # Creates the token file #
217 # CODE obtained from NSP #
218 ############################
219 sub set_token_regex
220 1 ▶{
Subroutine set_token_regex: called CC=10 SCAR=26.5
Pod undocumented
221 1 my $self = shift; my @tokenRegex = (); $tokenizerRegex = "";
222
223 1 ▶ if(-e $TOKEN_FILE) {
Branch: 1 branch
BranchTrueFalseTotal
if (-e $TOKEN_FILE) { } 1 0 1
224 1 ▶ open (TOKEN, $TOKEN_FILE) || die "Couldnt open $TOKEN_FILE\n";
Branch: 1 branch
BranchTrueFalseTotal
open TOKEN, $TOKEN_FILE or die "Couldnt open $TOKEN_FILE\n" 1 0 1
225
226 1 ▶ while(<TOKEN>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline TOKEN)) 1 1 2
227 1 chomp; s/^\s*//; s/\s*$//;
228 0 ▶ if (length($_) <= 0) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if (length $_ <= 0) 0 1 1
229 1 ▶ if (!(/^\//) || !(/\/$/))
Branch: 1 branch
BranchTrueFalseTotal
unless (m[^/] and m[/$]) 0 1 1
Condition33%1/3
m[^/] and m[/$]
!ll&&!rl&&r
001
MC/DC0%0/2
m[^/] and m[/$]
m[^/] m[/$]
Truth table33%1/3
A and B
A= m[^/]B= m[/$]
ABresult
0X0
100
111
230 {
231 0 print STDERR "Ignoring regex with no delimiters: $_\n"; next;
232 }
233 1 s/^\///; s/\/$//;
234 1 push @tokenRegex, $_;
235 }
236 1 close TOKEN;
237 }
238 else {
239 0 push @tokenRegex, "\\w+"; push @tokenRegex, "[\.,;:\?!]";
240 }
241
242 # create the complete token regex
243
244 1 foreach my $token (@tokenRegex)
245 {
246 1 ▶ if ( length($tokenizerRegex) > 0 )
Branch: 1 branch
BranchTrueFalseTotal
if (length $tokenizerRegex > 0) 0 1 1
247 {
248 0 $tokenizerRegex .= "|";
249 }
250 1 $tokenizerRegex .= "(";
251 1 $tokenizerRegex .= $token;
252 1 $tokenizerRegex .= ")";
253 }
254
255 # if you dont have any tokens to work with, abort
256 1 ▶ if ( $#tokenRegex < 0 )
Branch: 1 branch
BranchTrueFalseTotal
if ($#tokenRegex < 0) 0 1 1
257 {
258 0 print STDERR "No token definitions to work with.\n";
259 0 exit;
260 }
261 }
262  
263 ##########################################
264 # Set the non token regular expression #
265 # CODE Obtained from NSP #
266 ##########################################
267 sub set_nontoken_regex
268 0 ▶{
Subroutine set_nontoken_regex: not called CC=8 SCAR=42.8
Pod undocumented
269 0 $nontokenizerRegex = "";
270  
271 #check if the file exists
272 0 ▶ if($NONTOKEN_FILE)
Branch: 1 branch
BranchTrueFalseTotal
if ($NONTOKEN_FILE) { } 0 0 0
273 {
274 #open the non token file
275 0 ▶ open(NOTOK, $NONTOKEN_FILE) || die "Couldn't open Nontoken file $NONTOKEN_FILE.\n";
Branch: 1 branch
BranchTrueFalseTotal
open NOTOK, $NONTOKEN_FILE or die "Couldn't open Nontoken file $NONTOKEN_FILE.\n" 0 0 0
276  
277 0 ▶ while(<NOTOK>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline NOTOK)) 0 0 0
278 0 chomp;
279 0 s/^\s+//; s/\s+$//;
280
281 #handling a blank lines
282 0 ▶ if(/^\s*$/) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if (/^\s*$/) 0 0 0
283  
284 0 ▶ if(!(/^\//)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (m[^/]) 0 0 0
285 0 print STDERR "Nontoken regular expression $_ should start with '/'\n"; exit;
286 }
287
288 0 ▶ if(!(/\/$/)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (m[/$]) 0 0 0
289 0 print STDERR "Nontoken regular expression $_ should end with '/'\n"; exit;
290 }
291
292 #removing the / s from the beginning and the end
293 0 s/^\///;
294 0 s/\/$//;
295
296 #foorm a single regex
297 0 $nontokenizerRegex .="(".$_.")|";
298 }
299
300 # if no valid regexs are found in Nontoken file
301 0 ▶ if(length($nontokenizerRegex)<=0) {
Branch: 1 branch
BranchTrueFalseTotal
if (length $nontokenizerRegex <= 0) 0 0 0
302 0 print STDERR "No valid Perl Regular Experssion found in Nontoken file $NONTOKEN_FILE.\n";
303 0 exit;
304 }
305
306 0 chop $nontokenizerRegex;
307 }
308 else {
309 0 print STDERR "Nontoken file $NONTOKEN_FILE doesn't exist.\n";
310 0 exit;
311 }
312 }
313  
314 ##############################
315 # Create the stoplist hash #
316 # CODE obtained from NSP #
317 ##############################
318 sub create_stop_list
319 0 ▶{
Subroutine create_stop_list: not called CC=7 SCAR=40.3
Pod documented
320 0 my $self = shift;
321 0 my $file = shift;
322
323 0 $stop_regex = "";
324  
325 0 ▶ open(FILE, $file) || die "Could not open the Stoplist : $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open FILE, $file or die "Could not open the Stoplist : $!\n" 0 0 0
326
327 0 ▶ while(<FILE>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline FILE)) 0 0 0
328 0 chomp; # accepting Perl Regexs from Stopfile
329 0 s/^\s+//;
330 0 s/\s+$//;
331
332 #handling a blank lines
333 0 ▶ if(/^\s*$/) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if (/^\s*$/) 0 0 0
334
335 #check if a valid Perl Regex
336 0 ▶ if(!(/^\//)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (m[^/]) 0 0 0
337 0 print STDERR "Stop token regular expression <$_> should start with '/'\n";
338 0 exit;
339 }
340 0 ▶ if(!(/\/$/)) {
Branch: 1 branch
BranchTrueFalseTotal
unless (m[/$]) 0 0 0
341 0 print STDERR "Stop token regular expression <$_> should end with '/'\n";
342 0 exit;
343 }
344
345 #remove the / s from beginning and end
346 0 s/^\///;
347 0 s/\/$//;
348
349 #form a single big regex
350 0 $stop_regex.="(".$_.")|";
351 }
352
353 0 ▶ if(length($stop_regex)<=0) {
Branch: 1 branch
BranchTrueFalseTotal
if (length $stop_regex <= 0) 0 0 0
354 0 print STDERR "No valid Perl Regular Experssion found in Stop file.";
355 0 exit;
356 }
357
358 0 chop $stop_regex;
359
360 # Reset the stop flag to true
361 0 $stop_flag = 1;
362
363 0 close FILE;
364 }
365  
366 ###############################
367 # Load the vocabulary array #
368 ###############################
369 sub load_vocab_array
370 1 ▶{
Subroutine load_vocab_array: called CC=3 SCAR=11.0
Pod undocumented
371 1 ▶ open(VOCAB, $VOCAB_FILE) || die "Could not open the vocab file: $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open VOCAB, $VOCAB_FILE or die "Could not open the vocab file: $!\n" 1 0 1
372  
373 1 @vocab_array = ();
374 1 ▶ while(<VOCAB>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline VOCAB)) 18 1 19
375 18 chomp;
376 18 my $token = <VOCAB>; chomp $token;
377 18 $vocab_array[$_] = $token;
378 }
379  
380 }
381  
382 #################################
383 # Set the windowing parameter #
384 #################################
385 sub set_window_size
386 0 ▶{
Subroutine set_window_size: not called CC=1 SCAR=6.9
Pod documented
387 0 my $self = shift;
388 0 $window_size = shift;
389 }
390  
391 ##############################
392 # Set the remove parameter #
393 ##############################
394 sub set_remove
395 0 ▶{
Subroutine set_remove: not called CC=1 SCAR=6.9
Pod documented
396 0 my $self = shift;
397 0 $remove = shift;
398 }
399  
400 ################################
401 # Set the marginal parameter #
402 ################################
403 sub set_marginals
404 1 ▶{
Subroutine set_marginals: called CC=1 SCAR=0.0
Pod documented
405 1 $marginals = 1;
406 }
407  
408 ################################
409 # Set the new_line parameter #
410 ################################
411 sub set_new_line
412 0 ▶{
Subroutine set_new_line: not called CC=1 SCAR=6.9
Pod undocumented
413 0 $new_line = 1;
414 }
415  
416 #######################
417 # Set the frequency #
418 #######################
419 sub set_frequency
420 0 ▶{
Subroutine set_frequency: not called CC=1 SCAR=6.9
Pod documented
421 0 my $self = shift;
422 0 $frequency = shift;
423 }
424  
425 ############################
426 # Set minimum ngram size #
427 ############################
428 sub set_ngram_size
429 0 ▶{
Subroutine set_ngram_size: not called CC=1 SCAR=6.9
Pod documented
430 0 my $self = shift;
431 0 $ngram_size = shift;
432  
433 0 $min_ngram_size = $ngram_size;
434 0 $max_ngram_size = $ngram_size;
435 }
436  
437 #######################
438 # Set the stop mode #
439 #######################
440 sub set_stop_mode
441 0 ▶{
Subroutine set_stop_mode: not called CC=1 SCAR=6.9
Pod documented
442  
443 0 my $self = shift;
444 0 $stop_mode = shift;
445 }
446  
447 ########################
448 # Set the token file #
449 ########################
450 sub set_token_file
451 1 ▶{
Subroutine set_token_file: called CC=1 SCAR=0.0
Pod documented
452 1 my $self = shift;
453 1 $TOKEN_FILE = shift;
454 }
455  
456 ###########################
457 # Set the nontoken file #
458 ###########################
459 sub set_nontoken_file
460 0 ▶{
Subroutine set_nontoken_file: not called CC=1 SCAR=6.9
Pod documented
461 0 my $self = shift;
462 0 $NONTOKEN_FILE = shift;
463 }
464  
465 #############################
466 # Set the ngram file name #
467 #############################
468 sub set_destination_file
469 1 ▶{
Subroutine set_destination_file: called CC=1 SCAR=0.0
Pod documented
470 1 my $self = shift;
471 1 my $file = shift;
472  
473 1 $timestamp = time();
474  
475 # Set the file names of the internal files
476 # that will be used by the perl module.
477 1 $VOCAB_FILE = $file . ".vocab." . $timestamp;
478 1 $SNT_FILE = $file . ".snt." . $timestamp;
479 1 $SNTNGRAM_FILE = $file . ".sntngram." . $timestamp;
480  
481 # Set the ngram file
482 1 $NGRAM_FILE = $file;
483 }
484  
485 #################################
486 # Return the number of ngrams #
487 #################################
488 sub get_ngram_count
489 0 ▶{
Subroutine get_ngram_count: not called CC=1 SCAR=6.9
Pod documented
490 0 return $ngram_count;
491 }
492  
493 ###########################
494 # Return the ngram file #
495 ###########################
496 sub get_ngram_file
497 0 ▶{
Subroutine get_ngram_file: not called CC=1 SCAR=6.9
Pod undocumented
498 0 return $NGRAM_FILE;
499 }
500  
501 ##########################################################
502 # METHOD THAT CALLS THE FUNCTIONS TO OBTAIN THE NGRAMS #
503 ##########################################################
504 sub get_ngrams
505 1 ▶{
Subroutine get_ngrams: called CC=3 SCAR=11.2
Pod documented
506  
507 # Set the ngram count to zero
508 1 $ngram_count = 0;
509  
510 # Create the corpus array
511 1 corpus_array();
512
513 # If the window size is 0; set it equal to the
514 # size of the ngram
515 1 ▶ $window_size = $ngram_size if $window_size == 0;
Branch: 1 branch
BranchTrueFalseTotal
if $window_size == 0 1 0 1
516  
517 # check to make certain marginals are not set with
518 # the window size greater than the ngram size
519 1 ▶ $marginals = 0 if $window_size > $ngram_size;
Branch: 1 branch
BranchTrueFalseTotal
if $window_size > $ngram_size 0 1 1
520  
521 # create the window
522 1 create_window();
523
524 # print ngrams to the snt ngram file
525 1 print_sntngrams();
526
527 # print the token ngram to the ngram file
528 1 print_ngrams();
529 }
530  
531 #############################
532 # Create the corpus array #
533 #############################
534 sub corpus_array
535 1 ▶{
Subroutine corpus_array: called CC=4 SCAR=13.9
Pod undocumented
536 #open SNTFILE
537 1 ▶ open(SNT, $SNT_FILE) || die "Could not open the sntfile: $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open SNT, $SNT_FILE or die "Could not open the sntfile: $!\n" 1 0 1
538
539 # Initialize the variables
540 1 my $offset = 0; $corpus = ""; $N = 0;
541  
542 1 ▶ while(<SNT>){
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline SNT)) 3 1 4
543 3 chomp;
544 3 my @t = split/\s+/;
545 3 foreach (@t) { vec($corpus, $offset++, $bit) = $_; $N++; vec($unigrams, $_, $bit)++; }
546 }
547  
548 #decrement N by one to obtain the actual size of the corpus
549 1 $N--;
550 }
551  
552 #################################################
553 # WINDOWING METHODS DESCRIBED BY GIL AND DIAS #
554 #################################################
555 sub create_window
556 1 ▶{
Subroutine create_window: called CC=6 SCAR=17.9
Pod undocumented
557 1 my $doc = 0; @window_array = ();
558  
559 1 for my $i(0..$N) {
560  
561 22 for my $j(0..((2**$window_size)-1) ) {
562  
563 # determine the binary representation of one of the possible ngram combinations
564 88 my @bitarray = split//, unpack("B32", pack("N", $j));
565  
566 # ensure that it is not greater than the corpus size
567 88 ▶ if($i+$ngram_size > ($N+1) ) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if ($i + $ngram_size > $N + 1) 4 84 88
568
569 # Reduce the bit array from 32 to the window size
570 84 ▶ my @bits = @bitarray[$#bitarray-$window_size+1 .. $#bitarray]; if($bits[0] != 1) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if ($bits[0] != 1) 42 42 84
571
572 # Get the size of the possible ngram and if it to large or small - next
573 42 ▶ my $size=0; map{$size+=$_} @bits; if($size!=$ngram_size){ next; }
Branch: 1 branch
BranchTrueFalseTotal
if ($size != $ngram_size) 21 21 42
574  
575 # Looks like everything is correct so create the bit vec for the positional ngram
576 21 my $temp = ""; my $offset = 0;
577  
578 # Set document number and the start position
579 21 vec($temp, $offset++, $bit) = $doc; vec($temp, $offset++, $bit) = $i;
580
581 # Create the bit map
582 21 map{ vec($temp, $offset++, $win_bit) = $bits[$_] } 0..$#bits; push @window_array, $temp;
583 }
584 }
585 }
586  
587 ##############################################
588 # Prints the positional ngrams for testing #
589 ##############################################
590 sub print_window
591 0 ▶{
Subroutine print_window: not called CC=4 SCAR=30.0
Pod undocumented
592 0 print "$#window_array\n";
593 0 foreach my $win (@window_array) {
594 0 my $start = vec($win, 1, $bit);
595 0 my $index = $start;
596 0 my @ngram = ();
597 0 for my $j(2..$window_size+1) {
598 0 ▶ if( vec($win, $j, $win_bit) == 1 ) {
Branch: 1 branch
BranchTrueFalseTotal
if (vec($win, $j, $win_bit) == 1) 0 0 0
599 0 push @ngram, vec($corpus, $index, $bit);
600 }
601 0 $index++;
602 }
603 0 print "@ngram\n";
604 }
605 }
606  
607 #####################################
608 # Print the sntngrams to the file #
609 #####################################
610 sub print_sntngrams
611 1 ▶{
Subroutine print_sntngrams: called CC=11 SCAR=25.0
Pod undocumented
612 # Open the SNTGRAM File #
613 1 ▶ open(SNTNGRAM, ">$SNTNGRAM_FILE") ||
Branch: 1 branch
BranchTrueFalseTotal
open SNTNGRAM, ">$SNTNGRAM_FILE" or die "Could not open the SNTNGRAM file : $!\n" 1 0 1
614 die "Could not open the SNTNGRAM file : $!\n";
615
616 # Load the vocab hash if doesn;t exist
617 1 ▶ if (!@vocab_array) { load_vocab_array(); }
Branch: 1 branch
BranchTrueFalseTotal
unless (@vocab_array) 1 0 1
618  
619 1 my @prev = (); my $freq = 0;
620 1 foreach my $win (sort byvec @window_array) {
621  
622 21 my $index = vec($win, 1, $bit); my @ngram = (); my $zero_flag = 0;
623
624 # Get the ngram
625 21 for (2..$window_size+1) {
626 42 ▶ push @ngram, vec($corpus, $index, $bit) if vec($win, $_, $win_bit) == 1; $index++;
Branch: 1 branch
BranchTrueFalseTotal
if vec($win, $_, $win_bit) == 1 42 0 42
627 }
628
629 0 ▶ if($ngram[$#ngram] == 0) { next; }
Branch: 1 branch
BranchTrueFalseTotal
if ($ngram[$#ngram] == 0) 0 21 21
630
631 # First time around need to initialize the @prev
632 21 ▶ if($#prev == -1 ) { @prev = @ngram; $freq++;}
Branch: 2 branches
BranchTrueFalseTotal
if ($#prev == -1) { } 1 20 21
elsif (join(' ', @ngram) ne join(' ', @prev)) { } 20 0 20
633
634 # If @ngram and @prev are not equal print @prev and its freq.
635 # Reinitialize freq and @prev, increment the ngram count.
636 elsif( (join " ", @ngram) ne (join " ", @prev) ) {
637
638 # Check if the ngram is valid
639 20 my $return_value = valid( (join " ", @prev), $freq);
640
641 # If it is okay, print the ngram to the sntngram file
642 20 ▶ if($return_value == 1) { print SNTNGRAM "@prev $freq\n"; }
Branch: 1 branch
BranchTrueFalseTotal
if ($return_value == 1) 20 0 20
643  
644 # Initialize the freq and previous ngram
645 20 $freq = 1; @prev = @ngram;
646
647 0 } else { $freq++; }
648 }
649 # Print the last ngram to the sntngram file if it is valid
650 1 my $return_value = valid( (join " ", @prev), $freq );
651 1 ▶ if($return_value == 1) { print SNTNGRAM "@prev $freq\n"; }
Branch: 1 branch
BranchTrueFalseTotal
if ($return_value == 1) 1 0 1
652 }
653  
654 #####################################################
655 # Check to determine if the ngram is a valid ngram #
656 ######################################################
657 sub valid
658 21 ▶{
Subroutine valid: called CC=19 SCAR=50.1
Pod undocumented
659 21 my @ngram = split/\s+/, shift;
660 21 my $freq = shift;
661
662 # Initialize variables
663 21 my $doStop = 0; my $line = 0; my @token_ngram = ();
664
665 # Get the token ngram
666 21 map { push @token_ngram, $vocab_array[$ngram[$_]] } 0..$#ngram;
667
668 # If stoplist exists determine if the ngram is part of the stoplist
669 21 ▶ if($stop_flag) {
Branch: 1 branch
BranchTrueFalseTotal
if ($stop_flag) 0 21 21
670
671 # Set the doStop flag
672 0 ▶ if($stop_mode=~/OR|or/) { $doStop = 0; } else { $doStop = 1; }
Branch: 1 branch
BranchTrueFalseTotal
if ($stop_mode =~ /OR|or/) { } 0 0 0
673
674 0 for my $i(0..$#token_ngram) {
675 # if mode is OR, remove the current ngram if any word is a stop word
676 0 ▶ if($stop_mode=~/OR|or/) { if($token_ngram[$i]=~/$stop_regex/) { $doStop=1; last; } }
Branch: 2 branches
BranchTrueFalseTotal
if ($stop_mode =~ /OR|or/) { } 0 0 0
if ($token_ngram[$i] =~ /$stop_regex/) 0 0 0
677
678 # if mode is AND, accept the current ngram if any word is not a stop word
679 0 ▶ else { if(!($token_ngram[$i]=~/$stop_regex/)) { $doStop=0; last; } }
Branch: 1 branch
BranchTrueFalseTotal
unless ($token_ngram[$i] =~ /$stop_regex/) 0 0 0
680 }
681 # If counting the marginals add the adjustment to the remove_hash
682 0 ▶ if($doStop && $marginals) {
Branch: 1 branch
BranchTrueFalseTotal
if ($doStop and $marginals) 0 0 0
Condition0%0/3
$doStop and $marginals
!ll&&!rl&&r
000
MC/DC0%0/2
$doStop and $marginals
$doStop $marginals
Truth table0%0/3
A and B
A= $doStopB= $marginals
ABresult
0X0
100
111
683 0 for (0..$#ngram) {
684 0 ▶ if(exists $remove_hash{$_ . ":" . $ngram[$_]}) {
Branch: 1 branch
BranchTrueFalseTotal
if (exists $remove_hash{$_ . ':' . $ngram[$_]}) { } 0 0 0
685 0 $remove_hash{$_ . ":" . $ngram[$_]} += $freq;
686 }
687 0 else { $remove_hash{$_ . ":" . $ngram[$_]} = $freq; }
688 }
689 }
690 }
691
692 # If new line determine if the new line exists in the ngram
693 0 ▶ if($new_line) { map { if($_ == 1) { $line++; } } @ngram; }
Branch: 2 branches
BranchTrueFalseTotal
if ($new_line) 0 21 21
if ($_ == 1) 0 0 0
694  
695 # If the ngram frequency is greater or equal to a specified frequency, a new
696 # line flag is false and the ngram is not elimanted by the stop list then print
697 # the ngram in its integer form with its frequency to the snt ngram file
698 21 ▶ if($doStop == 0 && $line == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($doStop == 0 and $line == 0) 21 0 21
Condition33%1/3
$doStop == 0 and $line == 0
!ll&&!rl&&r
0021
MC/DC0%0/2
$doStop == 0 and $line == 0
$doStop == 0 $line == 0
Truth table33%1/3
A and B
A= $doStop == 0B= $line == 0
ABresult
0X0
100
111
699 21 ▶ if($remove <= $freq) {
Branch: 1 branch
BranchTrueFalseTotal
if ($remove <= $freq) { } 21 0 21
700 21 $ngram_count+=$freq;
701 21 ▶ if($frequency <= $freq) { return 1; }
Branch: 1 branch
BranchTrueFalseTotal
if ($frequency <= $freq) 21 0 21
702 }
703 else {
704 0 for (0..$#ngram) {
705 0 ▶ if(exists $remove_hash{$_ . ":" . $ngram[$_]}) {
Branch: 1 branch
BranchTrueFalseTotal
if (exists $remove_hash{$_ . ':' . $ngram[$_]}) { } 0 0 0
706 0 $remove_hash{$_ . ":" . $ngram[$_]} += $freq;
707 }
708 0 else { $remove_hash{$_ . ":" . $ngram[$_]} = $freq; }
709 }
710 }
711 }
712 0 return 0;
713 }
714  
715 #############################################
716 # Print the positional ngrams to the file #
717 #############################################
718 sub print_ngrams
719 1 ▶{
Subroutine print_ngrams: called CC=7 SCAR=20.0
Pod undocumented
720 #open the SNTNGRAM file
721 1 ▶ open(SNTNGRAM, $SNTNGRAM_FILE) || die "Could not open the sntngram file: $!\n";
Branch: 1 branch
BranchTrueFalseTotal
open SNTNGRAM, $SNTNGRAM_FILE or die "Could not open the sntngram file: $!\n" 1 0 1
722
723 #open the ngram file
724 1 ▶ open(NGRAM, ">$NGRAM_FILE") || die "Could not open the ngram file: $! \n";
Branch: 1 branch
BranchTrueFalseTotal
open NGRAM, ">$NGRAM_FILE" or die "Could not open the ngram file: $! \n" 1 0 1
725
726 # Load the vocab hash if doesn;t exist
727 0 ▶ if (!@vocab_array) { load_vocab_array(); }
Branch: 1 branch
BranchTrueFalseTotal
unless (@vocab_array) 0 1 1
728  
729 # Print the ngram count
730 1 print NGRAM "$ngram_count\n";
731
732 1 ▶ while(<SNTNGRAM>) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline SNTNGRAM)) 21 1 22
733 # get the ngram and its frequency
734 21 chomp; my @ngram = split/\s+/, $_; my @marginalFreqs = ();
735
736 21 my $freq = pop @ngram;
737  
738 21 ▶ if($marginals) { @marginalFreqs = Marginals(@ngram); }
Branch: 1 branch
BranchTrueFalseTotal
if ($marginals) 21 0 21
739  
740 # print the ngram
741 21 for (0..$#ngram) { print NGRAM "$vocab_array[$ngram[$_]]<>"; }
742  
743 # print the frequencies
744 21 print NGRAM "$freq @marginalFreqs \n";
745 }
746 }
747  
748  
749 # Gets the marginal counts for each individual word in the ngram
750 sub Marginals
751 21 ▶{
Subroutine Marginals: called CC=9 SCAR=26.3
Pod undocumented
752 21 my @marginalFreqs = ();
753
754 21 for my $i(0..$#_) {
755 42 push @marginalFreqs, vec($unigrams, $_[$i], $bit);
756
757 42 ▶ if($i == 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i == 0) 21 21 42
758 0 ▶ if($_[$i] == vec($corpus, $N, $bit)) { $marginalFreqs[$#marginalFreqs] -= 1; }
Branch: 1 branch
BranchTrueFalseTotal
if ($_[$i] == vec($corpus, $N, $bit)) 0 21 21
759 }
760 42 ▶ if($i == $#_) {
Branch: 1 branch
BranchTrueFalseTotal
if ($i == $#_) 21 21 42
761 0 ▶ if($_[$i] == vec($corpus, 0, $bit)) { $marginalFreqs[$#marginalFreqs] -= 1; }
Branch: 1 branch
BranchTrueFalseTotal
if ($_[$i] == vec($corpus, 0, $bit)) 0 21 21
762 }
763  
764 42 ▶ if($stop_flag || $remove > 0) {
Branch: 1 branch
BranchTrueFalseTotal
if ($stop_flag or $remove > 0) 0 42 42
Condition33%1/3
$stop_flag or $remove > 0
l!l&&r!l&&!r
0042
MC/DC0%0/2
$stop_flag or $remove > 0
$stop_flag $remove > 0
Truth table33%1/3
A or B
A= $stop_flagB= $remove > 0
ABresult
1X1
011
000
765 0 ▶ if(exists $remove_hash{$i . ":" . $_[$i]}) {
Branch: 1 branch
BranchTrueFalseTotal
if (exists $remove_hash{$i . ':' . $_[$i]}) 0 0 0
766 0 $marginalFreqs[$#marginalFreqs] -= $remove_hash{$i . ":" . $_[$i]};
767 }
768 }
769 }
770 21 return @marginalFreqs;
771 }
772  
773 #############################
774 # Windowing sort function #
775 #############################
776 sub byvec
777 60 ▶{
Subroutine byvec: called CC=8 SCAR=20.9
Pod undocumented
778 # Get the ngrams of the two elements
779 60 my @a_array = (); my @b_array = (); my $z = 0; my $x = 0; my $counter = 0;
780 60 my $a_index = vec($a, 1, $bit); my $b_index = vec($b, 1, $bit);
781 60 for my $i(2..$window_size+1) {
782 120 ▶ if(vec($a, $i, $win_bit) == 1) { push @a_array, vec($corpus, $a_index, $bit); } $a_index++;
Branch: 1 branch
BranchTrueFalseTotal
if (vec($a, $i, $win_bit) == 1) 120 0 120
783 120 ▶ if(vec($b, $i, $win_bit) == 1) { push @b_array, vec($corpus, $b_index, $bit); } $b_index++;
Branch: 1 branch
BranchTrueFalseTotal
if (vec($b, $i, $win_bit) == 1) 120 0 120
784 }
785
786 # Find the first occurence of a non equal token in the ngrams, if exists.
787 60 ▶ for $z(0..$#a_array) { if($a_array[$z]!=$b_array[$z]) { $x = $z; next; } }
Branch: 1 branch
BranchTrueFalseTotal
if ($a_array[$z] != $b_array[$z]) 115 5 120
788
789 60 ▶ return ( $a_array[$x] > $b_array[$x] ? 1 :
Branch: 2 branches
BranchTrueFalseTotal
$a_array[$x] > $b_array[$x] ? : 30 30 60
$a_array[$x] < $b_array[$x] ? : 30 0 30
790 ($a_array[$x] < $b_array[$x] ? -1 : 0) );
791  
792 }
793  
794  
795 1 1;
796  
797 __END__
↑ Summary text-positional-ngram-driver.pl »