text-positional-ngram-driver.pl untested

filter: Statementstmt 0.0% 0 / 114 Branchbran 0.0% 0 / 38 Conditioncond 0.0% 0 / 8 MC/DCmcdc 0.0% 0 / 5 Subroutinesub 0.0% 0 / 6 Podpod 0.0% 0 / 4 totaltotal 0.0% 0 / 175 CC 1 SCAR 6.9
CC
1
Cov
0%
askHelp
2.0
showVersion
2.0
minimalUsageNotes
2.0
CRAP 2.0
1 #!/usr/bin/perl
2
3 =head1 NAME
4
5 text-positional-ngram-driver.pl
6
7 =head1 SYNOPSIS
8
9 text-positional-ngram-driver.pl takes as input one or more text files
10 and calculates the ngram frequency for the whole corpus.
11
12 =head1 DESCRIPTION
13
14 See README
15
16 =head1 AUTHOR
17
18 Bridget Thomson McInnes, bthomson@d.umn.edu
19
20 Ted Pedersen, tpederse@d.umn.edu
21
22 =head1 BUGS
23
24 =head1 SEE ALSO
25
26 =head1 COPYRIGHT
27
28 Copyright (C) 2004-2007, Bridget T. McInnes
29
30 This program is free software; you can redistribute it and/or
31 modify it under the terms of the GNU General Public License
32 as published by the Free Software Foundation; either version 2
33 of the License, or (at your option) any later version.
34 This program is distributed in the hope that it will be useful,
35 but WITHOUT ANY WARRANTY; without even the implied warranty of
36 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
37 GNU General Public License for more details.
38
39 You should have received a copy of the GNU General Public License
40 along with this program; if not, write to the Free Software
41 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
42
43 =cut
44
45 #-----------------------------------------------------------------------------
46 # Start of program
47 #-----------------------------------------------------------------------------
48
49 # Set the Text::Positional::Ngram;
50 use Ngram;
51
52 # we have to use commandline options, so use the necessary package!
53 use Getopt::Long;
54
55 # Create the accessor
56 $text = Text::Positional::Ngram->new();
57
58 # first check if no commandline options have been provided... in which case
59 # print out the usage notes!
60 if ( $#ARGV == -1 )
61 {
62 &minimalUsageNotes();
63 exit;
64 }
65
66 # now get the options!
67 GetOptions( "version", "help", "frequency=i", "stop=s", "stopmode=s",
68 "newLine", "token=s", "nontoken=s", "ngram=i", "remove=i",
69 "marginals", "window=s");
70
71 # if help has been requested, print out help!
72 if ( defined $opt_help )
73 {
74 $opt_help = 1;
75 &showHelp();
76 exit;
77 }
78
79 # if version has been requested, show version!
80 if ( defined $opt_version )
81 {
82 $opt_version = 1;
83 &showVersion();
84 exit;
85 }
86
87 # check the windowing option
88 if ( defined $opt_window ) { $text->set_window_size($opt_window); }
89
90 # check the frequency option
91 if ( defined $opt_frequency ) { $text->set_frequency($opt_frequency); }
92 else { $text->set_frequency(0); }
93
94 # check the remove option
95 if ( defined $opt_remove ) { $text->set_remove($opt_remove); }
96 else { $text->set_remove(0); }
97
98 # check to see if the marginals are wanted
99 if ( defined $opt_marginals ) { $text->set_marginals(1); }
100
101 # check to see if the token option has been defined
102 if ( defined $opt_token ) { $text->set_token_file($opt_token); }
103
104 # check to see if the nontoken option has been defined
105 if ( defined $opt_nontoken ) { $text->set_nontoken_file($opt_nontoken); }
106
107 # check to see if the stop option has been defined
108 if ( defined $opt_stop ) { $text->create_stop_list($opt_stop); }
109
110 # check to see if the newline option has been defined
111 if ( defined $opt_newline ) { $text->set_new_line(); }
112
113 # check to see if the stop mode has been defined
114 if ( defined $opt_stopmode ) {
115
116 if( lc($opt_stopmode) ne "or" && lc($opt_stopmode) ne "and" ) {
117 print STDERR "Stop mode must be either: AND or OR.\n";
118 askHelp();
119 exit;
120 }
121 $text->set_stop_mode($opt_stopmode);
122 }
123 else { $text->set_stop_mode(OR); }
124
125 # check the ngram option
126 if ( defined $opt_ngram ) {
127
128 if($opt_ngram <= 0) {
129 print STDERR "Cannot have 'n' value of ngrams as less than 1\n";
130 askHelp();
131 exit();
132 }
133
134 $text->set_ngram_size($opt_ngram);
135 }
136 else { $text->set_ngram_size(2); }
137
138 # having stripped the commandline of all the options etc, we should now be
139 # left only with the source/destination files
140
141 # retrieve the destination file
142 $destination = shift;
143
144 # check to see if a destination has been supplied at all...
145 if ( !($destination ) )
146 {
147 print STDERR "No output file (DESTINATION) supplied.\n";
148 askHelp();
149 exit;
150 }
151
152 # check to see if destination exists, and if so, if we should overwrite...
153 if ( -e $destination )
154 {
155 print "Output file $destination already exists! Overwrite (Y/N)? ";
156 $reply = <STDIN>;
157 chomp $reply;
158 $reply = uc $reply;
159 exit 0 if ($reply ne "Y");
160 }
161
162 # set the destination file
163 $text->set_destination_file($destination);
164
165 # get the source files
166 @files = ();
167 foreach $element (@ARGV) {
168 if(-d $element) {
169 my @temp = ();
170 opendir(THISDIR, $element) || die "Can not open the $dirpath";
171 push(@temp, grep {$_ ne '.' and $_ ne '..' } readdir THISDIR);
172 closedir THISDIR;
173 foreach (@temp) { push @files, $element . $_; }
174 }
175 else { push @files, $element; }
176 }
177
178 # create the files necessary to run the module
179 $text->create_files(@files);
180
181 # get the ngrams
182 $text->get_ngrams();
183
184 # remove the miscellaneous file created by the Text::Positional::Ngram module
185 $text->remove_files();
186
187 # function to output "ask for help" message when the user's goofed up!
188 sub askHelp
189 {
190 print STDERR "Type text-positional-ngram-driver.pl --help for help.\n";
191 }
192
193 # function to output the version number
194 sub showVersion
195 {
196 print STDERR "text-positional-ngram-driver.pl - version 0.5\n";
197 print STDERR "Copyright (C) 2004-2007, Bridget Thomson McInnes\n";
198 print STDERR "Date of Last Update 28/08/07\n";
199
200 }
201
202 # function to output a minimal usage note when the user has not provided any
203 # commandline options
204 sub minimalUsageNotes
205 {
206 print STDERR "Usage: text-positional-nram-driver.pl [OPTIONS] DESTINATION SOURCE [[, SOURCE] ...]\n";
207 askHelp();
208 }
209
210 # function to output help messages for this program
211 sub showHelp
212 {
213 print "Usage: text-positional-ngram-driver.pl [OPTIONS] DESTINATION SOURCE [[, SOURCE] ...]\n\n";
214
215 print "Counts up the frequency of all n-grams occurring in SOURCE.\n";
216 print "Sends to DESTINATION the list of n-grams found and their frequencies.\n";
217
218 print "OPTIONS:\n\n";
219
220 print " --ngram N Creates n-grams of N tokens each. N = 2 by\n";
221 print " default.\n\n";
222
223 print " --token FILE Uses regular expressions in FILE to create\n";
224 print " tokens. By default two regular expressions\n";
225 print " are provided (see README).\n\n";
226
227 print " --nontoken FILE Removes all characters sequences that match\n";
228 print " Perl regular expressions specified in FILE.\n\n";
229
230 print " --stop FILE Removes n-grams containing at least one (in\n";
231 print " OR mode) or all stop words (in AND mode).\n";
232 print " Stop words should be declared as Perl Regular\n";
233 print " expressions in FILE.\n\n";
234
235 print " --stop_mode MODE Sets the stop mode to OR or AND. OR mode removes\n";
236 print " n-grams containing at least one stop word. AND \n";
237 print " mode removes n-grams that contain all stop words.\n\n";
238
239 print " --frequency N Does not display n-grams that occur less\n";
240 print " than N times.\n\n";
241
242 print " --remove N Ignores n-grams that occur less than N\n";
243 print " times. Ignored n-grams are not counted and\n";
244 print " so do not affect counts and frequencies.\n\n";
245
246 print " --newLine Prevents n-grams from spanning across the\n";
247 print " new-line character.\n\n";
248
249 print " --marginals Prints the marginal counts of the\n";
250 print " individual words in the n-gram\n\n";
251
252 print " --window N Sets the size of the window in which\n";
253 print " the positional ngram can be found in.\n\n";
254
255 print " --version Prints the version number.\n\n";
256
257 print " --help Prints this help message.\n\n";
258 }