| 1 | #!/usr/bin/perl | ||
| 2 | |||
| 3 | =head1 NAME | ||
| 4 | |||
| 5 | text-positional-ngram-driver.pl | ||
| 6 | |||
| 7 | =head1 SYNOPSIS | ||
| 8 | |||
| 9 | text-positional-ngram-driver.pl takes as input one or more text files | ||
| 10 | and calculates the ngram frequency for the whole corpus. | ||
| 11 | |||
| 12 | =head1 DESCRIPTION | ||
| 13 | |||
| 14 | See README | ||
| 15 | |||
| 16 | =head1 AUTHOR | ||
| 17 | |||
| 18 | Bridget Thomson McInnes, bthomson@d.umn.edu | ||
| 19 | |||
| 20 | Ted Pedersen, tpederse@d.umn.edu | ||
| 21 | |||
| 22 | =head1 BUGS | ||
| 23 | |||
| 24 | =head1 SEE ALSO | ||
| 25 | |||
| 26 | =head1 COPYRIGHT | ||
| 27 | |||
| 28 | Copyright (C) 2004-2007, Bridget T. McInnes | ||
| 29 | |||
| 30 | This program is free software; you can redistribute it and/or | ||
| 31 | modify it under the terms of the GNU General Public License | ||
| 32 | as published by the Free Software Foundation; either version 2 | ||
| 33 | of the License, or (at your option) any later version. | ||
| 34 | This program is distributed in the hope that it will be useful, | ||
| 35 | but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 36 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 37 | GNU General Public License for more details. | ||
| 38 | |||
| 39 | You should have received a copy of the GNU General Public License | ||
| 40 | along with this program; if not, write to the Free Software | ||
| 41 | Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
| 42 | |||
| 43 | =cut | ||
| 44 | |||
| 45 | #----------------------------------------------------------------------------- | ||
| 46 | # Start of program | ||
| 47 | #----------------------------------------------------------------------------- | ||
| 48 | |||
| 49 | # Set the Text::Positional::Ngram; | ||
| 50 | use Ngram; | ||
| 51 | |||
| 52 | # we have to use commandline options, so use the necessary package! | ||
| 53 | use Getopt::Long; | ||
| 54 | |||
| 55 | # Create the accessor | ||
| 56 | $text = Text::Positional::Ngram->new(); | ||
| 57 | |||
| 58 | # first check if no commandline options have been provided... in which case | ||
| 59 | # print out the usage notes! | ||
| 60 | if ( $#ARGV == -1 ) | ||
| 61 | { | ||
| 62 | &minimalUsageNotes(); | ||
| 63 | exit; | ||
| 64 | } | ||
| 65 | |||
| 66 | # now get the options! | ||
| 67 | GetOptions( "version", "help", "frequency=i", "stop=s", "stopmode=s", | ||
| 68 | "newLine", "token=s", "nontoken=s", "ngram=i", "remove=i", | ||
| 69 | "marginals", "window=s"); | ||
| 70 | |||
| 71 | # if help has been requested, print out help! | ||
| 72 | if ( defined $opt_help ) | ||
| 73 | { | ||
| 74 | $opt_help = 1; | ||
| 75 | &showHelp(); | ||
| 76 | exit; | ||
| 77 | } | ||
| 78 | |||
| 79 | # if version has been requested, show version! | ||
| 80 | if ( defined $opt_version ) | ||
| 81 | { | ||
| 82 | $opt_version = 1; | ||
| 83 | &showVersion(); | ||
| 84 | exit; | ||
| 85 | } | ||
| 86 | |||
| 87 | # check the windowing option | ||
| 88 | if ( defined $opt_window ) { $text->set_window_size($opt_window); } | ||
| 89 | |||
| 90 | # check the frequency option | ||
| 91 | if ( defined $opt_frequency ) { $text->set_frequency($opt_frequency); } | ||
| 92 | else { $text->set_frequency(0); } | ||
| 93 | |||
| 94 | # check the remove option | ||
| 95 | if ( defined $opt_remove ) { $text->set_remove($opt_remove); } | ||
| 96 | else { $text->set_remove(0); } | ||
| 97 | |||
| 98 | # check to see if the marginals are wanted | ||
| 99 | if ( defined $opt_marginals ) { $text->set_marginals(1); } | ||
| 100 | |||
| 101 | # check to see if the token option has been defined | ||
| 102 | if ( defined $opt_token ) { $text->set_token_file($opt_token); } | ||
| 103 | |||
| 104 | # check to see if the nontoken option has been defined | ||
| 105 | if ( defined $opt_nontoken ) { $text->set_nontoken_file($opt_nontoken); } | ||
| 106 | |||
| 107 | # check to see if the stop option has been defined | ||
| 108 | if ( defined $opt_stop ) { $text->create_stop_list($opt_stop); } | ||
| 109 | |||
| 110 | # check to see if the newline option has been defined | ||
| 111 | if ( defined $opt_newline ) { $text->set_new_line(); } | ||
| 112 | |||
| 113 | # check to see if the stop mode has been defined | ||
| 114 | if ( defined $opt_stopmode ) { | ||
| 115 | |||
| 116 | if( lc($opt_stopmode) ne "or" && lc($opt_stopmode) ne "and" ) { | ||
| 117 | print STDERR "Stop mode must be either: AND or OR.\n"; | ||
| 118 | askHelp(); | ||
| 119 | exit; | ||
| 120 | } | ||
| 121 | $text->set_stop_mode($opt_stopmode); | ||
| 122 | } | ||
| 123 | else { $text->set_stop_mode(OR); } | ||
| 124 | |||
| 125 | # check the ngram option | ||
| 126 | if ( defined $opt_ngram ) { | ||
| 127 | |||
| 128 | if($opt_ngram <= 0) { | ||
| 129 | print STDERR "Cannot have 'n' value of ngrams as less than 1\n"; | ||
| 130 | askHelp(); | ||
| 131 | exit(); | ||
| 132 | } | ||
| 133 | |||
| 134 | $text->set_ngram_size($opt_ngram); | ||
| 135 | } | ||
| 136 | else { $text->set_ngram_size(2); } | ||
| 137 | |||
| 138 | # having stripped the commandline of all the options etc, we should now be | ||
| 139 | # left only with the source/destination files | ||
| 140 | |||
| 141 | # retrieve the destination file | ||
| 142 | $destination = shift; | ||
| 143 | |||
| 144 | # check to see if a destination has been supplied at all... | ||
| 145 | if ( !($destination ) ) | ||
| 146 | { | ||
| 147 | print STDERR "No output file (DESTINATION) supplied.\n"; | ||
| 148 | askHelp(); | ||
| 149 | exit; | ||
| 150 | } | ||
| 151 | |||
| 152 | # check to see if destination exists, and if so, if we should overwrite... | ||
| 153 | if ( -e $destination ) | ||
| 154 | { | ||
| 155 | print "Output file $destination already exists! Overwrite (Y/N)? "; | ||
| 156 | $reply = <STDIN>; | ||
| 157 | chomp $reply; | ||
| 158 | $reply = uc $reply; | ||
| 159 | exit 0 if ($reply ne "Y"); | ||
| 160 | } | ||
| 161 | |||
| 162 | # set the destination file | ||
| 163 | $text->set_destination_file($destination); | ||
| 164 | |||
| 165 | # get the source files | ||
| 166 | @files = (); | ||
| 167 | foreach $element (@ARGV) { | ||
| 168 | if(-d $element) { | ||
| 169 | my @temp = (); | ||
| 170 | opendir(THISDIR, $element) || die "Can not open the $dirpath"; | ||
| 171 | push(@temp, grep {$_ ne '.' and $_ ne '..' } readdir THISDIR); | ||
| 172 | closedir THISDIR; | ||
| 173 | foreach (@temp) { push @files, $element . $_; } | ||
| 174 | } | ||
| 175 | else { push @files, $element; } | ||
| 176 | } | ||
| 177 | |||
| 178 | # create the files necessary to run the module | ||
| 179 | $text->create_files(@files); | ||
| 180 | |||
| 181 | # get the ngrams | ||
| 182 | $text->get_ngrams(); | ||
| 183 | |||
| 184 | # remove the miscellaneous file created by the Text::Positional::Ngram module | ||
| 185 | $text->remove_files(); | ||
| 186 | |||
| 187 | # function to output "ask for help" message when the user's goofed up! | ||
| 188 | sub askHelp | ||
| 189 | { | ||
| 190 | print STDERR "Type text-positional-ngram-driver.pl --help for help.\n"; | ||
| 191 | } | ||
| 192 | |||
| 193 | # function to output the version number | ||
| 194 | sub showVersion | ||
| 195 | { | ||
| 196 | print STDERR "text-positional-ngram-driver.pl - version 0.5\n"; | ||
| 197 | print STDERR "Copyright (C) 2004-2007, Bridget Thomson McInnes\n"; | ||
| 198 | print STDERR "Date of Last Update 28/08/07\n"; | ||
| 199 | |||
| 200 | } | ||
| 201 | |||
| 202 | # function to output a minimal usage note when the user has not provided any | ||
| 203 | # commandline options | ||
| 204 | sub minimalUsageNotes | ||
| 205 | { | ||
| 206 | print STDERR "Usage: text-positional-nram-driver.pl [OPTIONS] DESTINATION SOURCE [[, SOURCE] ...]\n"; | ||
| 207 | askHelp(); | ||
| 208 | } | ||
| 209 | |||
| 210 | # function to output help messages for this program | ||
| 211 | sub showHelp | ||
| 212 | { | ||
| 213 | print "Usage: text-positional-ngram-driver.pl [OPTIONS] DESTINATION SOURCE [[, SOURCE] ...]\n\n"; | ||
| 214 | |||
| 215 | print "Counts up the frequency of all n-grams occurring in SOURCE.\n"; | ||
| 216 | print "Sends to DESTINATION the list of n-grams found and their frequencies.\n"; | ||
| 217 | |||
| 218 | print "OPTIONS:\n\n"; | ||
| 219 | |||
| 220 | print " --ngram N Creates n-grams of N tokens each. N = 2 by\n"; | ||
| 221 | print " default.\n\n"; | ||
| 222 | |||
| 223 | print " --token FILE Uses regular expressions in FILE to create\n"; | ||
| 224 | print " tokens. By default two regular expressions\n"; | ||
| 225 | print " are provided (see README).\n\n"; | ||
| 226 | |||
| 227 | print " --nontoken FILE Removes all characters sequences that match\n"; | ||
| 228 | print " Perl regular expressions specified in FILE.\n\n"; | ||
| 229 | |||
| 230 | print " --stop FILE Removes n-grams containing at least one (in\n"; | ||
| 231 | print " OR mode) or all stop words (in AND mode).\n"; | ||
| 232 | print " Stop words should be declared as Perl Regular\n"; | ||
| 233 | print " expressions in FILE.\n\n"; | ||
| 234 | |||
| 235 | print " --stop_mode MODE Sets the stop mode to OR or AND. OR mode removes\n"; | ||
| 236 | print " n-grams containing at least one stop word. AND \n"; | ||
| 237 | print " mode removes n-grams that contain all stop words.\n\n"; | ||
| 238 | |||
| 239 | print " --frequency N Does not display n-grams that occur less\n"; | ||
| 240 | print " than N times.\n\n"; | ||
| 241 | |||
| 242 | print " --remove N Ignores n-grams that occur less than N\n"; | ||
| 243 | print " times. Ignored n-grams are not counted and\n"; | ||
| 244 | print " so do not affect counts and frequencies.\n\n"; | ||
| 245 | |||
| 246 | print " --newLine Prevents n-grams from spanning across the\n"; | ||
| 247 | print " new-line character.\n\n"; | ||
| 248 | |||
| 249 | print " --marginals Prints the marginal counts of the\n"; | ||
| 250 | print " individual words in the n-gram\n\n"; | ||
| 251 | |||
| 252 | print " --window N Sets the size of the window in which\n"; | ||
| 253 | print " the positional ngram can be found in.\n\n"; | ||
| 254 | |||
| 255 | print " --version Prints the version number.\n\n"; | ||
| 256 | |||
| 257 | print " --help Prints this help message.\n\n"; | ||
| 258 | } |