File Coverage

blib/lib/Algorithm/TicketClusterer.pm

Criterion	Covered	Total	%
statement	22	24	91.6
branch			n/a
condition			n/a
subroutine	8	8	100.0
pod			n/a
total	30	32	93.7

package Algorithm::TicketClusterer;

#---------------------------------------------------------------------------
# Copyright (c) 2014 Avinash Kak. All rights reserved.  This program is
# free software.  You may modify and/or distribute it under the same terms
# as Perl itself.  This copyright notice must remain attached to the file.
#
# Algorithm::TicketClusterer is a Perl module for retrieving Excel-stored
# past tickets that are most similar to a new ticket.  Tickets are commonly
# used in software services industry and customer support businesses to
# record requests for service, product complaints, user feedback, and so
# on.
# ---------------------------------------------------------------------------

use 5.10.0;
use strict;
use warnings;
use Carp;
use Storable;
use Spreadsheet::ParseExcel;
use Spreadsheet::XLSX;
use WordNet::QueryData;
use Text::Iconv;
use SDBM_File;
use Fcntl;

our $VERSION = '1.01';

############################### The Constructor #############################

sub new {
    my ($class, %args) = @_;
    my @params = keys %args;
    croak "\nYou have used a wrong name for a keyword argument " .
          "--- perhaps a misspelling\n"
          if _check_for_illegal_params(@params) == 0;
    bless {
        _excel_filename         =>   $args{excel_filename},
        _which_worksheet        =>   $args{which_worksheet},
        _raw_tickets_db         =>   $args{raw_tickets_db},
        _processed_tickets_db   =>   $args{processed_tickets_db},
        _synset_cache_db        =>   $args{synset_cache_db},
        _stemmed_tickets_db     =>   $args{stemmed_tickets_db},
        _inverted_index_db      =>   $args{inverted_index_db},
        _tickets_vocab_db       =>   $args{tickets_vocab_db},
        _idf_db                 =>   $args{idf_db},
        _tkt_doc_vecs_db        =>   $args{tkt_doc_vecs_db},
        _tkt_doc_vecs_normed_db =>   $args{tkt_doc_vecs_normed_db},
        _clustering_fieldname   =>   $args{clustering_fieldname},
        _unique_id_fieldname    =>   $args{unique_id_fieldname},
        _stop_words_file        =>   $args{stop_words_file},
        _misspelled_words_file  =>   $args{misspelled_words_file},
        _min_word_length        =>   $args{min_word_length} || 4,
        _add_synsets_to_tickets =>   $args{add_synsets_to_tickets} || 0,
        _want_stemming          =>   $args{want_stemming} || 0,
        _how_many_retrievals    =>   $args{how_many_retrievals} || 5,
        _min_idf_threshold      =>   $args{min_idf_threshold},
        _max_num_syn_words      =>   $args{max_num_syn_words} || 3,
        _want_synset_caching    =>   $args{want_synset_caching} || 0,
        _stop_words             =>   {},
        _all_tickets            =>   [],
        _column_headers         =>   [],
        _good_columns           =>   [],
        _tickets_by_ids         =>   {},
        _processed_tkts_by_ids  =>   {},
        _stemmed_tkts_by_ids    =>   {},
        _misspelled_words       =>   {},
        _total_num_tickets      =>   0,
        _synset_cache           =>   {},
        _vocab_hash             =>   {},
        _vocab_idf_hist         =>   {},
        _idf_t                  =>   {},
        _vocab_size             =>   undef,
        _doc_vector_template    =>   {},
        _tkt_doc_vecs           =>   {},
        _tkt_doc_vecs_normed    =>   {},
        _query_ticket_id        =>   undef,
        _inverted_index         =>   {},
        _debug1                 =>   $args{debug1} || 0, # for processing Excel
        _debug2                 =>   $args{debug2} || 0, # for modeling tickets
        _debug3                 =>   $args{debug3} || 0, # for retrieving similar tickets
        _wn                     =>   WordNet::QueryData->new( verbose => 0,
                                                              noload => 1 ),
    }, $class;
}

#############################  Extract info from Excel  #######################

sub get_tickets_from_excel {
    my $self = shift;
    unlink $self->{_raw_tickets_db} if -s $self->{_raw_tickets_db};
    unlink $self->{_processed_tickets_db} if -s $self->{_processed_tickets_db};
    unlink $self->{_synset_cache_db} if -s $self->{_synset_cache_db};
    unlink $self->{_stemmed_tickets_db} if -s $self->{_stemmed_tickets_db};
    unlink $self->{_inverted_index_db} if -s $self->{_inverted_index_db};
    unlink $self->{_tkt_doc_vecs_db} if -s $self->{_tkt_doc_vecs_db};
    unlink $self->{_tkt_doc_vecs_normed_db} if -s $self->{_tkt_doc_vecs_normed_db};
    unlink glob "$self->{_tickets_vocab_db}.*";
    unlink glob "$self->{_idf_db}.*";
    my $filename = $self->{_excel_filename} || die("Excel file required"),
    my $clustering_fieldname = $self->{_clustering_fieldname}
      || die("\nYou forgot to specify a value for the constructor parameter clustering_fieldname that points to the data to be clustered in your Excel sheet -- ");
    my $unique_id_fieldname = $self->{_unique_id_fieldname}
      || die("\nYou forgot to specify a value for the constructor parameter unique_id_fieldname that is a unique integer identifier for the rows of your Excel sheet -- ");
    my $workbook;
    if ($filename =~ /\.xls$/) {
        my $parser = Spreadsheet::ParseExcel->new();
        $workbook = $parser->parse($filename);
        die $parser->error() unless defined $workbook;
    } elsif ($filename =~ /\.xlsx$/) {
#        use Text::Iconv;
        my $converter = Text::Iconv->new("utf-8", "windows-1251");
        $workbook = Spreadsheet::XLSX->new($filename, $converter);
    } else {
        die "File suffix on the Excel file not recognized";
    }
    my @worksheets = $workbook->worksheets();
    my $which_worksheet = $self->{_which_worksheet} ||
        die "\nYou have not specified which Excel worksheet contains the tickets\n";
    my ( $row_min, $row_max ) = $worksheets[$which_worksheet-1]->row_range();
    my ( $col_min, $col_max ) = $worksheets[$which_worksheet-1]->col_range();
    my @good_columns;
    my $col_headers_row;
    my $col_headers_found = 0;
    my $col_index_for_unique_id;
    my $col_index_for_clustering_field;
    for my $row ( $row_min .. $row_max ) {
        last if $col_headers_found;
        @good_columns = ();
        for my $col ( $col_min .. $col_max ) {
            my $cell =
                   $worksheets[$which_worksheet-1]->get_cell( $row, $col );
            next unless $cell;
            my $cell_value = _get_rid_of_wide_chars($cell->value());
            push @good_columns, $col if $cell_value;
            if ($cell_value eq $unique_id_fieldname) {
                $col_index_for_unique_id = $col;
                $col_headers_row = $row;
                $col_headers_found = 1;
            }
            if ($cell_value eq $clustering_fieldname) {
                $col_index_for_clustering_field = $col;
            }
        }
    }
    $self->{_good_columns} = \@good_columns;
    print "\nThe unique id is in column: $col_index_for_unique_id\n"
        if $self->{_debug1};
    print "The clustering field is in column: " .
                "$col_index_for_clustering_field\n\n" if $self->{_debug1};
    my %Column_Headers;
    foreach my $field_index (0..@good_columns-1) {
        my $key = "field_" . $field_index;
        $Column_Headers{$key} = "";
    }
    my @col_headers = map {
        my $cell =
           $worksheets[$which_worksheet-1]->get_cell($col_headers_row, $_);
        $cell ? _get_rid_of_wide_chars($cell->value()) : '';
    } @good_columns;
    $self->{_column_headers} = \@col_headers;
    $self->_display_column_headers() if $self->{_debug1};
    my $unique_id_field_index_in_good_columns =
     _find_index_for_given_element( $col_index_for_unique_id, \@good_columns );
    my $clustering_field_index_in_good_columns =
     _find_index_for_given_element( $col_index_for_clustering_field,
                             \@good_columns );
    die "Something is wrong with the info extracted from Excel " .
        "as the index for the column with unique IDs is not one of " .
        "good columns\n\n"
        unless (defined $unique_id_field_index_in_good_columns) &&
               (defined $clustering_field_index_in_good_columns);
    for my $row_index ( $col_headers_row+1..$row_max-1) {
        my @values = map {
            my $cell =
              $worksheets[$which_worksheet-1]->get_cell($row_index, $_);
            $cell ? _get_rid_of_wide_chars($cell->value()) : '';
        } @good_columns;
        next unless $values[$unique_id_field_index_in_good_columns] =~ /\d+/;
        next unless $values[$clustering_field_index_in_good_columns] =~ /\w+/;
        my %onerow;
        foreach my $field_index (0..@good_columns-1) {
            my $key = "field_" . $field_index;
            die "The Columns Headers hash has no field for index " .
                   "$field_index\n    "
                unless exists $col_headers[$field_index];
            $onerow{$col_headers[$field_index]} = $values[$field_index];
        }
        push @{$self->{_all_tickets}}, \%onerow;
    }
    my @duplicates_for_id_field = @{$self->_check_unique_id_field()};
    if (@duplicates_for_id_field > 0) {
        print "Your supposedly unique ID field values for duplicates: @duplicates_for_id_field\n";
        die "\n\nYour unique id field for tickets contains duplicate id's";
    }
    foreach my $ticket (@{$self->{_all_tickets}}) {
        $self->{_tickets_by_ids}->{$ticket->{$unique_id_fieldname}} =
            lc($ticket->{$clustering_fieldname});
    }
    $self->{_total_num_tickets} = scalar @{$self->{_all_tickets}};
    $self->store_raw_tickets_on_disk();
}

sub _test_excel_for_tickets {
    my $self = shift;
    use Text::Iconv;
    my $converter = Text::Iconv->new("utf-8", "windows-1251");
    my $filename = $self->{_excel_filename} || die("Excel sheet needed for testing is missing");
    my $workbook = Spreadsheet::XLSX->new( $filename, $converter );
    my @worksheets = $workbook->worksheets();
    my ( $row_min, $row_max ) = $worksheets[0]->row_range();
    my ( $col_min, $col_max ) = $worksheets[0]->col_range();
    return ($row_min, $row_max, $col_min, $col_max);
}

sub _display_column_headers {
    my $self = shift;
    print "\nThe good columns are: @{$self->{_good_columns}}\n\n";
    my $overall_header_string = join '  <>  ', @{$self->{_column_headers}};
    print "The column headers are: $overall_header_string\n\n";
}

sub _check_unique_id_field {
    my $self = shift;
    my %check_hash;
    my @duplicates;
    foreach my $ticket (@{$self->{_all_tickets}}) {
        if (exists $ticket->{$self->{_unique_id_fieldname}}) {
            push @duplicates, $ticket->{$self->{_unique_id_fieldname}}
               if exists $check_hash{$ticket->{$self->{_unique_id_fieldname}}};
            $check_hash{$ticket->{$self->{_unique_id_fieldname}}} = 1;
        }
    }
    if ($self->{_debug1}) {
        my $num_of_tickets = @{$self->{_all_tickets}};
        my $num_entries_check_hash = keys %check_hash;
        print "Number of tickets: $num_of_tickets\n";
        print "Number of keys in check hash: $num_entries_check_hash\n";
    }
    return \@duplicates;
}

sub show_original_ticket_for_given_id {
    my $self = shift;
    my $id = shift;
    print "\n\nDisplaying the fields for the ticket $id:\n\n";
    foreach my $ticket (@{$self->{_all_tickets}}) {
        if ( $ticket->{$self->{_unique_id_fieldname}} == $id) {
            foreach my $key (sort keys %{$ticket}) {
                my $value = $ticket->{$key};
                $value =~ s/^\s+//;
                $value =~ s/\s+$//;
                printf("%20s  ==>  %s\n", $key, $value);
            }
        }
    }
}

sub show_raw_ticket_clustering_data_for_given_id {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_tickets_by_ids}->{$ticket_id};
    print "\n\nDISPLAYING THE RAW CLUSTERING DATA FOR TICKET $ticket_id:\n\n" .
        "$record\n\n";
    return $record;
}

# Needed by test.t
sub _raw_ticket_clustering_data_for_given_id {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_tickets_by_ids}->{$ticket_id};
    return $record;
}


sub show_processed_ticket_clustering_data_for_given_id {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_processed_tkts_by_ids}->{$ticket_id};
    print "\n\nDISPLAYING PROCESSED CLUSTERING DATA FOR TICKET $ticket_id:\n\n" .
        "$record\n\n";
}

sub show_stemmed_ticket_clustering_data_for_given_id {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_stemmed_tkts_by_ids}->{$ticket_id};
    print "\n\nDISPLAYING STEMMED CLUSTERING DATA FOR TICKET $ticket_id:\n\n" .
        "$record\n\n";
}

# The following function is a good diagnostic tool to look into the
# array stored in $self->{_all_tickets}.  Each element of this array
# is a record that represents one row of the Excel file.
sub _show_row {
    my $self = shift;
    my $row_num = shift;
    my $total_rows = @{$self->{_all_tickets}};
    print "There are $total_rows items in the \$all_tickets array\n";
    die "The row that you want to see does not exist"
        unless $row_num < $total_rows;
    my %record = %{$self->{_all_tickets}->[$row_num]};
    foreach my $field (sort keys %record) {
        my $value = $record{$field};
        no warnings;
        print "$field  ==>  $value\n";
    }
}

sub store_raw_tickets_on_disk {
    my $self = shift;
    $self->{_raw_tickets_db} = "raw_tickets.db" unless $self->{_raw_tickets_db};
    unlink $self->{_raw_tickets_db};
    eval {
        store( $self->{_all_tickets}, $self->{_raw_tickets_db} );
    };
    if ($@) {
        die "Something went wrong with disk storage of ticket data: $@";
    }
}

sub restore_raw_tickets_from_disk {
    my $self = shift;
    my $clustering_fieldname = $self->{_clustering_fieldname}
      || die("\nYou forgot to specify a value for the constructor parameter clustering_fieldname that points to the data to be clustered in your Excel sheet -- ");
    my $unique_id_fieldname = $self->{_unique_id_fieldname}
      || die("\nYou forgot to specify a value for the constructor parameter unique_id_fieldname that is a unique integer identifier for the rows of your Excel sheet -- ");
    eval {
        $self->{_all_tickets} = retrieve( $self->{_raw_tickets_db} );
    };
    if ($@) {
        die "Unable to retrieve raw tickets from disk: $@";
    }
    foreach my $ticket (@{$self->{_all_tickets}}) {
        $self->{_tickets_by_ids}->{$ticket->{$unique_id_fieldname}} =
            lc($ticket->{$clustering_fieldname});
    }
    $self->{_total_num_tickets} = scalar keys %{$self->{_tickets_by_ids}};
}

sub delete_markup_from_all_tickets {
    my $self = shift;
    foreach my $ticket (@{$self->{_all_tickets}}) {
        $self->_delete_markup_from_one_ticket($ticket->{$self->{_unique_id_fieldname}});
    }
}

sub _delete_markup_from_one_ticket {
    my $self = shift;
    my $ticket_id = shift;
    my $ticket_strings = $self->{_tickets_by_ids}->{$ticket_id};
    my @strings = grep $_, split /\s+/, $ticket_strings;
    my $cleaned_up_strings = join ' ', grep {$_ !~ /^<[^<>]+>$/} @strings;
    $self->{_tickets_by_ids}->{$ticket_id} = $cleaned_up_strings;
    foreach my $ticket (@{$self->{_all_tickets}}) {
        if ( $ticket->{$self->{_unique_id_fieldname}} == $ticket_id ) {
            $ticket->{$self->{_clustering_fieldname}} = $cleaned_up_strings;
            last;
        }
    }
}

sub apply_filter_to_all_tickets {
    my $self = shift;
    my $stop_words_file = $self->{_stop_words_file}
        || die("\nYou forgot to supply the name of the stop words file in your constructor call\n");
    my @stop_words = @{_fetch_words_from_file($stop_words_file)};
    my $misspelled_words_file = $self->{_misspelled_words_file}
        || die("\nYou forgot to supply the name of the misspelled words file in your constructor call\n");
    foreach my $word (@stop_words) {
        $self->{_stop_words}->{$word} = 1;
    }
    if ($self->{_misspelled_words_file}) {
        my @misspelled_word_pairs =
            @{_fetch_word_pairs_from_file($self->{_misspelled_words_file})};
        foreach my $wordpair (@misspelled_word_pairs) {
            my ($wrong_word, $good_word) = grep $_, split /\s+/, $wordpair;
            $self->{_misspelled_words}->{$wrong_word} = $good_word;
        }
    }
    my $i = 1;
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_tickets_by_ids}}) {
        print "\nApplying filter to ticket $ticket_id ($i out of $self->{_total_num_tickets})\n";
        $self->_apply_filter_to_one_ticket($ticket_id);
        $i++;
    }
}

sub _apply_filter_to_one_ticket {
    my $self = shift;
    my $ticket_id = shift;

    unless (keys %{$self->{_stop_words}} > 0) {
        my @stop_words = @{_fetch_words_from_file($self->{_stop_words_file})};
        foreach my $word (@stop_words) {
            $self->{_stop_words}->{$word} = 1;
        }
    }
    unless (keys %{$self->{_misspelled_words}} > 0) {
        if ($self->{_misspelled_words_file}) {
            my @misspelled_word_pairs =
                @{_fetch_word_pairs_from_file($self->{_misspelled_words_file})};
            foreach my $wordpair (@misspelled_word_pairs) {
                my ($wrong_word, $good_word) = grep $_, split /\s+/, $wordpair;
                $self->{_misspelled_words}->{$wrong_word} = $good_word;
            }
        }
    }
    my $record = $self->{_tickets_by_ids}->{$ticket_id};
    my $min = $self->{_min_word_length};
    my @words = split /\n|\r|\"|\'|\.|\,|\;|\?|\(|\)|\[|\]|\\|\/|\s+|\&/, $record;
    my @clean_words = grep $_, map { /([a-z0-9_]{$min,})/i;$1 } @words;
    return unless @clean_words;
    my @new_words;
    foreach my $word (@words) {
        $word =~ s/(.+)[.,:!-]$/$1/;
        unless (($word eq 'no') or ($word eq 'not')) {
            next if length($word) < $self->{_min_word_length};
        }
        if (exists $self->{_misspelled_words}->{lc($word)}) {
            push @new_words, $self->{_misspelled_words}->{$word};
            next;
        }
        push @new_words, $word unless exists $self->{_stop_words}->{lc($word)};
    }
    my $new_record = join ' ', @new_words;
    $self->{_processed_tkts_by_ids}->{$ticket_id} = $new_record;
}

sub _get_synonyms_for_word {
    my $self = shift;
    my $word = shift;
    my $no_sense_indicators = 1;
    my $wn = $self->{_wn};
    my @parts_of_speech = $wn->querySense("$word");

    my %noun_synonyms;
    my %verb_synonyms;
    my %adj_synonyms;
    my %adv_synonyms;
    foreach my $pos (@parts_of_speech) {
        if ($pos =~ /n$/) {
            my @all_noun_syn_sense_labels = $wn->querySense( $pos, "syns");
            my $how_many = @all_noun_syn_sense_labels;
            foreach my $noun_sense (@all_noun_syn_sense_labels) {
                my @noun_synonyms = $wn->querySense($noun_sense, "syns");
                my $answer = "";
                foreach my $noun_syn (@noun_synonyms) {
                    next if $noun_syn eq $noun_sense;
                    $noun_syn =~ s/\#.+$// if $no_sense_indicators;
                    $noun_synonyms{$noun_syn} = 1;
                    $answer .= " $noun_syn ";
                }
            }
        } elsif ($pos =~ /v$/) {
            my @all_verb_syn_sense_labels = $wn->querySense( $pos, "syns");
            my $how_many = @all_verb_syn_sense_labels;
            foreach my $verb_sense (@all_verb_syn_sense_labels) {
                my @verb_synonyms = $wn->querySense($verb_sense, "syns");
                my $answer = "";
                foreach my $verb_syn (@verb_synonyms) {
                    next if $verb_syn eq $verb_sense;
                    $verb_syn =~ s/\#.+$// if $no_sense_indicators;
                    $verb_synonyms{$verb_syn} = 1;
                    $answer .= " $verb_syn ";
                }
            }
        } elsif ($pos =~ /a$/) {
            my @all_adj_syn_sense_labels = $wn->querySense( $pos, "syns");
            my $how_many = @all_adj_syn_sense_labels;
            foreach my $adj_sense (@all_adj_syn_sense_labels) {
                my @adj_synonyms = $wn->querySense($adj_sense, "syns");
                my $answer = "";
                foreach my $adj_syn (@adj_synonyms) {
                    next if $adj_syn eq $adj_sense;
                    $adj_syn =~ s/\#.+$// if $no_sense_indicators;
                    $adj_synonyms{$adj_syn} = 1;
                    $answer .= " $adj_syn ";
                }
            }
        } elsif ($pos =~ /r$/) {
            my @all_adv_syn_sense_labels = $wn->querySense( $pos, "syns");
            my $how_many = @all_adv_syn_sense_labels;
            foreach my $adv_sense (@all_adv_syn_sense_labels) {
                my @adv_synonyms = $wn->querySense($adv_sense, "syns");
                my $answer = "";
                foreach my $adv_syn (@adv_synonyms) {
                    next if $adv_syn eq $adv_sense;
                    $adv_syn =~ s/\#.+$// if $no_sense_indicators;
                    $adv_synonyms{$adv_syn} = 1;
                    $answer .= " $adv_syn ";
                }
            }
        } else {
            die "\nThe Part of Speech $pos not recognized\n\n";
        }
    }
    my @all_synonyms;
    my @all_noun_synonyms = keys %noun_synonyms;
    my @all_verb_synonyms = keys %verb_synonyms;
    my @all_adj_synonyms =  keys %adj_synonyms;
    my @all_adv_synonyms =  keys %adv_synonyms;
    push @all_synonyms, @all_noun_synonyms if @all_noun_synonyms > 0;
    push @all_synonyms, @all_verb_synonyms if @all_verb_synonyms > 0;
    push @all_synonyms, @all_adj_synonyms  if @all_adj_synonyms > 0;
    push @all_synonyms, @all_adv_synonyms  if @all_adv_synonyms > 0;
    my %synonym_set;
    foreach my $synonym (@all_synonyms) {
        $synonym_set{$synonym} = 1;
    }
    my @synonym_set = sort keys %synonym_set;
    return \@synonym_set;
}

sub _get_antonyms_for_word {
    my $self = shift;
    my $word = shift;
    my $no_sense_indicators = 1;
    my $wn = $self->{_wn};
    my @parts_of_speech = $wn->querySense("$word");
    my %noun_antonyms;
    my %verb_antonyms;
    my %adj_antonyms;
    my %adv_antonyms;
    foreach my $pos (@parts_of_speech) {
        if ($pos =~ /n$/) {
            my @all_noun_ant_sense_labels = $wn->queryWord( $pos, "ants");
            my $how_many = @all_noun_ant_sense_labels;
            foreach my $noun_sense (@all_noun_ant_sense_labels) {
                my @noun_antonyms = $wn->queryWord($noun_sense, "ants");
                my $answer = "";
                foreach my $noun_ant (@noun_antonyms) {
                    next if $noun_ant eq $noun_sense;
                    $noun_ant =~ s/\#.+$// if $no_sense_indicators;
                    $noun_antonyms{$noun_ant} = 1;
                    $answer .= " $noun_ant ";
                }
            }
        } elsif ($pos =~ /v$/) {
            my @all_verb_ant_sense_labels = $wn->queryWord( $pos, "ants");
            my $how_many = @all_verb_ant_sense_labels;
            foreach my $verb_sense (@all_verb_ant_sense_labels) {
                my @verb_antonyms = $wn->queryWord($verb_sense, "ants");
                my $answer = "";
                foreach my $verb_ant (@verb_antonyms) {
                    next if $verb_ant eq $verb_sense;
                    $verb_ant =~ s/\#.+$// if $no_sense_indicators;
                    $verb_antonyms{$verb_ant} = 1;
                    $answer .= " $verb_ant ";
                }
            }
        } elsif ($pos =~ /a$/) {
            my @all_adj_ant_sense_labels = $wn->queryWord( $pos, "ants");
            my $how_many = @all_adj_ant_sense_labels;
            foreach my $adj_sense (@all_adj_ant_sense_labels) {
                my @adj_antonyms = $wn->queryWord($adj_sense, "ants");
                my $answer = "";
                foreach my $adj_ant (@adj_antonyms) {
                    next if $adj_ant eq $adj_sense;
                    $adj_ant =~ s/\#.+$// if $no_sense_indicators;
                    $adj_antonyms{$adj_ant} = 1;
                    $answer .= " $adj_ant ";
                }
            }
        } elsif ($pos =~ /r$/) {
            my @all_adv_ant_sense_labels = $wn->queryWord( $pos, "ants");
            my $how_many = @all_adv_ant_sense_labels;
            foreach my $adv_sense (@all_adv_ant_sense_labels) {
                my @adv_antonyms = $wn->queryWord($adv_sense, "ants");
                my $answer = "";
                foreach my $adv_ant (@adv_antonyms) {
                    next if $adv_ant eq $adv_sense;
                    $adv_ant =~ s/\#.+$// if $no_sense_indicators;
                    $adv_antonyms{$adv_ant} = 1;
                    $answer .= " $adv_ant ";
                }
            }
        } else {
            die "\nThe Part of Speech $pos not recognized\n\n";
        }
    }
    my @all_antonyms;
    my @all_noun_antonyms = keys %noun_antonyms;
    my @all_verb_antonyms = keys %verb_antonyms;
    my @all_adj_antonyms =  keys %adj_antonyms;
    my @all_adv_antonyms =  keys %adv_antonyms;
    push @all_antonyms, @all_noun_antonyms if @all_noun_antonyms > 0;
    push @all_antonyms, @all_verb_antonyms if @all_verb_antonyms > 0;
    push @all_antonyms, @all_adj_antonyms  if @all_adj_antonyms > 0;
    push @all_antonyms, @all_adv_antonyms  if @all_adv_antonyms > 0;
    my %antonym_set;
    foreach my $antonym (@all_antonyms) {
        $antonym_set{$antonym} = 1;
    }
    my @antonym_set = sort keys %antonym_set;
    return \@antonym_set;
}

sub expand_all_tickets_with_synonyms {
    my $self = shift;
    return unless $self->{_add_synsets_to_tickets};
    my $num_of_tickets = $self->{_total_num_tickets};
    if ($self->{_want_synset_caching}) {
        eval {
            $self->{_synset_cache} = retrieve( $self->{_synset_cache_db} );
        } if -s $self->{_synset_cache_db};
        if ($@) {
           print "Something went wrong with restoration of synset cache: $@";
        }
    }
    my $i = 1;
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_processed_tkts_by_ids}}) {
        $self->_expand_one_ticket_with_synonyms($ticket_id);
        print "Finished syn expansion of ticket $ticket_id ($i out of $num_of_tickets)\n";
        $i++;
    }
    if ($self->{_want_synset_caching}) {
        $self->{_synset_cache_db} = "synset_cache.db" unless $self->{_synset_cache_db};
        eval {
            store( $self->{_synset_cache}, $self->{_synset_cache_db} );
        };
        if ($@) {
           die "Something went wrong with disk storage of synset cache: $@";
        }
    }
}

sub _expand_one_ticket_with_synonyms {
    my $self = shift;
    my $ticket_id = shift;
    print "\n\nEXPANDING TICKET $ticket_id WITH SYN-SETS:\n\n"
                                              if $self->{_debug2};
    $self->_replace_negated_words_with_antonyms_one_ticket( $ticket_id );
    $self->_add_to_words_their_synonyms_one_ticket( $ticket_id );
}

sub _replace_negated_words_with_antonyms_one_ticket {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_processed_tkts_by_ids}->{$ticket_id};
    my @words_negated_with_not = $record =~ /\bnot\s+(\w+)/ig;
    foreach my $word (@words_negated_with_not) {
        next unless (($word =~ /^\w+$/) &&
                     (length($word) > $self->{_min_word_length}));
        my @antonym_words = @{$self->_get_antonyms_for_word( $word )};
        next unless @antonym_words > 0;
        $#antonym_words = $self->{_max_num_syn_words} - 1
              if @antonym_words > $self->{_max_num_syn_words};
        my $antonym_replacement_string = join ' ', @antonym_words;
        print "Antonym for $word is $antonym_replacement_string\n"
            if $self->{_debug2};
        $record =~ s/not\s+$word/$antonym_replacement_string/g;
    }
    my @words_negated_with_no = $record =~ /\bno\s+(\w+)/ig;
    foreach my $word (@words_negated_with_no) {
        next unless (($word =~ /^\w+$/) &&
                    (length($word) > $self->{_min_word_length}));
        my @antonym_words = @{$self->_get_antonyms_for_word( $word )};
        next unless @antonym_words > 0;
        $#antonym_words = $self->{_max_num_syn_words} - 1
              if @antonym_words > $self->{_max_num_syn_words};
        my $antonym_replacement_string = join ' ', @antonym_words;
        print "Antonym for $word is $antonym_replacement_string\n"
            if $self->{_debug2};
        $record =~ s/no\s+$word/$antonym_replacement_string/g;
    }
    $self->{_processed_tkts_by_ids}->{$ticket_id} = $record;
}

sub _add_to_words_their_synonyms_one_ticket {
    my $self = shift;
    my $ticket_id = shift;
    my $record = $self->{_processed_tkts_by_ids}->{$ticket_id};
    my @words = split /\s+/, $record;
    my @synonym_bag;
    foreach my $word (@words) {
        next if $word eq 'no';
        next if $word eq 'not';
        next unless $word =~ /^\w+$/ &&
                    length($word) > $self->{_min_word_length};
        my @synonym_words;
        @synonym_words = @{$self->{_synset_cache}->{$word}}
                      if exists $self->{_synset_cache}->{$word};
        unless (exists $self->{_synset_cache}->{$word}) {
            @synonym_words = @{$self->_get_synonyms_for_word( $word )};
            print "syn-set for $word  =>   @synonym_words\n\n"
                if $self->{_debug2};
            my $word_root;
            if (@synonym_words == 0) {
                if ((length($word) > 4) && ($word =~ /(.+)s$/)) {
                    $word_root = $1;
                    @synonym_words = @{$self->_get_synonyms_for_word( $word_root )}
                        if length($word_root) >= $self->{_min_word_length};
                } elsif ((length($word) > 6) && ($word =~ /(.+)ing$/)) {
                    $word_root = $1;
                    @synonym_words = @{$self->_get_synonyms_for_word( $word_root )}
                        if length($word_root) >= $self->{_min_word_length};
                }
            }
            print "syn-set for word root $word_root  =>   @synonym_words\n\n"
                if ( $self->{_debug2} && defined $word_root );
            _fisher_yates_shuffle( \@synonym_words ) if @synonym_words > 0;
            $#synonym_words = $self->{_max_num_syn_words} - 1
                  if @synonym_words > $self->{_max_num_syn_words};
            print "Retained syn-set for $word  =>   @synonym_words\n\n"
                if $self->{_debug2};
            $self->{_synset_cache}->{$word} = \@synonym_words;
            push @synonym_bag, @synonym_words;
        }
    }
    foreach my $syn_word (@synonym_bag) {
        push @words, lc($syn_word)
            unless ((exists $self->{_stop_words}->{$syn_word}) ||
                        (length($syn_word) <= $self->{_min_word_length}));
    }
    my @sorted_words = sort @words;
    my $new_record = join ' ', @sorted_words;
    $self->{_processed_tkts_by_ids}->{$ticket_id} = $new_record;
}

sub store_processed_tickets_on_disk {
    my $self = shift;
    $self->{_processed_tickets_db} = "processed_tickets.db" unless $self->{_processed_tickets_db};
    unlink $self->{_processed_tickets_db};
    eval {
        store( $self->{_processed_tkts_by_ids}, $self->{_processed_tickets_db} );
    };
    if ($@) {
       die "Something went wrong with disk storage of processed tickets: $@";
    }
}

sub store_stemmed_tickets_and_inverted_index_on_disk {
    my $self = shift;
    $self->{_stemmed_tickets_db} = "stemmed_tickets.db" unless $self->{_stemmed_tickets_db};
    unlink $self->{_stemmed_tickets_db};
    eval {
        print "\n\nStoring stemmed tickets on disk\n\n";
        store( $self->{_stemmed_tkts_by_ids}, $self->{_stemmed_tickets_db} );
    };
    if ($@) {
       die "Something went wrong with disk storage of stemmed tickets: $@";
    }
    $self->{_inverted_index_db} = "inverted_index.db" unless $self->{_inverted_index_db};
    unlink $self->{_inverted_index_db};
    eval {
        print "\nStoring inverted index on disk\n\n";
        store( $self->{_inverted_index}, $self->{_inverted_index_db} );
    };
    if ($@) {
       die "Something went wrong with disk storage of the inverted index: $@";
    }
}

sub restore_processed_tickets_from_disk {
    my $self = shift;
    eval {
        $self->{_processed_tkts_by_ids} = retrieve( $self->{_processed_tickets_db} );
    };
    if ($@) {
       die "Something went wrong with restoration of processed tickets: $@";
    }
}

sub restore_stemmed_tickets_from_disk {
    my $self = shift;
    eval {
        $self->{_stemmed_tkts_by_ids} = retrieve( $self->{_stemmed_tickets_db} );
    };
    if ($@) {
       die "Something went wrong with restoration of stemmed tickets: $@";
    }
}

####################  Get Ticket Vocabulary and Word Counts #################

sub get_ticket_vocabulary_and_construct_inverted_index {
    my $self = shift;
    my $total_num_of_tickets = keys %{$self->{_processed_tkts_by_ids}};
    $self->{_tickets_vocab_db} = "tickets_vocab.db" unless $self->{_tickets_vocab_db};
    unlink glob "$self->{_tickets_vocab_db}.*";
    my %vocab_hist_on_disk;
    tie %vocab_hist_on_disk, 'SDBM_File',
             $self->{_tickets_vocab_db}, O_RDWR|O_CREAT, 0640
            or die "Can't create DBM files: $!";
    my %inverted_index;
    my $min = $self->{_min_word_length};
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_processed_tkts_by_ids}}) {
        my %uniques = ();
        my $record = $self->{_processed_tkts_by_ids}->{$ticket_id};
        my @brokenup = split /\n|\r|\"|\'|\.|\(|\)|\[|\]|\\|\/|\s+/, $record;
        my @clean_words = grep $_, map { /([a-z0-9_]{$min,})/i;$1 } @brokenup;
        next unless @clean_words;
        @clean_words = grep $_, map &_simple_stemmer($_, $self->{_debug2}),
                                                                 @clean_words;
        map { $vocab_hist_on_disk{"\L$_"}++ } grep $_, @clean_words;
        for (@clean_words) { $uniques{"\L$_"}++ };
        map { $self->{_vocab_idf_hist}->{"\L$_"}++ } keys %uniques;
        map { push @{$self->{_inverted_index}->{"\L$_"}}, $ticket_id }
                                                            keys %uniques;
        $self->{_stemmed_tkts_by_ids}->{$ticket_id} = join ' ', @clean_words;
    }
    foreach (keys %vocab_hist_on_disk) {
        $self->{_vocab_hist}->{$_} = $vocab_hist_on_disk{$_};
    }
    untie %vocab_hist_on_disk;
    $self->{_tkt_vocab_done} = 1;
    $self->{_vocab_size} = scalar( keys %{$self->{_vocab_hist}} );
    print "\n\nVocabulary size:  $self->{_vocab_size}\n\n"
        if $self->{_debug2};
    # Calculate idf(t):
    $self->{_idf_db} = "idf.db" unless $self->{_idf_db};
    unlink glob "$self->{_idf_db}.*";
    tie my %idf_t_on_disk, 'SDBM_File', $self->{_idf_db}, O_RDWR|O_CREAT, 0640
                                            or die "Can't create DBM files: $!";
    foreach (keys %{$self->{_vocab_idf_hist}}) {
        $idf_t_on_disk{$_} = abs( (1 + log($total_num_of_tickets
                                           /
                                           (1 + $self->{_vocab_idf_hist}->{$_})))
                                           / log(10) );
    }
    foreach (keys %idf_t_on_disk) {
        $self->{_idf_t}->{$_} = $idf_t_on_disk{$_};
    }
    untie %idf_t_on_disk;
}

sub display_tickets_vocab {
    my $self = shift;
    die "tickets vocabulary not yet constructed"
        unless keys %{$self->{_vocab_hist}};
    print "\n\nDisplaying tickets vocabulary (the number shown against each word is the number of times each word appears in ALL the tickets):\n\n";
    foreach (sort keys %{$self->{_vocab_hist}}){
        my $outstring = sprintf("%30s     %d", $_,$self->{_vocab_hist}->{$_});
        print "$outstring\n";
    }
    my $vocab_size = scalar( keys %{$self->{_vocab_hist}} );
    print "\nSize of the tickets vocabulary: $vocab_size\n\n";
}

sub display_inverse_document_frequencies {
    my $self = shift;
    die "tickets vocabulary not yet constructed"
        unless keys %{$self->{_vocab_idf_hist}};
    print "\n\nDisplaying inverse document frequencies (the number of tickets in which each word appears):\n\n";
    foreach ( sort keys %{$self->{_vocab_idf_hist}} ) {
        my $outstring = sprintf("%30s     %d",
                       $_, $self->{_vocab_idf_hist}->{$_});
        print "$outstring\n";
    }
    print "\nDisplaying idf(t) = log(D/d(t)) where D is total number of tickets and d(t) the number of tickets with the word t:\n";
    foreach ( sort keys %{$self->{_idf_t}} ) {
        my $outstring = sprintf("%30s     %f", $_,$self->{_idf_t}->{$_});
        print "$outstring\n";
    }
}

# The following subroutine is useful for diagnostic purposes.  It
# lists the number of tickets that a word appears in and also lists
# the tickets.  But be careful in interpreting its results.  Note
# if you invoke this subroutine after the synsets have been added
# to the tickets, you may find words being attributed to tickets
# that do not actually contain them in the original Excel sheet.
sub list_processed_tickets_for_a_word {
    my $self = shift;
    while (my $word = ) {    #enter ctrl-D to exit the loop
        chomp $word;
        my @ticket_list;
        foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_processed_tkts_by_ids}}) {
            my $record = $self->{_processed_tkts_by_ids}->{$ticket_id};
            push @ticket_list, $ticket_id if $record =~ /\b$word\b/i;
        }
        my $num = @ticket_list;
        print "\nThe number of processed tickets that mention the word `$word': $num\n\n";
        print "The processed tickets: @ticket_list\n\n";
    }
}

sub list_stemmed_tickets_for_a_word {
    my $self = shift;
    while (my $word = ) {    #enter ctrl-D to exit the loop
        chomp $word;
        my @ticket_list;
        foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_stemmed_tkts_by_ids}}) {
            my $record = $self->{_stemmed_tkts_by_ids}->{$ticket_id};
            push @ticket_list, $ticket_id if $record =~ /\b$word\b/i;
        }
        my $num = @ticket_list;
        print "\nThe number of stemmed tickets that mention the word `$word': $num\n\n";
        print "The stemmed tickets: @ticket_list\n\n";
    }
}

##############  Generate Document Vectors for Tickets  ####################

sub construct_doc_vectors_for_all_tickets {
    my $self = shift;
    foreach ( sort keys %{$self->{_vocab_hist}} ) {
        $self->{_doc_vector_template}->{$_} = 0;
    }
    my $num_of_tickets = keys %{$self->{_stemmed_tkts_by_ids}};
    my $i = 1;
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_stemmed_tkts_by_ids}}) {
        my $doc_vec_ref =
            $self->_construct_doc_vector_for_one_ticket($ticket_id);
        print "Finished constructing two doc vecs for ticket $ticket_id ($i out of $num_of_tickets)\n";
        $i++;
    }
}

sub _construct_doc_vector_for_one_ticket {
    my $self = shift;
    my $ticket_id = shift;
    unless (keys %{$self->{_doc_vector_template}}) {
        foreach ( sort keys %{$self->{_vocab_hist}} ) {
            $self->{_doc_vector_template}->{$_} = 0;
        }
    }
    my %doc_vector = %{_deep_copy_hash($self->{_doc_vector_template})};
    foreach ( sort keys %{$self->{_doc_vector_template}} ) {
        $doc_vector{$_} = 0;
    }
    my $min = $self->{_min_word_length};
    my $total_words_in_ticket = 0;
    my $record = $self->{_stemmed_tkts_by_ids}->{$ticket_id};
    my @clean_words = split /\s+/, $record;
    map { $doc_vector{"\L$_"}++ }
            grep {exists $self->{_vocab_hist}->{"\L$_"}} @clean_words;
    die "Something went wrong. Doc vector size unequal to vocab size"
        unless $self->{_vocab_size} == scalar(keys %doc_vector);
    foreach (keys %doc_vector) {
        $total_words_in_ticket += $doc_vector{$_};
    }
    my %normalized_doc_vector;
    foreach (keys %doc_vector) {
        $normalized_doc_vector{$_} = $doc_vector{$_}
                                     *
                                     $self->{_idf_t}->{$_}
                                     /
                                     $total_words_in_ticket;
    }
    $self->{_tkt_doc_vecs}->{$ticket_id} = \%doc_vector;
    $self->{_tkt_doc_vecs_normed}->{$ticket_id} = \%normalized_doc_vector;
}

sub store_ticket_vectors {
    my $self = shift;
    die "You have not yet created doc vectors for tickets"
        unless keys %{$self->{_tkt_doc_vecs}};
    $self->{_tkt_doc_vecs_db} = "tkt_doc_vecs.db" unless $self->{_tkt_doc_vecs_db};
    $self->{_tkt_doc_vecs_normed_db} = "tkt_doc_vecs_normed.db"
                                           unless $self->{_tkt_doc_vecs_normed_db};
    unlink $self->{_tkt_doc_vecs_db};
    unlink $self->{_tkt_doc_vecs_normed_db};
    print "\nStoring the ticket doc vecs on disk. This could take a while.\n\n";
    eval {
        store( $self->{_tkt_doc_vecs}, $self->{_tkt_doc_vecs_db} );
    };
    if ($@) {
        die "Something went wrong with disk storage of ticket doc vectors: $@";
    }
    print "\nStoring normalized ticket doc vecs on disk. This could take a while.\n\n";
    eval {
        store($self->{_tkt_doc_vecs_normed}, $self->{_tkt_doc_vecs_normed_db});
    };
    if ($@) {
        die "Something wrong with disk storage of normalized doc vecs: $@";
    }
}

sub restore_ticket_vectors_and_inverted_index {
    my $self = shift;
    $self->restore_raw_tickets_from_disk();
    $self->restore_processed_tickets_from_disk();
    $self->restore_stemmed_tickets_from_disk();
    tie my %vocab_hist_on_disk, 'SDBM_File', $self->{_tickets_vocab_db}, O_RDONLY, 0640
            or die "Can't connect with DBM file: $!";
    foreach (keys %vocab_hist_on_disk) {
        $self->{_vocab_hist}->{$_} = $vocab_hist_on_disk{$_};
    }
    untie %vocab_hist_on_disk;
    tie my %idf_t_on_disk, 'SDBM_File', $self->{_idf_db}, O_RDONLY, 0640
            or die "Can't connect with DBM file: $!";
    foreach (keys %idf_t_on_disk) {
        $self->{_idf_t}->{$_} = $idf_t_on_disk{$_};
    }
    untie %idf_t_on_disk;
    eval {
        $self->{_tkt_doc_vecs} = retrieve( $self->{_tkt_doc_vecs_db} );
    };
    if ($@) {
       print "Something went wrong with retrieval of ticket doc vectors: $@";
    }
    eval {
        $self->{_tkt_doc_vecs_normed} = retrieve( $self->{_tkt_doc_vecs_normed_db} );
    };
    if ($@) {
       print "Something went wrong with retrieval of normed ticket doc vectors: $@";
    }
    eval {
        $self->{_inverted_index} =
                   retrieve( $self->{_inverted_index_db} );
    };
    if ($@) {
       print "Something went wrong with retrieval of inverted_index: $@";
    }
}

sub display_all_doc_vectors {
    my $self = shift;
    die "Ticket doc vectors not yet constructed"
        unless keys %{$self->{_tkt_doc_vecs}};
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_tkt_doc_vecs}}) {
        $self->_display_a_single_ticket_doc_vector($ticket_id);
    }
}

sub _display_a_single_ticket_doc_vector {
    my $self = shift;
    my $ticket_id = shift;
    die "Ticket doc vectors not yet constructed"
        unless keys %{$self->{_tkt_doc_vecs}};
    print "\n\nDISPLAYING Doc Vec FOR TICKET $ticket_id:\n\n";
    foreach ( sort keys %{$self->{_tkt_doc_vecs}->{$ticket_id}} ) {
        print "$_  =>   $self->{_tkt_doc_vecs}->{$ticket_id}->{$_}\n";
    }
    my $docvec_size = keys %{$self->{_tkt_doc_vecs}->{$ticket_id}};
    print "\nSize of vector for ticket $ticket_id: $docvec_size\n";
}

sub display_all_normalized_doc_vectors {
    my $self = shift;
    die "normalized document vectors not yet constructed"
        unless keys %{$self->{_tkt_doc_vecs_normed}};
    foreach my $ticket_id (sort {$a <=> $b} keys %{$self->{_tkt_doc_vecs_normed}}) {
        $self->_display_a_single_normalized_doc_vector($ticket_id);
    }
}

sub _display_a_single_normalized_doc_vector {
    my $self = shift;
    my $ticket_id = shift;
    die "Normalized ticket doc vectors not yet constructed"
        unless keys %{$self->{_tkt_doc_vecs_normed}};
    print "\n\nDISPLAYING Normalized Doc Vec FOR TICKET $ticket_id:\n\n";
    foreach ( sort keys %{$self->{_tkt_doc_vecs_normed}->{$ticket_id}} ) {
        print "$_  =>   $self->{_tkt_doc_vecs_normed}->{$ticket_id}->{$_}\n";
    }
    my $docvec_size = keys %{$self->{_tkt_doc_vecs_normed}->{$ticket_id}};
    print "\nSize of normalized vector for ticket $ticket_id: $docvec_size\n";
}

##########################  Display Inverted Index  ###########################

sub display_inverted_index {
    my $self = shift;
    print "\n\nDisplaying inverted index:\n\n";
    foreach my $word (sort keys %{$self->{_vocab_hist}}) {
        $self->display_inverted_index_for_given_word($word);
    }
}

sub display_inverted_index_for_given_word {
    my $self = shift;
    my $word = shift;
    defined $self->{_inverted_index}->{$word} ?
        print "$word =>  @{$self->{_inverted_index}->{$word}}\n"  :
        die "Something is wrong with your inverted index\n";
}

sub display_inverted_index_for_given_query {
    my $self = shift;
    my $query_ticket_id = shift;
    my $query_record = $self->{_stemmed_tkts_by_ids}->{$query_ticket_id};
    my @query_words = grep $_, split /\s+/, $query_record;
    foreach my $qword (@query_words) {
        my $idf_t = $self->{_idf_t}->{$qword};
        my @relevant_tickets = @{$self->{_inverted_index}->{$qword}};
        print "\n$qword ($idf_t)  ===>  @relevant_tickets\n\n";
    }
}

#############  Retrieve Most Similar Tickets with VSM Model  ###################

sub retrieve_similar_tickets_with_vsm {
    my $self = shift;
    $self->{_query_ticket_id} = shift;
    die "\nFirst generate normalized doc vectors for tickets before you can call retrieve with vsm function()\n"
        unless scalar(keys %{$self->{_vocab_hist}})
                  && scalar(keys %{$self->{_tkt_doc_vecs_normed}});
    print "\nCalculating the similarity set for query ticket $self->{_query_ticket_id}\n\n";
    my $query_record = $self->{_stemmed_tkts_by_ids}->{$self->{_query_ticket_id}};
    my @query_words = grep $_, split /\s+/, $query_record;
    my %relevant_tickets_set;
    die "\n\nYou did not set a value for the constructor parameter min_idf_threshold -- "
        unless $self->{_min_idf_threshold};
    foreach my $qword (@query_words) {
        map {$relevant_tickets_set{$_} = 1} @{$self->{_inverted_index}->{$qword}}
            if $self->{_idf_t}->{$qword} > $self->{_min_idf_threshold};
    }
    my @relevant_tickets = sort {$a <=> $b} keys %relevant_tickets_set;
    print "The relevant tickets for query: @relevant_tickets"
        if $self->{_debug3};
    my $num_relevant_tkts = @relevant_tickets;
    print "\nThe number of tickets relevant to the query: $num_relevant_tkts\n\n";
    my %retrievals;
    my $rank = 0;
    foreach (sort {$self->_doc_vec_comparator} @relevant_tickets ) {
        $retrievals{$_} = $self->_similarity_to_query_ticket($_);
        $rank++;
        last if $rank == $self->{_how_many_retrievals};
    }
    if ($self->{_debug3}) {
        print "\n\nShowing the VSM retrievals and the similarity scores:\n\n";
        foreach (sort {$retrievals{$b} <=> $retrievals{$a}} keys %retrievals) {
            print "$_   =>   $retrievals{$_}\n";
        }
    }
    return \%retrievals;
}

sub _doc_vec_comparator {
    my $self = shift;
    my %query_ticket_data_normed =
           %{$self->{_tkt_doc_vecs_normed}->{$self->{_query_ticket_id}}};
    my $vec1_hash_ref = $self->{_tkt_doc_vecs_normed}->{$a};
    my $vec2_hash_ref = $self->{_tkt_doc_vecs_normed}->{$b};
    my @vec1 = ();
    my @vec2 = ();
    my @qvec = ();
    foreach my $word (sort keys %{$self->{_vocab_hist}}) {
        push @vec1, $vec1_hash_ref->{$word};
        push @vec2, $vec2_hash_ref->{$word};
        push @qvec, $query_ticket_data_normed{$word};
    }
    my $vec1_mag = _vec_magnitude(\@vec1);
    my $vec2_mag = _vec_magnitude(\@vec2);
    my $qvec_mag = _vec_magnitude(\@qvec);
    my $product1 = _vec_scalar_product(\@vec1, \@qvec);
    $product1 /= $vec1_mag * $qvec_mag;
    my $product2 = _vec_scalar_product(\@vec2, \@qvec);
    $product2 /= $vec2_mag * $qvec_mag;
    return 1 if $product1 < $product2;
    return 0  if $product1 == $product2;
    return -1  if $product1 > $product2;
}

sub _similarity_to_query_ticket {
    my $self = shift;
    my $ticket_id = shift;
    my $ticket_data_normed = $self->{_tkt_doc_vecs_normed}->{$ticket_id};
    my @vec = ();
    my @qvec = ();
    foreach my $word (sort keys %$ticket_data_normed) {
        push @vec, $ticket_data_normed->{$word};
        push @qvec,
            $self->{_tkt_doc_vecs_normed}->{$self->{_query_ticket_id}}->{$word};
    }
    my $vec_mag = _vec_magnitude(\@vec);
    my $qvec_mag = _vec_magnitude(\@qvec);
    die "\nThe query ticket appears to be empty\n" if $qvec_mag == 0;
    my $product = _vec_scalar_product(\@vec, \@qvec);
    $product /= $vec_mag * $qvec_mag;
    return $product;
}


########################  Utility Subroutines  ##########################

sub _simple_stemmer {
    my $word = shift;
    my $debug = shift;
    print "\nStemming the word:        $word\n" if $debug;
    $word =~ s/(.*[a-z]t)ted$/$1/i;
    $word =~ s/(.*[a-z]t)ting$/$1/i;
    $word =~ s/(.*[a-z]l)ling$/$1/i;
    $word =~ s/(.*[a-z]g)ging$/$1/i;
    $word =~ s/(.*[a-z]ll)ed$/$1/i;
    $word =~ s/(.*[a-z][^aeious])s$/$1/i;
    $word =~ s/(.*[a-z])ies$/$1y/i;
    $word =~ s/(.*[a-z]s)es$/$1/i;
    $word =~ s/(.*[a-z][ck])es$/$1e/i;
    $word =~ s/(.*[a-z]+)tions$/$1tion/i;
    $word =~ s/(.*[a-z]+)mming$/$1m/i;
    $word =~ s/(.*[a-z]+[^rl])ing$/$1/i;
    $word =~ s/(.*[a-z]+o[sn])ing$/$1e/i;
    $word =~ s/(.*[a-z]+)tices$/$1tex/i;
    $word =~ s/(.*[a-z]+)pes$/$1pe/i;
    $word =~ s/(.*[a-z]+)sed$/$1se/i;
    $word =~ s/(.*[a-z]+)ed$/$1/i;
    $word =~ s/(.*[a-z]+)tation$/$1t/i;
    print "Stemmed word:                           $word\n\n" if $debug;
    return $word;
}

sub _exists {
    my $element = shift;
    my $array   = shift;
    my %hash;
    for my $item (@$array) {
        $hash{$item} = 1;
    }
    return exists $hash{$element};
}

sub _fetch_words_from_file {
    my $file = shift;
    my @words;
    open( IN, "$file" ) or die "unable to open the file $file: $!";
    while () {
        next if /^#/;
        next if /^[ ]*\r?\n?$/;
        $_ =~ s/\r?\n?$//;
        my @how_many_in_line = grep $_, split /\s+/, $_;
        die "File $file: Exactly one word allowed in each line  -- "
                    unless @how_many_in_line == 1;
        push @words, $_;
    }
    close IN;
    return \@words;
}

sub _fetch_word_pairs_from_file {
    my $file = shift;
    my @word_pairs;
    open( IN, "$file" ) or die "unable to open the file $file: $!";
    while () {
        next if /^#/;
        next if /^[ ]*$/;
        chomp;
        my @how_many_in_line = grep $_, split /\s+/, $_;
        die "File: $file --- Exactly two words must be in each non-comment or not-empty line -- "
                    unless @how_many_in_line == 2;
        push @word_pairs, $_;
    }
    close IN;
    return \@word_pairs;
}

sub _get_rid_of_wide_chars {
    my $string = shift;
    $string =~ s/[^[:ascii:]]+//g;
#    $string =~ s/\x{FEFF}//g;           to get rid of wide characters
    return $string;
}

sub _find_index_for_given_element {
    my $ele = shift;
    my $array_ref = shift;
    foreach my $i (0..@{$array_ref}-1) {
        return $i if $ele == $array_ref->[$i];
    }
}

sub _check_for_illegal_params {
    my @params = @_;
    my @legal_params = qw / excel_filename
                            which_worksheet
                            raw_tickets_db
                            processed_tickets_db
                            stemmed_tickets_db
                            inverted_index_db
                            tickets_vocab_db
                            idf_db
                            tkt_doc_vecs_db
                            tkt_doc_vecs_normed_db
                            synset_cache_db
                            want_synset_caching
                            add_synsets_to_tickets
                            clustering_fieldname
                            min_word_length
                            min_idf_threshold
                            max_num_syn_words
                            stop_words_file
                            misspelled_words_file
                            unique_id_fieldname
                            want_stemming
                            how_many_retrievals
                            debug1
                            debug2
                            debug3
                          /;
    my $found_match_flag;
    foreach my $param (@params) {

        foreach my $legal (@legal_params) {
            $found_match_flag = 0;
            if ($param eq $legal) {
                $found_match_flag = 1;
                last;
            }
        }
        last if $found_match_flag == 0;
    }
    return $found_match_flag;
}

# Meant only for an un-nested hash:
sub _deep_copy_hash {
    my $ref_in = shift;
    my $ref_out = {};
    foreach ( keys %{$ref_in} ) {
        $ref_out->{$_} = $ref_in->{$_};
    }
    return $ref_out;
}

# from perl docs:
sub _fisher_yates_shuffle {
    my $arr =  shift;
    my $i = @$arr;
    while (--$i) {
        my $j = int rand( $i + 1 );
        @$arr[$i, $j] = @$arr[$j, $i];
    }
}

sub _vec_scalar_product {
    my $vec1 = shift;
    my $vec2 = shift;
    die "Something is wrong --- the two vectors are of unequal length"
        unless @$vec1 == @$vec2;
    my $product;
    for my $i (0..@$vec1-1) {
        $product += $vec1->[$i] * $vec2->[$i];
    }
    return $product;
}

sub _vec_magnitude {
    my $vec = shift;
    my $mag_squared = 0;
    foreach my $num (@$vec) {
        $mag_squared += $num ** 2;
    }
    return sqrt $mag_squared;
}

1;

__END__