| 1 | package Lingua::EN::WSD::CorpusBased::Corpus; | ||
| 2 | |||
| 3 | our $VERSION = '0.11'; | ||
| 4 | |||
| 5 | |||
| 6 | use stem; | ||
| 7 | use warnings; | ||
| 8 | use strict; | ||
| 9 | |||
| 10 | # The demo- and test corpus | ||
| 11 | my %democorpus = ('1' => ['hello','world'], | ||
| 12 | '2' => ['application','e-mail'], | ||
| 13 | '3' => ['world','peace','love'], | ||
| 14 | '4' => ['e-mail','program'], | ||
| 15 | '5' => ['e-mail','job']); | ||
| 16 | |||
| 17 | sub new { | ||
| 18 | my $class = shift; | ||
| 19 | my %args = ('debug' => 0, | ||
| 20 | 'wnref' => 0, | ||
| 21 | @_); | ||
| 22 | |||
| 23 | return -1 if (! exists($args{'corpus'})); | ||
| 24 | |||
| 25 | my $this = { 'filename' => $args{'corpus'}, | ||
| 26 | 'debug' => $args{'debug'}, | ||
| 27 | 'initialized' => 0, | ||
| 28 | 'wnref' => $args{'wnref'}, | ||
| 29 | 'stemmer' => stem->new($args{'wnref'}) }; | ||
| 30 | |||
| 31 | if ($this->{'filename'} eq '_democorpus_') { | ||
| 32 | $this->{'corpus'} = \%democorpus; | ||
| 33 | } else { | ||
| 34 | open FH, $this->{'filename'}; | ||
| 35 | my $linenumber = 1; | ||
| 36 | while(<FH>) { | ||
| 37 | my $line = $_; | ||
| 38 | $line = $this->{'stemmer'}->stemString($_) if ($this->{'wnref'}); | ||
| 39 | my @line = split(/ /, $line); | ||
| 40 | #pop @line; | ||
| 41 | $this->{'corpus'}->{$linenumber} = \@line; | ||
| 42 | $linenumber++; | ||
| 43 | }; | ||
| 44 | close FH; | ||
| 45 | }; | ||
| 46 | |||
| 47 | $this->{'cache'} = {}; | ||
| 48 | print STDERR "Corpus object created.\n" if ($this->{'debug'} > 0); | ||
| 49 | return bless $this, $class; | ||
| 50 | } | ||
| 51 | |||
| 52 | sub init { | ||
| 53 | my ($self) = @_; | ||
| 54 | foreach my $linenumber (keys %{$self->{'corpus'}}) { | ||
| 55 | my @sentence = @{ $self->{'corpus'}->{$linenumber} }; | ||
| 56 | foreach my $word (@sentence) { | ||
| 57 | $self->{'corpusptr'}->{$word}->{$linenumber} = 1; | ||
| 58 | }; | ||
| 59 | }; | ||
| 60 | $self->{'initialized'} = 1; | ||
| 61 | print STDERR "Corpus object initialized.\n" if ($self->{'debug'} > 0); | ||
| 62 | } | ||
| 63 | |||
| 64 | sub sentences { | ||
| 65 | my $self = shift; | ||
| 66 | my @words = @_; | ||
| 67 | |||
| 68 | if ($self->{'initialized'}) { | ||
| 69 | my @gsen = keys %{$self->{'corpus'}}; | ||
| 70 | my $good = \@gsen; | ||
| 71 | foreach my $word (@words) { | ||
| 72 | #print ref $self->{'corpusptr'}->{$word}; | ||
| 73 | my @t = keys %{$self->{'corpusptr'}->{$word}}; | ||
| 74 | $good = $self->merge_lists($good, \@t); | ||
| 75 | }; | ||
| 76 | print 'Words ('.join(',',@words).') are in sentences '.join(',',@$good)."\n" | ||
| 77 | if ($self->{'debug'} > 1); | ||
| 78 | return @$good; | ||
| 79 | }; | ||
| 80 | return (); | ||
| 81 | } | ||
| 82 | |||
| 83 | # The function returns the number of sentences containing these words | ||
| 84 | sub count { | ||
| 85 | my $self = shift; | ||
| 86 | # @_ contains a list of words. | ||
| 87 | |||
| 88 | my @words = @_; | ||
| 89 | |||
| 90 | if (exists $self->{'cache'}->{join('-',@words)}) { | ||
| 91 | my $c = $self->{'cache'}->{join('-',@words)}; | ||
| 92 | print STDERR " ".join(",",@words).": $c (cached)\n" if ($self->{'debug'} > 0); | ||
| 93 | return $c; | ||
| 94 | }; | ||
| 95 | |||
| 96 | my $c = 0; | ||
| 97 | if ($self->{'initialized'}) { | ||
| 98 | $c = scalar $self->sentences(@words); | ||
| 99 | } else { | ||
| 100 | foreach my $sentence (values %{$self->{'corpus'}}) { | ||
| 101 | my $one = 1; | ||
| 102 | #print $sentence; | ||
| 103 | foreach my $word (@words) { | ||
| 104 | my $regexp = $word; #.($self->{'flexibility'}?'s?':''); | ||
| 105 | $one = 0 if (! grep { /^$regexp$/i } @$sentence); | ||
| 106 | } | ||
| 107 | $c++ if ($one); | ||
| 108 | }; | ||
| 109 | |||
| 110 | }; | ||
| 111 | print STDERR " ".join(",",@words).": $c\n" if ($self->{'debug'} > 0); | ||
| 112 | $self->{'cache'}->{join('-',@words)} = $c; | ||
| 113 | return $c; | ||
| 114 | |||
| 115 | } | ||
| 116 | |||
| 117 | sub merge_lists { | ||
| 118 | my ($self,$list1,$list2) = @_; | ||
| 119 | my @result = (); | ||
| 120 | foreach my $item (@$list1) { | ||
| 121 | push (@result, $item) if (grep /$item/, @$list2); | ||
| 122 | } | ||
| 123 | return \@result; | ||
| 124 | } | ||
| 125 | |||
| 126 | sub empty_cache { | ||
| 127 | my ($self) = @_; | ||
| 128 | $self->{'cache'} = {}; | ||
| 129 | } | ||
| 130 | |||
| 131 | sub line { | ||
| 132 | my ($self,$num) = @_; | ||
| 133 | return $self->{'corpus'}->{$num}; | ||
| 134 | } | ||
| 135 | |||
| 136 | 1; | ||
| 137 | |||
| 138 | |||
| 139 | __END__ |