blib/lib/Text/TFIDF/Ngram.pm

filter: Statementstmt 100% 93 / 93 Branchbran 86.6% 26 / 30 Conditioncond 64.7% 11 / 17 MC/DCmcdc 50.0% 5 / 10 Subroutinesub 100% 20 / 20 Podpod 80.0% 4 / 5 totaltotal 90.8% 159 / 175 CC 27 SCAR 14.6
CC
27
Cov
93%
_process_ngrams
9.0
__ANON__
6.7
tfidf_by_file
5.0
CRAP 4.3
↑ Summary
1 package Text::TFIDF::Ngram;
2 2 our $AUTHORITY = 'cpan:GENE';
3  
4 # ABSTRACT: Compute the TF-IDF measure for ngram phrases
5  
6 2 our $VERSION = '0.0509';
7  
8 2 ▶use strictures 2;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 2 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 2 ▶use Lingua::EN::Ngram;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 2 ▶use Lingua::StopWords qw( getStopWords );
Subroutine BEGIN: called CC=1 SCAR=0.0
12 2 ▶use List::Util qw( sum0 );
Subroutine BEGIN: called CC=1 SCAR=0.0
13 2 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
14 2 ▶use namespace::clean;
Subroutine BEGIN: called CC=1 SCAR=0.0
15  
16  
17 has files => (
18 is => 'ro',
19 7 ▶ isa => sub { croak 'Invalid files list' unless ref $_[0] eq 'ARRAY' },
Subroutine __ANON__: called CC=2 SCAR=7.6
Branch: 1 branch
BranchTrueFalseTotal
unless ref $_[0] eq "ARRAY" 0 7 7
20 2 );
21  
22  
23 has size => (
24 is => 'ro',
25 8 ▶ isa => sub { croak 'Invalid integer' unless $_[0] && $_[0] =~ /^\d+$/ && $_[0] > 0 },
Subroutine __ANON__: called CC=4 SCAR=19.1
Branch: 1 branch
BranchTrueFalseTotal
unless $_[0] and $_[0] =~ /^\d+$/ and $_[0] > 0 0 8 8
Condition33%2/6
$_[0] and $_[0] =~ /^\d+$/ and $_[0] > 0
!ll&&!rl&&r
008
$_[0] and $_[0] =~ /^\d+$/
!ll&&!rl&&r
008
MC/DC0%0/3
$_[0] and $_[0] =~ /^\d+$/ and $_[0] > 0
$_[0] $_[0] =~ /^\d+$/ $_[0] > 0
Truth table25%1/4
A and B and C
A= $_[0]B= $_[0] =~ /^\d+$/C= $_[0] > 0
ABCresult
0XX0
10X0
1100
1111
26 1 ▶ default => sub { 1 },
Subroutine __ANON__: called CC=1 SCAR=0.0
27 2 );
28  
29  
30 has stopwords => (
31 is => 'ro',
32 8 ▶ isa => sub { croak 'Invalid Boolean' unless defined $_[0] },
Subroutine __ANON__: called CC=2 SCAR=7.6
Branch: 1 branch
BranchTrueFalseTotal
unless defined $_[0] 0 8 8
33 1 ▶ default => sub { 1 },
Subroutine __ANON__: called CC=1 SCAR=0.0
34 2 );
35  
36  
37 has punctuation => (
38 is => 'ro',
39 7 ▶ default => sub { qr/(?!')[[:punct:]]/ },
Subroutine __ANON__: called CC=1 SCAR=0.0
40 2 );
41  
42  
43 has lowercase => (
44 is => 'ro',
45 6 ▶ default => sub { 0 },
Subroutine __ANON__: called CC=1 SCAR=0.0
46 2 );
47  
48  
49 2 has counts => (
50 is => 'ro',
51 init_arg => undef,
52 );
53  
54  
55 2 has file_tfidf => (
56 is => 'ro',
57 init_arg => undef,
58 );
59  
60  
61 8 ▶sub BUILD {
Subroutine BUILD: called CC=4 SCAR=13.9
Pod undocumented
62 8 my ( $self, $args ) = @_;
63  
64 8 ▶ return unless $args->{files};
Branch: 1 branch
BranchTrueFalseTotal
unless $args->{"files"} 1 7 8
65  
66 7 ▶ $args->{size} ||= 1;
Condition50%1/2
$args->{"size"} ||= 1
l!l
70
MC/DC0%0/1
$args->{"size"} ||= 1
$args->{"size"}
Truth table50%1/2
A
A= $args->{"size"}
Aresult
11
00
67  
68 7 for my $file ( @{ $args->{files} } ) {
69 11 $self->_process_ngrams( $file, $args->{size} );
70 }
71 }
72  
73 11 ▶sub _process_ngrams {
Subroutine _process_ngrams: called CC=9 SCAR=22.0
74 11 my ( $self, $file, $size ) = @_;
75  
76 11 my $ngram = Lingua::EN::Ngram->new( file => $file );
77 11 my $phrase = $ngram->ngram($size);
78  
79 11 ▶ if ( $self->lowercase ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->lowercase) 2 9 11
80 2 $phrase = { map { lc $_ => $phrase->{$_} } keys %$phrase };
81 }
82  
83 11 my $stop = getStopWords('en');
84  
85 11 my $counts;
86  
87 11 for my $p ( keys %$phrase ) {
88 next if $self->stopwords
89 91 ▶ && grep { $stop->{$_} } split /\s+/, $p; # Exclude stopwords
Branch: 1 branch
BranchTrueFalseTotal
if $self->stopwords and grep {$stop->{$_};} split(/\s+/, $p, 0) 5 86 91
Condition100%3/3
$self->stopwords and grep {$stop->{$_};} split(/\s+/, $p, 0)
!ll&&!rl&&r
8335
MC/DC100%2/2
$self->stopwords and grep {$stop->{$_};} split(/\s+/, $p, 0)
$self->stopwords grep {$stop->{$_};} split(/\s+/, $p, 0)
Truth table100%3/3
A and B
A= $self->stopwordsB= grep {$stop->{$_};} split(/\s+/, $p, 0)
ABresult
0X0
100
111
90  
91 86 my $pat = $self->punctuation;
92 86 ▶ $p =~ s/$pat//g if $pat; # Remove unwanted punctuation
Branch: 1 branch
BranchTrueFalseTotal
if $pat 73 13 86
93  
94 # Skip if we don't have an ngram of the requested size anymore
95 86 my @p = grep { $_ } split /\s+/, $p;
96 86 ▶ next unless @p == $size;
Branch: 1 branch
BranchTrueFalseTotal
unless @p == $size 19 67 86
97  
98 # Skip an ngram with a lone single quote (allowed by the default punctuation)
99 67 ▶ next if grep { $_ eq "'" } @p;
Branch: 1 branch
BranchTrueFalseTotal
if grep {$_ eq "'";} @p 0 67 67
100  
101 67 ▶ $p = lc $p if $self->lowercase;
Branch: 1 branch
BranchTrueFalseTotal
if $self->lowercase 17 50 67
102  
103 67 $counts->{$p} = $phrase->{$p};
104 }
105  
106 11 $self->{counts}{$file} = $counts;
107 }
108  
109  
110 15 ▶sub tf {
Subroutine tf: called CC=3 SCAR=11.0
Pod documented
111 15 my ( $self, $file, $phrase ) = @_;
112 15 ▶ return 0 unless exists $self->counts->{$file} && exists $self->counts->{$file}{$phrase};
Branch: 1 branch
BranchTrueFalseTotal
unless exists $self->counts->{$file} and exists $self->counts->{$file}{$phrase} 1 14 15
Condition66%2/3
exists $self->counts->{$file} and exists $self->counts->{$file}{$phrase}
!ll&&!rl&&r
0114
MC/DC50%1/2
exists $self->counts->{$file} and exists $self->counts->{$file}{$phrase}
exists $self->counts->{$file} exists $self->counts->{$file}{$phrase}
Truth table66%2/3
A and B
A= exists $self->counts->{$file}B= exists $self->counts->{$file}{$phrase}
ABresult
0X0
100
111
113 14 return $self->counts->{$file}{$phrase} / sum0( values %{ $self->counts->{$file} } );
114 }
115  
116  
117 20 ▶sub idf {
Subroutine idf: called CC=4 SCAR=13.9
Pod documented
118 20 my ( $self, $phrase ) = @_;
119  
120 20 my $count = 0;
121  
122 20 for my $file ( keys %{ $self->counts } ) {
123 40 ▶ $count++ if exists $self->counts->{$file}{$phrase};
Branch: 1 branch
BranchTrueFalseTotal
if exists $self->counts->{$file}{$phrase} 24 16 40
124 }
125  
126 20 ▶ unless ( $count ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($count) 2 18 20
127 2 carp "'$phrase' is not present in any document";
128 2 return undef;
129 }
130  
131 18 return - log( $count / keys %{ $self->counts } ) / log(10) + 0;
132 }
133  
134  
135 14 ▶sub tfidf {
Subroutine tfidf: called CC=2 SCAR=6.9
Pod documented
136 14 my ( $self, $file, $phrase ) = @_;
137 14 my $idf = $self->idf($phrase);
138 14 ▶ return undef unless $idf;
Branch: 1 branch
BranchTrueFalseTotal
unless $idf 6 8 14
139 8 return $self->tf( $file, $phrase ) * $idf;
140 }
141  
142  
143 1 ▶sub tfidf_by_file {
Subroutine tfidf_by_file: called CC=5 SCAR=16.1
Pod documented
144 1 my ($self) = @_;
145  
146 1 my %seen;
147  
148 1 for my $file ( keys %{ $self->counts } ) {
149 2 for my $phrase ( keys %{ $self->counts->{$file} } ) {
150 8 my $tfidf = $self->tfidf( $file, $phrase );
151  
152 8 ▶ next if $seen{$phrase}++ || !defined $tfidf;
Branch: 1 branch
BranchTrueFalseTotal
if $seen{$phrase}++ or not defined $tfidf 4 4 8
Condition100%3/3
$seen{$phrase}++ or not defined $tfidf
l!l&&r!l&&!r
224
MC/DC100%2/2
$seen{$phrase}++ or not defined $tfidf
$seen{$phrase}++ not defined $tfidf
Truth table100%3/3
A or not(B)
A= $seen{$phrase}++B= not defined $tfidf
ABresult
1X1
011
000
153  
154 4 $self->{file_tfidf}{$file}{$phrase} = $tfidf;
155 }
156 }
157  
158 1 return $self->file_tfidf;
159 }
160  
161 2 1;
162  
163 __END__
↑ Summary