TT/Document.pm

filter: Statementstmt 18.9% 21 / 111 Branchbran 0.0% 0 / 20 Conditioncond 0.0% 0 / 17 MC/DCmcdc 0.0% 0 / 11 Subroutinesub 33.3% 9 / 27 Podpod 0.0% 0 / 18 totaltotal 14.7% 30 / 204 CC 24 SCAR 33.2
CC
24
Cov
14%
splitN
110.0
fromFile
20.0
toFile
20.0
CRAP 27.6
« TT/Unigrams.pm ↑ Summary TT/DBx/Document.pm »
1 ## -*- Mode: CPerl -*-
2 ## File: Lingua::TT::Document.pm
3 ## Author: Bryan Jurish <TT/IO.pm>
4 ## Descript: TT I/O: Documents
5  
6  
7 package Lingua::TT::Document;
8 1 ▶use Lingua::TT::Token;
Subroutine BEGIN: called CC=1 SCAR=0.0
9 1 ▶use Lingua::TT::Sentence;
Subroutine BEGIN: called CC=1 SCAR=0.0
10 1 ▶use Lingua::TT::Persistent;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 1 ▶use Encode qw(encode decode);
Subroutine BEGIN: called CC=1 SCAR=0.0
12 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
13  
14 ##==============================================================================
15 ## Globals & Constants
16  
17 1 our @ISA = qw(Lingua::TT::Persistent);
18  
19 ##==============================================================================
20 ## Constructors etc.
21  
22 ## $doc = CLASS_OR_OBJECT->new(@sents)
23 ## + $doc: ARRAY-ref
24 ## [$sent1, $sent2, ..., $sentN]
25 0 ▶sub new {
Subroutine new: not called CC=2 SCAR=17.9
Pod undocumented
26 0 my $that = shift;
27 0 ▶ return bless([@_], ref($that)||$that);
Condition0%0/3
ref $that || $that
l!l&&r!l&&!r
000
MC/DC0%0/2
ref $that || $that
ref $that $that
Truth table0%0/3
A || B
A= ref $thatB= $that
ABresult
1X1
011
000
28 }
29  
30 ## $doc = CLASS_OR_OBJECT->newFromString($str)
31 ## + should be equivalent to CLASS_OR_OBJECT->new()->fromString($str)
32 0 ▶sub newFromString {
Subroutine newFromString: not called CC=1 SCAR=6.9
Pod undocumented
33 0 return $_[0]->new()->fromString($_[1]);
34 }
35  
36 ## $doc2 = $doc->copy($depth)
37 ## + creates a copy of $doc
38 ## + if $deep is 0, only a shallow copy is created (sentences & tokens are shared)
39 ## + if $deep is >=1 (or <0), sentences are copied as well (tokens are still shared)
40 ## + if $deep is >=2 (or <0), tokens are copied as well
41 0 ▶sub copy {
Subroutine copy: not called CC=2 SCAR=17.9
Pod undocumented
42 0 my ($doc,$deep) = @_;
43 0 my $doc2 = bless([],ref($doc));
44 0 ▶ @$doc2 = $deep ? (map {$_->copy($deep-1)} @$doc) : @$doc;
Branch: 1 branch
BranchTrueFalseTotal
$deep ? : 0 0 0
45 0 return $doc2;
46 }
47  
48 ##==============================================================================
49 ## Methods: Access & Manipulation
50  
51 ## $nSents = $doc->nSentences()
52 0 ▶sub nSentences {
Subroutine nSentences: not called CC=1 SCAR=6.9
Pod undocumented
53 0 return scalar(@{$_[0]});
54 }
55  
56 ## $nToks = $doc->nTokens()
57 0 ▶sub nTokens {
Subroutine nTokens: not called CC=2 SCAR=17.9
Pod undocumented
58 0 my $n = 0;
59 0 $n += scalar(@$_) foreach (@{$_[0]});
60 0 return $n;
61 }
62  
63 ## $bool = $doc->isEmpty()
64 ## + true iff $sent has no non-empty sentences
65 0 ▶sub isEmpty {
Subroutine isEmpty: not called CC=1 SCAR=6.9
Pod undocumented
66 0 return !grep {!$_->isEmpty} @{$_[0]};
67 }
68  
69 ## $doc = $doc->rmEmptySentences()
70 ## + removes empty & undefined sentences from @$doc
71 0 ▶sub rmEmptySentences {
Subroutine rmEmptySentences: not called CC=2 SCAR=17.9
Pod undocumented
72 0 ▶ @{$_[0]} = grep {defined($_) && !$_->isEmpty} @{$_[0]};
Condition0%0/3
defined $_ && !$_->isEmpty
!ll&&!rl&&r
000
MC/DC0%0/2
defined $_ && !$_->isEmpty
defined $_ !$_->isEmpty
Truth table0%0/3
A && not(B)
A= defined $_B= !$_->isEmpty
ABresult
0X0
100
111
73 0 return $_[0];
74 }
75  
76 ## $doc = $doc->rmEmptyTokens()
77 ## + removes all empty tokens from all sentences in @$doc
78 0 ▶sub rmEmptyTokens {
Subroutine rmEmptyTokens: not called CC=2 SCAR=17.9
Pod undocumented
79 0 $_->rmEmptyTokens foreach (@{$_[0]});
80 0 return $_[0];
81 }
82  
83 ## $doc = $doc->rmComments()
84 ## + removes all comment pseudo-tokens from all sentences in @$doc
85 0 ▶sub rmComments {
Subroutine rmComments: not called CC=2 SCAR=17.9
Pod undocumented
86 0 $_->rmComments foreach (@{$_[0]});
87 0 return $_[0];
88 }
89  
90 ## $doc = $doc->rmNonVanilla()
91 ## + removes non-vanilla tokens from all sentences in @$doc
92 0 ▶sub rmNonVanilla {
Subroutine rmNonVanilla: not called CC=2 SCAR=17.9
Pod undocumented
93 0 $_->rmNonVanilla foreach (@{$_[0]});
94 0 return $_[0];
95 }
96  
97 ## $doc = $doc->canonicalize()
98 ## + removes all non-vanilla tokens and empty setences from @$doc
99 0 ▶sub canonicalize {
Subroutine canonicalize: not called CC=1 SCAR=6.9
Pod undocumented
100 0 $_[0]->rmNonVanilla;
101 0 $_[0]->rmEmptySentences;
102 0 return $_[0];
103 }
104  
105 ## $tokens = $doc->flat()
106 ## + returns flat list of pseudo-tokens (sentence boundaries replaced with "empty" tokens)
107 0 ▶sub flat {
Subroutine flat: not called CC=1 SCAR=6.9
Pod undocumented
108 0 my $doc = shift;
109 0 my $sflat = Lingua::TT::Sentence->new;
110 0 my $eos = Lingua::TT::Token->new('');
111 0 @$sflat = map {(@$_,$eos)} @$doc;
112 0 pop(@$sflat); ##-- remove final $eos
113 #@$doc = ($sflat);
114 #return $doc;
115 ##--
116 0 return $sflat;
117 }
118  
119 ##==============================================================================
120 ## Methods: I/O
121  
122 ##--------------------------------------------------------------
123 ## Methods: I/O: TT
124  
125 ## $str = $doc->toString()
126 ## + returns string representing $doc
127 1 ▶BEGIN { *saveNativeString = \&toString; }
Subroutine BEGIN: called CC=1 SCAR=0.0
128 0 ▶sub toString {
Subroutine toString: not called CC=1 SCAR=6.9
Pod undocumented
129 0 return join("\n", map {$_->toString} @{$_[0]})."\n";
130 }
131  
132 ## $doc = $doc->fromString($str)
133 ## + parses $doc from string $str
134 1 ▶BEGIN { *loadNativeString = \&fromString; }
Subroutine BEGIN: called CC=1 SCAR=0.0
135 0 ▶sub fromString {
Subroutine fromString: not called CC=1 SCAR=6.9
Pod undocumented
136 #my ($sent,$str) = @_;
137 0 @{$_[0]} = map {Lingua::TT::Sentence->newFromString($_)} split(/(?:\r?\n){2}/,$_[1]);
138 0 return $_[0];
139 }
140  
141 ## $doc = $CLASS_OR_OBJECT->fromFile($filename_or_fh,%opts)
142 ## + parses $doc from file
143 1 ▶BEGIN { *load = *loadNativeFile = \&fromFile; }
Subroutine BEGIN: called CC=1 SCAR=0.0
144 0 ▶sub fromFile {
Subroutine fromFile: not called CC=4 SCAR=30.0
Pod undocumented
145 0 my ($doc,$file,%opts) = @_;
146 0 ▶ my $ttio = Lingua::TT::IO->fromFile($file,%opts)
Branch: 1 branch
BranchTrueFalseTotal
my $ttio = "Lingua::TT::IO"->fromFile($file, %opts) or die +(ref $doc || $doc) . "::fromFile(): open failed for '${file}': $!" 0 0 0
Condition0%0/3
ref $doc || $doc
l!l&&r!l&&!r
000
MC/DC0%0/2
ref $doc || $doc
ref $doc $doc
Truth table0%0/3
A || B
A= ref $docB= $doc
ABresult
1X1
011
000
147 or die((ref($doc)||$doc)."::fromFile(): open failed for '$file': $!");
148 0 my $got = $ttio->getDocument;
149 0 $ttio->close();
150 0 ▶ return $got if (!ref($doc));
Branch: 1 branch
BranchTrueFalseTotal
unless ref $doc 0 0 0
151 0 @$doc = @$got;
152 0 return $doc;
153 }
154  
155 ## $doc = $CLASS_OR_OBJECT->toFile($filename_or_fh,%opts)
156 ## + saves $doc to file
157 1 ▶BEGIN { *save = *saveNativeFile = \&toFile; }
Subroutine BEGIN: called CC=1 SCAR=0.0
158 0 ▶sub toFile {
Subroutine toFile: not called CC=4 SCAR=30.0
Pod undocumented
159 0 my ($doc,$file,%opts) = @_;
160 0 ▶ my $ttio = Lingua::TT::IO->toFile($file,%opts)
Branch: 1 branch
BranchTrueFalseTotal
my $ttio = "Lingua::TT::IO"->toFile($file, %opts) or die +(ref $doc || $doc) . "::toFile(): open failed for '${file}': $!" 0 0 0
Condition0%0/3
ref $doc || $doc
l!l&&r!l&&!r
000
MC/DC0%0/2
ref $doc || $doc
ref $doc $doc
Truth table0%0/3
A || B
A= ref $docB= $doc
ABresult
1X1
011
000
161 or die((ref($doc)||$doc)."::toFile(): open failed for '$file': $!");
162 0 my $rc = $ttio->putDocument($doc);
163 0 $ttio->close();
164 0 ▶ return $rc ? $doc : undef;
Branch: 1 branch
BranchTrueFalseTotal
$rc ? : 0 0 0
165 }
166  
167  
168 ##==============================================================================
169 ## Methods: Shuffle & Split
170  
171 ## $doc = $doc->shuffle(%opts)
172 ## + randomly re-orders sentences in @$doc to @$doc2
173 ## + %opts:
174 ## seed => $seed, ##-- calls srand($seed) if defined
175 0 ▶sub shuffle {
Subroutine shuffle: not called CC=2 SCAR=17.9
Pod undocumented
176 0 my ($doc,%opts) = @_;
177 0 ▶ srand($opts{seed}) if (defined($opts{seed}));
Branch: 1 branch
BranchTrueFalseTotal
if defined $opts{"seed"} 0 0 0
178 0 my @keys = map {rand} @$doc;
179 0 @$doc = @$doc[sort {$keys[$a]<=>$keys[$b]} (0..$#$doc)];
180 0 return $doc;
181 }
182  
183 ## @docs = $doc->splitN($n,%opts) ##-- array context
184 ## \@docs = $doc->splitN($n,%opts) ##-- scalar context
185 ## + splits $doc deterministically into $n roughly equally-sized @docs
186 ## + sentence data is shared (refs) between $doc and @docs
187 ## + for a random split, call $doc->shuffle(seed=>$seed)->splitN($n)
188 ## + %opts:
189 ## contiguous => $bool, ##-- if true, output @docs will represent contiguous sections of input (alias: 'contig')
190 0 ▶sub splitN {
Subroutine splitN: not called CC=10 SCAR=47.0
Pod undocumented
191 0 my ($doc,$n,%opts) = @_;
192 0 my @odocs = map {$doc->new} (1..$n);
193 0 my @osizes = map {0} @odocs;
194 0 ▶ if ($opts{contiguous} || $opts{contig}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($opts{'contiguous'} or $opts{'contig'}) { } 0 0 0
Condition0%0/3
$opts{"contiguous"} or $opts{"contig"}
l!l&&r!l&&!r
000
MC/DC0%0/2
$opts{"contiguous"} or $opts{"contig"}
$opts{"contiguous"} $opts{"contig"}
Truth table0%0/3
A or B
A= $opts{"contiguous"}B= $opts{"contig"}
ABresult
1X1
011
000
195 ##-- contiguous mode
196 0 my $oi = 0;
197 0 ▶ my $osize = $doc->nTokens / ($n || 1);
Condition0%0/2
$n || 1
l!l
00
MC/DC0%0/1
$n || 1
$n
Truth table0%0/2
A
A= $n
Aresult
11
00
198 0 my ($sent);
199 0 foreach $sent (@$doc) {
200 0 push(@{$odocs[$oi]}, $sent);
201 0 $osizes[$oi] += scalar(@$sent);
202 0 ▶ ++$oi if ($osizes[$oi] >= $osize);
Branch: 1 branch
BranchTrueFalseTotal
if $osizes[$oi] >= $osize 0 0 0
203 }
204 } else {
205 ##-- best-split mode
206 0 my ($sent,$oi,$oi_min);
207 0 foreach $sent (@$doc) {
208 ##-- find smallest @odoc
209 0 $oi_min = 0;
210 0 foreach $oi (1..$#odocs) {
211 0 ▶ $oi_min = $oi if ($osizes[$oi] < $osizes[$oi_min]);
Branch: 1 branch
BranchTrueFalseTotal
if $osizes[$oi] < $osizes[$oi_min] 0 0 0
212 }
213 0 push(@{$odocs[$oi_min]}, $sent);
214 0 $osizes[$oi_min] += scalar(@$sent);
215 }
216 }
217 0 ▶ return wantarray ? @odocs : \@odocs;
Branch: 1 branch
BranchTrueFalseTotal
wantarray ? : 0 0 0
218 }
219  
220  
221 ##==============================================================================
222 ## Footer
223 1 1;
224  
225 __END__
« TT/Unigrams.pm ↑ Summary TT/DBx/Document.pm »