CAB/Format/CONLLU.pm

filter: Statementstmt 15.4% 24 / 155 Branchbran 0.0% 0 / 104 Conditioncond 0.0% 0 / 94 MC/DCmcdc 0.0% 0 / 47 Subroutinesub 38.8% 7 / 18 Podpod 100% 11 / 11 totaltotal 9.7% 42 / 429 CC 96 SCAR 71.5
CC
96
Cov
7%
parseConlluFh
1980.0
putToken
1482.0
putSentence
90.0
CRAP 1278.2
« CAB/Format/TT.pm ↑ Summary CAB/Format/TCF.pm »
1 ## -*- Mode: CPerl -*-
2 ##
3 ## File: DTA::CAB::Format::CONLLU.pm
4 ## Author: Bryan Jurish <moocow@cpan.org>
5 ## Description: Datum parser/formatter: CONLL-U format
6 ## + see https://universaldependencies.org/format.html
7 ##
8 ## Format fields:
9 ## ID: Word index, integer starting at 1 for each new sentence;
10 ## FORM: Word form or punctuation symbol.
11 ## LEMMA: Lemma or stem of word form.
12 ## UPOS: Universal part-of-speech tag.
13 ## XPOS: Language-specific part-of-speech tag; underscore if not available.
14 ## FEATS: List of morphological features from the universal feature inventory or underscore if not available.
15 ## HEAD: Head of the current word, which is either a value of ID or zero (0).
16 ## DEPREL: Universal dependency relation to the HEAD (root iff HEAD = 0) or a defined language-specific subtype of one.
17 ## DEPS: Enhanced dependency graph in the form of a list of head-deprel pairs.
18 ## MISC: Any other annotation, split by '|'
19 ##
20 ## Local format conventions for MISC ("MISC1|...|MISCn")
21 ## + MISC fields "MISC$i" is of the form 'ATTR=VALUE' are handled specially for the following ATTRs:
22 ## id=TOKID # sets $tok->{id}
23 ## loc=OFFSET LENGTH # sets $tok->{loc}
24 ## xlit=XTEXT # sets $tok->{xlit}{latin1Text}; also honors CONLL-U "Translit=XTEXT"
25 ## norm=NORM # sets $tok->{moot}{word}
26 ## details=DETAILS # sets $tok->{moot}{details}{details}
27 ## json=JSON # clobbers %$tok with JSON a la Format::TJ
28 ## + VALUEs of specially handled attributes containing literal '%' or '|'
29 ## should have these 2 characters (and only these 2 characters) URI-escaped ('%25', '%7C' respectively)
30  
31 package DTA::CAB::Format::CONLLU;
32 1 ▶use DTA::CAB::Format;
Subroutine BEGIN: called CC=1 SCAR=0.0
33 1 ▶use DTA::CAB::Format::TJ;
Subroutine BEGIN: called CC=1 SCAR=0.0
34 1 ▶use DTA::CAB::Datum ':all';
Subroutine BEGIN: called CC=1 SCAR=0.0
35 1 ▶use IO::File;
Subroutine BEGIN: called CC=1 SCAR=0.0
36 1 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
37 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
38  
39 ##==============================================================================
40 ## Globals
41 ##==============================================================================
42  
43 1 our @ISA = qw(DTA::CAB::Format::TJ);
44  
45 1 ▶BEGIN {
Subroutine BEGIN: called CC=2 SCAR=6.9
46 1 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, filenameRegex=>qr/\.(?i:conllu|conll[_-]u|cab[\.-]connlu|cab[\.-]conll[\.-]u)$/);
47 DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_)
48 1 foreach (qw(conllu conll-u cab-conllu cab-conll-u));
49 }
50  
51 ##==============================================================================
52 ## Constructors etc.
53 ##==============================================================================
54  
55 ## $fmt = CLASS_OR_OBJ->new(%args)
56 ## + object structure: assumed HASH
57 ## {
58 ## ##-- Input
59 ## doc => $doc, ##-- INHERITED: buffered input document
60 ## cuMiscIn => $bool, ##-- NEW: parse special MISC attrs (default=true)
61 ##
62 ## ##-- Output
63 ## outbuf => $stringBuffer, ##-- INHERITED: buffered output
64 ## level => $formatLevel, ##-- OVERRIDE: <0:omit-misc ; 0:default:include-misc,exclude-json, >=1:include-json, >=2:canonical-json
65 ## tagset => $tagset, ##-- auto-convert XPOS->UPOS for $tagset (known values: 'stts' (default))
66 ##
67 ## ##-- Common (INHERITED from Format::TT)
68 ## raw => $bool, ##-- INHERITED: attempt to load/save raw data
69 ## fh => $fh, ##-- INHERITED: IO::Handle for read/write
70 ## utf8 => $bool, ##-- INHERITED: read/write utf8?
71 ## tloc => $attr, ##-- INHERITED: if non-empty, parseTokenizerString() sets $w->{$attr}="$off $len"; default=0
72 ## #defaultFieldName => $name, ##-- INHERITED: default name for unnamed misc-fields; parsed into @{$tok->{other}{$name}}; default=''
73 ## }
74  
75 0 ▶sub new {
Subroutine new: not called CC=1 SCAR=6.9
Pod documented
76 0 my $that = shift;
77 0 my $fmt = $that->SUPER::new(
78 ##-- input
79 doc => undef,
80 cuMiscIn=>1,
81  
82 ##-- output
83 #level => 0,
84 tagset => 'stts',
85  
86 ##-- common
87 utf8 => 1,
88 #defaultFieldName => '',
89 #tloc => undef,
90  
91 ##-- user args
92 @_
93 );
94 0 return $fmt;
95 }
96  
97 ##==============================================================================
98 ## Methods: Persistence
99 ##==============================================================================
100  
101 ## @keys = $class_or_obj->noSaveKeys()
102 ## + returns list of keys not to be saved
103 ## + default just returns empty list
104 ## + INHERITED from Format::TJ
105  
106 ##==============================================================================
107 ## Methods: I/O: Generic
108 ##==============================================================================
109  
110 ## $jxs = $fmt->jsonxs()
111 ## + INHERITED from Format::TJ
112  
113 ## $str = unescapeConllu($str)
114 ## + un-escapes CONLLU value strings using URI-escape sequences ('%7C' => '|', '%25'=>'%')
115 0 ▶sub unescapeConllu {
Subroutine unescapeConllu: not called CC=1 SCAR=6.9
Pod documented
116 0 my $str = shift;
117 0 $str =~ s/%7C/\|/gi;
118 0 $str =~ s/%25/\%/gi;
119 0 return $str;
120 }
121  
122 ## $str = escapeConllu($str)
123 ## + escapes CONLLU value strings using URI-escape sequences ('|'=>'%7C', '%'=>'%25')
124 0 ▶sub escapeConllu {
Subroutine escapeConllu: not called CC=1 SCAR=6.9
Pod documented
125 0 my $str = shift;
126 0 $str =~ s/\%/%25/gi;
127 0 $str =~ s/\|/%7C/gi;
128 0 return $str;
129 }
130  
131 ##--------------------------------------------------------------
132 ## Methods & Data: tagset conversions
133  
134 ## %XPOS2UPOS => ($tagset => $CODE_OR_HASHREF, ...)
135 1 our %XPOS2UPOS =
136 (
137 ##-- xpos2upos:stts: see http://universaldependencies.org/tagset-conversion/de-stts-uposf.html
138 stts => {
139 '$(' => 'PUNCT',
140 '$,' => 'PUNCT',
141 '$.' => 'PUNCT',
142 'ADJA' => 'ADJ',
143 'ADJD' => 'ADJ',
144 'ADV' => 'ADV',
145 'APPO' => 'ADP',
146 'APPR' => 'ADP',
147 'APPRART' => 'ADP',
148 'APZR' => 'ADP',
149 'ART' => 'DET',
150 'CARD' => 'NUM',
151 'FM' => 'X',
152 'ITJ' => 'INTJ',
153 'KOKOM' => 'CCONJ',
154 'KON' => 'CCONJ',
155 'KOUI' => 'SCONJ',
156 'KOUS' => 'SCONJ',
157 'NE' => 'PROPN',
158 'NN' => 'NOUN',
159 'PAV' => 'ADV',
160 'PDAT' => 'DET',
161 'PDS' => 'PRON',
162 'PIAT' => 'DET',
163 'PIDAT' => 'DET',
164 'PIS' => 'PRON',
165 'PPER' => 'PRON',
166 'PPOSAT' => 'DET',
167 'PPOSS' => 'PRON',
168 'PRELAT' => 'DET',
169 'PRELS' => 'PRON',
170 'PRF' => 'PRON',
171 'PTKA' => 'PART',
172 'PTKANT' => 'PART',
173 'PTKNEG' => 'PART',
174 'PTKVZ' => 'ADP',
175 'PTKZU' => 'PART',
176 'PWAT' => 'DET',
177 'PWAV' => 'ADV',
178 'PWS' => 'PRON',
179 'TRUNC' => 'X',
180 'VAFIN' => 'AUX',
181 'VAIMP' => 'AUX',
182 'VAINF' => 'AUX',
183 'VAPP' => 'AUX',
184 'VMFIN' => 'VERB',
185 'VMINF' => 'VERB',
186 'VMPP' => 'VERB',
187 'VVFIN' => 'VERB',
188 'VVIMP' => 'VERB',
189 'VVINF' => 'VERB',
190 'VVIZU' => 'VERB',
191 'VVPP' => 'VERB',
192 'XY' => 'X',
193 },
194 );
195  
196 ##==============================================================================
197 ## Methods: I/O: Block-wise
198 ## + mostly INHERITED from Format::TT
199 ##==============================================================================
200  
201 ##--------------------------------------------------------------
202 ## Methods: I/O: Block-wise: Generic
203  
204 ## %blockOpts = $CLASS_OR_OBJECT->blockDefaults()
205 ## + returns default block options as for blockOptions()
206 ## + INHERITED default just returns as for $CLASS_OR_OBJECT->blockOptions('128k@w')
207  
208 ##--------------------------------------------------------------
209 ## Methods: I/O: Block-wise: Input
210  
211 ## \%head = blockScanHead(\$buf,$io,\%opts)
212 ## + gets header offset, length from (mmaped) \$buf
213 ## + %opts are as for blockScan()
214 ## + OVERRIDE scans for CONLL-U "# newdoc" comment
215 0 ▶sub blockScanHead {
Subroutine blockScanHead: not called CC=2 SCAR=17.9
Pod documented
216 0 my ($fmt,$bufr,$io,$opts) = @_;
217 0 ▶ return [0,$+[0]] if ($$bufr =~ m(\A\n*+(?:#\s*newdoc\b.*\n++)?));
Branch: 1 branch
BranchTrueFalseTotal
if $$bufr =~ /\A\n*+(?:#\s*newdoc\b.*\n++)?/ 0 0 0
218 0 return [0,0];
219 }
220  
221 ## \%head = blockScanFoot(\$buf,$io,\%opts)
222 ## + gets footer offset, length from (mmaped) \$buf
223 ## + %opts are as for blockScan()
224 ## + override INHERITED from Format::TT returns empty
225  
226 ## \@blocks = $fmt->blockScanBody(\$buf,\%opts)
227 ## + scans $filename for block boundaries according to \%opts
228 ## + INHERITED from Format::TT
229  
230 ##--------------------------------------------------------------
231 ## Methods: I/O: Block-wise: Output
232  
233 ## $blk = $fmt->blockStore(\$odata,$blk,$bopt)
234 ## + store output buffer \$buf in $blk->{odata}
235 ## + additionally store keys qw(ofmt ohead odata ofoot) relative to $blk->{odata}
236 ## + override truncates trailing newlines according to $blk->{eos} before calling inherited method
237 ## + INHERITED from Format::TT
238  
239 ## $fmt_or_undef = $fmt->blockAppend($block,$filename)
240 ## + append a block $block to a file $filename
241 ## + $block is a HASH-ref as returned by blockScan()
242 ## + INHERITED from DTA::CAB::Format
243  
244  
245 ##==============================================================================
246 ## Methods: Input
247 ##==============================================================================
248  
249 ##--------------------------------------------------------------
250 ## Methods: Input: Input selection
251  
252 ## $fmt = $fmt->fromFh($filename_or_handle)
253 ## + new override calls Format::fromFh
254 0 ▶sub fromFh {
Subroutine fromFh: not called CC=2 SCAR=17.9
Pod documented
255 #return $_[0]->fromFh_str(@_[1..$#_]);
256 0 my $fmt = shift;
257 0 ▶ $fmt->DTA::CAB::Format::fromFh(@_)
Branch: 1 branch
BranchTrueFalseTotal
$fmt->DTA::CAB::Format::fromFh(@_) or $fmt->logconfess("fromFh(): inherited Format::fromFh() failed: $!") 0 0 0
258 or $fmt->logconfess("fromFh(): inherited Format::fromFh() failed: $!");
259 0 return $fmt->parseConlluFh($_[0]);
260 }
261  
262 ## $fmt = $fmt->fromString(\$string)
263 ## + select input from string $string
264 ## + INHERITED from Format::TJ
265  
266 ##--------------------------------------------------------------
267 ## Methods: Input: Local
268  
269 ## $fmt = $fmt->parseConlluFH($fh)
270 ## + guts for fromFh(): parse handle $fh into local document buffer.
271 0 ▶sub parseConlluFh {
Subroutine parseConlluFh: not called CC=44 SCAR=75.9
Pod documented
272 0 my ($fmt,$fh) = @_;
273 0 $fmt->setLayers($fh);
274 0 my $jxs = $fmt->jsonxs();
275  
276 ##-- ye olde loope
277 0 my (%sa,%doca);
278 0 my $toks = [];
279 0 my @body = qw();
280 0 my ($tok,%cu,@misc,$json);
281 0 ▶ while (defined($_=<$fh>)) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined($_ = readline $fh)) 0 0 0
282 0 ▶ if (/^\#\s*newdoc\s+id\s*=\s*(.*)$/) {
Branch: 9 branches
BranchTrueFalseTotal
if (/^\#\s*newdoc\s+id\s*=\s*(.*)$/) { } 0 0 0
elsif ($_ =~ /^#\s*sent_id\s*=\s*(.*)$/) { } 0 0 0
elsif ($_ =~ /^#\s*text\s*=\s*(.*)$/) { } 0 0 0
elsif ($_ =~ /^#\s*\$TJ:DOC=(.+)$/) { } 0 0 0
elsif ($_ =~ /^#\s*\$TJ:SENT=(.+)$/) { } 0 0 0
elsif ($_ =~ /^# (?:xml\:)?base=(.*)$/) { } 0 0 0
elsif ($_ =~ /^# Sentence (.*)$/) { } 0 0 0
elsif ($_ =~ /^\#(.*)$/) { } 0 0 0
elsif ($_ =~ /^$/) { } 0 0 0
283 ##-- conllu comment: document attribute: doc id (-> "id", "base")
284 0 $doca{'id'} = $doca{'base'} = $1;
285 }
286 elsif ($_ =~ /^#\s*sent_id\s*=\s*(.*)$/) {
287 ##-- connlu comment: sentence attribute: xml:id (-> "sent_id", "id")
288 0 $sa{sent_id} = $sa{'id'} = $1;
289 }
290 elsif ($_ =~ /^#\s*text\s*=\s*(.*)$/) {
291 ##-- connl-u comment: sentence attribute: raw text (-> "stxt", "text")
292 0 $sa{'stxt'} = $sa{'text'} = $1;
293 }
294 elsif ($_ =~ /^#\s*\$TJ:DOC=(.+)$/) {
295 ##-- tj directive: document attributes
296 0 ▶ $json = defined($1) && $1 ? $jxs->decode($1) : {};
Branch: 1 branch
BranchTrueFalseTotal
defined $1 && $1 ? : 0 0 0
Condition0%0/3
defined $1 and $1
!ll&&!rl&&r
000
MC/DC0%0/2
defined $1 and $1
defined $1 $1
Truth table0%0/3
A and B
A= defined $1B= $1
ABresult
0X0
100
111
297 0 @doca{keys %$json} = values %$json;
298 }
299 elsif ($_ =~ /^#\s*\$TJ:SENT=(.+)$/) {
300 ##-- tj directive: setence attributes
301 0 ▶ $json = defined($1) && $1 ? $jxs->decode($1) : {};
Branch: 1 branch
BranchTrueFalseTotal
defined $1 && $1 ? : 0 0 0
Condition0%0/3
defined $1 and $1
!ll&&!rl&&r
000
MC/DC0%0/2
defined $1 and $1
defined $1 $1
Truth table0%0/3
A and B
A= defined $1B= $1
ABresult
0X0
100
111
302 0 @sa{keys %$json} = values %$json;
303 }
304 elsif ($_ =~ /^# (?:xml\:)?base=(.*)$/) {
305 ##-- (tt-compat) special comment: document attribute: xml:base
306 0 $doca{'base'} = $1;
307 }
308 elsif ($_ =~ /^# Sentence (.*)$/) {
309 ##-- (tt-compat) special comment: sentence attribute: xml:id
310 0 $sa{'id'} = $1;
311 }
312 elsif ($_ =~ /^\#(.*)$/) {
313 ##-- generic conllu- comment line: add to '_cmts' attribute of current sentence
314 0 push(@{$sa{_cmts}},$1);
315 }
316 elsif ($_ =~ /^$/) {
317 ##-- empty line: EOS
318 0 ▶ if (%sa || @$toks) {
Branch: 1 branch
BranchTrueFalseTotal
if (%sa or @$toks) 0 0 0
Condition0%0/3
%sa or @$toks
l!l&&r!l&&!r
000
MC/DC0%0/2
%sa or @$toks
%sa @$toks
Truth table0%0/3
A or B
A= %saB= @$toks
ABresult
1X1
011
000
319 0 push(@body,{%sa,tokens=>$toks});
320 0 $toks = [];
321 0 %sa = qw();
322 }
323 }
324 else {
325 ##-- vanilla token
326 0 chomp;
327 @cu{qw(id form lemma upos xpos feats head deprel deps misc)}
328 0 ▶ = map { ($_//'_') eq '_' ? undef : $_ } split(/\t/,$_,10);
Branch: 1 branch
BranchTrueFalseTotal
($_ // '_') eq '_' ? : 0 0 0
Condition0%0/2
$_ // "_"
l!l
00
MC/DC0%0/1
$_ // "_"
$_
Truth table0%0/2
A
A= $_
Aresult
11
00
329 0 $tok = { text=>$cu{form} };
330  
331 ##-- parse: feats
332 ## + example: 'Case=Acc,Dat|Number=Sing' --> {'Case'=>'Acc,Dat', 'Number'=>'Sing'}
333 0 $cu{feats} = { map {split('=',$_,2)} split('|',$cu{feats}) }
334 0 ▶ if ($cu{feats});
Branch: 1 branch
BranchTrueFalseTotal
if $cu{"feats"} 0 0 0
335  
336 ##-- parse: deps (extended dependency graph)
337 ## + example: '0:root|2:conj|4:conj' --> [[0,'root'],[2,'conj'],[4,'conj']]
338 0 $cu{deps} = [ map {[split(':',$_,2)]} split('|',$cu{deps}) ]
339 0 ▶ if ($cu{deps});
Branch: 1 branch
BranchTrueFalseTotal
if $cu{"deps"} 0 0 0
340  
341 ##-- parse: misc
342 0 ▶ if ($cu{misc} && ($fmt->{cuMiscIn}//1)) {
Branch: 2 branches
BranchTrueFalseTotal
if ($cu{'misc'} and $fmt->{'cuMiscIn'} // 1) { } 0 0 0
elsif ($cu{'misc'}) { } 0 0 0
Condition0%0/5
$cu{"misc"} and $fmt->{"cuMiscIn"} // 1
!ll&&!rl&&r
000
$fmt->{"cuMiscIn"} // 1
l!l
00
MC/DC0%0/2
$cu{"misc"} and $fmt->{"cuMiscIn"} // 1
$cu{"misc"} $fmt->{"cuMiscIn"}
Truth table0%0/3
A and B
A= $cu{"misc"}B= $fmt->{"cuMiscIn"}
ABresult
0X0
100
111
343 0 @misc = qw();
344 0 foreach (split(/\|/,$cu{misc})) {
345 0 ▶ if (m/^loc=(?:off=)?([0-9]+) (?:len=)?([0-9]+)$/) {
Branch: 6 branches
BranchTrueFalseTotal
if (/^loc=(?:off=)?([0-9]+) (?:len=)?([0-9]+)$/) { } 0 0 0
elsif (/(?:xml\:?)?id=(.*)$/) { } 0 0 0
elsif (s/^(?:xlit|[Ll]?[Tt]ranslit)=//) { } 0 0 0
elsif (/^norm=(.*)$/) { } 0 0 0
elsif (/^details=(\S*)?(?:\s\@\s(\S+))?\s(?:\~\s)?(.*?)(?: <([0-9\.\+\-eE]+)>)?$/) { } 0 0 0
elsif (/^json=(.+)$/) { } 0 0 0
346 ##-- misc: loc=OFFSET LENGTH (sets $tok->{loc})
347 0 $tok->{loc} = { off=>$1, len=>$2 };
348 }
349 elsif (m/(?:xml\:?)?id=(.*)$/) {
350 ##-- misc: id=TOKID # sets $tok->{id}
351 0 $tok->{id} = unescapeConllu($1);
352 }
353 elsif (s/^(?:xlit|[Ll]?[Tt]ranslit)=//) {
354 ##-- misc: xlit=XTEXT (sets $tok->{xlit}{latin1Text}; also honors CONLL-U "Translit=XTEXT")
355 0 ▶ if (m/^(?:isLatin1|l1)=([01]) (?:isLatinExt|lx)=([01]) (?:latin1Text|l1s)=(.*)$/) {
Branch: 1 branch
BranchTrueFalseTotal
if (/^(?:isLatin1|l1)=([01]) (?:isLatinExt|lx)=([01]) (?:latin1Text|l1s)=(.*)$/) { } 0 0 0
356 0 ▶ $tok->{xlit} = { isLatin1=>($1||0), isLatinExt=>($2||0), latin1Text=>unescapeConllu($3) };
Condition0%0/4
$1 || 0
l!l
00
$2 || 0
l!l
00
MC/DC0%0/1
$1 || 0
$1
MC/DC0%0/1
$2 || 0
$2
Truth table0%0/2
A
A= $1
Aresult
11
00
Truth table0%0/2
A
A= $2
Aresult
11
00
357 } else {
358 0 $tok->{xlit} = { isLatin1=>'', isLatinExt=>'', latin1Text=>unescapeConllu($_) };
359 }
360 }
361 elsif (m/^norm=(.*)$/) {
362 ## misc: norm=NORM (sets $tok->{moot}{word})
363 0 $tok->{moot}{word} = unescapeConllu($1);
364 }
365 elsif (m/^details=(\S*)?(?:\s\@\s(\S+))?\s(?:\~\s)?(.*?)(?: <([0-9\.\+\-eE]+)>)?$/) {
366 ## misc: details=DETAILS (sets $tok->{moot}{details}
367 $tok->{moot}{details} = {
368 0 ▶ (($1//'') ne '' ? (tag=>unescapeConllu($1)) : qw()),
Branch: 2 branches
BranchTrueFalseTotal
($1 // '') ne '' ? : 0 0 0
($2 // '') ne '' ? : 0 0 0
Condition0%0/4
$1 // ""
l!l
00
$2 // ""
l!l
00
MC/DC0%0/1
$1 // ""
$1
MC/DC0%0/1
$2 // ""
$2
Truth table0%0/2
A
A= $1
Aresult
11
00
Truth table0%0/2
A
A= $2
Aresult
11
00
369 (($2//'') ne '' ? (lemma=>unescapeConllu($2)) : qw()),
370 details=>unescapeConllu($3),
371 prob=>$4,
372 };
373 }
374 elsif (m/^json=(.+)$/) {
375 ## misc: json=JSON (clobbers %$tok with JSON a la Format::TJ)
376 0 $json = $jxs->decode(unescapeConllu($1));
377 0 @$tok{keys %$json} = values %$json;
378 }
379 else {
380 ## misc: extra attribute, add to $cu{misc}
381 0 push(@misc, $_);
382 }
383 }
384 0 $cu{misc} = [@misc];
385 }
386 elsif ($cu{misc}) {
387 ##-- MISC: don't parse special attributes, just split
388 0 $cu{misc} = [split(/\|/,$cu{misc})];
389 }
390  
391 ##-- store token
392 0 $tok->{conllu} = {%cu};
393 0 ▶ $tok->{moot}{tag} //= ($cu{xpos} // $cu{upos});
Condition0%0/6
$tok->{"moot"}{"tag"} //= $cu{"xpos"} // $cu{"upos"}
l!l&&r!l&&!r
000
$cu{"xpos"} // $cu{"upos"}
l!l&&r!l&&!r
000
MC/DC0%0/3
$tok->{"moot"}{"tag"} //= $cu{"xpos"} // $cu{"upos"}
$tok->{"moot"}{"tag"} $cu{"xpos"} $cu{"upos"}
Truth table0%0/4
A //= B // C
A= $tok->{"moot"}{"tag"}B= $cu{"xpos"}C= $cu{"upos"}
ABCresult
1XX1
01X1
0011
0000
394 0 ▶ $tok->{moot}{lemma} //= $cu{lemma};
Condition0%0/3
$tok->{"moot"}{"lemma"} //= $cu{"lemma"}
l!l&&r!l&&!r
000
MC/DC0%0/2
$tok->{"moot"}{"lemma"} //= $cu{"lemma"}
$tok->{"moot"}{"lemma"} $cu{"lemma"}
Truth table0%0/3
A //= B
A= $tok->{"moot"}{"lemma"}B= $cu{"lemma"}
ABresult
1X1
011
000
395 0 push(@$toks, $tok)
396 }
397 }
398 0 ▶ push(@body, {%sa,tokens=>$toks}) if (%sa || @$toks); ##-- handle missing EOS at EOF
Branch: 1 branch
BranchTrueFalseTotal
if %sa or @$toks 0 0 0
Condition0%0/3
%sa or @$toks
l!l&&r!l&&!r
000
MC/DC0%0/2
%sa or @$toks
%sa @$toks
Truth table0%0/3
A or B
A= %saB= @$toks
ABresult
1X1
011
000
399  
400 ##-- construct & buffer output document
401 0 $fmt->{doc} = bless({%doca,body=>\@body}, 'DTA::CAB::Document');
402 0 return $fmt;
403 }
404  
405  
406 ##--------------------------------------------------------------
407 ## Methods: Input: Generic API
408  
409 ## $doc = $fmt->parseDocument()
410 ## + INHERITED from Format::TJ
411  
412 ##==============================================================================
413 ## Methods: Output
414 ##==============================================================================
415  
416 ##--------------------------------------------------------------
417 ## Methods: Output: Generic
418  
419 ## $type = $fmt->mimeType()
420 ## + INHERITED default returns text/plain
421  
422 ## $ext = $fmt->defaultExtension()
423 ## + returns default filename extension for this format
424 0 ▶sub defaultExtension { return '.conllu'; }
Subroutine defaultExtension: not called CC=1 SCAR=6.9
Pod documented
425  
426 ##--------------------------------------------------------------
427 ## Methods: Output: Generic API
428  
429 ## $fmt = $fmt->putToken($tok)
430 ## $fmt = $fmt->putToken($tok,$conllu_id)
431 ## + honors $fmt->{level} : <0:omit-misc ; 0:default:include-misc,exclude-json, >=1:include-json, >=2:canonical-json
432 0 ▶sub putToken {
Subroutine putToken: not called CC=38 SCAR=73.0
Pod documented
433 0 my ($fmt,$tok,$id) = @_;
434  
435 ##-- conllu fields ($id,$text,$lemma,$upos,$xpos,$feats,$head,$deprel,$deps) ... everything but MISC
436 0 ▶ my %cu = %{$tok->{conllu} // {}};
Condition0%0/2
$tok->{"conllu"} // {}
l!l
00
MC/DC0%0/1
$tok->{"conllu"} // {}
$tok->{"conllu"}
Truth table0%0/2
A
A= $tok->{"conllu"}
Aresult
11
00
437 0 ▶ $cu{id} //= $id // '_';
Condition0%0/4
$cu{"id"} //= $id // "_"
l!l
00
$id // "_"
l!l
00
MC/DC0%0/1
$cu{"id"} //= $id // "_"
$cu{"id"}
Truth table0%0/2
A
A= $cu{"id"}
Aresult
11
00
438 0 ▶ $cu{form} //= $tok->{text};
Condition0%0/3
$cu{"form"} //= $tok->{"text"}
l!l&&r!l&&!r
000
MC/DC0%0/2
$cu{"form"} //= $tok->{"text"}
$cu{"form"} $tok->{"text"}
Truth table0%0/3
A //= B
A= $cu{"form"}B= $tok->{"text"}
ABresult
1X1
011
000
439 0 ▶ $cu{xpos} //= $tok->{pos} // ($tok->{moot} ? $tok->{moot}{tag} : undef);
Branch: 1 branch
BranchTrueFalseTotal
$tok->{'moot'} ? : 0 0 0
Condition0%0/6
$cu{"xpos"} //= $tok->{"pos"} // ($tok->{"moot"} ? $tok->{"moot"}{"tag"} : undef)
l!l&&r!l&&!r
000
$tok->{"pos"} // ($tok->{"moot"} ? $tok->{"moot"}{"tag"} : undef)
l!l&&r!l&&!r
000
MC/DC0%0/3
$cu{"xpos"} //= $tok->{"pos"} // ($tok->{"moot"} ? $tok->{"moot"}{"tag"} : undef)
$cu{"xpos"} $tok->{"pos"} ($tok->{"moot"} ? $tok->{"moot"}{"tag"} : undef)
Truth table0%0/4
A //= B // C
A= $cu{"xpos"}B= $tok->{"pos"}C= ($tok->{"moot"} ? $tok->{"moot"}{"tag"} : undef)
ABCresult
1XX1
01X1
0011
0000
440 0 ▶ $cu{lemma} //= $tok->{lemma} // ($tok->{moot} ? $tok->{moot}{lemma} : undef);
Branch: 1 branch
BranchTrueFalseTotal
$tok->{'moot'} ? : 0 0 0
Condition0%0/6
$cu{"lemma"} //= $tok->{"lemma"} // ($tok->{"moot"} ? $tok->{"moot"}{"lemma"} : undef)
l!l&&r!l&&!r
000
$tok->{"lemma"} // ($tok->{"moot"} ? $tok->{"moot"}{"lemma"} : undef)
l!l&&r!l&&!r
000
MC/DC0%0/3
$cu{"lemma"} //= $tok->{"lemma"} // ($tok->{"moot"} ? $tok->{"moot"}{"lemma"} : undef)
$cu{"lemma"} $tok->{"lemma"} ($tok->{"moot"} ? $tok->{"moot"}{"lemma"} : undef)
Truth table0%0/4
A //= B // C
A= $cu{"lemma"}B= $tok->{"lemma"}C= ($tok->{"moot"} ? $tok->{"moot"}{"lemma"} : undef)
ABCresult
1XX1
01X1
0011
0000
441  
442 ##-- implicit xpos->upos conversion
443 0 ▶ if (!$cu{upos} && $fmt->{tagset} && defined(my $x2u=$XPOS2UPOS{$fmt->{tagset}})) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $cu{"upos"} and $fmt->{"tagset"} and defined(my $x2u = $DTA::CAB::Format::CONLLU::XPOS2UPOS{$fmt->{"tagset"}})) 0 0 0
Condition0%0/6
not $cu{"upos"} and $fmt->{"tagset"} and defined(my $x2u = $DTA::CAB::Format::CONLLU::XPOS2UPOS{$fmt->{"tagset"}})
!ll&&!rl&&r
000
not $cu{"upos"} and $fmt->{"tagset"}
!ll&&!rl&&r
000
MC/DC0%0/3
not $cu{"upos"} and $fmt->{"tagset"} and defined(my $x2u = $DTA::CAB::Format::CONLLU::XPOS2UPOS{$fmt->{"tagset"}})
not $cu{"upos"} $fmt->{"tagset"} defined(my $x2u = $DTA::CAB::Format::CONLLU::XPOS2UPOS{$fmt->{"tagset"}})
Truth table0%0/4
not(A) and B and C
A= not $cu{"upos"}B= $fmt->{"tagset"}C= defined(my $x2u = $DTA::CAB::Format::CONLLU::XPOS2UPOS{$fmt->{"tagset"}})
ABCresult
0XX0
10X0
1100
1111
444 0 ▶ if (UNIVERSAL::isa($x2u,'HASH')) {
Branch: 2 branches
BranchTrueFalseTotal
if (UNIVERSAL::isa($x2u, 'HASH')) { } 0 0 0
elsif (UNIVERSAL::isa($x2u, 'CODE')) { } 0 0 0
445 0 ▶ $cu{upos} = $x2u->{$cu{xpos}//''} // 'X';
Condition0%0/4
$$x2u{$cu{"xpos"} // ""} // "X"
l!l
00
$cu{"xpos"} // ""
l!l
00
MC/DC0%0/1
$$x2u{$cu{"xpos"} // ""} // "X"
$$x2u{$cu{"xpos"} // ""}
MC/DC0%0/1
$cu{"xpos"} // ""
$cu{"xpos"}
Truth table0%0/2
A
A= $$x2u{$cu{"xpos"} // ""}
Aresult
11
00
Truth table0%0/2
A
A= $cu{"xpos"}
Aresult
11
00
446 }
447 elsif (UNIVERSAL::isa($x2u,'CODE')) {
448 0 ▶ $cu{upos} = $x2u->($cu{xpos}) // 'X';
Condition0%0/2
&$x2u($cu{"xpos"}) // "X"
l!l
00
MC/DC0%0/1
&$x2u($cu{"xpos"}) // "X"
&$x2u($cu{"xpos"})
Truth table0%0/2
A
A= &$x2u($cu{"xpos"})
Aresult
11
00
449 }
450 else {
451 0 confess(__PACKAGE__, "::putToken(): PoS-translation table must be a HASH- or CODE-ref");
452 }
453 }
454  
455 ##-- special MISC ATTRS
456 0 my @misc = qw();
457 0 ▶ if (($fmt->{level}//0) >= 0) {
Branch: 1 branch
BranchTrueFalseTotal
if (($fmt->{"level"} // 0) >= 0) 0 0 0
Condition0%0/2
$fmt->{"level"} // 0
l!l
00
MC/DC0%0/1
$fmt->{"level"} // 0
$fmt->{"level"}
Truth table0%0/2
A
A= $fmt->{"level"}
Aresult
11
00
458 ##-- include special misc ATTRS
459 0 ▶ push(@misc, "id=$tok->{id}") if ($tok->{id});
Branch: 1 branch
BranchTrueFalseTotal
if $tok->{"id"} 0 0 0
460 0 ▶ push(@misc, "loc=$tok->{loc}{off} $tok->{loc}{len}") if ($tok->{loc});
Branch: 1 branch
BranchTrueFalseTotal
if $tok->{"loc"} 0 0 0
461 0 ▶ push(@misc, "Translit=$tok->{xlit}{latin1Text}") if ($tok->{xlit});
Branch: 1 branch
BranchTrueFalseTotal
if $tok->{"xlit"} 0 0 0
462 0 ▶ if ($tok->{moot}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($tok->{"moot"}) 0 0 0
463 push(@misc, "norm=$tok->{moot}{word}")
464 0 ▶ if (defined($tok->{moot}{word}));
Branch: 1 branch
BranchTrueFalseTotal
if defined $tok->{"moot"}{"word"} 0 0 0
465 push(@misc, "details=$tok->{moot}{details}{details} <".($tok->{moot}{details}{prob}||$tok->{moot}{details}{cost}||0).">")
466 0 ▶ if (defined($tok->{moot}{details}{details}));
Branch: 1 branch
BranchTrueFalseTotal
if defined $tok->{"moot"}{"details"}{"details"} 0 0 0
Condition0%0/5
$tok->{'moot'}{'details'}{'prob'} || $tok->{'moot'}{'details'}{'cost'} || 0
l!l
00
$tok->{"moot"}{"details"}{"prob"} or $tok->{"moot"}{"details"}{"cost"}
l!l&&r!l&&!r
000
MC/DC0%0/2
$tok->{'moot'}{'details'}{'prob'} || $tok->{'moot'}{'details'}{'cost'} || 0
$tok->{"moot"}{"details"}{"prob"} $tok->{"moot"}{"details"}{"cost"}
Truth table0%0/3
A or B
A= $tok->{"moot"}{"details"}{"prob"}B= $tok->{"moot"}{"details"}{"cost"}
ABresult
1X1
011
000
467 }
468  
469 0 ▶ if (($fmt->{level}//0) >= 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (($fmt->{"level"} // 0) >= 1) 0 0 0
Condition0%0/2
$fmt->{"level"} // 0
l!l
00
MC/DC0%0/1
$fmt->{"level"} // 0
$fmt->{"level"}
Truth table0%0/2
A
A= $fmt->{"level"}
Aresult
11
00
470 ##-- include misc/json
471 0 push(@misc, "json=".$fmt->jsonxs->encode($tok));
472 }
473  
474 0 $_ = escapeConllu($_) foreach (@misc)
475 }
476  
477 $fmt->{fh}->print
478 (
479 ##-- comments
480 0 ($tok->{_cmts} ? join('', map {"#$_\n"} map {split(/\n/,$_)} @{$tok->{_cmts}}) : ''),
481 ##
482 join("\t",
483 ##-- conllu fixed fields ($id,$text,$lemma,$upos,$xpos,$feats,$head,$deprel,$deps) ... everything but MISC
484 0 ▶ map { ($_//'') eq '' ? '_' : $_ }
Branch: 2 branches
BranchTrueFalseTotal
($_ // '') eq '' ? : 0 0 0
UNIVERSAL::isa($cu{'feats'}, 'HASH') ? : 0 0 0
Condition0%0/2
$_ // ""
l!l
00
MC/DC0%0/1
$_ // ""
$_
Truth table0%0/2
A
A= $_
Aresult
11
00
485 @cu{qw(id form lemma upos xpos)},
486 (UNIVERSAL::isa($cu{feats},'HASH')
487 0 ▶ ? join('|',map {"$_:$cu{feats}{$_}"} sort keys %{$cu{feats}})
Branch: 1 branch
BranchTrueFalseTotal
UNIVERSAL::isa($cu{'deps'}, 'ARRAY') ? : 0 0 0
488 : $cu{feats}),
489 @cu{qw(head deprel)},
490 (UNIVERSAL::isa($cu{deps},'ARRAY')
491 0 ? join('|',map {"$_->[0]:$_->[1]"} @{$cu{deps}})
492 : $cu{deps}),
493 ##-- conllu MISC (may be empty, depending on $fmt->{level})
494 0 ▶ join('|', @{$cu{misc}//[]}, @misc),
Branch: 1 branch
BranchTrueFalseTotal
$tok->{'_cmts'} ? : 0 0 0
Condition0%0/2
$cu{"misc"} // []
l!l
00
MC/DC0%0/1
$cu{"misc"} // []
$cu{"misc"}
Truth table0%0/2
A
A= $cu{"misc"}
Aresult
11
00
495 ),
496 "\n",
497 );
498  
499 0 return $fmt;
500 }
501  
502  
503 ## $fmt = $fmt->putSentence($sent)
504 ## + concatenates formatted tokens, adding sentence-id comment if available
505 0 ▶sub putSentence {
Subroutine putSentence: not called CC=9 SCAR=45.0
Pod documented
506 #my ($fmt,$sent) = @_;
507 0 ▶ my $sh = {(map {$_ eq 'tokens' ? qw() : ($_=>$_[1]{$_})} keys %{$_[1]})};
Branch: 1 branch
BranchTrueFalseTotal
$_ eq 'tokens' ? : 0 0 0
508 0 ▶ $_[0]{fh}->print(join('', map {"#$_\n"} map {split(/\n/,$_)} @{$_[1]{_cmts}})) if ($_[1]{_cmts});
Branch: 1 branch
BranchTrueFalseTotal
if $_[1]{"_cmts"} 0 0 0
509 0 ▶ $_[0]{fh}->print("# sent_id = ", ($_[1]{id}//''), "\n");
Condition0%0/2
$_[1]{"id"} // ""
l!l
00
MC/DC0%0/1
$_[1]{"id"} // ""
$_[1]{"id"}
Truth table0%0/2
A
A= $_[1]{"id"}
Aresult
11
00
510 0 ▶ $_[0]{fh}->print("# text = $_[1]{stxt}\n") if (defined($_[1]{stxt}));
Branch: 1 branch
BranchTrueFalseTotal
if defined $_[1]{"stxt"} 0 0 0
511 0 ▶ $_[0]{fh}->print('# $TJ:SENT=', $_[0]->jsonxs->encode($sh), "\n") if (%$sh && ($_[0]{level}//0) >= 1);
Branch: 1 branch
BranchTrueFalseTotal
if %$sh and ($_[0]{"level"} // 0) >= 1 0 0 0
Condition0%0/5
%$sh and ($_[0]{"level"} // 0) >= 1
!ll&&!rl&&r
000
$_[0]{"level"} // 0
l!l
00
MC/DC0%0/2
%$sh and ($_[0]{"level"} // 0) >= 1
%$sh $_[0]{"level"}
Truth table0%0/3
A and B
A= %$shB= $_[0]{"level"}
ABresult
0X0
100
111
512 0 my $i = 0;
513 0 $_[0]->putToken($_,++$i) foreach (@{toSentence($_[1])->{tokens}});
514 0 $_[0]{fh}->print("\n");
515 0 return $_[0];
516 }
517  
518 ## $fmt = $fmt->putDocument($doc)
519 ## + concatenates formatted sentences, adding document 'xmlbase' comment if available
520 1 our %TJ_BAD_DOC_KEYS = %DTA::CAB::Format::TJ::TJ_BAD_DOC_KEYS;
521 0 ▶sub putDocument {
Subroutine putDocument: not called CC=5 SCAR=34.0
Pod documented
522 #my ($fmt,$doc) = @_;
523 0 my $dh = { (map {($_=>$_[1]{$_})} grep {!exists($TJ_BAD_DOC_KEYS{$_})} keys %{$_[1]}) };
524 0 ▶ $_[0]{fh}->print('# $TJ:DOC=', $_[0]->jsonxs->encode($dh), "\n") if (%$dh && ($_[0]{level}//0) >= 1);
Branch: 1 branch
BranchTrueFalseTotal
if %$dh and ($_[0]{"level"} // 0) >= 1 0 0 0
Condition0%0/5
%$dh and ($_[0]{"level"} // 0) >= 1
!ll&&!rl&&r
000
$_[0]{"level"} // 0
l!l
00
MC/DC0%0/2
%$dh and ($_[0]{"level"} // 0) >= 1
%$dh $_[0]{"level"}
Truth table0%0/3
A and B
A= %$dhB= $_[0]{"level"}
ABresult
0X0
100
111
525 0 $_[0]->putSentence($_) foreach (@{toDocument($_[1])->{body}});
526 0 return $_[0];
527 }
528  
529 ## $fmt = $fmt->putData($data)
530 ## + puts raw data (uses forceDocument())
531 ## + OVERRIDE uses Format::TT implementation
532 0 ▶sub putData {
Subroutine putData: not called CC=1 SCAR=6.9
Pod documented
533 0 return $_[0]->DTA::CAB::Format::TT->putData($_[1]);
534 }
535  
536  
537 1 1; ##-- be happy
538  
539 __END__
« CAB/Format/TT.pm ↑ Summary CAB/Format/TCF.pm »