lib/App/Greple/xlate/Text.pm

filter: Statementstmt 100% 66 / 66 Branchbran 82.1% 23 / 28 Conditioncond 33.3% 1 / 3 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 100% 10 / 10 Podpod 50.0% 4 / 8 totaltotal 88.8% 104 / 117 CC 19 SCAR 13.8
CC
19
Cov
93%
__ANON__
7.0
__ANON__
4.0
strip
4.0
CRAP 4.0
« blib/lib/App/Greple/xlate/llm.pm ↑ Summary blib/lib/App/Greple/xlate/Filter.pm »
1 package App::Greple::xlate::Text;
2  
3 =encoding utf-8
4  
5 =head1 NAME
6  
7 App::Greple::xlate::Text - text normalization interface
8  
9 =head1 SYNOPSIS
10  
11 my $obj = App::Greple::xlate::Text->new($text, paragraph => 1);
12 my $normalized = $obj->normalized;
13  
14 $result = process($normalized);
15  
16 $obj->unstrip($result);
17  
18 =head1 DESCRIPTION
19  
20 This is an interface used within L<App::Greple::xlate> to normalize
21 text.
22  
23 To get the normalized text, use the C<normalized> method.
24  
25 During normalization process, any whitespace at the beginning and the
26 end of the line is removed. Therefore, the result of processing the
27 normalized text does not preserve the whitespace in the original
28 string; the C<unstrip> method can be used to restore the removed
29 whitespace.
30  
31 =head1 METHODS
32  
33 =over 7
34  
35 =item B<new>
36  
37 Creates an object. The first parameter is the original string; the
38 second and subsequent parameters are pairs of attribute name and values.
39  
40 =over 4
41  
42 =item B<paragraph>
43  
44 Specifies whether or not the text should be treated as a paragraph.
45  
46 If true, multiple lines are concatenated into a single line.
47  
48 If false, multiple strings are processed as they are.
49  
50 In both cases, leading and trailing whitespace is stripped from each
51 line.
52  
53 =back
54  
55 =item B<normalized>()
56  
57 Returns a normalized string.
58  
59 =item B<unstrip>(I<$text>)
60  
61 Recover removed white spaces from normalized text or corresponding
62 cooked text.
63  
64 If not in paragraph mode, the string to be processed must have the
65 same number of lines as the original string.
66  
67 =item B<text>
68  
69 Retrieve original text.
70  
71 =back
72  
73 =cut
74  
75 use v5.14;
76 use warnings;
77 use utf8;
78  
79 use Data::Dumper;
80 use Unicode::EastAsianWidth;
81 use Hash::Util qw(lock_keys);
82  
83 sub new {
84 67 ▶ my $class = shift;
Subroutine new: called CC=1 SCAR=0.0
Pod documented
85 67 my $obj = bless {
86 ATTR => {},
87 TEXT => undef,
88 STRIPPED => undef,
89 NORMALIZED => undef,
90 UNSTRIP => undef,
91 }, $class;
92 67 lock_keys %{$obj};
93 67 $obj->text = shift;
94 67 %{$obj->{ATTR}} = (%{$obj->{ATTR}}, @_);
95 67 $obj;
96 }
97  
98 67 ▶sub text :lvalue { +shift->{TEXT} }
Subroutine text: called CC=1 SCAR=0.0
Pod documented
99 67 ▶sub attr :lvalue { +shift->{ATTR} }
Subroutine attr: called CC=1 SCAR=0.0
Pod undocumented
100  
101 sub normalize {
102 67 ▶ my $obj = shift;
Subroutine normalize: called CC=2 SCAR=6.9
Pod undocumented
103 67 my $paragraph = $obj->attr->{paragraph};
104 67 local $_ = $obj->text;
105 67 my $normalized = do {
106 67 ▶ if (not $paragraph) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $paragraph) { } 8 279 287
107 67 s{^.+}{
108 287 local $_ = ${^MATCH};
109 287 s/\A(\h*)(.*?)(\h*?)\z/$2/;
110 287 $_;
111 }pmger;
112 } else {
113 287 s{^.+(?:\n.+)*}{
114 287 local $_ = ${^MATCH};
115 # remove leading/trailing spaces
116 287 s/\A(\h*)(.*?)(\h*?)\z/$2/;
117 # remove newline after Japanese Punct char
118 287 s/(?<=\p{InFullwidth})(?<=\pP)\n//g;
119 # join Japanese lines without space
120 287 s/(?<=\p{InFullwidth})\n(?=\p{InFullwidth})//g;
121 # join ASCII lines with single space
122 287 s/\s+/ /g;
123 858 $_;
124 }pmger;
125 }
126 };
127 429 return $normalized;
128 }
129  
130 sub normalized {
131 287 ▶ my $obj = shift;
Subroutine normalized: called CC=2 SCAR=8.1
Pod documented
132 287 ▶ $obj->{NORMALIZED} //= $obj->normalize;
Condition33%1/3
$obj->{"NORMALIZED"} //= $obj->normalize
l!l&&r!l&&!r
02870
MC/DC0%0/2
$obj->{"NORMALIZED"} //= $obj->normalize
$obj->{"NORMALIZED"} $obj->normalize
Truth table33%1/3
A //= B
A= $obj->{"NORMALIZED"}B= $obj->normalize
ABresult
1X1
011
000
133 }
134  
135 sub strip {
136 287 ▶ my $obj = shift;
Subroutine strip: called CC=4 SCAR=13.9
Pod undocumented
137 287 my $text = $obj->text;
138 287 ▶ if ($obj->attr->{paragraph}) {
Branch: 1 branch
BranchTrueFalseTotal
if ($obj->attr->{"paragraph"}) 134 8 142
139 8 return $obj->paragraph_strip;
140 }
141 14 my $line_re = qr/.*\n|.+\z/;
142 14 my @text = $text =~ /$line_re/g;
143 my @space = map {
144 14 ▶ [ s/\A(\s+)// ? $1 : '', s/(\h+)$// ? $1 : '' ]
Branch: 2 branches
BranchTrueFalseTotal
s/\A(\s+)//u ? : 11 3 14
s/(\h+)$//u ? : 5 9 14
145 } @text;
146 279 $obj->{STRIPPED} = join '', @text;
147 $obj->{UNSTRIP} = sub {
148 279 ▶ for (@_) {
Subroutine __ANON__: called CC=7 SCAR=19.5
149 279 my @text = /.*\n|.+\z/g;
150 279 ▶ if (@space == @text + 1) {
Branch: 1 branch
BranchTrueFalseTotal
if (@space == @text + 1) 2 6 8
151 287 push @text, '';
152 }
153 287 ▶ die "UNMATCH:\n".Dumper(\@text, \@space) if @text != @space;
Branch: 1 branch
BranchTrueFalseTotal
if @text != @space 0 8 8
154 287 for my $i (keys @text) {
155 142 my($head, $tail) = @{$space[$i]};
156 142 ▶ $text[$i] =~ s/\A/$head/ if length $head > 0;
Branch: 1 branch
BranchTrueFalseTotal
if length $head > 0 11 3 14
157 134 ▶ $text[$i] =~ s/\Z/$tail/ if length $tail > 0;
Branch: 1 branch
BranchTrueFalseTotal
if length $tail > 0 5 9 14
158 }
159 8 $_ = join '', @text;
160 }
161 279 };
162 279 $obj;
163 }
164  
165 sub paragraph_strip {
166 8 ▶ my $obj = shift;
Subroutine paragraph_strip: called CC=3 SCAR=11.0
Pod undocumented
167 8 local *_ = \($obj->{STRIPPED} = $obj->text);
168 14 ▶ my $head = s/\A(\s+)// ? $1 : '' ;
Branch: 1 branch
BranchTrueFalseTotal
s/\A(\s+)//u ? : 4 130 134
169 8 ▶ my $tail = s/(\h+)$// ? $1 : '' ;
Branch: 1 branch
BranchTrueFalseTotal
s/(\h+)$//u ? : 0 134 134
170 $obj->{UNSTRIP} = sub {
171 8 ▶ for (@_) {
Subroutine __ANON__: called CC=4 SCAR=14.0
172 8 ▶ s/\A/$head/ if length $head;
Branch: 1 branch
BranchTrueFalseTotal
if length $head 4 130 134
173 8 ▶ s/\Z/$tail/ if length $tail;
Branch: 1 branch
BranchTrueFalseTotal
if length $tail 0 134 134
174 }
175 8 };
176 8 $obj;
177 }
178  
179 sub unstrip {
180 2 ▶ my $obj = shift;
Subroutine unstrip: called CC=3 SCAR=11.3
Pod documented
181 8 ▶ $obj->strip if not $obj->{UNSTRIP};
Branch: 1 branch
BranchTrueFalseTotal
unless $obj->{"UNSTRIP"} 142 0 142
182 8 ▶ if (my $unstrip = $obj->{UNSTRIP}) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $unstrip = $obj->{"UNSTRIP"}) 142 0 142
183 14 $unstrip->(@_);
184 }
185 14 $obj;
186 }
187  
188 1;
« blib/lib/App/Greple/xlate/llm.pm ↑ Summary blib/lib/App/Greple/xlate/Filter.pm »