Skullplot/ClassifyColumns.pm

filter: Statementstmt 91.3% 21 / 23 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 8 / 8 Podpod n/a totaltotal 93.5% 29 / 31 CC 1 SCAR 0.4
CC
1
Cov
91%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.0
« Skullplot.pm ↑ Summary
1 package Graphics::Skullplot::ClassifyColumns;
2 3 ▶use Moo;
Subroutine BEGIN: called CC=1 SCAR=0.0
3 3 ▶use MooX::Types::MooseLike::Base qw(:all);
Subroutine BEGIN: called CC=1 SCAR=0.0
4  
5 =head1 NAME
6  
7 Graphics::Skullplot::ClassifyColumns - simple type inference of columns of tabular data
8  
9 =head1 VERSION
10  
11 Version 0.01
12  
13 =cut
14  
15 # TODO revise these before shipping
16 our $VERSION = '0.02';
17 my $DEBUG = 1;
18  
19 =head1 SYNOPSIS
20  
21 use Graphics::Skullplot::ClassifyColumns;
22  
23 my $cc = Graphics::Skullplot::ClassifyColumns->new( data => $data );
24 my $plot_cols =
25 $cc->classify_columns_simple( { indie_count => $indie_count, } );
26  
27 =head1 DESCRIPTION
28  
29 Graphics::Skullplot::ClassifyColumns is a stripped down version
30 of an old experimental module I was developing I called Data::Classify.
31 I expect to go back to that project and develop a more elaborate
32 system of plug-ins to target different kinds of databases and so on,
33 most likely named Table::TypeInference.
34  
35 This particular module just needs a "classify_columns_simple" routine
36 that works well enough to figure out how to plot some data via
37 ggplot2 in R (i.e. the "Graphics::Skullplot" project).
38  
39 =cut
40  
41 3 ▶use 5.10.0;
Subroutine BEGIN: called CC=1 SCAR=0.0
42 3 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
43 3 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
44 3 ▶use Carp;
Subroutine BEGIN: called CC=1 SCAR=0.0
45 3 ▶use Data::Dumper;
Subroutine BEGIN: called CC=1 SCAR=0.0
46  
47 0 ▶use Scalar::Classify qw();
Subroutine BEGIN: called CC=1 SCAR=2.6
48  
49 =over
50  
51 =item new
52  
53 Creates a new Graphics::Skullplot::ClassifyColumns object.
54  
55 Takes a hashref as an argument, with named fields identical
56 to the names of the object attributes. These attributes are:
57  
58 =over
59  
60 =item data
61  
62 A required field, columns of data as an array of array references,
63 with a header in the first row.
64  
65 =back
66  
67 =cut
68  
69 # Example attribute:
70 # has is_loop => ( is => 'rw', isa => Int, default => 0 );
71 # Tempted to use Mouse over Moo so I can do my usual "isa => 'Int'"
72  
73 has data => ( is => 'ro', isa => ArrayRef );
74  
75 has patterns => ( is => 'ro', isa => HashRef, builder => "define_regxeps" );
76  
77 # $DB::single = 1;
78  
79  
80 =item classify_columns_simple
81  
82 Note: here "simple" might be thought of as "stub":
83 This does the simplest possible categorization using only
84 a single numeric hint for the number of independent fields.
85  
86 The presumption here is the incoming data is organized like
87 the output of a typical sql group by select, x-axis in the
88 first column a number of columns of dependent data as the
89 end, and (possibly) a certain number of categorical variables
90 (ones with a small number of allowed values) in-between.
91  
92 This returns a hash indicating how different columns should be
93 handled in the plotting stage, the keys are:
94  
95 x (rename: indie_x )
96 y but just for when there's only one dependent
97 gb_cats
98 dep_fields (rename: dependents_y }
99  
100 Example usage:
101  
102 my $cc = Graphics::Skullplot::ClassifyColumns->new( data => $data );
103 my $opt = { indie_count => 1, };
104 my $plot_cols_href =
105 $cc->classify_columns_simple( $opt );
106  
107 =cut
108  
109 sub classify_columns_simple {
110 my $self = shift;
111  
112 my $opt = shift;
113 my $indie_count = $opt->{ indie_count } // 1;
114  
115 my %field_data; # return values
116  
117 my $dependent_requested = $opt->{ dependent_requested };
118 my $independent_requested = $opt->{ independent_requested };
119  
120 my $data = $self->data;
121 my @header = @{ $data->[0] };
122  
123 # when we're told what to do there's no need to guess
124 if ( $dependent_requested && $independent_requested ) {
125  
126 # TODO might be better to just use the empty set
127 # my @gb_cats = grep{ !/^$dependent_requested$/ } grep{ !/^$independent_requested$/ } @header;
128 my @gb_cats = ();
129  
130 %field_data =
131 ( indie_x => $independent_requested,
132 y => $dependent_requested, # redundant with dependents_y
133 gb_cats => [ @gb_cats ],
134 dependents_y => [ $dependent_requested ],
135 );
136 } else {
137 # use first col as the default independent variable (x-axis)
138 my $independent_default = $header[ 0 ];
139 my $x_field = $independent_default;
140
141 # to start might presume we're just plotting the last column as the y-axis
142 my $dependent_default = $header[ -1 ];
143  
144 # the middle columns, excluding first and the trailing dependents
145 my @gb_cats = @header[ 1 .. ( $#header - $indie_count) ];
146  
147 # the trailing dependents
148 my @dependents_y = @header[ ( ( $#header - $indie_count ) + 1 ) .. $#header ];
149  
150 my $y_field = '';
151 if ($indie_count == 1 ) {
152 $y_field = $dependent_default;
153 }
154  
155 %field_data =
156 ( indie_x => $x_field || '',
157 y => $y_field || '', # redundant with dependents_y
158 gb_cats => [ @gb_cats ],
159 dependents_y => [ @dependents_y ],
160 );
161 }
162 return \%field_data;
163 }
164  
165  
166  
167  
168  
169 =item column_types
170  
171 Given a reference to tabular data in an array-of-arrays format-
172 with a header expected in the first row- tries to infer the
173 rough data type of each column.
174  
175 Returns a list (or aref) of the type codes, in sequence.
176  
177 =cut
178  
179 sub column_types {
180 my $self = shift;
181 my $data = $self->data;
182  
183 my @header = @{ $data->[0] };
184  
185 my @types = ();
186 foreach my $col ( 0 .. $#header ) {
187 # initialize vars to summarize the column parameters
188 my %type_count = ();
189 foreach my $idx ( 1 .. $#{ $data } ) {
190 my $row = $data->[ $idx ];
191 my $item = $row->[ $col ];
192  
193 my $overall_type = $self->classify( $item );
194 $type_count{ $overall_type } += 1;
195 }
196 my $type = $self->most_common( \%type_count );
197 push @types, $type;
198 }
199 return wantarray ? @types : \@types;
200 }
201  
202  
203 =item classify
204  
205 A wrapper around Scalar::Classify's "classify", which also
206 subdivides the string category, looking for datetime types.
207  
208 The type is most often (but not limited to) one of the following:
209  
210 ARRAY
211 HASH
212 :NUMBER:
213 :STRING:
214  
215 This code examines any string values to see if a date/time code
216 is more appropriate:
217  
218 :DATE:
219 :DATETIME:
220 :TIME:
221  
222 =cut
223  
224 sub classify {
225 my $self = shift;
226 my $item = shift;
227  
228 # my $yyyymmdd_pat = qr{ \d\d\d\d - \d\d - \d\d }x;
229 my $pats = $self->patterns;
230 my $yyyymmdd_pat = $pats->{ yyyymmdd };
231 my $datetime_pat = $pats->{ datetime };
232 my $time_pat = $pats->{ time };
233  
234 my $type = '';
235 my $overall_type = ( Scalar::Classify::classify( $item ) )[0];
236 if ( $overall_type eq ':STRING:' ) {
237 # check if this looks like a date, time, or date-time
238 if( $item =~ m{ $yyyymmdd_pat }x ){
239 $type = ':DATE:';
240 } elsif( $item =~ m{ $datetime_pat }x ){
241 $type = ':DATETIME:';
242 } elsif( $item =~ m{ $time_pat }x ){
243 $type = ':TIME:';
244 } else {
245 $type = ':STRING:';
246 };
247 } elsif ($overall_type eq ':NUMBER:' ) {
248 $type = ':NUMBER:';
249 } else {
250 warn "$overall_type may not be handled well.";
251 $type = $overall_type;
252 }
253 return $type;
254 }
255  
256  
257 =item most_common
258  
259 Given a hash of numeric counts, returns the key of the maximum count.
260  
261 In the case of a tie, the return will be one of the tie values,
262 which one is undefined.
263  
264 =cut
265  
266 sub most_common {
267 my $self = shift;
268 my $counts = shift;
269  
270 my $max = 0;
271 my $pick = '';
272 foreach my $k ( keys %{ $counts } ) {
273 my $c = $counts->{ $k };
274 if ( $c > $max ) {
275 $pick = $k;
276 }
277 }
278 return $pick;
279 }
280  
281  
282 =item define_regxeps
283  
284 Generates a hashref of locally useful regexps.
285  
286 These are mostly intended to identify dates and times.
287 TODO just look up existing solutions, e.g. Regexp::Common.
288  
289 =cut
290  
291 sub define_regxeps {
292 my $self = shift;
293  
294 my $d = qr{ [0-9] }x;
295 my $d01 = qr{ [0-1] }x;
296 my $d03 = qr{ [0-3] }x;
297 my $d02 = qr{ [0-2] }x;
298 my $d04 = qr{ [0-4] }x;
299 my $d06 = qr{ [0-6] }x;
300  
301 # None of these patterns allow for leading zeros, trailing am/pm or timezones
302 my %patterns =
303 ( yyyymmdd1 => qr{ ^ \d\d\d\d - \d\d - \d\d $ }x,
304 yyyymmdd2 => qr{ ^ $d$d$d$d - $d$d - $d$d $ }x,
305 yyyymmdd => qr{ ^ $d$d$d$d - $d01$d - $d03$d $ }x,
306  
307 hhmm => qr{ ^ $d02$d : $d06$d }x, # note: unpinned at end
308 hhmmss => qr{ ^ $d02$d : $d06$d : $d06$d $ }x,
309  
310 time => qr{ ^ $d02$d : $d06$d [:\d]* $}x,
311  
312 datetime => qr{ ^ $d$d$d$d - $d01$d - $d03$d \s+ $d02$d : $d06$d [:\d]* $ }x,
313  
314 );
315 return \%patterns;
316 }
317  
318  
319  
320  
321  
322 =back
323  
324 =head1 AUTHOR
325  
326 Joseph Brenner, E<lt>doom@kzsu.stanford.eduE<gt>,
327 22 May 2018
328  
329 =head1 COPYRIGHT AND LICENSE
330  
331 Copyright (C) 2018 by Joseph Brenner
332  
333 This program is free software; you can redistribute it and/or modify it
334 under the terms of either: the GNU General Public License as published
335 by the Free Software Foundation; or the Artistic License.
336  
337 No warranty is provided with this code.
338  
339 See http://dev.perl.org/licenses/ for more information.
340  
341  
342 =cut
343  
344 1;
« Skullplot.pm ↑ Summary