| 1 | package Graphics::Skullplot::ClassifyColumns; | ||
| 2 | 3 | ▶ | use Moo; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 3 | ▶ | use MooX::Types::MooseLike::Base qw(:all); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | |||
| 5 | =head1 NAME | ||
| 6 | |||
| 7 | Graphics::Skullplot::ClassifyColumns - simple type inference of columns of tabular data | ||
| 8 | |||
| 9 | =head1 VERSION | ||
| 10 | |||
| 11 | Version 0.01 | ||
| 12 | |||
| 13 | =cut | ||
| 14 | |||
| 15 | # TODO revise these before shipping | ||
| 16 | our $VERSION = '0.02'; | ||
| 17 | my $DEBUG = 1; | ||
| 18 | |||
| 19 | =head1 SYNOPSIS | ||
| 20 | |||
| 21 | use Graphics::Skullplot::ClassifyColumns; | ||
| 22 | |||
| 23 | my $cc = Graphics::Skullplot::ClassifyColumns->new( data => $data ); | ||
| 24 | my $plot_cols = | ||
| 25 | $cc->classify_columns_simple( { indie_count => $indie_count, } ); | ||
| 26 | |||
| 27 | =head1 DESCRIPTION | ||
| 28 | |||
| 29 | Graphics::Skullplot::ClassifyColumns is a stripped down version | ||
| 30 | of an old experimental module I was developing I called Data::Classify. | ||
| 31 | I expect to go back to that project and develop a more elaborate | ||
| 32 | system of plug-ins to target different kinds of databases and so on, | ||
| 33 | most likely named Table::TypeInference. | ||
| 34 | |||
| 35 | This particular module just needs a "classify_columns_simple" routine | ||
| 36 | that works well enough to figure out how to plot some data via | ||
| 37 | ggplot2 in R (i.e. the "Graphics::Skullplot" project). | ||
| 38 | |||
| 39 | =cut | ||
| 40 | |||
| 41 | 3 | ▶ | use 5.10.0; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 42 | 3 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 43 | 3 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 44 | 3 | ▶ | use Carp; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 45 | 3 | ▶ | use Data::Dumper; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 46 | |||
| 47 | 0 | ▶ | use Scalar::Classify qw(); |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 48 | |||
| 49 | =over | ||
| 50 | |||
| 51 | =item new | ||
| 52 | |||
| 53 | Creates a new Graphics::Skullplot::ClassifyColumns object. | ||
| 54 | |||
| 55 | Takes a hashref as an argument, with named fields identical | ||
| 56 | to the names of the object attributes. These attributes are: | ||
| 57 | |||
| 58 | =over | ||
| 59 | |||
| 60 | =item data | ||
| 61 | |||
| 62 | A required field, columns of data as an array of array references, | ||
| 63 | with a header in the first row. | ||
| 64 | |||
| 65 | =back | ||
| 66 | |||
| 67 | =cut | ||
| 68 | |||
| 69 | # Example attribute: | ||
| 70 | # has is_loop => ( is => 'rw', isa => Int, default => 0 ); | ||
| 71 | # Tempted to use Mouse over Moo so I can do my usual "isa => 'Int'" | ||
| 72 | |||
| 73 | has data => ( is => 'ro', isa => ArrayRef ); | ||
| 74 | |||
| 75 | has patterns => ( is => 'ro', isa => HashRef, builder => "define_regxeps" ); | ||
| 76 | |||
| 77 | # $DB::single = 1; | ||
| 78 | |||
| 79 | |||
| 80 | =item classify_columns_simple | ||
| 81 | |||
| 82 | Note: here "simple" might be thought of as "stub": | ||
| 83 | This does the simplest possible categorization using only | ||
| 84 | a single numeric hint for the number of independent fields. | ||
| 85 | |||
| 86 | The presumption here is the incoming data is organized like | ||
| 87 | the output of a typical sql group by select, x-axis in the | ||
| 88 | first column a number of columns of dependent data as the | ||
| 89 | end, and (possibly) a certain number of categorical variables | ||
| 90 | (ones with a small number of allowed values) in-between. | ||
| 91 | |||
| 92 | This returns a hash indicating how different columns should be | ||
| 93 | handled in the plotting stage, the keys are: | ||
| 94 | |||
| 95 | x (rename: indie_x ) | ||
| 96 | y but just for when there's only one dependent | ||
| 97 | gb_cats | ||
| 98 | dep_fields (rename: dependents_y } | ||
| 99 | |||
| 100 | Example usage: | ||
| 101 | |||
| 102 | my $cc = Graphics::Skullplot::ClassifyColumns->new( data => $data ); | ||
| 103 | my $opt = { indie_count => 1, }; | ||
| 104 | my $plot_cols_href = | ||
| 105 | $cc->classify_columns_simple( $opt ); | ||
| 106 | |||
| 107 | =cut | ||
| 108 | |||
| 109 | sub classify_columns_simple { | ||
| 110 | my $self = shift; | ||
| 111 | |||
| 112 | my $opt = shift; | ||
| 113 | my $indie_count = $opt->{ indie_count } // 1; | ||
| 114 | |||
| 115 | my %field_data; # return values | ||
| 116 | |||
| 117 | my $dependent_requested = $opt->{ dependent_requested }; | ||
| 118 | my $independent_requested = $opt->{ independent_requested }; | ||
| 119 | |||
| 120 | my $data = $self->data; | ||
| 121 | my @header = @{ $data->[0] }; | ||
| 122 | |||
| 123 | # when we're told what to do there's no need to guess | ||
| 124 | if ( $dependent_requested && $independent_requested ) { | ||
| 125 | |||
| 126 | # TODO might be better to just use the empty set | ||
| 127 | # my @gb_cats = grep{ !/^$dependent_requested$/ } grep{ !/^$independent_requested$/ } @header; | ||
| 128 | my @gb_cats = (); | ||
| 129 | |||
| 130 | %field_data = | ||
| 131 | ( indie_x => $independent_requested, | ||
| 132 | y => $dependent_requested, # redundant with dependents_y | ||
| 133 | gb_cats => [ @gb_cats ], | ||
| 134 | dependents_y => [ $dependent_requested ], | ||
| 135 | ); | ||
| 136 | } else { | ||
| 137 | # use first col as the default independent variable (x-axis) | ||
| 138 | my $independent_default = $header[ 0 ]; | ||
| 139 | my $x_field = $independent_default; | ||
| 140 | |||
| 141 | # to start might presume we're just plotting the last column as the y-axis | ||
| 142 | my $dependent_default = $header[ -1 ]; | ||
| 143 | |||
| 144 | # the middle columns, excluding first and the trailing dependents | ||
| 145 | my @gb_cats = @header[ 1 .. ( $#header - $indie_count) ]; | ||
| 146 | |||
| 147 | # the trailing dependents | ||
| 148 | my @dependents_y = @header[ ( ( $#header - $indie_count ) + 1 ) .. $#header ]; | ||
| 149 | |||
| 150 | my $y_field = ''; | ||
| 151 | if ($indie_count == 1 ) { | ||
| 152 | $y_field = $dependent_default; | ||
| 153 | } | ||
| 154 | |||
| 155 | %field_data = | ||
| 156 | ( indie_x => $x_field || '', | ||
| 157 | y => $y_field || '', # redundant with dependents_y | ||
| 158 | gb_cats => [ @gb_cats ], | ||
| 159 | dependents_y => [ @dependents_y ], | ||
| 160 | ); | ||
| 161 | } | ||
| 162 | return \%field_data; | ||
| 163 | } | ||
| 164 | |||
| 165 | |||
| 166 | |||
| 167 | |||
| 168 | |||
| 169 | =item column_types | ||
| 170 | |||
| 171 | Given a reference to tabular data in an array-of-arrays format- | ||
| 172 | with a header expected in the first row- tries to infer the | ||
| 173 | rough data type of each column. | ||
| 174 | |||
| 175 | Returns a list (or aref) of the type codes, in sequence. | ||
| 176 | |||
| 177 | =cut | ||
| 178 | |||
| 179 | sub column_types { | ||
| 180 | my $self = shift; | ||
| 181 | my $data = $self->data; | ||
| 182 | |||
| 183 | my @header = @{ $data->[0] }; | ||
| 184 | |||
| 185 | my @types = (); | ||
| 186 | foreach my $col ( 0 .. $#header ) { | ||
| 187 | # initialize vars to summarize the column parameters | ||
| 188 | my %type_count = (); | ||
| 189 | foreach my $idx ( 1 .. $#{ $data } ) { | ||
| 190 | my $row = $data->[ $idx ]; | ||
| 191 | my $item = $row->[ $col ]; | ||
| 192 | |||
| 193 | my $overall_type = $self->classify( $item ); | ||
| 194 | $type_count{ $overall_type } += 1; | ||
| 195 | } | ||
| 196 | my $type = $self->most_common( \%type_count ); | ||
| 197 | push @types, $type; | ||
| 198 | } | ||
| 199 | return wantarray ? @types : \@types; | ||
| 200 | } | ||
| 201 | |||
| 202 | |||
| 203 | =item classify | ||
| 204 | |||
| 205 | A wrapper around Scalar::Classify's "classify", which also | ||
| 206 | subdivides the string category, looking for datetime types. | ||
| 207 | |||
| 208 | The type is most often (but not limited to) one of the following: | ||
| 209 | |||
| 210 | ARRAY | ||
| 211 | HASH | ||
| 212 | :NUMBER: | ||
| 213 | :STRING: | ||
| 214 | |||
| 215 | This code examines any string values to see if a date/time code | ||
| 216 | is more appropriate: | ||
| 217 | |||
| 218 | :DATE: | ||
| 219 | :DATETIME: | ||
| 220 | :TIME: | ||
| 221 | |||
| 222 | =cut | ||
| 223 | |||
| 224 | sub classify { | ||
| 225 | my $self = shift; | ||
| 226 | my $item = shift; | ||
| 227 | |||
| 228 | # my $yyyymmdd_pat = qr{ \d\d\d\d - \d\d - \d\d }x; | ||
| 229 | my $pats = $self->patterns; | ||
| 230 | my $yyyymmdd_pat = $pats->{ yyyymmdd }; | ||
| 231 | my $datetime_pat = $pats->{ datetime }; | ||
| 232 | my $time_pat = $pats->{ time }; | ||
| 233 | |||
| 234 | my $type = ''; | ||
| 235 | my $overall_type = ( Scalar::Classify::classify( $item ) )[0]; | ||
| 236 | if ( $overall_type eq ':STRING:' ) { | ||
| 237 | # check if this looks like a date, time, or date-time | ||
| 238 | if( $item =~ m{ $yyyymmdd_pat }x ){ | ||
| 239 | $type = ':DATE:'; | ||
| 240 | } elsif( $item =~ m{ $datetime_pat }x ){ | ||
| 241 | $type = ':DATETIME:'; | ||
| 242 | } elsif( $item =~ m{ $time_pat }x ){ | ||
| 243 | $type = ':TIME:'; | ||
| 244 | } else { | ||
| 245 | $type = ':STRING:'; | ||
| 246 | }; | ||
| 247 | } elsif ($overall_type eq ':NUMBER:' ) { | ||
| 248 | $type = ':NUMBER:'; | ||
| 249 | } else { | ||
| 250 | warn "$overall_type may not be handled well."; | ||
| 251 | $type = $overall_type; | ||
| 252 | } | ||
| 253 | return $type; | ||
| 254 | } | ||
| 255 | |||
| 256 | |||
| 257 | =item most_common | ||
| 258 | |||
| 259 | Given a hash of numeric counts, returns the key of the maximum count. | ||
| 260 | |||
| 261 | In the case of a tie, the return will be one of the tie values, | ||
| 262 | which one is undefined. | ||
| 263 | |||
| 264 | =cut | ||
| 265 | |||
| 266 | sub most_common { | ||
| 267 | my $self = shift; | ||
| 268 | my $counts = shift; | ||
| 269 | |||
| 270 | my $max = 0; | ||
| 271 | my $pick = ''; | ||
| 272 | foreach my $k ( keys %{ $counts } ) { | ||
| 273 | my $c = $counts->{ $k }; | ||
| 274 | if ( $c > $max ) { | ||
| 275 | $pick = $k; | ||
| 276 | } | ||
| 277 | } | ||
| 278 | return $pick; | ||
| 279 | } | ||
| 280 | |||
| 281 | |||
| 282 | =item define_regxeps | ||
| 283 | |||
| 284 | Generates a hashref of locally useful regexps. | ||
| 285 | |||
| 286 | These are mostly intended to identify dates and times. | ||
| 287 | TODO just look up existing solutions, e.g. Regexp::Common. | ||
| 288 | |||
| 289 | =cut | ||
| 290 | |||
| 291 | sub define_regxeps { | ||
| 292 | my $self = shift; | ||
| 293 | |||
| 294 | my $d = qr{ [0-9] }x; | ||
| 295 | my $d01 = qr{ [0-1] }x; | ||
| 296 | my $d03 = qr{ [0-3] }x; | ||
| 297 | my $d02 = qr{ [0-2] }x; | ||
| 298 | my $d04 = qr{ [0-4] }x; | ||
| 299 | my $d06 = qr{ [0-6] }x; | ||
| 300 | |||
| 301 | # None of these patterns allow for leading zeros, trailing am/pm or timezones | ||
| 302 | my %patterns = | ||
| 303 | ( yyyymmdd1 => qr{ ^ \d\d\d\d - \d\d - \d\d $ }x, | ||
| 304 | yyyymmdd2 => qr{ ^ $d$d$d$d - $d$d - $d$d $ }x, | ||
| 305 | yyyymmdd => qr{ ^ $d$d$d$d - $d01$d - $d03$d $ }x, | ||
| 306 | |||
| 307 | hhmm => qr{ ^ $d02$d : $d06$d }x, # note: unpinned at end | ||
| 308 | hhmmss => qr{ ^ $d02$d : $d06$d : $d06$d $ }x, | ||
| 309 | |||
| 310 | time => qr{ ^ $d02$d : $d06$d [:\d]* $}x, | ||
| 311 | |||
| 312 | datetime => qr{ ^ $d$d$d$d - $d01$d - $d03$d \s+ $d02$d : $d06$d [:\d]* $ }x, | ||
| 313 | |||
| 314 | ); | ||
| 315 | return \%patterns; | ||
| 316 | } | ||
| 317 | |||
| 318 | |||
| 319 | |||
| 320 | |||
| 321 | |||
| 322 | =back | ||
| 323 | |||
| 324 | =head1 AUTHOR | ||
| 325 | |||
| 326 | Joseph Brenner, E<lt>doom@kzsu.stanford.eduE<gt>, | ||
| 327 | 22 May 2018 | ||
| 328 | |||
| 329 | =head1 COPYRIGHT AND LICENSE | ||
| 330 | |||
| 331 | Copyright (C) 2018 by Joseph Brenner | ||
| 332 | |||
| 333 | This program is free software; you can redistribute it and/or modify it | ||
| 334 | under the terms of either: the GNU General Public License as published | ||
| 335 | by the Free Software Foundation; or the Artistic License. | ||
| 336 | |||
| 337 | No warranty is provided with this code. | ||
| 338 | |||
| 339 | See http://dev.perl.org/licenses/ for more information. | ||
| 340 | |||
| 341 | |||
| 342 | =cut | ||
| 343 | |||
| 344 | 1; | ||