lib/Convert/Pheno/IO/CSVHandler.pm

filter: Statementstmt 97.6% 334 / 342 Branchbran 74.2% 101 / 136 Conditioncond 53.6% 22 / 41 MC/DCmcdc 29.6% 8 / 27 Subroutinesub 95.3% 41 / 43 Podpod 0.0% 0 / 20 totaltotal 83.0% 506 / 609 CC 91 SCAR 21.0
CC
91
Cov
88%
read_sqldump_stream
15.1
read_csv
14.5
read_sqldump
14.2
CRAP 8.2
« lib/Convert/Pheno/PXF/ToBFF/Individuals.pm ↑ Summary lib/Convert/Pheno/Source.pm »
1 package Convert::Pheno::IO::CSVHandler;
2  
3 138 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 138 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 138 ▶use autodie;
Subroutine BEGIN: called CC=1 SCAR=0.0
6 138 ▶use feature qw(say);
Subroutine BEGIN: called CC=1 SCAR=0.0
7 138 ▶use File::Basename;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 138 ▶use Text::CSV_XS qw(csv);
Subroutine BEGIN: called CC=1 SCAR=0.0
9 138 ▶use Sort::Naturally qw(nsort);
Subroutine BEGIN: called CC=1 SCAR=0.0
10 138 ▶use List::Util qw(any);
Subroutine BEGIN: called CC=1 SCAR=0.0
11 138 ▶use File::Spec::Functions qw(catdir);
Subroutine BEGIN: called CC=1 SCAR=0.0
12 138 ▶use IO::Compress::Gzip qw($GzipError);
Subroutine BEGIN: called CC=1 SCAR=0.0
13 138 ▶use IO::Uncompress::Gunzip qw($GunzipError);
Subroutine BEGIN: called CC=1 SCAR=0.0
14  
15 138 ▶use Data::Dumper;
Subroutine BEGIN: called CC=1 SCAR=0.0
16 138 ▶use Devel::Size qw(total_size);
Subroutine BEGIN: called CC=1 SCAR=0.0
17 138 ▶use Convert::Pheno::Mapping::Compiler qw(
Subroutine BEGIN: called CC=1 SCAR=0.0
18 assert_mapping_version
19 load_mapping_document
20 138 );
21 138 ▶use Convert::Pheno::Tabular::REDCap::Dictionary;
Subroutine BEGIN: called CC=1 SCAR=0.0
22 138 ▶use Convert::Pheno::OMOP::Definitions;
Subroutine BEGIN: called CC=1 SCAR=0.0
23 138 ▶use Convert::Pheno::Utils::Schema;
Subroutine BEGIN: called CC=1 SCAR=0.0
24 138 ▶use Convert::Pheno::Mapping::Shared;
Subroutine BEGIN: called CC=1 SCAR=0.0
25 138 ▶use Exporter 'import';
Subroutine BEGIN: called CC=1 SCAR=0.0
26 138 our @EXPORT =
27 qw(read_csv read_csv_stream read_redcap_dict_file read_mapping_file read_sqldump read_sqldump_stream sqldump2csv transpose_omop_data_structure write_csv open_filehandle load_exposures get_headers convert_table_aoh_to_hoh);
28  
29 138 ▶use constant DEVEL_MODE => 0;
Subroutine BEGIN: called CC=1 SCAR=0.0
30  
31 #########################
32 #########################
33 # SUBROUTINES FOR CSV #
34 #########################
35 #########################
36  
37 14 ▶sub read_redcap_dictionary {
Subroutine read_redcap_dictionary: called CC=1 SCAR=0.0
Pod undocumented
38 14 my $filepath = shift;
39 14 return Convert::Pheno::Tabular::REDCap::Dictionary->from_file($filepath);
40 }
41  
42 14 ▶sub read_redcap_dict_file {
Subroutine read_redcap_dict_file: called CC=1 SCAR=0.0
Pod undocumented
43 14 my $arg = shift;
44  
45 # Read and load REDCap CSV dictionary
46 14 return read_redcap_dictionary( $arg->{redcap_dictionary} );
47 }
48  
49 66 ▶sub read_mapping_file {
Subroutine read_mapping_file: called CC=1 SCAR=0.0
Pod undocumented
50 66 my $arg = shift;
51  
52 # Read and load mapping file
53 66 my $data_mapping_file = load_mapping_document( $arg->{mapping_file} );
54 63 assert_mapping_version($data_mapping_file);
55  
56 # Validate mapping file against JSON schema
57 my $jv = Convert::Pheno::Utils::Schema->new(
58 {
59 data => $data_mapping_file,
60 debug => $arg->{self_validate_schema},
61 schema_file => $arg->{schema_file}
62 }
63 63 );
64 62 $jv->json_validate;
65  
66 # Return if succesful
67 56 return $data_mapping_file;
68 }
69  
70 9 ▶sub read_sqldump {
Subroutine read_sqldump: called CC=14 SCAR=26.5
Pod undocumented
71 9 my $arg = shift;
72 9 my $filepath = $arg->{in};
73 9 my $self = $arg->{self};
74  
75 # Before resorting to writting this subroutine I performed an exhaustive search on CPAN:
76 # - Tested MySQL::Dump::Parser::XS but I could not make it work...
77 # - App-MysqlUtils-0.022 has a CLI utility (mysql-sql-dump-extract-tables)
78 # - Of course one can always use *nix tools (sed, grep, awk, etc) or other programming languages....
79 # Anyway, I ended up writting the parser myself...
80  
81 # Define variables that modify what we load
82 9 my $max_lines_sql = $self->{max_lines_sql};
83 9 my @omop_tables = @{ $self->{omop_tables} };
84  
85 #COPY "OMOP_cdm_eunomia".attribute_definition (attribute_definition_id, attribute_name, attribute_description, attribute_type_concept_id, attribute_syntax) FROM stdin;
86 # ......
87 # \.
88  
89 # Verbose
90 9 ▶ say "Reading the SQL dump...\n" if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 9 9
91  
92 # Start reading the SQL dump
93 9 my $fh = open_filehandle( $filepath, 'r' );
94  
95 # Determine the print interval based on file size
96 9 my $print_interval = get_print_interval($filepath);
97  
98 # We'll store the data in the hashref $data
99 9 my $data = {};
100  
101 # Now we we start processing line by line
102 9 my $switch = 0;
103 9 my $local_count = 0;
104 9 my $total_count = 0;
105 9 my @headers;
106 my $headers_data_structure;
107 9 my $table_name;
108  
109 9 ▶ while ( my $line = <$fh> ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh)) 4316655 9 4316664
110  
111 4316655 ▶ if ( $line =~ m/^COPY/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($line =~ /^COPY/) 351 4316304 4316655
112  
113 351 chomp $line;
114  
115 # First line contains the headers
116 #COPY "OMOP_cdm_eunomia".attribute_definition (attribute_definition_id, attribute_name, ..., attribute_syntax) FROM stdin;
117  
118 # Create an array to hold the column names for this table
119 351 $line =~ s/[\(\),]//g; # getting rid of (),
120 351 @headers = split /\s+/, $line;
121  
122 351 $table_name = uc( ( split /\./, $headers[1] )[1] ); # ATTRIBUTE_DEFINITI
123  
124 # Discarding non @$omop_tables:
125 # This step improves RAM consumption
126 351 ▶ next unless any { $_ eq $table_name } @omop_tables;
Subroutine __ANON__: called CC=1 SCAR=0.0
Branch: 1 branch
BranchTrueFalseTotal
unless &any(sub { $_ eq $table_name; } , @omop_tables) 282 69 351
127  
128 # Discarding headers which are not terms/variables
129 69 @headers = @headers[ 2 .. $#headers - 2 ];
130  
131 # Turning on the switch for later
132 69 $switch = 1;
133  
134 # Reset count
135 69 $local_count = 0;
136  
137 # Initializing $data>key as empty arrayref
138 69 $data->{$table_name} = [];
139  
140 # Loading headers
141 69 $headers_data_structure->{$table_name} = [@headers];
142  
143 # Jump one line
144 69 $line = <$fh>;
145  
146 # Say if verbose
147 say "Loading <$table_name> in memory..."
148 69 ▶ if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 69 69
149  
150 }
151  
152 # Loading the data if $switch
153 4316373 ▶ if ($switch) {
Branch: 1 branch
BranchTrueFalseTotal
if ($switch) 36036 4280337 4316373
154  
155 36036 chomp $line;
156  
157 # Order matters. We exit before loading
158 36036 ▶ if ( $local_count == $max_lines_sql || $line =~ /^\\\.$/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($local_count == $max_lines_sql or $line =~ /^\\\.$/) 69 35967 36036
Condition100%3/3
$local_count == $max_lines_sql or $line =~ /^\\\.$/
l!l&&r!l&&!r
492035967
MC/DC100%2/2
$local_count == $max_lines_sql or $line =~ /^\\\.$/
$local_count == $max_lines_sql $line =~ /^\\\.$/
Truth table100%3/3
A or B
A= $local_count == $max_lines_sqlB= $line =~ /^\\\.$/
ABresult
1X1
011
000
159 69 $switch = 0;
160 print "==============\nRows read(total): $local_count\n\n"
161 69 ▶ if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 69 69
162 69 next;
163 }
164 35967 $local_count++;
165 35967 $total_count++;
166  
167 # Columns are separated by \t
168 # NB: 'split' and 'Text::CSV' split to strings
169 # We go with 'split'. Coercing a posteriori
170 35967 my @fields = split /\t/, $line;
171  
172 # Loading the fields like this:
173 #
174 # $VAR1 = {
175 # 'PERSON' => [ # NB: This is the table name
176 # {
177 # 'person_id' => 123,
178 # 'test' => 'abc'
179 # },
180 # {
181 # 'person_id' => 456,
182 # 'test' => 'def'
183 # }
184 # ]
185 # };
186  
187 # Using tmp hashref to load all fields at once with slice
188 35967 my $hash_slice;
189 35967 @{$hash_slice}{@headers} =
190 35967 map { dotify_and_coerce_number($_) } @fields;
191  
192 # Adding them as an array element (AoH)
193 35967 push @{ $data->{$table_name} }, $hash_slice;
194  
195 # Print
196 say "Rows read: $local_count"
197 35967 ▶ if ( $self->{verbose} && $local_count % $print_interval == 0 );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} and $local_count % $print_interval == 0 0 35967 35967
Condition33%1/3
$self->{"verbose"} and $local_count % $print_interval == 0
!ll&&!rl&&r
3596700
MC/DC0%0/2
$self->{"verbose"} and $local_count % $print_interval == 0
$self->{"verbose"} $local_count % $print_interval == 0
Truth table33%1/3
A and B
A= $self->{"verbose"}B= $local_count % $print_interval == 0
ABresult
0X0
100
111
198 }
199 }
200 9 close $fh;
201  
202 # Print if verbose
203 print
204 "==========================\nRows read (sqldump-total): $total_count\n==========================\n\n"
205 9 ▶ if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 9 9
206  
207 # RAM Usage
208 say ram_usage_str( 'read_sqldump', $data )
209 9 ▶ if ( DEVEL_MODE || $self->{verbose} );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 9 9
210  
211 9 return ( $data, $headers_data_structure );
212 }
213  
214 2 ▶sub read_sqldump_stream {
Subroutine read_sqldump_stream: called CC=14 SCAR=27.1
Pod undocumented
215 2 my $arg = shift;
216 2 my $filein = $arg->{in};
217 2 my $self = $arg->{self};
218 2 my $person = $arg->{person};
219 2 my $table_name = $self->{omop_tables}[0];
220 2 my $table_name_lc = lc($table_name);
221 2 my $entity_stream_mode =
222 Convert::Pheno::omop_streams_multiple_entities_wrapper($self);
223  
224 # Define variables that modify what we load
225 2 my $max_lines_sql = $self->{max_lines_sql};
226  
227 # Open filehandles
228 2 my $fh_in = open_filehandle( $filein, 'r' );
229 my $fh_out =
230 2 ▶ $entity_stream_mode ? undef : open_filehandle( $self->{out_file}, 'a' );
Branch: 1 branch
BranchTrueFalseTotal
$entity_stream_mode ? : 0 2 2
231  
232 # Determine the print interval based on file size
233 2 my $print_interval = get_print_interval($filein);
234  
235 # Start printing the array
236 #say $fh_out "[";
237  
238 # Now we we start processing line by line
239 2 my $count = 0;
240 2 my $switch = 0;
241 2 my @headers;
242  
243 2 ▶ while ( my $line = <$fh_in> ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined(my $line = readline $fh_in)) 642664 0 642664
244  
245 # Only parsing $table_name_lc and discarding others
246 # Note that double quotes are optional
247 # - COPY "OMOP_cdm_eunomia".person
248 # . COPY omop_cdm_eunomia_2.person
249 642664 ▶ if ( $line =~ /^COPY \"?(\w+)\"?\.$table_name_lc / ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($line =~ /^COPY \"?(\w+)\"?\.$table_name_lc /) 2 642662 642664
250 2 chomp $line;
251  
252 # Create an array to hold the column names for this table
253 2 $line =~ s/[\(\),]//g; # getting rid of (),
254 2 @headers = split /\s+/, $line;
255  
256 # Discarding headers which are not terms/variables
257 2 @headers = @headers[ 2 .. $#headers - 2 ];
258  
259 # Turning on the switch for later
260 2 $switch++;
261  
262 # Jump one line
263 2 $line = <$fh_in>;
264  
265 }
266  
267 # Loading the data if $switch
268 642664 ▶ if ($switch) {
Branch: 1 branch
BranchTrueFalseTotal
if ($switch) 5400 637264 642664
269  
270 # get rid of \n
271 5400 chomp $line;
272  
273 # Order matters. We exit before loading
274 5400 ▶ last if $line =~ /^\\\.$/;
Branch: 1 branch
BranchTrueFalseTotal
if $line =~ /^\\\.$/ 0 5400 5400
275  
276 # Splitting by tab, it's ok
277 5400 my @fields = split /\t/, $line;
278  
279 # Using tmp hashref to load all fields at once with slice
280 5400 my $hash_slice;
281 5400 @{$hash_slice}{@headers} =
282 5400 map { dotify_and_coerce_number($_) } @fields;
283  
284 # Error related to -max-lines-sqlError related to -max-lines-sql
285 die
286 "We could not find person_id:$hash_slice->{person_id}. Try increasing the #lines with --max-lines-sql\n"
287 5400 ▶ unless exists $person->{ $hash_slice->{person_id} };
Branch: 1 branch
BranchTrueFalseTotal
unless exists $$person{$hash_slice->{"person_id"}} 0 5400 5400
288  
289 # Increase counter
290 5400 $count++;
291  
292 # Encode data
293 5400 my $encoded_data =
294 encode_omop_stream( $table_name, $hash_slice, $person, $count,
295 $self );
296  
297 # Only after encoding we are able to discard 'null'
298 5400 ▶ say $fh_out $encoded_data
Branch: 1 branch
BranchTrueFalseTotal
if defined $encoded_data and $encoded_data ne "null" 5400 0 5400
Condition33%1/3
defined $encoded_data and $encoded_data ne "null"
!ll&&!rl&&r
005400
MC/DC0%0/2
defined $encoded_data and $encoded_data ne "null"
defined $encoded_data $encoded_data ne "null"
Truth table33%1/3
A and B
A= defined $encoded_dataB= $encoded_data ne "null"
ABresult
0X0
100
111
299 if defined $encoded_data
300 && $encoded_data ne 'null';
301  
302 # adhoc filter to speed-up development
303 5400 ▶ last if $count == $max_lines_sql;
Branch: 1 branch
BranchTrueFalseTotal
if $count == $max_lines_sql 2 5398 5400
304  
305 # Print if verbose
306 say "Rows processed: $count"
307 5398 ▶ if ( $self->{verbose} && $count % $print_interval == 0 );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} and $count % $print_interval == 0 0 5398 5398
Condition33%1/3
$self->{"verbose"} and $count % $print_interval == 0
!ll&&!rl&&r
539800
MC/DC0%0/2
$self->{"verbose"} and $count % $print_interval == 0
$self->{"verbose"} $count % $print_interval == 0
Truth table33%1/3
A and B
A= $self->{"verbose"}B= $count % $print_interval == 0
ABresult
0X0
100
111
308 }
309 }
310 2 ▶ say "==============\nRows processed(total): $count\n" if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 2 2
311  
312 #say $fh_out "]"; # not needed
313  
314 # Closing filehandles
315 2 close $fh_in;
316 2 ▶ close $fh_out if $fh_out;
Branch: 1 branch
BranchTrueFalseTotal
if $fh_out 2 0 2
317 2 return 1;
318 }
319  
320 6936 ▶sub encode_omop_stream {
Subroutine encode_omop_stream: called CC=3 SCAR=11.0
Pod undocumented
321 6936 my ( $table_name, $hash_slice, $person, $count, $self ) = @_;
322  
323 # *** IMPORTANT ***
324 # Table PERSON only has 1 individual
325 6936 my $person_id = $hash_slice->{person_id};
326 my $data = {
327 $table_name => [$hash_slice],
328 PERSON => $count == 1
329 ? $person->{$person_id}
330 : {
331 6936 ▶ map { $_ => $person->{$person_id}{$_} }
Branch: 1 branch
BranchTrueFalseTotal
$count == 1 ? : 8 6928 6936
332 qw(person_id gender_concept_id birth_datetime)
333 }
334 };
335  
336 # Obtain
337 6936 my $stream = Convert::Pheno::omop2bff_stream_processing( $self, $data );
338  
339 6936 ▶ if ( Convert::Pheno::omop_streams_multiple_entities_wrapper($self) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (Convert::Pheno::omop_streams_multiple_entities_wrapper($self)) 36 6900 6936
340 36 Convert::Pheno::omop_stream_targets_write_wrapper( $self, $stream );
341 36 return undef;
342 }
343  
344 # Return JSON string
345 # - canonical has some overhead but needed for t/)
346 # - $fh is already utf-8, no need to encode again here
347 6900 return JSON::XS->new->canonical->encode($stream);
348 }
349  
350 1 ▶sub sqldump2csv {
Subroutine sqldump2csv: called CC=5 SCAR=17.0
Pod undocumented
351 1 my ( $data, $dir, $sql_headers_data_structure ) = @_;
352  
353 # CSV separator (tab character)
354 1 my $sep = "\t";
355  
356 # Natural sort flag: set to 0 (off) by default.
357 1 my $sort = 0;
358  
359 # Iterate over each table in the data hash.
360 1 for my $table ( keys %{$data} ) {
361  
362 # Build the file path for the CSV file.
363 9 my $filepath = catdir( $dir, "$table.csv" );
364  
365 # Retrieve header fields for the current table.
366 9 my $table_headers = $sql_headers_data_structure->{$table};
367 9 ▶ die "No header data found for table '$table'"
Branch: 1 branch
BranchTrueFalseTotal
unless defined $table_headers and @$table_headers 0 9 9
Condition33%1/3
defined $table_headers and @$table_headers
!ll&&!rl&&r
009
MC/DC0%0/2
defined $table_headers and @$table_headers
defined $table_headers @$table_headers
Truth table33%1/3
A and B
A= defined $table_headersB= @$table_headers
ABresult
0X0
100
111
368 unless defined $table_headers && @$table_headers;
369  
370 # Determine header order: natural sort if enabled, or as stored.
371 my $headers =
372 $sort
373 0 ▶ ? [ nsort( @{$table_headers} ) ]
Branch: 1 branch
BranchTrueFalseTotal
$sort ? : 0 9 9
374 : $table_headers;
375  
376 # Write the CSV file using the specified separator, headers, and data.
377 write_csv(
378 {
379 sep => $sep,
380 filepath => $filepath,
381 headers => $headers,
382 9 data => $data->{$table},
383 }
384 );
385 }
386 1 return 1;
387 }
388  
389 33 ▶sub transpose_omop_data_structure {
Subroutine transpose_omop_data_structure: called CC=10 SCAR=23.4
Pod undocumented
390 33 my ( $self, $data ) = @_;
391  
392 # The situation is the following, $data comes in format:
393 #
394 #$VAR1 = {
395 # 'MEASUREMENT' => [
396 # {
397 # 'measurement_concept_id' => 1,
398 # 'person_id' => 666
399 # },
400 # {
401 # 'measurement_concept_id' => 2,
402 # 'person_id' => 666
403 # }
404 # ],
405 # 'PERSON' => [
406 # {
407 # 'person_id' => 666
408 # },
409 # {
410 # 'person_id' => 1
411 # }
412 # ]
413 # };
414  
415 # where all 'person_id' are together inside the TABLE_NAME.
416 # But, BFF "ideally" works at the individual level so we are going to
417 # transpose the data structure to end up into something like this
418 # NB: MEASUREMENT and OBSERVATION (among others, i.e., CONDITION_OCCURRENCE, PROCEDURE_OCCURRENCE)
419 # can have multiple values for one 'person_id' so they will be loaded as arrays
420 #
421 #
422 #$VAR1 = {
423 # '1' => {
424 # 'PERSON' => {
425 # 'person_id' => 1
426 # }
427 # },
428 # '666' => {
429 # 'MEASUREMENT' => [
430 # {
431 # 'measurement_concept_id' => 1,
432 # 'person_id' => 666
433 # },
434 # {
435 # 'measurement_concept_id' => 2,
436 # 'person_id' => 666
437 # }
438 # ],
439 # 'PERSON' => {
440 # 'person_id' => 666
441 # }
442 # }
443 # };
444  
445 # Debug messages
446 33 ▶ say "> Transposing OMOP data..." if $self->{debug};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"debug"} 0 33 33
447 33 my $omop_person_id = {};
448  
449 # Only performed for $omop_main_table
450 33 for my $table ( @{ $omop_main_table->{$omop_version} } ) { # global
451  
452 # We void the table when reading to avoid data duplication in RAM
453 495 ▶ while ( my $item = shift @{ $data->{$table} } ) { # We want to keep order (!pop)
Branch: 1 branch
BranchTrueFalseTotal
if (my $item = shift @{$$data{$table};}) 37767 495 38262
454  
455 37767 ▶ if ( exists $item->{person_id} && $item->{person_id} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (exists $item->{"person_id"} and $item->{"person_id"}) 37767 0 37767
Condition33%1/3
exists $item->{"person_id"} and $item->{"person_id"}
!ll&&!rl&&r
0037767
MC/DC0%0/2
exists $item->{"person_id"} and $item->{"person_id"}
exists $item->{"person_id"} $item->{"person_id"}
Truth table33%1/3
A and B
A= exists $item->{"person_id"}B= $item->{"person_id"}
ABresult
0X0
100
111
456 37767 my $person_id = $item->{person_id};
457  
458 # {person_id} can have multiple rows in @omop_array_tables
459 37767 ▶ if ( any { $_ eq $table } @omop_array_tables ) {
Subroutine __ANON__: called CC=1 SCAR=0.0
Branch: 1 branch
BranchTrueFalseTotal
if (&any(sub { $_ eq $table; } , @Convert::Pheno::IO::CSVHandler::omop_array_tables)) { } 25550 12217 37767
460 25550 push @{ $omop_person_id->{$person_id}{$table} }, $item; # array
461 }
462  
463 # {person_id} only has one value in a given table
464 else {
465 12217 $omop_person_id->{$person_id}{$table} = $item; # scalar
466 }
467 }
468 }
469 }
470  
471 # To get any unused memory back to Perl
472 33 $data = undef;
473  
474 # Finally we get rid of the 'person_id' key and return values as an array
475 #
476 #$VAR1 = [
477 # {
478 # 'PERSON' => {
479 # 'person_id' => 1
480 # }
481 # },
482 # ------------------------------------------------
483 # {
484 # 'MEASUREMENT' => [
485 # {
486 # 'measurement_concept_id' => 1,
487 # 'person_id' => 666
488 # },
489 # {
490 # 'measurement_concept_id' => 2,
491 # 'person_id' => 666
492 # }
493 # ],
494 # 'PERSON' => {
495 # 'person_id' => 666
496 # }
497 # }
498 # ];
499 # NB: We nsort keys to always have the same result but it's not needed
500 33 ▶ say "> Sorting OMOP data by <person_id>..." if $self->{debug};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"debug"} 0 33 33
501  
502 33 my $aoh;
503 33 for my $key ( nsort keys %{$omop_person_id} ) {
504 16736 push @{$aoh}, $omop_person_id->{$key};
505 16736 delete $omop_person_id->{$key}; # To avoid data duplication
506 }
507  
508 # RAM usage
509 say ram_usage_str( 'transpose_omop_data_structure', $aoh )
510 33 ▶ if ( DEVEL_MODE || $self->{verbose} );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 33 33
511  
512 33 return $aoh;
513 }
514  
515 128 ▶sub read_csv {
Subroutine read_csv: called CC=14 SCAR=26.7
Pod undocumented
516 128 my $arg = shift;
517 128 my $filepath = $arg->{in};
518 128 my $sep = $arg->{sep};
519 128 ▶ my $self = exists $arg->{self} ? $arg->{self} : { verbose => 0 };
Branch: 1 branch
BranchTrueFalseTotal
exists $arg->{'self'} ? : 64 64 128
520 my $coerce_numbers =
521 128 ▶ exists $arg->{coerce_numbers} ? $arg->{coerce_numbers} : 1;
Branch: 1 branch
BranchTrueFalseTotal
exists $arg->{'coerce_numbers'} ? : 23 105 128
522  
523 # Define split record separator from file extension
524 128 my ($separator) = define_separator( $filepath, $sep );
525  
526 # *** IMPORTANT ***
527 # Text::CSV_XS functional interface
528 # duplicates RAM <= DEPRECATED
529 #my $aoh = csv(
530 # in => $filepath,
531 # sep_char => $separator,
532 # headers => "auto",
533 # encoding => $encoding,
534 # auto_diag => 1
535 #);
536  
537 # Create a new Text::CSV_XS object
538 128 my $csv = Text::CSV_XS->new(
539 {
540 sep_char => $separator,
541 binary => 1,
542 auto_diag => 1,
543 }
544 );
545  
546 # Open fh
547 128 my $fh = open_filehandle( $filepath, 'r' );
548  
549 # Determine the print interval based on file size
550 127 my $print_interval = get_print_interval($filepath);
551  
552 # Get headers
553 127 my $headers = $csv->getline($fh);
554 127 $csv->column_names(@$headers);
555  
556 # Check for too many occurrences of separators
557 127 ▶ die
Branch: 1 branch
BranchTrueFalseTotal
if is_separator_incorrect($headers) 0 127 127
558 "Are you sure you are using the right --sep <$separator> for your data?\n"
559 if is_separator_incorrect($headers);
560  
561 # Load data
562 127 my @aoh;
563 127 my $count = 0;
564 127 ▶ while ( my $row = $csv->getline_hr($fh) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $csv->getline_hr($fh)) 31619 127 31746
565 31619 push @aoh, $row;
566 31619 $count++;
567  
568 say "Rows read: $count"
569 31619 ▶ if ( $self->{verbose} && $count % $print_interval == 0 );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} and $count % $print_interval == 0 0 31619 31619
Condition33%1/3
$self->{"verbose"} and $count % $print_interval == 0
!ll&&!rl&&r
3161900
MC/DC0%0/2
$self->{"verbose"} and $count % $print_interval == 0
$self->{"verbose"} $count % $print_interval == 0
Truth table33%1/3
A and B
A= $self->{"verbose"}B= $count % $print_interval == 0
ABresult
0X0
100
111
570  
571 }
572  
573 # Close fh
574 127 close $fh;
575  
576 # Print if verbose
577 print
578 "==========================\nRows read (total): $count\n==========================\n\n"
579 127 ▶ if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 127 127
580  
581 # Generic CSV loading defaults to numeric coercion because OMOP-style tabular
582 # inputs are largely typed. Mapping-file workflows opt out so raw strings such
583 # as REDCap choice codes, identifiers, or leading-zero values survive until a
584 # source-aware mapper decides how to interpret them.
585 127 for my $item (@aoh) {
586 31619 for my $key ( @{$headers} ) {
587 612286 ▶ if ($coerce_numbers) {
Branch: 2 branches
BranchTrueFalseTotal
if ($coerce_numbers) { } 513501 98785 612286
elsif (defined $item->{$key} and $item->{$key} eq '') { } 45310 53475 98785
588 513501 $item->{$key} = dotify_and_coerce_number( $item->{$key} );
589 }
590 ▶ elsif ( defined $item->{$key} && $item->{$key} eq '' ) {
Condition66%2/3
defined $item->{$key} and $item->{$key} eq ""
!ll&&!rl&&r
05347545310
MC/DC50%1/2
defined $item->{$key} and $item->{$key} eq ""
defined $item->{$key} $item->{$key} eq ""
Truth table66%2/3
A and B
A= defined $item->{$key}B= $item->{$key} eq ""
ABresult
0X0
100
111
591 45310 $item->{$key} = undef;
592 }
593 }
594 }
595  
596 # RAM usage
597 say ram_usage_str( "read_csv($filepath)", \@aoh )
598 127 ▶ if ( DEVEL_MODE || $self->{verbose} );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 127 127
599  
600 # Return data
601 127 return \@aoh;
602 }
603  
604 127 ▶sub is_separator_incorrect {
Subroutine is_separator_incorrect: called CC=2 SCAR=7.0
Pod undocumented
605 127 my $keys = shift;
606 127 my $max_delimiters = 5;
607  
608 # Count the number of delimiters (comma, semicolon, or tab) in the first key
609 127 my $sep_count = ( $keys->[0] =~ tr/,;\t// );
610  
611 # Return true (1) if the number of delimiters exceeds the maximum allowed, otherwise false (0)
612 127 ▶ return $sep_count > $max_delimiters ? 1 : 0;
Branch: 1 branch
BranchTrueFalseTotal
$sep_count > $max_delimiters ? : 0 127 127
613 }
614  
615 6 ▶sub read_csv_stream {
Subroutine read_csv_stream: called CC=9 SCAR=22.2
Pod undocumented
616 6 my $arg = shift;
617 6 my $filein = $arg->{in};
618 6 my $self = $arg->{self};
619 6 my $sep = $arg->{sep};
620 6 my $person = $arg->{person};
621 6 my $entity_stream_mode =
622 Convert::Pheno::omop_streams_multiple_entities_wrapper($self);
623  
624 # Define split record separator
625 6 my ( $separator, undef, $table_name ) = define_separator( $filein, $sep );
626 6 my $table_name_lc = lc($table_name);
627  
628 # Create a new Text::CSV_XS object
629 6 my $csv = Text::CSV_XS->new(
630 {
631 sep_char => $separator,
632 binary => 1,
633 auto_diag => 1,
634 }
635 );
636  
637 # Open filehandles
638 6 my $fh_in = open_filehandle( $filein, 'r' );
639 my $fh_out =
640 6 ▶ $entity_stream_mode ? undef : open_filehandle( $self->{out_file}, 'a' );
Branch: 1 branch
BranchTrueFalseTotal
$entity_stream_mode ? : 3 3 6
641  
642 # Get headers
643 6 my $headers = $csv->getline($fh_in);
644  
645 # *** IMPORTANT ***
646 # On Feb-19-2023 I tested Parallel::ForkManager and:
647 # 1 - The performance was by far slower than w/o it
648 # 2 - We hot SQLite errors for concurring fh
649 # Thus, it was not implemented
650  
651 6 my $count = 0;
652  
653 6 ▶ while ( my $row = $csv->getline($fh_in) ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $row = $csv->getline($fh_in)) 1536 6 1542
654  
655 # Load the values as a hash slice
656 1536 my %hash_slice;
657 1536 @hash_slice{@$headers} = map { dotify_and_coerce_number($_) } @$row;
658  
659 # Encode data
660 1536 my $encoded_data =
661 encode_omop_stream( $table_name, \%hash_slice, $person, $count,
662 $self );
663  
664 # Only after encoding we are able to discard 'null'
665 1536 ▶ say $fh_out $encoded_data
Branch: 1 branch
BranchTrueFalseTotal
if defined $encoded_data and $encoded_data ne "null" 1500 36 1536
Condition66%2/3
defined $encoded_data and $encoded_data ne "null"
!ll&&!rl&&r
3601500
MC/DC50%1/2
defined $encoded_data and $encoded_data ne "null"
defined $encoded_data $encoded_data ne "null"
Truth table66%2/3
A and B
A= defined $encoded_dataB= $encoded_data ne "null"
ABresult
0X0
100
111
666 if defined $encoded_data
667 && $encoded_data ne 'null';
668  
669 # Increment $count
670 1536 $count++;
671  
672 # Verbose logging every 10,000 rows
673 1536 ▶ if ( $self->{verbose} && $count % 10_000 == 0 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($self->{"verbose"} and $count % 10000 == 0) 0 1536 1536
Condition33%1/3
$self->{"verbose"} and $count % 10000 == 0
!ll&&!rl&&r
153600
MC/DC0%0/2
$self->{"verbose"} and $count % 10000 == 0
$self->{"verbose"} $count % 10000 == 0
Truth table33%1/3
A and B
A= $self->{"verbose"}B= $count % 10000 == 0
ABresult
0X0
100
111
674 0 say "Rows processed: $count";
675 }
676 }
677 6 ▶ say "==============\nRows total: $count\n" if $self->{verbose};
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} 0 6 6
678  
679 6 close $fh_in;
680 6 ▶ close $fh_out if $fh_out;
Branch: 1 branch
BranchTrueFalseTotal
if $fh_out 3 3 6
681 6 return 1;
682 }
683  
684 29 ▶sub write_csv {
Subroutine write_csv: called CC=7 SCAR=19.8
Pod undocumented
685 29 my $arg = shift;
686 29 my $sep = $arg->{sep};
687 29 my $data = $arg->{data};
688 29 my $filepath = $arg->{filepath};
689 29 my $headers = $arg->{headers};
690  
691 # Ensure $data is an array reference of hashes
692 29 ▶ if ( ref $data eq 'HASH' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $data eq "HASH") 0 29 29
693 0 $data = [$data]; # Convert to an array reference containing one hash
694 }
695  
696 29 my @exts = qw(.csv .tsv .csv.gz .tsv.gz);
697 29 my $msg =
698 qq(Can't recognize <$filepath> extension. Extensions allowed are: )
699 . ( join ',', @exts ) . "\n";
700 29 my ( undef, undef, $ext ) = fileparse( $filepath, @exts );
701 29 ▶ die $msg unless any { $_ eq $ext } @exts;
Subroutine __ANON__: called CC=1 SCAR=0.2
Branch: 1 branch
BranchTrueFalseTotal
unless &any(sub { $_ eq $ext; } , @exts) 0 29 29
702  
703 29 ▶ $headers ||= get_headers($data);
Condition33%1/3
$headers ||= get_headers($data)
l!l&&r!l&&!r
2900
MC/DC0%0/2
$headers ||= get_headers($data)
$headers get_headers($data)
Truth table33%1/3
A ||= B
A= $headersB= get_headers($data)
ABresult
1X1
011
000
704  
705 29 my $fh = open_filehandle( $filepath, 'w' );
706 29 ▶ my $csv = Text::CSV_XS->new(
Branch: 1 branch
BranchTrueFalseTotal
my $csv = "Text::CSV_XS"->new({"binary", 1, "eol", "\n", "sep_char", $sep}) or die "Cannot initialize CSV writer for <$filepath>" 29 0 29
707 {
708 binary => 1,
709 eol => "\n",
710 sep_char => $sep,
711 }
712 ) or die "Cannot initialize CSV writer for <$filepath>";
713  
714 29 $csv->print( $fh, $headers );
715 29 for my $row ( @{$data} ) {
716 $csv->print(
717 $fh,
718 4044 ▶ [ map { exists $row->{$_} ? $row->{$_} : undef } @{$headers} ]
Branch: 1 branch
BranchTrueFalseTotal
exists $row->{$_} ? : 72078 14098 86176
719 );
720 }
721  
722 29 close $fh;
723 29 return 1;
724 }
725  
726 209 ▶sub open_filehandle {
Subroutine open_filehandle: called CC=9 SCAR=22.0
Pod undocumented
727 209 my ( $filepath, $mode ) = @_;
728 209 ▶ my $handle = $mode eq 'a' ? '>>' : $mode eq 'w' ? '>' : '<';
Branch: 2 branches
BranchTrueFalseTotal
$mode eq 'a' ? : 19 190 209
$mode eq 'w' ? : 43 147 190
729 209 my $fh;
730 209 ▶ if ( $filepath =~ /\.gz$/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($filepath =~ /\.gz$/) { } 26 183 209
731 26 ▶ if ( $mode eq 'a' || $mode eq 'w' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($mode eq 'a' or $mode eq 'w') { } 10 16 26
Condition100%3/3
$mode eq "a" or $mode eq "w"
l!l&&r!l&&!r
5516
MC/DC100%2/2
$mode eq "a" or $mode eq "w"
$mode eq "a" $mode eq "w"
Truth table100%3/3
A or B
A= $mode eq "a"B= $mode eq "w"
ABresult
1X1
011
000
732 # Gzip ignores binmode, so encode character strings here instead.
733 10 my %gzip_args = (Encode => 'UTF-8');
734 10 ▶ $gzip_args{Append} = 1 if ( $mode eq 'a' && -e $filepath );
Branch: 1 branch
BranchTrueFalseTotal
if $mode eq "a" and -e $filepath 2 8 10
Condition100%3/3
$mode eq "a" and -e $filepath
!ll&&!rl&&r
532
MC/DC100%2/2
$mode eq "a" and -e $filepath
$mode eq "a" -e $filepath
Truth table100%3/3
A and B
A= $mode eq "a"B= -e $filepath
ABresult
0X0
100
111
735 10 $fh = IO::Compress::Gzip->new( $filepath, %gzip_args );
736 }
737 else {
738 16 $fh = IO::Uncompress::Gunzip->new( $filepath, MultiStream => 1 );
739 }
740 26 binmode( $fh, ":encoding(UTF-8)" );
741 }
742 else {
743 # Generated records use LF on every OS; embedded source newlines must
744 # also pass through unchanged. Input retains its existing read layers.
745 183 ▶ my $layers = $mode eq 'r' ? ':encoding(UTF-8)' : ':raw:encoding(UTF-8)';
Branch: 1 branch
BranchTrueFalseTotal
$mode eq 'r' ? : 131 52 183
746 183 open $fh, qq($handle$layers), $filepath;
747 }
748 208 return $fh;
749 }
750  
751 134 ▶sub define_separator {
Subroutine define_separator: called CC=6 SCAR=18.3
Pod undocumented
752 134 my ( $filepath, $sep ) = @_;
753  
754 # Define split record separator from file extension
755 134 my @exts = map { $_, $_ . '.gz' } qw(.csv .tsv .sql .txt);
756 134 my ( $table_name, undef, $ext ) = fileparse( $filepath, @exts );
757  
758 # Defining separator character
759 134 ▶ my $separator =
Branch: 5 branches
BranchTrueFalseTotal
$sep ? : 94 40 134
$ext eq '.csv' ? : 39 1 40
$ext eq '.csv.gz' ? : 0 1 1
$ext eq '.tsv' ? : 0 1 1
$ext eq '.tsv.gz' ? : 0 1 1
760 $sep
761 ? $sep
762 : $ext eq '.csv' ? ';' # Note we don't use comma but semicolon
763 : $ext eq '.csv.gz' ? ';' # idem
764 : $ext eq '.tsv' ? "\t"
765 : $ext eq '.tsv.gz' ? "\t"
766 : "\t";
767  
768 # Return 3 but some callers only use 1 or 2
769 134 return ( $separator, undef, $table_name );
770 }
771  
772 0 ▶sub to_gb {
Subroutine to_gb: not called CC=1 SCAR=6.9
Pod undocumented
773 0 my $bytes = shift;
774  
775 # base 2 => 1,073,741,824
776 0 my $gb = $bytes / 1_073_741_824;
777 0 return sprintf( '%9.4f', $gb ) . ' GB';
778 }
779  
780 0 ▶sub ram_usage_str {
Subroutine ram_usage_str: not called CC=1 SCAR=6.9
Pod undocumented
781 0 my ( $func, $data ) = @_;
782 0 return qq/***RAM Usage***($func):/ . to_gb( total_size($data) ) . "\n";
783 }
784  
785 40 ▶sub load_exposures {
Subroutine load_exposures: called CC=1 SCAR=0.0
Pod undocumented
786 40 my $data = read_csv( { in => shift, sep => "\t" } );
787  
788 # We will only use the key 'concept_id' and discard the rest
789 #$VAR1 = {
790 # '4138352' => 1
791 # };
792 40 my %hash = map { $_->{concept_id} => 1 } @$data;
793  
794 # Returning hashref
795 40 return \%hash;
796 }
797  
798 32 ▶sub get_headers {
Subroutine get_headers: called CC=5 SCAR=16.1
Pod undocumented
799 32 my $data = shift;
800  
801 # Ensure $data is an array reference, wrap it in an array if it's a hash reference.
802 32 ▶ $data = [$data] unless ref $data eq 'ARRAY';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $data eq "ARRAY" 0 32 32
803  
804 # Step 1 & 2: Collect all unique keys from all hashes, ignoring hash references.
805 32 my %all_keys;
806 32 foreach my $row (@$data) {
807 576 foreach my $key ( keys %$row ) {
808  
809 # Skip any key where the value is a reference (including hash references)
810 # Why?
811 # In pxf2csv I encountered HASH(foobarbaz) as header. This is actually
812 # a deeper issue I have to investigate
813 107469 ▶ next if ref $row->{$key};
Branch: 1 branch
BranchTrueFalseTotal
if ref $row->{$key} 1 107468 107469
814 107468 $all_keys{$key} = ();
815 }
816 }
817  
818 # Step 3: Sort keys for consistency.
819 32 my @headers = sort keys %all_keys;
820 32 return \@headers;
821 }
822  
823 60 ▶sub convert_table_aoh_to_hoh {
Subroutine convert_table_aoh_to_hoh: called CC=4 SCAR=13.9
Pod undocumented
824 60 my ( $data, $table, $self ) = @_;
825  
826 my %table_cursor =
827 60 map { $_ => $data->{$_} } qw(CONCEPT PERSON VISIT_OCCURRENCE);
828 60 my %table_id = (
829 CONCEPT => 'concept_id',
830 PERSON => 'person_id',
831 VISIT_OCCURRENCE => 'visit_occurrence_id'
832 );
833 60 my $array_ref = $table_cursor{$table};
834 60 my $id = $table_id{$table};
835  
836 ###########
837 # CONCEPT #
838 ###########
839  
840 # $VAR1 = [
841 # {
842 # 'concept_class_id' => '4-char billing code',
843 # 'concept_code' => 'K92.2',
844 # 'concept_id' => 35208414,
845 # 'concept_name' => 'Gastrointestinal hemorrhage, unspecified',
846 # 'domain_id' => 'Condition',
847 # 'invalid_reason' => undef,
848 # 'standard_concept' => undef,
849 # 'valid_end_date' => '2099-12-31',
850 # 'valid_start_date' => '2007-01-01',
851 # 'vocabulary_id' => 'ICD10CM'
852 # },
853 #
854 # and we convert it to hash to allow for quick searches by 'concept_id'
855 #
856 # $VAR1 = {
857 # '1107830' => {
858 # 'concept_class_id' => 'Ingredient',
859 # 'concept_code' => 28889,
860 # 'concept_id' => 1107830,
861 # 'concept_name' => 'Loratadine',
862 # 'domain_id' => 'Drug',
863 # 'invalid_reason' => undef,
864 # 'standard_concept' => 'S',
865 # 'valid_end_date' => '2099-12-31',
866 # 'valid_start_date' => '1970-01-01',
867 # 'vocabulary_id' => 'RxNorm'
868 # },
869 #
870 # NB: We store all columns yet we'll use 4:
871 # 'concept_id', 'concept_code', 'concept_name', 'vocabulary_id'
872  
873 ####################
874 # VISIT_OCCURRENCE #
875 ####################
876  
877 # Going from
878 #$VAR1 = [
879 # {
880 # 'admitting_source_concept_id' => 0,
881 # 'visit_occurrence_id' => 85,
882 # ...
883 # }
884 # ];
885  
886 # To
887 #$VAR1 = {
888 # '85' => {
889 # 'admitting_source_concept_id' => 0,
890 # 'visit_occurrence_id' => 85,
891 # ...
892 # }
893 # };
894  
895 # Initialize the new hash for transformed data
896 60 my $hoh = {};
897  
898 # Iterate over the array and build the hash while clearing the array
899 60 ▶ while ( my $item = pop @{$array_ref} ) { #faster than shift (order irrelevant here)
Branch: 1 branch
BranchTrueFalseTotal
if (my $item = pop @{$array_ref;}) 25010 60 25070
900 25010 my $key = $item->{$id}; # avoid stringfication
901 25010 $hoh->{$key} = $item;
902 }
903  
904 # The original array @{ $self->{data}{$table} is now empty
905 # RAM Usage
906 say ram_usage_str( "convert_table_aoh_to_hoh($table)", $hoh )
907 60 ▶ if ( $self->{verbose} || DEVEL_MODE );
Branch: 1 branch
BranchTrueFalseTotal
if $self->{"verbose"} or 0 0 60 60
Condition50%1/2
$self->{"verbose"} or 0
l!l
060
MC/DC0%0/1
$self->{"verbose"} or 0
$self->{"verbose"}
Truth table50%1/2
A
A= $self->{"verbose"}
Aresult
11
00
908  
909 60 return $hoh;
910 }
911  
912 138 ▶sub get_print_interval {
Subroutine get_print_interval: called CC=2 SCAR=6.9
Pod undocumented
913 138 my $filepath = shift;
914  
915 # Determine file size
916 138 my $file_size = -s $filepath;
917  
918 # Set print interval based on file size (threshold: 10 MB)
919 138 ▶ my $print_interval = $file_size > 10 * 1024 * 1024 ? 10_000 : 1_000;
Branch: 1 branch
BranchTrueFalseTotal
$file_size > 10485760 ? : 7 131 138
920  
921 138 return $print_interval;
922 }
923  
924 138 1;
« lib/Convert/Pheno/PXF/ToBFF/Individuals.pm ↑ Summary lib/Convert/Pheno/Source.pm »