line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Datahub::Factory; |
2
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
143781
|
use Datahub::Factory::Sane; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
6
|
|
4
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
our $VERSION = '1.76'; |
6
|
|
|
|
|
|
|
|
7
|
1
|
|
|
1
|
|
414
|
use Datahub::Factory::Env; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
29
|
|
8
|
1
|
|
|
1
|
|
371
|
use Datahub::Factory::Config; |
|
1
|
|
|
|
|
4
|
|
|
1
|
|
|
|
|
31
|
|
9
|
1
|
|
|
1
|
|
461
|
use Datahub::Factory::Pipeline; |
|
1
|
|
|
|
|
4
|
|
|
1
|
|
|
|
|
30
|
|
10
|
1
|
|
|
1
|
|
6
|
use File::Spec; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
25
|
|
11
|
1
|
|
|
1
|
|
5
|
use Sub::Exporter::Util qw(curry_method); |
|
1
|
|
|
|
|
1
|
|
|
1
|
|
|
|
|
11
|
|
12
|
1
|
|
|
|
|
5
|
use Sub::Exporter -setup => { |
13
|
|
|
|
|
|
|
exports => [ |
14
|
|
|
|
|
|
|
log => curry_method, |
15
|
|
|
|
|
|
|
cfg => curry_method, |
16
|
|
|
|
|
|
|
importer => curry_method, |
17
|
|
|
|
|
|
|
fixer => curry_method, |
18
|
|
|
|
|
|
|
store => curry_method, |
19
|
|
|
|
|
|
|
exporter => curry_method, |
20
|
|
|
|
|
|
|
indexer => curry_method, |
21
|
|
|
|
|
|
|
pipeline => curry_method, |
22
|
|
|
|
|
|
|
module => curry_method, |
23
|
|
|
|
|
|
|
], |
24
|
|
|
|
|
|
|
collectors => {'-load' => \'_import_load', ':load' => \'_import_load'}, |
25
|
1
|
|
|
1
|
|
268
|
}; |
|
1
|
|
|
|
|
2
|
|
26
|
1
|
|
|
1
|
|
635
|
use namespace::clean; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
3
|
|
27
|
|
|
|
|
|
|
|
28
|
|
|
|
|
|
|
sub _import_load { |
29
|
0
|
|
|
0
|
|
|
my ($self, $value, $data) = @_; |
30
|
|
|
|
|
|
|
|
31
|
0
|
0
|
|
|
|
|
if (is_array_ref $value) { |
32
|
0
|
|
|
|
|
|
self->load(@$value); |
33
|
|
|
|
|
|
|
} |
34
|
|
|
|
|
|
|
else { |
35
|
0
|
|
|
|
|
|
$self->load; |
36
|
|
|
|
|
|
|
} |
37
|
|
|
|
|
|
|
|
38
|
0
|
|
|
|
|
|
1; |
39
|
|
|
|
|
|
|
} |
40
|
|
|
|
|
|
|
|
41
|
|
|
|
|
|
|
sub load { |
42
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
43
|
0
|
|
|
|
|
|
my $env = Datahub::Factory::Env->new(); |
44
|
0
|
|
|
|
|
|
$class->_env($env); |
45
|
0
|
|
|
|
|
|
$class; |
46
|
|
|
|
|
|
|
} |
47
|
|
|
|
|
|
|
|
48
|
|
|
|
|
|
|
sub _env { |
49
|
0
|
|
|
0
|
|
|
my ($class, $env) = @_; |
50
|
0
|
|
|
|
|
|
state $loaded_env; |
51
|
0
|
0
|
|
|
|
|
$loaded_env = $env if defined $env; |
52
|
0
|
|
0
|
|
|
|
$loaded_env |
53
|
|
|
|
|
|
|
||= Datahub::Factory::Env->new(load_paths => $class->default_load_path); |
54
|
|
|
|
|
|
|
} |
55
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
sub default_load_path { # TODO move to Catmandu::Env |
57
|
0
|
|
|
0
|
0
|
|
my ($class, $path) = @_; |
58
|
0
|
|
|
|
|
|
state $default_path; |
59
|
0
|
0
|
|
|
|
|
$default_path = $path if defined $path; |
60
|
0
|
|
0
|
|
|
|
$default_path //= do { |
61
|
0
|
|
|
|
|
|
my $script = File::Spec->rel2abs($0); |
62
|
0
|
|
|
|
|
|
my ($script_vol, $script_path, $script_name) |
63
|
|
|
|
|
|
|
= File::Spec->splitpath($script); |
64
|
0
|
|
|
|
|
|
my @dirs = grep length, File::Spec->splitdir($script_path); |
65
|
0
|
0
|
|
|
|
|
if ($dirs[-1] eq 'bin') { |
66
|
0
|
|
|
|
|
|
pop @dirs; |
67
|
0
|
|
|
|
|
|
File::Spec->catdir(File::Spec->rootdir, @dirs); |
68
|
|
|
|
|
|
|
} |
69
|
|
|
|
|
|
|
else { |
70
|
0
|
|
|
|
|
|
$script_path; |
71
|
|
|
|
|
|
|
} |
72
|
|
|
|
|
|
|
}; |
73
|
|
|
|
|
|
|
} |
74
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
sub config { |
76
|
0
|
|
|
0
|
0
|
|
my ($class, $config) = @_; |
77
|
|
|
|
|
|
|
|
78
|
0
|
0
|
|
|
|
|
if ($config) { |
79
|
0
|
|
|
|
|
|
my $env = Datahub::Factory::Env->new(load_paths => $class->_env->load_paths); |
80
|
0
|
|
|
|
|
|
$env->_set_config($config); |
81
|
0
|
|
|
|
|
|
$class->_env($env); |
82
|
|
|
|
|
|
|
} |
83
|
|
|
|
|
|
|
|
84
|
0
|
|
|
|
|
|
$class->_env->config; |
85
|
|
|
|
|
|
|
} |
86
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
sub importer { |
88
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
89
|
0
|
|
|
|
|
|
$class->_env->importer(@_); |
90
|
|
|
|
|
|
|
} |
91
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
sub fixer { |
93
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
94
|
0
|
|
|
|
|
|
$class->_env->fixer(@_); |
95
|
|
|
|
|
|
|
} |
96
|
|
|
|
|
|
|
|
97
|
|
|
|
|
|
|
sub exporter { |
98
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
99
|
0
|
|
|
|
|
|
$class->_env->exporter(@_); |
100
|
|
|
|
|
|
|
} |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
sub indexer { |
103
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
104
|
0
|
|
|
|
|
|
$class->_env->indexer(@_); |
105
|
|
|
|
|
|
|
} |
106
|
|
|
|
|
|
|
|
107
|
|
|
|
|
|
|
sub log { |
108
|
0
|
|
|
0
|
0
|
|
$_[0]->_env->log; |
109
|
|
|
|
|
|
|
} |
110
|
|
|
|
|
|
|
|
111
|
|
|
|
|
|
|
sub cfg { |
112
|
0
|
|
|
0
|
0
|
|
my $cfg = Datahub::Factory::Config->new(); |
113
|
0
|
|
|
|
|
|
return $cfg->config; |
114
|
|
|
|
|
|
|
} |
115
|
|
|
|
|
|
|
|
116
|
|
|
|
|
|
|
sub pipeline { |
117
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
118
|
0
|
|
|
|
|
|
return $class->_env->pipeline(@_); |
119
|
|
|
|
|
|
|
} |
120
|
|
|
|
|
|
|
|
121
|
|
|
|
|
|
|
sub module { |
122
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
123
|
0
|
|
|
|
|
|
return $class->_env->module(@_); |
124
|
|
|
|
|
|
|
} |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
1; |
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
__END__ |
129
|
|
|
|
|
|
|
|
130
|
|
|
|
|
|
|
=pod |
131
|
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
=head1 NAME |
133
|
|
|
|
|
|
|
|
134
|
|
|
|
|
|
|
Datahub::Factory - A conveyor belt which transports data from a data source to a data sink. |
135
|
|
|
|
|
|
|
|
136
|
|
|
|
|
|
|
=head1 SYNOPSIS |
137
|
|
|
|
|
|
|
|
138
|
|
|
|
|
|
|
dhconveyor command OPTIONS |
139
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
=head1 DESCRIPTION |
141
|
|
|
|
|
|
|
|
142
|
|
|
|
|
|
|
Datahub::Factory is a command line conveyor belt which automates three tasks: |
143
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
=over |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
=item Data is fetched automatically from a local or remote data source. |
147
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
=item Data is converted to an exchange format. |
149
|
|
|
|
|
|
|
|
150
|
|
|
|
|
|
|
=item The output is pushed to a data sink. |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
=back |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
Datahub::Factory fetches data from several sources as specified by the Importer settings, |
155
|
|
|
|
|
|
|
executes a L<Catmandu::Fix> and sends it to a data sink, set via an Exporter. |
156
|
|
|
|
|
|
|
Several importer and exporter modules are provided, but developers can extend the functionality with their own modules. |
157
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
Datahub::Factory contains Log4perl support. |
159
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
=head1 CONFIGURATION |
161
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
=head2 Command line options |
163
|
|
|
|
|
|
|
|
164
|
|
|
|
|
|
|
All commands share the following switches: |
165
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
=over |
167
|
|
|
|
|
|
|
|
168
|
|
|
|
|
|
|
=item --log_level |
169
|
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
Set the log_level. Takes a numeric parameter. Supported levels are: 1 (WARN), |
171
|
|
|
|
|
|
|
2 (INFO), 3 (DEBUG). WARN (1) is the default. |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
=item --log_output |
174
|
|
|
|
|
|
|
|
175
|
|
|
|
|
|
|
Selects an output for the log messages. By default, it will send them to STDERR |
176
|
|
|
|
|
|
|
(pass STDERR as parameter), but STDOUT (STDOUT) and a log file. |
177
|
|
|
|
|
|
|
|
178
|
|
|
|
|
|
|
=item --verbose |
179
|
|
|
|
|
|
|
|
180
|
|
|
|
|
|
|
Set verbosity. Invoking the command with the --verbose, -v flag will render |
181
|
|
|
|
|
|
|
verbose output to the terminal. |
182
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
=back |
184
|
|
|
|
|
|
|
|
185
|
|
|
|
|
|
|
=head1 COMMANDS |
186
|
|
|
|
|
|
|
|
187
|
|
|
|
|
|
|
=head2 help COMMAND |
188
|
|
|
|
|
|
|
|
189
|
|
|
|
|
|
|
Documentation about command line options. |
190
|
|
|
|
|
|
|
|
191
|
|
|
|
|
|
|
=head2 L<transport OPTIONS|https://metacpan.org/pod/Datahub::Factory::Command::transport> |
192
|
|
|
|
|
|
|
|
193
|
|
|
|
|
|
|
Fetch data from a local or remote source, convert it to an exchange format and export the data. |
194
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
=head2 L<transport OPTIONS|https://metacpan.org/pod/Datahub::Factory::Command::index> |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
Fetch data from a local source, and push it to an enterprise search engine via a bulk API. |
198
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
|
200
|
|
|
|
|
|
|
=head1 API |
201
|
|
|
|
|
|
|
|
202
|
|
|
|
|
|
|
Datahub::Factory uses a plugin-based architecture, making it easy to extend with new functionality. |
203
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
New commands can be added by creating a Perl module that contains a `command_name.pm` file in the `lib/Datahub/Factory/Command` path. |
205
|
|
|
|
|
|
|
Datahub::Factory uses the L<Datahub::Factory::Command> namespace and leverages L<App::Cmd> internally. |
206
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
New L<Datahub::Factory::Importer>, L<Datahub::Factory::Exporter> and L<Datahub::Factory::Fixer> plugins can be added in the same way. |
208
|
|
|
|
|
|
|
|
209
|
|
|
|
|
|
|
=head1 AUTHORS |
210
|
|
|
|
|
|
|
|
211
|
|
|
|
|
|
|
Matthias Vandermaesen <matthias.vandermaesen@vlaamsekunstcollectie.be> |
212
|
|
|
|
|
|
|
Pieter De Praetere <pieter@packed.be> |
213
|
|
|
|
|
|
|
|
214
|
|
|
|
|
|
|
=head1 COPYRIGHT AND LICENSE |
215
|
|
|
|
|
|
|
|
216
|
|
|
|
|
|
|
This software is copyright (c) 2016 by PACKED, vzw, Vlaamse Kunstcollectie, vzw. |
217
|
|
|
|
|
|
|
|
218
|
|
|
|
|
|
|
This is free software; you can redistribute it and/or modify it under the terms |
219
|
|
|
|
|
|
|
of the GNU General Public License, Version 3, June 2007. |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
=cut |