line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Data::News; |
2
|
1
|
|
|
1
|
|
6488
|
use Moose; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
3
|
|
|
|
|
|
|
use Text::CSV_XS; |
4
|
|
|
|
|
|
|
use DateTime; |
5
|
|
|
|
|
|
|
use Digest::SHA1 qw(sha1_hex); |
6
|
|
|
|
|
|
|
use HTML::Entities; |
7
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
has filename_csv => ( |
9
|
|
|
|
|
|
|
is => 'rw', |
10
|
|
|
|
|
|
|
isa => 'Str', |
11
|
|
|
|
|
|
|
default => sub { |
12
|
|
|
|
|
|
|
my ($self) = @_; |
13
|
|
|
|
|
|
|
my $today = DateTime->now( time_zone => 'local' ); |
14
|
|
|
|
|
|
|
#defines a name for our csv. |
15
|
|
|
|
|
|
|
my $filename = $today->dmy('-').'_' . $today->hms( '-' ) . '.csv'; |
16
|
|
|
|
|
|
|
$self->filename_csv($filename); |
17
|
|
|
|
|
|
|
}, |
18
|
|
|
|
|
|
|
); |
19
|
|
|
|
|
|
|
|
20
|
|
|
|
|
|
|
has site_name => ( |
21
|
|
|
|
|
|
|
is => 'rw', |
22
|
|
|
|
|
|
|
isa => 'Str', |
23
|
|
|
|
|
|
|
default => '', |
24
|
|
|
|
|
|
|
); |
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
after 'site_name' => sub { |
27
|
|
|
|
|
|
|
my ( $self, $value, $skip_verify ) = @_; |
28
|
|
|
|
|
|
|
return if ! $value; |
29
|
|
|
|
|
|
|
if ( ! $skip_verify ) { |
30
|
|
|
|
|
|
|
$value =~ s{::}{-}g; |
31
|
|
|
|
|
|
|
$self->site_name( $value, 1 ); |
32
|
|
|
|
|
|
|
} |
33
|
|
|
|
|
|
|
} ; |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
has [ qw/title author content webpage meta_keywords meta_description/ ] => ( |
36
|
|
|
|
|
|
|
is => 'rw', |
37
|
|
|
|
|
|
|
isa => 'Any', |
38
|
|
|
|
|
|
|
); |
39
|
|
|
|
|
|
|
|
40
|
|
|
|
|
|
|
has images => ( |
41
|
|
|
|
|
|
|
is => 'rw', |
42
|
|
|
|
|
|
|
isa => 'ArrayRef', |
43
|
|
|
|
|
|
|
default => sub { return []; } , |
44
|
|
|
|
|
|
|
); |
45
|
|
|
|
|
|
|
|
46
|
|
|
|
|
|
|
has data => ( |
47
|
|
|
|
|
|
|
is => 'rw', |
48
|
|
|
|
|
|
|
isa => 'Data::News', |
49
|
|
|
|
|
|
|
default => sub { |
50
|
|
|
|
|
|
|
my ($self) = @_; |
51
|
|
|
|
|
|
|
return $self; |
52
|
|
|
|
|
|
|
}, |
53
|
|
|
|
|
|
|
); |
54
|
|
|
|
|
|
|
|
55
|
|
|
|
|
|
|
has csv => ( |
56
|
|
|
|
|
|
|
is => 'ro', |
57
|
|
|
|
|
|
|
isa => 'Text::CSV_XS', |
58
|
|
|
|
|
|
|
default => sub { |
59
|
|
|
|
|
|
|
my $csv = Text::CSV_XS->new() |
60
|
|
|
|
|
|
|
or die "Cannot use CSV: " . Text::CSV_XS->error_diag(); |
61
|
|
|
|
|
|
|
$csv->eol("\r\n"); |
62
|
|
|
|
|
|
|
return $csv; |
63
|
|
|
|
|
|
|
}, |
64
|
|
|
|
|
|
|
); |
65
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
sub save { #saves the data to csv |
67
|
|
|
|
|
|
|
my ($self) = @_; |
68
|
|
|
|
|
|
|
my @rows = ( |
69
|
|
|
|
|
|
|
[ |
70
|
|
|
|
|
|
|
sha1_hex( $self->webpage ), |
71
|
|
|
|
|
|
|
$self->webpage, |
72
|
|
|
|
|
|
|
decode_entities( $self->data->title ), |
73
|
|
|
|
|
|
|
decode_entities( $self->data->author ), |
74
|
|
|
|
|
|
|
decode_entities( $self->data->content ), |
75
|
|
|
|
|
|
|
decode_entities( $self->data->meta_keywords ), |
76
|
|
|
|
|
|
|
decode_entities( $self->data->meta_description ), |
77
|
|
|
|
|
|
|
join( '|' , @{ $self->images } ), |
78
|
|
|
|
|
|
|
], |
79
|
|
|
|
|
|
|
); |
80
|
|
|
|
|
|
|
my $file = './data/NEWS-' . $self->site_name. '-' . $self->filename_csv; |
81
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
open my $fh, ">>:encoding(utf8)", "$file" or die "$file: $!"; |
83
|
|
|
|
|
|
|
$self->csv->print( $fh, $_ ) for @rows; |
84
|
|
|
|
|
|
|
close $fh or die "Error on file $file: $!"; |
85
|
|
|
|
|
|
|
} |
86
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
1; |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
|
90
|
|
|
|
|
|
|
=head1 NAME |
91
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
Data::News - Handles the extracted data and saves it |
93
|
|
|
|
|
|
|
|
94
|
|
|
|
|
|
|
=head1 DESCRIPTION |
95
|
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
Data::News is an example class as to how the extracted data should |
97
|
|
|
|
|
|
|
be handled. |
98
|
|
|
|
|
|
|
|
99
|
|
|
|
|
|
|
In this case lib/Sites/XYZ will populate Data::News which will |
100
|
|
|
|
|
|
|
then handle the saving of this data. |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
=head1 AUTHOR |
103
|
|
|
|
|
|
|
|
104
|
|
|
|
|
|
|
Hernan Lopes |
105
|
|
|
|
|
|
|
CPAN ID: HERNAN |
106
|
|
|
|
|
|
|
Hernan |
107
|
|
|
|
|
|
|
hernanlopes@gmail.com |
108
|
|
|
|
|
|
|
http://github.com/hernan |
109
|
|
|
|
|
|
|
|
110
|
|
|
|
|
|
|
=head1 COPYRIGHT |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
This program is free software; you can redistribute |
113
|
|
|
|
|
|
|
it and/or modify it under the same terms as Perl itself. |
114
|
|
|
|
|
|
|
|
115
|
|
|
|
|
|
|
The full text of the license can be found in the |
116
|
|
|
|
|
|
|
LICENSE file included with this module. |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
|
119
|
|
|
|
|
|
|
=head1 SEE ALSO |
120
|
|
|
|
|
|
|
|
121
|
|
|
|
|
|
|
perl(1). |
122
|
|
|
|
|
|
|
|
123
|
|
|
|
|
|
|
=cut |
124
|
|
|
|
|
|
|
|