NewsSpider/Terra.pm untested

filter: Statementstmt 0.0% 0 / 58 Branchbran 0.0% 0 / 4 Conditioncond n/a MC/DCmcdc n/a Subroutinesub 0.0% 0 / 8 Podpod 0.0% 0 / 7 totaltotal 0.0% 0 / 77 CC 3 SCAR 12.8
CC
3
Cov
0%
search
6.0
on_link
6.0
on_start
2.0
CRAP 3.6
1 package NewsSpider::Terra; #has charset iso-8859-1
2 use Moose;
3 with qw/Jungle::Spider/;
4  
5 has startpage => (
6 is => 'rw',
7 isa => 'Str',
8 default => 'http://noticias.terra.com.br/ultimasnoticias/0,,EI188,00.html',
9 );
10  
11 sub on_start {
12 my ( $self ) = @_;
13 }
14  
15 sub search {
16 my ( $self ) = @_;
17 my $news = $self->tree->findnodes( '//div[@class="list articles"]/ol/li/a' );
18 foreach my $item ( $news->get_nodelist ) {
19 my $url = $item->attr( 'href' );
20 if ( $url =~ m{br\.invertia\.com}ig ) {
21 $self->prepend( details_invertia => $url );
22 } else {
23 $self->prepend( details => $url );
24 }
25 }
26 }
27  
28 sub on_link {
29 my ( $self, $url ) = @_;
30 if ( $url =~ m{http://noticias.terra.com.br/ultimasnoticias/0,,EI188-PI(1|2),00.html}ig ) { # only the first 2 pages to test
31 $self->prepend( search => $url ); # append url on end of list
32 }
33 }
34  
35  
36 sub details_invertia {
37 my ( $self ) = @_;
38 my $page_title = $self->tree->findvalue( '//title' );
39 my $author_nodes = $self->tree->findnodes( '//dl/dd' );
40 my $author = '';
41 foreach my $node ( $author_nodes->get_nodelist ) {
42 $author .= $node->as_text . "\n";
43 }
44 my $content_nodes = $self->tree->findnodes( '//div[@id="SearchKey_Text1"]' );
45 my $content;
46 foreach my $node ( $content_nodes->get_nodelist ) {
47 $content .= $node->as_HTML;
48 }
49 $self->data->author( $author );
50 $self->data->content( $content );
51 $self->data->title( $page_title );
52 $self->data->webpage( $self->current_page );
53 $self->grab_meta;
54 $self->grab_images;
55  
56 $self->data->save;
57 }
58  
59 sub grab_images {
60 my ( $self ) = @_;
61 my $images_nodes = $self->tree->findnodes( '//div[contains(@class,"img-article")]/img' );
62 my @images = ();
63 foreach my $im ( $images_nodes->get_nodelist ) {
64 push ( @images, $self->normalize_url( $im->attr( 'src' ) ) );
65 }
66 $self->data->images( \@images );
67 }
68  
69  
70 sub details {
71 my ( $self ) = @_;
72 my $author_nodes = $self->tree->findnodes( '//dl/dd' );
73 my $author = '';
74 foreach my $node ( $author_nodes->get_nodelist ) {
75 $author .= $node->as_text . "\n";
76 }
77 my $content_nodes = $self->tree->findnodes( '//div[@id="SearchKey_Text1"]//p' );
78 my $content = '';
79 foreach my $node ( $content_nodes->get_nodelist ) {
80 $content .= $node->as_text."\n";
81 }
82 $self->data->author( $author );
83 $self->data->content( $content );
84 $self->data->title( $self->tree->findvalue( '//title' ) );
85 $self->data->webpage( $self->current_page );
86 $self->grab_meta;
87 $self->grab_images;
88  
89 $self->data->save;
90 }
91  
92 sub grab_meta {
93 my ( $self ) = @_;
94 $self->data->meta_keywords( $self->tree->findvalue( '//meta[@name="keywords"]/@content' ) );
95 $self->data->meta_description( $self->tree->findvalue( '//meta[@name="description"]/@content' ) );
96 }
97  
98 1;
99