| 1 | package NewsSpider::Terra; #has charset iso-8859-1 | ||
| 2 | use Moose; | ||
| 3 | with qw/Jungle::Spider/; | ||
| 4 | |||
| 5 | has startpage => ( | ||
| 6 | is => 'rw', | ||
| 7 | isa => 'Str', | ||
| 8 | default => 'http://noticias.terra.com.br/ultimasnoticias/0,,EI188,00.html', | ||
| 9 | ); | ||
| 10 | |||
| 11 | sub on_start { | ||
| 12 | my ( $self ) = @_; | ||
| 13 | } | ||
| 14 | |||
| 15 | sub search { | ||
| 16 | my ( $self ) = @_; | ||
| 17 | my $news = $self->tree->findnodes( '//div[@class="list articles"]/ol/li/a' ); | ||
| 18 | foreach my $item ( $news->get_nodelist ) { | ||
| 19 | my $url = $item->attr( 'href' ); | ||
| 20 | if ( $url =~ m{br\.invertia\.com}ig ) { | ||
| 21 | $self->prepend( details_invertia => $url ); | ||
| 22 | } else { | ||
| 23 | $self->prepend( details => $url ); | ||
| 24 | } | ||
| 25 | } | ||
| 26 | } | ||
| 27 | |||
| 28 | sub on_link { | ||
| 29 | my ( $self, $url ) = @_; | ||
| 30 | if ( $url =~ m{http://noticias.terra.com.br/ultimasnoticias/0,,EI188-PI(1|2),00.html}ig ) { # only the first 2 pages to test | ||
| 31 | $self->prepend( search => $url ); # append url on end of list | ||
| 32 | } | ||
| 33 | } | ||
| 34 | |||
| 35 | |||
| 36 | sub details_invertia { | ||
| 37 | my ( $self ) = @_; | ||
| 38 | my $page_title = $self->tree->findvalue( '//title' ); | ||
| 39 | my $author_nodes = $self->tree->findnodes( '//dl/dd' ); | ||
| 40 | my $author = ''; | ||
| 41 | foreach my $node ( $author_nodes->get_nodelist ) { | ||
| 42 | $author .= $node->as_text . "\n"; | ||
| 43 | } | ||
| 44 | my $content_nodes = $self->tree->findnodes( '//div[@id="SearchKey_Text1"]' ); | ||
| 45 | my $content; | ||
| 46 | foreach my $node ( $content_nodes->get_nodelist ) { | ||
| 47 | $content .= $node->as_HTML; | ||
| 48 | } | ||
| 49 | $self->data->author( $author ); | ||
| 50 | $self->data->content( $content ); | ||
| 51 | $self->data->title( $page_title ); | ||
| 52 | $self->data->webpage( $self->current_page ); | ||
| 53 | $self->grab_meta; | ||
| 54 | $self->grab_images; | ||
| 55 | |||
| 56 | $self->data->save; | ||
| 57 | } | ||
| 58 | |||
| 59 | sub grab_images { | ||
| 60 | my ( $self ) = @_; | ||
| 61 | my $images_nodes = $self->tree->findnodes( '//div[contains(@class,"img-article")]/img' ); | ||
| 62 | my @images = (); | ||
| 63 | foreach my $im ( $images_nodes->get_nodelist ) { | ||
| 64 | push ( @images, $self->normalize_url( $im->attr( 'src' ) ) ); | ||
| 65 | } | ||
| 66 | $self->data->images( \@images ); | ||
| 67 | } | ||
| 68 | |||
| 69 | |||
| 70 | sub details { | ||
| 71 | my ( $self ) = @_; | ||
| 72 | my $author_nodes = $self->tree->findnodes( '//dl/dd' ); | ||
| 73 | my $author = ''; | ||
| 74 | foreach my $node ( $author_nodes->get_nodelist ) { | ||
| 75 | $author .= $node->as_text . "\n"; | ||
| 76 | } | ||
| 77 | my $content_nodes = $self->tree->findnodes( '//div[@id="SearchKey_Text1"]//p' ); | ||
| 78 | my $content = ''; | ||
| 79 | foreach my $node ( $content_nodes->get_nodelist ) { | ||
| 80 | $content .= $node->as_text."\n"; | ||
| 81 | } | ||
| 82 | $self->data->author( $author ); | ||
| 83 | $self->data->content( $content ); | ||
| 84 | $self->data->title( $self->tree->findvalue( '//title' ) ); | ||
| 85 | $self->data->webpage( $self->current_page ); | ||
| 86 | $self->grab_meta; | ||
| 87 | $self->grab_images; | ||
| 88 | |||
| 89 | $self->data->save; | ||
| 90 | } | ||
| 91 | |||
| 92 | sub grab_meta { | ||
| 93 | my ( $self ) = @_; | ||
| 94 | $self->data->meta_keywords( $self->tree->findvalue( '//meta[@name="keywords"]/@content' ) ); | ||
| 95 | $self->data->meta_description( $self->tree->findvalue( '//meta[@name="description"]/@content' ) ); | ||
| 96 | } | ||
| 97 | |||
| 98 | 1; | ||
| 99 |