| 1 | # $Id: /mirror/gungho/lib/Gungho/Component/RobotRules.pm 31300 2007-11-29T11:51:03.339151Z lestrrat $ | ||
| 2 | # | ||
| 3 | # Copyright (c) 2007 Daisuke Maki <daisuke@endeworks.jp> | ||
| 4 | |||
| 5 | package Gungho::Component::RobotRules; | ||
| 6 | use strict; | ||
| 7 | use warnings; | ||
| 8 | use base qw(Gungho::Component); | ||
| 9 | use Gungho::Component::RobotRules::Rule; | ||
| 10 | use WWW::RobotRules::Parser; | ||
| 11 | |||
| 12 | __PACKAGE__->mk_classdata($_) for qw(pending_robots_txt robot_rules_parser robot_rules_storage pending_count); | ||
| 13 | |||
| 14 | sub setup | ||
| 15 | { | ||
| 16 | my $c = shift; | ||
| 17 | $c->pending_robots_txt({}); | ||
| 18 | $c->pending_count(0); | ||
| 19 | $c->setup_robot_rules_storage(); | ||
| 20 | $c->setup_robot_rules_parser(); | ||
| 21 | $c->next::method(@_); | ||
| 22 | } | ||
| 23 | |||
| 24 | sub send_request | ||
| 25 | { | ||
| 26 | my ($c, $request) = @_; | ||
| 27 | |||
| 28 | my $allowed = | ||
| 29 | $request->uri->path eq '/robots.txt' || | ||
| 30 | $c->allowed($request) | ||
| 31 | ; | ||
| 32 | if ($allowed == -2) { | ||
| 33 | $c->log->debug("[ROBOT RULES] Fetch for /robots.txt already scheduled for " . $request->original_uri->host_port); | ||
| 34 | } elsif ($allowed == -1) { | ||
| 35 | $c->log->debug("[ROBOT RULES] No robot rules found for " . $request->original_uri->host_port . ", going to fetch one"); | ||
| 36 | } elsif ($allowed) { | ||
| 37 | return $c->next::method($request); | ||
| 38 | } else { | ||
| 39 | $c->log->debug("[ROBOT RULES] " . $request->uri . " is disallowed by robot rules"); | ||
| 40 | } | ||
| 41 | return 0; | ||
| 42 | } | ||
| 43 | |||
| 44 | sub allowed | ||
| 45 | { | ||
| 46 | my ($c, $request) = @_; | ||
| 47 | |||
| 48 | my $rule = $c->robot_rules_storage->get_rule( $c, $request ); | ||
| 49 | if (! $rule) { | ||
| 50 | if ($c->push_pending_robots_txt($request) == 0) { | ||
| 51 | return -2; | ||
| 52 | } | ||
| 53 | |||
| 54 | my $uri = $request->original_uri; | ||
| 55 | $uri->path('/robots.txt'); | ||
| 56 | $uri->query(undef); | ||
| 57 | $uri->fragment(undef); | ||
| 58 | my $req = Gungho::Request->new(GET => $uri); | ||
| 59 | $req->notes('auto_robot_rules' => 1); | ||
| 60 | $c->pushback_request( $req ); | ||
| 61 | return -1; | ||
| 62 | } else { | ||
| 63 | return $rule->allowed( $c, $request->uri ); | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | sub handle_response | ||
| 68 | { | ||
| 69 | my $c = shift; | ||
| 70 | my ($request, $response) = @_; | ||
| 71 | |||
| 72 | if ($request->uri->path eq '/robots.txt' && $request->notes('auto_robot_rules')) { | ||
| 73 | $c->log->debug("Handling robots.txt response for " . $request->uri); | ||
| 74 | $c->parse_robot_rules($request, $response); | ||
| 75 | $c->dispatch_pending_robots_txt($request); | ||
| 76 | return; | ||
| 77 | } | ||
| 78 | |||
| 79 | $c->next::method(@_); | ||
| 80 | } | ||
| 81 | |||
| 82 | sub push_pending_robots_txt | ||
| 83 | { | ||
| 84 | my ($c, $request) = @_; | ||
| 85 | $c->pending_count( $c->pending_count + 1 ); | ||
| 86 | $c->log->debug( | ||
| 87 | "[ROBOT RULES]: Requests still pending: " . $c->pending_count | ||
| 88 | ); | ||
| 89 | return $c->robot_rules_storage->push_pending_robots_txt( $c, $request ); | ||
| 90 | } | ||
| 91 | |||
| 92 | sub dispatch_pending_robots_txt | ||
| 93 | { | ||
| 94 | my ($c, $request) = @_; | ||
| 95 | |||
| 96 | my $pending = $c->robot_rules_storage->get_pending_robots_txt($c, $request); | ||
| 97 | if ($pending && ref $pending eq 'HASH') { | ||
| 98 | foreach my $request (values %$pending) { | ||
| 99 | $c->pending_count( $c->pending_count - 1 ); | ||
| 100 | $c->pushback_request( $request ); | ||
| 101 | } | ||
| 102 | } | ||
| 103 | $c->log->debug( | ||
| 104 | "[ROBOT RULES]: Requests still pending: " . $c->pending_count | ||
| 105 | ); | ||
| 106 | } | ||
| 107 | |||
| 108 | sub setup_robot_rules_storage | ||
| 109 | { | ||
| 110 | my $c = shift; | ||
| 111 | |||
| 112 | my $config = $c->config->{robotrules}{storage} || {}; | ||
| 113 | |||
| 114 | my $pkg = $config->{module} || 'DB_File'; | ||
| 115 | my $pkg_config = $config->{config} || {}; | ||
| 116 | $pkg = $c->load_gungho_module($pkg, 'Component::RobotRules::Storage'); | ||
| 117 | my $storage = $pkg->new(%$config); | ||
| 118 | $storage->setup($c); | ||
| 119 | $c->robot_rules_storage( $storage ); | ||
| 120 | } | ||
| 121 | |||
| 122 | sub setup_robot_rules_parser | ||
| 123 | { | ||
| 124 | my $c = shift; | ||
| 125 | |||
| 126 | my $config = $c->config->{robotrules}{parser} || {}; | ||
| 127 | |||
| 128 | my $pkg = $config->{module} || '+WWW::RobotRules::Parser'; | ||
| 129 | my $pkg_config = $config->{config} || {}; | ||
| 130 | $pkg = $c->load_gungho_module($pkg, 'Component'); | ||
| 131 | my $parser = $pkg->new($config); | ||
| 132 | $c->robot_rules_parser( $parser ); | ||
| 133 | } | ||
| 134 | |||
| 135 | sub parse_robot_rules | ||
| 136 | { | ||
| 137 | my ($c, $request, $response) = @_; | ||
| 138 | |||
| 139 | my $h = ($request && $response && $response->is_success && $response->content) ? | ||
| 140 | $c->robot_rules_parser->parse($request->original_uri, $response->content) : | ||
| 141 | {} | ||
| 142 | ; | ||
| 143 | $c->log->debug("Parse robot rules " . $request->uri . ": " . keys(%$h) . " rules"); | ||
| 144 | my $rule = Gungho::Component::RobotRules::Rule->new($h); | ||
| 145 | $c->robot_rules_storage->put_rule($c, $request, $rule); | ||
| 146 | } | ||
| 147 | |||
| 148 | 1; | ||
| 149 | |||
| 150 | =head1 NAME | ||
| 151 | |||
| 152 | Gungho::Component::RobotRules - Respect robots.txt | ||
| 153 | |||
| 154 | =head1 SYNOPSIS | ||
| 155 | |||
| 156 | --- | ||
| 157 | components: | ||
| 158 | - RobotRules | ||
| 159 | |||
| 160 | =head1 METHODS | ||
| 161 | |||
| 162 | =head2 setup | ||
| 163 | |||
| 164 | =head2 setup_robot_rules_parser | ||
| 165 | |||
| 166 | =head2 setup_robot_rules_storage | ||
| 167 | |||
| 168 | =head2 handle_response | ||
| 169 | |||
| 170 | =head2 send_request | ||
| 171 | |||
| 172 | =head2 allowed($request) | ||
| 173 | |||
| 174 | Returns 1 if request is allowed to be fetched, 0 if not. -1 and -2 are returned | ||
| 175 | when there is a pending request to fetch /robots.txt | ||
| 176 | |||
| 177 | =head2 dispatch_pending_robots_txt | ||
| 178 | |||
| 179 | Dispatches requests that were pending because of a missing robots.txt entry | ||
| 180 | |||
| 181 | =head2 push_pending_robots_txt | ||
| 182 | |||
| 183 | Pushes a request in the wait queue for a robots.txt | ||
| 184 | |||
| 185 | =head2 parse_robot_rules | ||
| 186 | |||
| 187 | Parses the robot rule and stores it | ||
| 188 | |||
| 189 | =cut |