| 1 | ############################################################################### | ||
| 2 | package CrawlerCommons::ParseState; | ||
| 3 | |||
| 4 | # MODULE IMPORTS | ||
| 5 | ######################################## | ||
| 6 | # Pragmas | ||
| 7 | #------------------# | ||
| 8 | |||
| 9 | # CPAN/Core | ||
| 10 | #------------------# | ||
| 11 | 2 | ▶ | use Const::Fast; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 12 | 2 | ▶ | use Try::Tiny; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 13 | |||
| 14 | # Moose Setup | ||
| 15 | #------------------# | ||
| 16 | 2 | ▶ | use Moose; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 17 | 2 | ▶ | use namespace::autoclean; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 18 | |||
| 19 | # Moose Pragmas | ||
| 20 | #------------------# | ||
| 21 | |||
| 22 | # Custom Modules | ||
| 23 | #------------------# | ||
| 24 | 2 | ▶ | use CrawlerCommons::RobotRules; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 25 | |||
| 26 | |||
| 27 | # VARIABLES/CONSTANTS | ||
| 28 | ######################################## | ||
| 29 | # Constants | ||
| 30 | #------------------# | ||
| 31 | |||
| 32 | # Variables | ||
| 33 | #------------------# | ||
| 34 | 2 | our $VERSION = '0.03'; | |
| 35 | |||
| 36 | |||
| 37 | # MOOSE ATTRIBUTES | ||
| 38 | ######################################## | ||
| 39 | # Class | ||
| 40 | #-----------------------------------------------------------------------------# | ||
| 41 | #-----------------------------------------------------------------------------# | ||
| 42 | |||
| 43 | # Instance | ||
| 44 | #-----------------------------------------------------------------------------# | ||
| 45 | has 'current_rules' => ( | ||
| 46 | 70 | ▶ | default => sub { |
|
Subroutine
__ANON__: called CC=1 SCAR=0.0
| |||
| 47 | 70 | CrawlerCommons::RobotRules->new( | |
| 48 | _mode => $CrawlerCommons::RobotRules::ALLOW_SOME); | ||
| 49 | }, | ||
| 50 | 2 | is => 'ro', | |
| 51 | isa => 'CrawlerCommons::RobotRules', | ||
| 52 | ); | ||
| 53 | #-----------------------------------------------------------------------------# | ||
| 54 | 2 | has 'is_adding_rules' => ( | |
| 55 | default => 0, | ||
| 56 | is => 'rw', | ||
| 57 | isa => 'Bool', | ||
| 58 | ); | ||
| 59 | #-----------------------------------------------------------------------------# | ||
| 60 | 2 | has 'is_finished_agent_fields' => ( | |
| 61 | default => 0, | ||
| 62 | is => 'rw', | ||
| 63 | isa => 'Bool', | ||
| 64 | ); | ||
| 65 | #-----------------------------------------------------------------------------# | ||
| 66 | 2 | has 'is_matched_real_name' => ( | |
| 67 | default => 0, | ||
| 68 | is => 'rw', | ||
| 69 | isa => 'Bool', | ||
| 70 | ); | ||
| 71 | #-----------------------------------------------------------------------------# | ||
| 72 | 2 | has 'is_matched_wildcard' => ( | |
| 73 | default => 0, | ||
| 74 | is => 'rw', | ||
| 75 | isa => 'Bool', | ||
| 76 | ); | ||
| 77 | #-----------------------------------------------------------------------------# | ||
| 78 | 2 | has 'is_skip_agents' => ( | |
| 79 | default => 0, | ||
| 80 | is => 'rw', | ||
| 81 | isa => 'Bool', | ||
| 82 | ); | ||
| 83 | #-----------------------------------------------------------------------------# | ||
| 84 | 2 | has 'target_name' => ( | |
| 85 | is => 'ro', | ||
| 86 | isa => 'Str', | ||
| 87 | required => 1, | ||
| 88 | ); | ||
| 89 | #-----------------------------------------------------------------------------# | ||
| 90 | 2 | has 'url' => ( | |
| 91 | is => 'ro', | ||
| 92 | isa => 'Str', | ||
| 93 | required => 1, | ||
| 94 | ); | ||
| 95 | #-----------------------------------------------------------------------------# | ||
| 96 | |||
| 97 | # METHODS | ||
| 98 | ######################################## | ||
| 99 | # Construction | ||
| 100 | #------------------# | ||
| 101 | #-----------------------------------------------------------------------------# | ||
| 102 | #-----------------------------------------------------------------------------# | ||
| 103 | |||
| 104 | # Class Methods | ||
| 105 | #------------------# | ||
| 106 | #-----------------------------------------------------------------------------# | ||
| 107 | #-----------------------------------------------------------------------------# | ||
| 108 | |||
| 109 | # Instance Methods | ||
| 110 | #------------------# | ||
| 111 | #-----------------------------------------------------------------------------# | ||
| 112 | 254 | ▶ | sub add_rule { |
|
Subroutine
add_rule: called CC=1 SCAR=0.0
Pod
undocumented
| |||
| 113 | 254 | my ($self, $path, $allow) = @_; | |
| 114 | 254 | $self->current_rules->add_rule( $path, $allow ); | |
| 115 | } | ||
| 116 | #-----------------------------------------------------------------------------# | ||
| 117 | 20 | ▶ | sub add_sitemap { |
|
Subroutine
add_sitemap: called CC=1 SCAR=0.0
Pod
undocumented
| |||
| 118 | 20 | my ($self, $sitemap) = @_; | |
| 119 | 20 | $self->current_rules->add_sitemap( $sitemap ); | |
| 120 | } | ||
| 121 | #-----------------------------------------------------------------------------# | ||
| 122 | 44 | ▶ | sub clear_rules {$_[0]->current_rules->clear_rules;} |
|
Subroutine
clear_rules: called CC=1 SCAR=0.0
Pod
undocumented
| |||
| 123 | #-----------------------------------------------------------------------------# | ||
| 124 | 9 | ▶ | sub set_crawl_delay { |
|
Subroutine
set_crawl_delay: called CC=1 SCAR=0.0
Pod
undocumented
| |||
| 125 | 9 | my ($self, $delay_ms) = @_; | |
| 126 | 9 | $self->current_rules->set_crawl_delay( $delay_ms ); | |
| 127 | } | ||
| 128 | #-----------------------------------------------------------------------------# | ||
| 129 | |||
| 130 | # Private Methods | ||
| 131 | #------------------# | ||
| 132 | #-----------------------------------------------------------------------------# | ||
| 133 | #-----------------------------------------------------------------------------# | ||
| 134 | |||
| 135 | ############################################################################### | ||
| 136 | |||
| 137 | 2 | __PACKAGE__->meta->make_immutable; | |
| 138 | |||
| 139 | ############################################################################### | ||
| 140 | |||
| 141 | 2 | 1; | |
| 142 | |||
| 143 | __END__ | ||