| 1 | # Generated by default/object.tt | ||
| 2 | package Paws::Kendra::WebCrawlerConfiguration; | ||
| 3 | use Moose; | ||
| 4 | has AuthenticationConfiguration => (is => 'ro', isa => 'Paws::Kendra::AuthenticationConfiguration'); | ||
| 5 | has CrawlDepth => (is => 'ro', isa => 'Int'); | ||
| 6 | has MaxContentSizePerPageInMegaBytes => (is => 'ro', isa => 'Num'); | ||
| 7 | has MaxLinksPerPage => (is => 'ro', isa => 'Int'); | ||
| 8 | has MaxUrlsPerMinuteCrawlRate => (is => 'ro', isa => 'Int'); | ||
| 9 | has ProxyConfiguration => (is => 'ro', isa => 'Paws::Kendra::ProxyConfiguration'); | ||
| 10 | has UrlExclusionPatterns => (is => 'ro', isa => 'ArrayRef[Str|Undef]'); | ||
| 11 | has UrlInclusionPatterns => (is => 'ro', isa => 'ArrayRef[Str|Undef]'); | ||
| 12 | has Urls => (is => 'ro', isa => 'Paws::Kendra::Urls', required => 1); | ||
| 13 | |||
| 14 | 1; | ||
| 15 | |||
| 16 | ### main pod documentation begin ### | ||
| 17 | |||
| 18 | =head1 NAME | ||
| 19 | |||
| 20 | Paws::Kendra::WebCrawlerConfiguration | ||
| 21 | |||
| 22 | =head1 USAGE | ||
| 23 | |||
| 24 | This class represents one of two things: | ||
| 25 | |||
| 26 | =head3 Arguments in a call to a service | ||
| 27 | |||
| 28 | Use the attributes of this class as arguments to methods. You shouldn't make instances of this class. | ||
| 29 | Each attribute should be used as a named argument in the calls that expect this type of object. | ||
| 30 | |||
| 31 | As an example, if Att1 is expected to be a Paws::Kendra::WebCrawlerConfiguration object: | ||
| 32 | |||
| 33 | $service_obj->Method(Att1 => { AuthenticationConfiguration => $value, ..., Urls => $value }); | ||
| 34 | |||
| 35 | =head3 Results returned from an API call | ||
| 36 | |||
| 37 | Use accessors for each attribute. If Att1 is expected to be an Paws::Kendra::WebCrawlerConfiguration object: | ||
| 38 | |||
| 39 | $result = $service_obj->Method(...); | ||
| 40 | $result->Att1->AuthenticationConfiguration | ||
| 41 | |||
| 42 | =head1 DESCRIPTION | ||
| 43 | |||
| 44 | Provides the configuration information required for Amazon Kendra web | ||
| 45 | crawler. | ||
| 46 | |||
| 47 | =head1 ATTRIBUTES | ||
| 48 | |||
| 49 | |||
| 50 | =head2 AuthenticationConfiguration => L<Paws::Kendra::AuthenticationConfiguration> | ||
| 51 | |||
| 52 | Provides configuration information required to connect to websites | ||
| 53 | using authentication. | ||
| 54 | |||
| 55 | You can connect to websites using basic authentication of user name and | ||
| 56 | password. | ||
| 57 | |||
| 58 | You must provide the website host name and port number. For example, | ||
| 59 | the host name of https://a.example.com/page1.html is "a.example.com" | ||
| 60 | and the port is 443, the standard port for HTTPS. You use a secret in | ||
| 61 | AWS Secrets Manager | ||
| 62 | (https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html) | ||
| 63 | to store your authentication credentials. | ||
| 64 | |||
| 65 | |||
| 66 | =head2 CrawlDepth => Int | ||
| 67 | |||
| 68 | Specifies the number of levels in a website that you want to crawl. | ||
| 69 | |||
| 70 | The first level begins from the website seed or starting point URL. For | ||
| 71 | example, if a website has 3 levels E<ndash> index level (i.e. seed in | ||
| 72 | this example), sections level, and subsections level E<ndash> and you | ||
| 73 | are only interested in crawling information up to the sections level | ||
| 74 | (i.e. levels 0-1), you can set your depth to 1. | ||
| 75 | |||
| 76 | The default crawl depth is set to 2. | ||
| 77 | |||
| 78 | |||
| 79 | =head2 MaxContentSizePerPageInMegaBytes => Num | ||
| 80 | |||
| 81 | The maximum size (in MB) of a webpage or attachment to crawl. | ||
| 82 | |||
| 83 | Files larger than this size (in MB) are skipped/not crawled. | ||
| 84 | |||
| 85 | The default maximum size of a webpage or attachment is set to 50 MB. | ||
| 86 | |||
| 87 | |||
| 88 | =head2 MaxLinksPerPage => Int | ||
| 89 | |||
| 90 | The maximum number of URLs on a webpage to include when crawling a | ||
| 91 | website. This number is per webpage. | ||
| 92 | |||
| 93 | As a websiteE<rsquo>s webpages are crawled, any URLs the webpages link | ||
| 94 | to are also crawled. URLs on a webpage are crawled in order of | ||
| 95 | appearance. | ||
| 96 | |||
| 97 | The default maximum links per page is 100. | ||
| 98 | |||
| 99 | |||
| 100 | =head2 MaxUrlsPerMinuteCrawlRate => Int | ||
| 101 | |||
| 102 | The maximum number of URLs crawled per website host per minute. | ||
| 103 | |||
| 104 | A minimum of one URL is required. | ||
| 105 | |||
| 106 | The default maximum number of URLs crawled per website host per minute | ||
| 107 | is 300. | ||
| 108 | |||
| 109 | |||
| 110 | =head2 ProxyConfiguration => L<Paws::Kendra::ProxyConfiguration> | ||
| 111 | |||
| 112 | Provides configuration information required to connect to your internal | ||
| 113 | websites via a web proxy. | ||
| 114 | |||
| 115 | You must provide the website host name and port number. For example, | ||
| 116 | the host name of https://a.example.com/page1.html is "a.example.com" | ||
| 117 | and the port is 443, the standard port for HTTPS. | ||
| 118 | |||
| 119 | Web proxy credentials are optional and you can use them to connect to a | ||
| 120 | web proxy server that requires basic authentication. To store web proxy | ||
| 121 | credentials, you use a secret in AWS Secrets Manager | ||
| 122 | (https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html). | ||
| 123 | |||
| 124 | |||
| 125 | =head2 UrlExclusionPatterns => ArrayRef[Str|Undef] | ||
| 126 | |||
| 127 | The regular expression pattern to exclude certain URLs to crawl. | ||
| 128 | |||
| 129 | If there is a regular expression pattern to include certain URLs that | ||
| 130 | conflicts with the exclude pattern, the exclude pattern takes | ||
| 131 | precedence. | ||
| 132 | |||
| 133 | |||
| 134 | =head2 UrlInclusionPatterns => ArrayRef[Str|Undef] | ||
| 135 | |||
| 136 | The regular expression pattern to include certain URLs to crawl. | ||
| 137 | |||
| 138 | If there is a regular expression pattern to exclude certain URLs that | ||
| 139 | conflicts with the include pattern, the exclude pattern takes | ||
| 140 | precedence. | ||
| 141 | |||
| 142 | |||
| 143 | =head2 B<REQUIRED> Urls => L<Paws::Kendra::Urls> | ||
| 144 | |||
| 145 | Specifies the seed or starting point URLs of the websites or the | ||
| 146 | sitemap URLs of the websites you want to crawl. | ||
| 147 | |||
| 148 | You can include website subdomains. You can list up to 100 seed URLs | ||
| 149 | and up to three sitemap URLs. | ||
| 150 | |||
| 151 | I<When selecting websites to index, you must adhere to the Amazon | ||
| 152 | Acceptable Use Policy (https://aws.amazon.com/aup/) and all other | ||
| 153 | Amazon terms. Remember that you must only use the Amazon Kendra web | ||
| 154 | crawler to index your own webpages, or webpages that you have | ||
| 155 | authorization to index.> | ||
| 156 | |||
| 157 | |||
| 158 | |||
| 159 | =head1 SEE ALSO | ||
| 160 | |||
| 161 | This class forms part of L<Paws>, describing an object used in L<Paws::Kendra> | ||
| 162 | |||
| 163 | =head1 BUGS and CONTRIBUTIONS | ||
| 164 | |||
| 165 | The source code is located here: L<https://github.com/pplu/aws-sdk-perl> | ||
| 166 | |||
| 167 | Please report bugs to: L<https://github.com/pplu/aws-sdk-perl/issues> | ||
| 168 | |||
| 169 | =cut | ||
| 170 |