blib/lib/WWW/Scraper/ISBN/AmazonUK_Driver.pm | |||
---|---|---|---|
Criterion | Covered | Total | % |
statement | 88 | 95 | 92.6 |
branch | 32 | 44 | 72.7 |
condition | 10 | 24 | 41.6 |
subroutine | 9 | 9 | 100.0 |
pod | 1 | 1 | 100.0 |
total | 140 | 173 | 80.9 |
line | stmt | bran | cond | sub | pod | time | code |
---|---|---|---|---|---|---|---|
1 | package WWW::Scraper::ISBN::AmazonUK_Driver; | ||||||
2 | |||||||
3 | 6 | 6 | 122940 | use strict; | |||
6 | 20 | ||||||
6 | 342 | ||||||
4 | 6 | 6 | 42 | use warnings; | |||
6 | 19 | ||||||
6 | 272 | ||||||
5 | |||||||
6 | 6 | 6 | 72 | use vars qw($VERSION); | |||
6 | 14 | ||||||
6 | 548 | ||||||
7 | $VERSION = '0.40'; | ||||||
8 | |||||||
9 | #-------------------------------------------------------------------------- | ||||||
10 | |||||||
11 | =head1 NAME | ||||||
12 | |||||||
13 | WWW::Scraper::ISBN::AmazonUK_Driver - Search driver for Amazon.co.uk | ||||||
14 | |||||||
15 | =head1 SYNOPSIS | ||||||
16 | |||||||
17 | See parent class documentation (L |
||||||
18 | |||||||
19 | =head1 DESCRIPTION | ||||||
20 | |||||||
21 | Searches for book information from the (UK) Amazon online catalog. | ||||||
22 | |||||||
23 | =cut | ||||||
24 | |||||||
25 | #-------------------------------------------------------------------------- | ||||||
26 | |||||||
27 | ########################################################################### | ||||||
28 | # Inheritence | ||||||
29 | |||||||
30 | 6 | 6 | 35 | use base qw(WWW::Scraper::ISBN::Driver); | |||
6 | 21 | ||||||
6 | 2016 | ||||||
31 | |||||||
32 | ########################################################################### | ||||||
33 | # Modules | ||||||
34 | |||||||
35 | 6 | 6 | 4729 | use WWW::Mechanize; | |||
6 | 498765 | ||||||
6 | 200 | ||||||
36 | 6 | 6 | 1322 | use JSON; | |||
6 | 13377 | ||||||
6 | 69 | ||||||
37 | |||||||
38 | ########################################################################### | ||||||
39 | # Variables | ||||||
40 | |||||||
41 | my $AMA_SEARCH = 'http://www.amazon.co.uk/s/ref=nb_sb_noss?url=search-alias%3Daps&x=18&y=16&field-keywords='; | ||||||
42 | my $AMA_URL = 'http://www.amazon.co.uk/[^/]+/dp/[\dX]+/ref=sr_1_1/'; | ||||||
43 | my $IN2MM = 0.0393700787; # number of inches in a millimetre (mm) | ||||||
44 | my $LB2G = 0.00220462; # number of pounds (lbs) in a gram | ||||||
45 | my $OZ2G = 0.035274; # number of ounces (oz) in a gram | ||||||
46 | |||||||
47 | #-------------------------------------------------------------------------- | ||||||
48 | |||||||
49 | ########################################################################### | ||||||
50 | # Public Interface | ||||||
51 | |||||||
52 | =head1 METHODS | ||||||
53 | |||||||
54 | =over 4 | ||||||
55 | |||||||
56 | =item C |
||||||
57 | |||||||
58 | Creates a query string, then passes the appropriate form fields to the | ||||||
59 | Amazon (UK) server. | ||||||
60 | |||||||
61 | The returned page should be the correct catalog page for that ISBN. If not the | ||||||
62 | function returns zero and allows the next driver in the chain to have a go. If | ||||||
63 | a valid page is returned, the following fields are returned via the book hash: | ||||||
64 | |||||||
65 | isbn (now returns isbn13) | ||||||
66 | isbn10 | ||||||
67 | isbn13 | ||||||
68 | ean13 (industry name) | ||||||
69 | author | ||||||
70 | title | ||||||
71 | book_link | ||||||
72 | thumb_link | ||||||
73 | image_link | ||||||
74 | pubdate | ||||||
75 | publisher | ||||||
76 | binding (if known) | ||||||
77 | pages (if known) | ||||||
78 | weight (if known) (in grams) | ||||||
79 | width (if known) (in millimetres) | ||||||
80 | height (if known) (in millimetres) | ||||||
81 | depth (if known) (in millimetres) | ||||||
82 | |||||||
83 | The book_link, thumb_link and image_link refer back to the Amazon (UK) website. | ||||||
84 | |||||||
85 | =back | ||||||
86 | |||||||
87 | =cut | ||||||
88 | |||||||
89 | sub search { | ||||||
90 | 3 | 3 | 1 | 23901 | my $self = shift; | ||
91 | 3 | 8 | my $isbn = shift; | ||||
92 | 3 | 17 | $self->found(0); | ||||
93 | 3 | 59 | $self->book(undef); | ||||
94 | |||||||
95 | # validate and convert into EAN13 format | ||||||
96 | 3 | 37 | my $ean = $self->convert_to_ean13($isbn); | ||||
97 | 3 | 50 | 66 | 157 | return $self->handler("Invalid ISBN specified [$isbn]") | ||
33 | |||||||
66 | |||||||
33 | |||||||
98 | if(!$ean || (length $isbn == 13 && $isbn ne $ean) | ||||||
99 | || (length $isbn == 10 && $isbn ne $self->convert_to_isbn10($ean))); | ||||||
100 | |||||||
101 | 3 | 66 | my $mech = WWW::Mechanize->new(); | ||||
102 | 3 | 19729 | $mech->agent_alias( 'Linux Mozilla' ); | ||||
103 | |||||||
104 | 3 | 206 | my $search = $AMA_SEARCH . $ean; | ||||
105 | |||||||
106 | 3 | 7 | eval { $mech->get( $search ) }; | ||||
3 | 14 | ||||||
107 | 3 | 50 | 33 | 1487234 | return $self->handler("Amazon UK website appears to be unavailable.") | ||
33 | |||||||
108 | if($@ || !$mech->success() || !$mech->content()); | ||||||
109 | |||||||
110 | 3 | 213 | my $content = $mech->content(); | ||||
111 | #print STDERR "\n# content=[$content]\n"; | ||||||
112 | 3 | 428 | my ($link) = $content =~ m!($AMA_URL)!s; | ||||
113 | 3 | 50 | 13 | return $self->handler("Failed to find that book on Amazon UK website.") | |||
114 | unless($link); | ||||||
115 | |||||||
116 | 3 | 8 | eval { $mech->get( $link ) }; | ||||
3 | 16 | ||||||
117 | 3 | 50 | 33 | 6396971 | return $self->handler("Amazon UK website appears to be unavailable.") | ||
33 | |||||||
118 | if($@ || !$mech->success() || !$mech->content()); | ||||||
119 | |||||||
120 | 3 | 583 | return $self->_parse($mech); | ||||
121 | } | ||||||
122 | |||||||
123 | sub _parse { | ||||||
124 | 5 | 5 | 1503 | my $self = shift; | |||
125 | 5 | 11 | my $mech = shift; | ||||
126 | |||||||
127 | # The Book page | ||||||
128 | 5 | 20 | my $html = $mech->content; | ||||
129 | 5 | 538 | my $data = {}; | ||||
130 | |||||||
131 | #print STDERR "\n# html=[$html]\n"; | ||||||
132 | |||||||
133 | 5 | 1 | 94454 | my @size = $html =~ m! |
|||
1 | 21 | ||||||
1 | 3 | ||||||
1 | 27 | ||||||
134 | 5 | 100 | 31319 | @size = $html =~ m! |
|||
135 | 5 | 100 | 24 | if(@size) { | |||
136 | 4 | 12 | my $type = pop @size; | ||||
137 | 4 | 53 | ($data->{depth},$data->{width},$data->{height}) = sort @size; | ||||
138 | 4 | 50 | 23 | if($type eq 'cm') { | |||
0 | |||||||
139 | 4 | 57 | $data->{$_} = int($data->{$_} * 10) for(qw( height width depth )); | ||||
140 | } elsif($type eq 'inches') { | ||||||
141 | 0 | 0 | $data->{$_} = int($data->{$_} / $IN2MM) for(qw( height width depth )); | ||||
142 | } | ||||||
143 | } | ||||||
144 | |||||||
145 | 5 | 23891 | ($data->{binding},$data->{pages}) = $html =~ m! |
||||
146 | 5 | 41319 | ($data->{weight}) = $html =~ m! |
||||
147 | 5 | 23814 | ($data->{published}) = $html =~ m! |
||||
148 | 5 | 23835 | ($data->{isbn10}) = $html =~ m! |
||||
149 | 5 | 23157 | ($data->{isbn13}) = $html =~ m! |
||||
150 | 5 | 6519 | ($data->{content}) = $html =~ m! | ||||
151 | 5 | 46891 | ($data->{description}) = $html =~ m!From the Back Cover\s* \s* (.*?) | ||||
152 | 5 | 100 | 25698 | ($data->{description}) = $html =~ m! ]*>.*?{description}); |
|||
153 | |||||||
154 | 5 | 100 | 42 | $data->{weight} = int($data->{weight} / $OZ2G) if($data->{weight}); | |||
155 | |||||||
156 | 5 | 100 | 23 | if($data->{description}) { | |||
157 | 4 | 184 | $data->{description} =~ s!<[^>]+>!!g; | ||||
158 | 4 | 342 | $data->{description} =~ s! +! !g; | ||||
159 | } | ||||||
160 | |||||||
161 | # The images | ||||||
162 | 5 | 41058 | my ($json) = $html =~ /var colorImages = ([^;]+);/si; | ||||
163 | 5 | 50 | 32 | if($json) { | |||
164 | 0 | 0 | my $code = decode_json($json); | ||||
165 | 0 | 0 | my @order = grep {$_} $code->{initial}[0]{thumb}, $code->{initial}[0]{landing}, @{$code->{initial}[0]{main}}, $code->{initial}[0]{large}; | ||||
0 | 0 | ||||||
0 | 0 | ||||||
166 | 0 | 0 | 0 | $data->{thumb_link} = $order[0] if(@order); | |||
167 | 0 | 0 | 0 | $data->{image_link} = $order[-1] if(@order); | |||
168 | |||||||
169 | #use Data::Dumper; | ||||||
170 | #print STDERR "\n# code=[".Dumper($code)."]\n"; | ||||||
171 | } else { | ||||||
172 | 5 | 15187 | my ($code) = $html =~ /'imageGalleryData'\s*:\s*([^;]+);/si; | ||||
173 | 5 | 100 | 33 | if($code) { | |||
174 | 4 | 58 | ($data->{thumb_link}) = $code =~ /"thumbUrl":\s*"([^+]+)"/; | ||||
175 | 4 | 73 | ($data->{image_link}) = $code =~ /"mainUrl":\s*"([^+]+)"/; | ||||
176 | } | ||||||
177 | #use Data::Dumper; | ||||||
178 | #print STDERR "\n# code=[".Dumper($code)."]\n"; | ||||||
179 | } | ||||||
180 | |||||||
181 | |||||||
182 | # {\"initial\":[{\"large\":\"http://ecx.images-amazon.com/images/I/31cLTIXHKgL.jpg\",\"landing\":[\"http://ecx.images-amazon.com/images/I/31cLTIXHKgL._SY300_.jpg\"],\"thumb\":\"http://ecx.images-amazon.com/images/I/31cLTIXHKgL._SS40_.jpg\",\"main\":[\"http://ecx.images-amazon.com/images/I/31cLTIXHKgL._SX342_.jpg\",\"http://ecx.images-amazon.com/images/I/31cLTIXHKgL._SX385_.jpg\"]}]}; | ||||||
183 | |||||||
184 | 5 | 100 | 30 | if($data->{content}) { | |||
185 | 4 | 37 | $data->{content} =~ s/Amazon\.co\.uk.*?://i; | ||||
186 | 4 | 35 | $data->{content} =~ s/: Books.*//i; | ||||
187 | 4 | 54 | ($data->{title},$data->{author}) = split(/\s+by\s+/,$data->{content}); | ||||
188 | 4 | 34 | $data->{title} =~ s/^Buy\s+//; | ||||
189 | 4 | 40 | $data->{author} =~ s/\s*\(.*//; | ||||
190 | } | ||||||
191 | |||||||
192 | 5 | 100 | 77 | ($data->{publisher},$data->{pubdate}) = ($data->{published} =~ /\s*(.*?)(?:;.*?)?\s+\((.*?)\)/) if($data->{published}); | |||
193 | 5 | 100 | 24 | $data->{isbn10} =~ s/[^\dX]+//g if($data->{isbn10}); | |||
194 | 5 | 100 | 33 | $data->{isbn13} =~ s/\D+//g if($data->{isbn13}); | |||
195 | 5 | 100 | 22 | $data->{pubdate} =~ s/^.*?\(// if($data->{pubdate}); | |||
196 | |||||||
197 | 5 | 100 | 30 | return $self->handler("Could not extract data from Amazon UK result page.") | |||
198 | unless(defined $data->{isbn13}); | ||||||
199 | |||||||
200 | # trim top and tail | ||||||
201 | 4 | 100 | 30 | foreach (keys %$data) { next unless(defined $data->{$_});$data->{$_} =~ s/^\s+//;$data->{$_} =~ s/\s+$//; } | |||
68 | 133 | ||||||
65 | 131 | ||||||
65 | 282 | ||||||
202 | |||||||
203 | #use Data::Dumper; | ||||||
204 | #print STDERR "\n# data=[".Dumper($data)."]\n"; | ||||||
205 | |||||||
206 | 4 | 86 | my $bk = { | ||||
207 | 'ean13' => $data->{isbn13}, | ||||||
208 | 'isbn13' => $data->{isbn13}, | ||||||
209 | 'isbn10' => $data->{isbn10}, | ||||||
210 | 'isbn' => $data->{isbn13}, | ||||||
211 | 'author' => $data->{author}, | ||||||
212 | 'title' => $data->{title}, | ||||||
213 | 'image_link' => $data->{image_link}, | ||||||
214 | 'thumb_link' => $data->{thumb_link}, | ||||||
215 | 'publisher' => $data->{publisher}, | ||||||
216 | 'pubdate' => $data->{pubdate}, | ||||||
217 | 'book_link' => $mech->uri(), | ||||||
218 | 'content' => $data->{content}, | ||||||
219 | 'binding' => $data->{binding}, | ||||||
220 | 'pages' => $data->{pages}, | ||||||
221 | 'weight' => $data->{weight}, | ||||||
222 | 'width' => $data->{width}, | ||||||
223 | 'height' => $data->{height}, | ||||||
224 | 'depth' => $data->{depth}, | ||||||
225 | 'description' => $data->{description}, | ||||||
226 | 'html' => $html | ||||||
227 | }; | ||||||
228 | 4 | 242 | $self->book($bk); | ||||
229 | 4 | 78 | $self->found(1); | ||||
230 | 4 | 39 | return $self->book; | ||||
231 | } | ||||||
232 | |||||||
233 | q{currently reading: 'Torn Apart: The Life of Ian Curtis' by Mick Middles and Lindsay Reade}; | ||||||
234 | |||||||
235 | __END__ |