| 1 | package Lyrics::Fetcher::Google; | ||
| 2 | use strict; | ||
| 3 | use warnings; | ||
| 4 | use Exporter; | ||
| 5 | use Net::Google; | ||
| 6 | use LWP::UserAgent; | ||
| 7 | use String::Similarity; | ||
| 8 | use HTML::LinkExtractor; | ||
| 9 | use HTML::TokeParser::Simple; | ||
| 10 | |||
| 11 | our @ISA = qw(Exporter); | ||
| 12 | our $VERSION = '0.02'; | ||
| 13 | |||
| 14 | use constant MIN_STRING_LENGTH => 200; | ||
| 15 | use constant MAX_BLOCKS_TO_SEARCH => 4; | ||
| 16 | |||
| 17 | my ($class) = @_; | ||
| 18 | my $ua = new LWP::UserAgent; | ||
| 19 | my $lx = new HTML::LinkExtractor; | ||
| 20 | $Lyrics::Fetcher::Error = 'OK'; | ||
| 21 | |||
| 22 | sub fetch($$$) { | ||
| 23 | my $self = shift; | ||
| 24 | my ( $artist, $song ) = @_; | ||
| 25 | my @links = &links( $artist, $song ); | ||
| 26 | |||
| 27 | $Lyrics::Fetcher::Error = | ||
| 28 | "Could not get any results from google. Did you supply a gid?" | ||
| 29 | unless (@links); | ||
| 30 | |||
| 31 | my $totaltext; | ||
| 32 | |||
| 33 | foreach my $link (@links) { | ||
| 34 | $totaltext .= &get($link); | ||
| 35 | } | ||
| 36 | |||
| 37 | my @biggest = &biggest_blocks($totaltext); | ||
| 38 | my %songs = &most_similar( splice( @biggest, 0, MAX_BLOCKS_TO_SEARCH ) ); | ||
| 39 | |||
| 40 | #@results contains multiple entries. Only the first(highest weighted) entry is returned. | ||
| 41 | my @results = sort { $songs{$b} <=> $songs{$a} } keys %songs; | ||
| 42 | return shift(@results); | ||
| 43 | } | ||
| 44 | |||
| 45 | sub links { | ||
| 46 | my ( $artist, $song ) = @_; | ||
| 47 | my $google = Net::Google->new( key => $Lyrics::Fetcher::gid ); | ||
| 48 | my $search = $google->search(); | ||
| 49 | $search->max_results(5); | ||
| 50 | $search->query( $artist, $song, 'lyrics' ); | ||
| 51 | return map { $_ = $_->{__URL} } @{ $search->results() }; | ||
| 52 | } | ||
| 53 | |||
| 54 | sub get { | ||
| 55 | my ($url) = @_; | ||
| 56 | $ua->timeout(6); | ||
| 57 | $ua->agent("Mozilla/5.0"); | ||
| 58 | my $res = $ua->get($url); | ||
| 59 | if ( $res->is_success ) { | ||
| 60 | return $res->content; | ||
| 61 | } | ||
| 62 | } | ||
| 63 | |||
| 64 | sub biggest_blocks { | ||
| 65 | my ($html) = @_; | ||
| 66 | my $p = HTML::TokeParser::Simple->new( \$html ); | ||
| 67 | my @blocks = (); | ||
| 68 | |||
| 69 | my $tc = ''; | ||
| 70 | while ( my $token = $p->get_token ) { | ||
| 71 | |||
| 72 | if ( $token->is_tag('br') ) { $tc .= "\n"; next; } | ||
| 73 | |||
| 74 | if ( $token->is_text ) { | ||
| 75 | my $t = $token->as_is; | ||
| 76 | $t =~ s/\&\#\d+\;//gs; | ||
| 77 | $t =~ s/<\/*.*?>//gs; | ||
| 78 | $t =~ s/([a-z])([A-Z])/$1\n$2/gs; | ||
| 79 | $t =~ s/\s+/ /gs; | ||
| 80 | $tc .= " $t"; | ||
| 81 | } | ||
| 82 | else { | ||
| 83 | push( @blocks, $tc ) if ( length($tc) > MIN_STRING_LENGTH ); | ||
| 84 | $tc = ''; | ||
| 85 | } | ||
| 86 | } | ||
| 87 | return sort { length $b <=> length $a } @blocks; | ||
| 88 | } | ||
| 89 | |||
| 90 | sub most_similar { | ||
| 91 | my (@strings) = @_; | ||
| 92 | my %rank; | ||
| 93 | foreach my $outside (@strings) { | ||
| 94 | foreach my $inside (@strings) { | ||
| 95 | $rank{$outside} += similarity( $outside, $inside ); | ||
| 96 | } | ||
| 97 | } | ||
| 98 | return %rank; | ||
| 99 | } | ||
| 100 | |||
| 101 | 1; | ||
| 102 | |||
| 103 | =pod | ||
| 104 | |||
| 105 | =head1 NAME | ||
| 106 | |||
| 107 | Lyrics::Fetcher::Google - Get some lyrics. Maybe. | ||
| 108 | |||
| 109 | |||
| 110 | =head1 SYNOPSIS | ||
| 111 | |||
| 112 | use Lyrics::Fetcher; | ||
| 113 | |||
| 114 | $Lyrics::Fetcher::gid = '<your google API id>'; | ||
| 115 | |||
| 116 | print Lyrics::Fetcher->fetch("<artist>","<song>","Google"); | ||
| 117 | |||
| 118 | |||
| 119 | =head1 DESCRIPTION | ||
| 120 | |||
| 121 | This module tries to find lyrics on the web. | ||
| 122 | Sometimes it works. But it probably won't. | ||
| 123 | |||
| 124 | It searches google for an initial set. It then | ||
| 125 | finds the largest block of plain text in the top | ||
| 126 | 5 results. Those results are then compared to | ||
| 127 | one another and weighted. The idea being that | ||
| 128 | a large block of text on one site may be a bunch | ||
| 129 | of poo, but a large area of similar text on multiple | ||
| 130 | sites most likely is the lyrics for which you are | ||
| 131 | looking. | ||
| 132 | |||
| 133 | |||
| 134 | =head1 BUGS | ||
| 135 | |||
| 136 | Yes. I would be happy to hear that this worked for someone. | ||
| 137 | Let me know if it does. I may even respond if you let me | ||
| 138 | know that it doesn't. | ||
| 139 | |||
| 140 | =head1 COPYRIGHT | ||
| 141 | |||
| 142 | This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. | ||
| 143 | |||
| 144 | The full text of the license can be found in the LICENSE file included with this module. | ||
| 145 | |||
| 146 | =head1 AUTHOR | ||
| 147 | |||
| 148 | John Lifsey <nebulous@crashed.net> | ||
| 149 | |||
| 150 | =cut | ||
| 151 | |||
| 152 | __END__ |