| 1 | package WWW::YouTube::Download::Channel; | ||
| 2 | 1 | ▶ | use utf8; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 3 | 1 | ▶ | use Moose; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 1 | ▶ | use WWW::Mechanize; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 0 | ▶ | use XML::XPath; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 6 | use XML::XPath::XMLParser; | ||
| 7 | use WWW::YouTube::Download; | ||
| 8 | use Perl6::Form; | ||
| 9 | use DateTime; | ||
| 10 | use Try::Tiny; | ||
| 11 | |||
| 12 | our $VERSION = '0.09'; | ||
| 13 | our $VER = $VERSION; | ||
| 14 | |||
| 15 | has agent => ( | ||
| 16 | is => 'rw', | ||
| 17 | isa => 'WWW::Mechanize', | ||
| 18 | default => sub { | ||
| 19 | my $mech = WWW::Mechanize->new(); | ||
| 20 | $mech->agent_alias('Windows IE 6'); | ||
| 21 | return $mech; | ||
| 22 | }, | ||
| 23 | ); | ||
| 24 | |||
| 25 | has xmlxpath => ( | ||
| 26 | is => 'rw', | ||
| 27 | isa => 'XML::XPath', | ||
| 28 | ); | ||
| 29 | |||
| 30 | has video_list_ids => ( | ||
| 31 | is => 'rw', | ||
| 32 | isa => 'ArrayRef', | ||
| 33 | default => sub { | ||
| 34 | my @arr; | ||
| 35 | return \@arr; | ||
| 36 | }, | ||
| 37 | ); | ||
| 38 | |||
| 39 | has total_user_videos => ( | ||
| 40 | is => 'rw', | ||
| 41 | isa => 'Int', | ||
| 42 | default => 0, | ||
| 43 | ); | ||
| 44 | |||
| 45 | has total_download_videos => ( | ||
| 46 | is => 'rw', | ||
| 47 | isa => 'Int', | ||
| 48 | default => 0, | ||
| 49 | ); | ||
| 50 | |||
| 51 | has entry_url => ( | ||
| 52 | is => 'rw', | ||
| 53 | isa => 'Str', | ||
| 54 | ); | ||
| 55 | |||
| 56 | has channel => ( | ||
| 57 | is => 'rw', | ||
| 58 | isa => 'Str', | ||
| 59 | default => '', | ||
| 60 | ); | ||
| 61 | |||
| 62 | has url_next => ( | ||
| 63 | is => 'rw', | ||
| 64 | isa => 'Str', | ||
| 65 | default => '', | ||
| 66 | ); | ||
| 67 | |||
| 68 | has page_video_found => ( | ||
| 69 | is => 'rw', | ||
| 70 | isa => 'Int', | ||
| 71 | default => 0, | ||
| 72 | ); | ||
| 73 | |||
| 74 | has start_index => ( #page index | ||
| 75 | is => 'rw', | ||
| 76 | isa => 'Int', | ||
| 77 | default => 1, | ||
| 78 | ); | ||
| 79 | |||
| 80 | has max_results => ( #limit results per page retrieved | ||
| 81 | is => 'ro', | ||
| 82 | isa => 'Int', | ||
| 83 | default => 50, #youtube limit | ||
| 84 | ); | ||
| 85 | |||
| 86 | has target_directory => ( | ||
| 87 | is => 'rw', | ||
| 88 | isa => 'Str', | ||
| 89 | ); | ||
| 90 | |||
| 91 | has filter_title_regex => ( | ||
| 92 | is => 'rw', | ||
| 93 | isa => 'Str', | ||
| 94 | ); | ||
| 95 | |||
| 96 | has skip_title_regex => ( | ||
| 97 | is => 'rw', | ||
| 98 | isa => 'Str', | ||
| 99 | |||
| 100 | # default => '', | ||
| 101 | ); | ||
| 102 | |||
| 103 | has date_filter_newer => ( | ||
| 104 | is => 'rw', | ||
| 105 | isa => 'DateTime', | ||
| 106 | ); | ||
| 107 | |||
| 108 | has debug => ( | ||
| 109 | is => 'rw', | ||
| 110 | isa => 'Int', | ||
| 111 | default => 0, | ||
| 112 | ); | ||
| 113 | |||
| 114 | has errors_download => ( | ||
| 115 | is => 'rw', | ||
| 116 | isa => 'ArrayRef', | ||
| 117 | default => sub { [] }, | ||
| 118 | ); | ||
| 119 | |||
| 120 | sub prepare_nfo { | ||
| 121 | my ( $self, $item ) = @_; | ||
| 122 | |||
| 123 | my $info = form | ||
| 124 | "===============================================================================", | ||
| 125 | "--[ WWW::YouTube::Download::Channel ]------------------------------------------", | ||
| 126 | " ", | ||
| 127 | " {||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||} ", | ||
| 128 | "[" . $item->{title} . "]", | ||
| 129 | " ", | ||
| 130 | " .............Title: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 131 | $item->{title}, | ||
| 132 | " .........Video Url: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 133 | $item->{video_url}, | ||
| 134 | " ............Author: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 135 | $item->{author}, | ||
| 136 | " ........Author Url: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 137 | $item->{author_url}, | ||
| 138 | " ....Date Published: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 139 | $item->{published_date}, | ||
| 140 | " ......Date Updated: {<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<} ", | ||
| 141 | $item->{updated_date}, | ||
| 142 | " ", | ||
| 143 | " [ REVIEW ] ", | ||
| 144 | " ", | ||
| 145 | " {[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[} ", | ||
| 146 | $item->{content}, | ||
| 147 | " ", | ||
| 148 | "---------------------------------------------------[ version $VER by HERNAN ]--", | ||
| 149 | "===============================================================================" | ||
| 150 | ,; | ||
| 151 | return $info; | ||
| 152 | } | ||
| 153 | |||
| 154 | sub newer_than { | ||
| 155 | my ( $self, $date ) = @_; | ||
| 156 | return | ||
| 157 | if ref $date ne 'HASH' | ||
| 158 | and !$date->{day} | ||
| 159 | and !$date->{month} | ||
| 160 | and !$date->{year}; | ||
| 161 | $self->date_filter_newer( DateTime->new($date) ); | ||
| 162 | } | ||
| 163 | |||
| 164 | sub prepare_item { | ||
| 165 | my ( $self, $html ) = @_; | ||
| 166 | my $xml_details = XML::XPath->new( xml => $html ); | ||
| 167 | my $video_id = $xml_details->findvalue('//id'); | ||
| 168 | $video_id =~ s{http://gdata.youtube.com/feeds/api/videos/}{}i; | ||
| 169 | |||
| 170 | my $published_date = | ||
| 171 | $self->transform_youtube_date( $xml_details->findvalue('//published') ); | ||
| 172 | |||
| 173 | my $updated_date = | ||
| 174 | $self->transform_youtube_date( $xml_details->findvalue('//updated') ); | ||
| 175 | |||
| 176 | my $content = $xml_details->findvalue('//content'); | ||
| 177 | my $author = $xml_details->findvalue('//author//name'); | ||
| 178 | my $video_url = "http://www.youtube.com/watch?v=$video_id"; | ||
| 179 | my $author_url = "http://www.youtube.com/user/$author"; | ||
| 180 | |||
| 181 | my $video_title = $xml_details->findvalue('//title'); | ||
| 182 | |||
| 183 | my $filename = | ||
| 184 | $self->title_to_filename( $video_title . '-' . $published_date ); | ||
| 185 | |||
| 186 | $filename = | ||
| 187 | ( defined $self->target_directory ) | ||
| 188 | ? $self->target_directory . '/' . $filename | ||
| 189 | : $filename; | ||
| 190 | my $filename_nfo = $filename . '.nfo'; | ||
| 191 | my $item = { | ||
| 192 | id => $video_id, | ||
| 193 | title => $video_title, | ||
| 194 | published_date => $published_date, | ||
| 195 | published_datetime => $self->string_to_datetime($published_date), | ||
| 196 | updated_date => $updated_date, | ||
| 197 | url => $video_url, | ||
| 198 | filename => $filename, | ||
| 199 | filename_nfo => $filename_nfo, | ||
| 200 | video_url => $video_url, | ||
| 201 | author_url => $author_url, | ||
| 202 | author => $author, | ||
| 203 | content => $content, | ||
| 204 | }; | ||
| 205 | undef $xml_details; | ||
| 206 | return $item; | ||
| 207 | } | ||
| 208 | |||
| 209 | sub parse_page { | ||
| 210 | my ( $self, $html_content ) = @_; | ||
| 211 | my $xml = XML::XPath->new( xml => $html_content ); | ||
| 212 | $self->page_video_found(0); | ||
| 213 | my $nodeset = $xml->findnodes('//entry'); | ||
| 214 | foreach my $node_html ( $nodeset->get_nodelist ) { | ||
| 215 | if ( $node_html->string_value =~ | ||
| 216 | m{^http://gdata.youtube.com/feeds/api/videos} ) | ||
| 217 | { | ||
| 218 | $self->page_video_found( $self->page_video_found + 1 ); | ||
| 219 | $self->total_user_videos( $self->total_user_videos + 1 ); | ||
| 220 | |||
| 221 | my $item = $self->prepare_item( | ||
| 222 | XML::XPath::XMLParser::as_string($node_html) ); | ||
| 223 | |||
| 224 | my $regex = $self->filter_title_regex | ||
| 225 | if defined $self->filter_title_regex; | ||
| 226 | |||
| 227 | if ( !$regex || $item->{title} =~ m/$regex/ig ) { | ||
| 228 | my $regex_skip = $self->skip_title_regex | ||
| 229 | if defined $self->skip_title_regex; | ||
| 230 | |||
| 231 | # warn "skipping regex: " . $regex_skip ; | ||
| 232 | if ( !$regex_skip || $item->{title} !~ m/$regex_skip/i ) { | ||
| 233 | if ( | ||
| 234 | !$self->date_filter_newer #skips filter by date | ||
| 235 | || DateTime->compare( | ||
| 236 | $item->{published_datetime}, | ||
| 237 | $self->date_filter_newer | ||
| 238 | ) == 1 #date1 > date2 | ||
| 239 | || DateTime->compare( | ||
| 240 | $item->{published_datetime}, | ||
| 241 | $self->date_filter_newer | ||
| 242 | ) == 0 #date1 == date2 | ||
| 243 | ) | ||
| 244 | { | ||
| 245 | warn "Video_id: " . $item->{id} if $self->debug; | ||
| 246 | warn "Title: " . $item->{title} if $self->debug; | ||
| 247 | warn "Filename: " . $item->{filename} if $self->debug; | ||
| 248 | $self->total_download_videos( | ||
| 249 | $self->total_download_videos + 1 ); | ||
| 250 | |||
| 251 | $item->{nfo} = $self->prepare_nfo($item); | ||
| 252 | push( @{ $self->video_list_ids }, $item ); | ||
| 253 | } | ||
| 254 | } | ||
| 255 | } | ||
| 256 | } | ||
| 257 | } | ||
| 258 | undef($xml); | ||
| 259 | } | ||
| 260 | |||
| 261 | sub string_to_datetime { | ||
| 262 | my ( $self, $date_string ) = @_; | ||
| 263 | my @date_parts = split( '-', $date_string ); | ||
| 264 | my $date = DateTime->new( | ||
| 265 | { | ||
| 266 | day => $date_parts[2], | ||
| 267 | month => $date_parts[1], | ||
| 268 | year => $date_parts[0], | ||
| 269 | } | ||
| 270 | ); | ||
| 271 | return $date; | ||
| 272 | } | ||
| 273 | |||
| 274 | sub transform_youtube_date { | ||
| 275 | my ( $self, $date ) = @_; | ||
| 276 | if ( $date =~ m/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})/i ) { | ||
| 277 | my $year = $1; | ||
| 278 | my $month = $2; | ||
| 279 | my $day = $3; | ||
| 280 | my $hour = $4; | ||
| 281 | my $min = $5; | ||
| 282 | my $sec = $6; | ||
| 283 | return "$year-$month-$day"; | ||
| 284 | } | ||
| 285 | else { | ||
| 286 | return "0000-00-00"; | ||
| 287 | } | ||
| 288 | } | ||
| 289 | |||
| 290 | sub title_to_filename { | ||
| 291 | my ( $self, $title ) = @_; | ||
| 292 | $title =~ s/\W/-/ig; | ||
| 293 | $title =~ s/--{1,}/-/ig; | ||
| 294 | $title =~ s/^-|-$//ig; | ||
| 295 | $title =~ | ||
| 296 | tr/àáâãäçèéêëìíîïñòóôõöùúûüýÿÀÁÂÃÄÇÈÉÊËÌÍÎÏÑÒÓÔÕÖÙÚÛÜÝ/aaaaaceeeeiiiinooooouuuuyyAAAAACEEEEIIIINOOOOOUUUUY/; | ||
| 297 | return $title; | ||
| 298 | } | ||
| 299 | |||
| 300 | sub entry { | ||
| 301 | my ( $self, $url ) = @_; | ||
| 302 | $self->entry_url($url); | ||
| 303 | $self->define_next_url(); | ||
| 304 | $self->list_videos(); | ||
| 305 | } | ||
| 306 | |||
| 307 | sub leech_channel { | ||
| 308 | my ( $self, $channel ) = @_; | ||
| 309 | $self->channel($channel); | ||
| 310 | $self->entry( | ||
| 311 | 'https://gdata.youtube.com/feeds/api/users/' . $channel . '/uploads' ) | ||
| 312 | if defined $channel; | ||
| 313 | } | ||
| 314 | |||
| 315 | sub define_next_url { | ||
| 316 | my ($self) = @_; | ||
| 317 | my $uri = URI->new( $self->entry_url ); | ||
| 318 | $uri->query_form( | ||
| 319 | 'start-index' => $self->start_index, | ||
| 320 | 'max-results' => $self->max_results, | ||
| 321 | ); | ||
| 322 | $self->url_next( $uri->as_string ); | ||
| 323 | } | ||
| 324 | |||
| 325 | sub list_videos { | ||
| 326 | my ($self) = @_; | ||
| 327 | $self->agent->get( $self->url_next ); | ||
| 328 | $self->parse_page( $self->agent->content ); | ||
| 329 | while ( $self->page_video_found > 0 ) { | ||
| 330 | $self->start_index( $self->start_index + $self->max_results ); | ||
| 331 | $self->define_next_url(); | ||
| 332 | $self->list_videos(); | ||
| 333 | } | ||
| 334 | } | ||
| 335 | |||
| 336 | sub download_all { | ||
| 337 | my ($self) = @_; | ||
| 338 | my $client = WWW::YouTube::Download->new; | ||
| 339 | |||
| 340 | my $counter = 0; | ||
| 341 | warn 'Total ' | ||
| 342 | . $self->total_user_videos | ||
| 343 | . ' videos found for channel ' | ||
| 344 | . $self->channel; | ||
| 345 | foreach my $item ( @{ $self->video_list_ids } ) { | ||
| 346 | $counter++; | ||
| 347 | warn $counter . '/' | ||
| 348 | . $self->total_download_videos | ||
| 349 | . ' - Downloading: ' | ||
| 350 | . $item->{title} | ||
| 351 | . ' into ' | ||
| 352 | . $item->{filename}; | ||
| 353 | |||
| 354 | try { | ||
| 355 | if ( !-e $item->{filename} ) { | ||
| 356 | $client->download( $item->{id}, | ||
| 357 | { ( file_name => $item->{filename} ), } ); | ||
| 358 | $self->save_nfo( $item->{nfo}, $item->{filename_nfo} ); | ||
| 359 | } | ||
| 360 | } | ||
| 361 | catch { | ||
| 362 | warn "caught error: $_"; # not $@ | ||
| 363 | push( | ||
| 364 | @{ $self->errors_download }, | ||
| 365 | { | ||
| 366 | item => $item, | ||
| 367 | error => $_, | ||
| 368 | } | ||
| 369 | ); | ||
| 370 | }; | ||
| 371 | } | ||
| 372 | } | ||
| 373 | |||
| 374 | after 'download_all' => sub { | ||
| 375 | my ( $self, $c ) = @_; | ||
| 376 | $self->show_download_errors; | ||
| 377 | }; | ||
| 378 | |||
| 379 | sub show_download_errors { | ||
| 380 | my ( $self, $c ) = @_; | ||
| 381 | my $i = 0; | ||
| 382 | foreach my $err ( @{ $self->errors_download } ) { | ||
| 383 | $i++; | ||
| 384 | warn "Error $i " | ||
| 385 | . $err->{error} | ||
| 386 | . ', for video: ' | ||
| 387 | . $err->{item}->{title} . ' ( ' | ||
| 388 | . $err->{item}->{filename} . ' )'; | ||
| 389 | } | ||
| 390 | $self->errors_download( [] ); | ||
| 391 | } | ||
| 392 | |||
| 393 | sub save_nfo { | ||
| 394 | my ( $self, $nfo, $filename ) = @_; | ||
| 395 | open FH, ">$filename"; | ||
| 396 | print FH $nfo; | ||
| 397 | close FH; | ||
| 398 | } | ||
| 399 | |||
| 400 | sub apply_regex_filter { | ||
| 401 | my ( $self, $regex ) = @_; | ||
| 402 | $self->filter_title_regex($regex); | ||
| 403 | } | ||
| 404 | |||
| 405 | sub apply_regex_skip { | ||
| 406 | my ( $self, $regex ) = @_; | ||
| 407 | $self->skip_title_regex($regex); | ||
| 408 | } | ||
| 409 | |||
| 410 | =head1 NAME | ||
| 411 | |||
| 412 | WWW::YouTube::Download::Channel - Downloads all/every/some of the videos from any youtube user channel | ||
| 413 | |||
| 414 | =head1 SYNOPSIS | ||
| 415 | |||
| 416 | use WWW::YouTube::Download::Channel; | ||
| 417 | my $yt = WWW::YouTube::Download::Channel->new(); | ||
| 418 | |||
| 419 | $yt->target_directory('/youtuve/thiers48'); #OPTIONAL. default is current dir | ||
| 420 | $yt->apply_regex_filter('24 horas|24H'); #OPTIONAL apply regex filters by title.. | ||
| 421 | $yt->apply_regex_skip( 'skip|this|title' ); #OPTIONAL skip some titles | ||
| 422 | $yt->newer_than( { #OPTIONAL filter videos by dates | ||
| 423 | day => 1, month => 12, year => 2000 } ); | ||
| 424 | $yt->leech_channel('thiers48'); #REQ | ||
| 425 | $yt->download_all; #REQ find and download youtube videos | ||
| 426 | $yt->show_download_errors; #display list of errors if any | ||
| 427 | |||
| 428 | warn "total user vids: " . $yt->total_user_videos; | ||
| 429 | warn "total downloads: " . $yt->total_download_videos; | ||
| 430 | |||
| 431 | #use Data::Dumper; | ||
| 432 | #warn Dumper $yt->video_list_ids; | ||
| 433 | |||
| 434 | =head1 DESCRIPTION | ||
| 435 | |||
| 436 | Use WWW::YouTube::Download::Channel to download a complete youtube channel / user videos. | ||
| 437 | Just pass the channel id and download all the flv directly onto your hdd for later usage. | ||
| 438 | Enjoy! | ||
| 439 | |||
| 440 | =head1 AUTHOR | ||
| 441 | |||
| 442 | Hernan Lopes | ||
| 443 | CPAN ID: HERNAN | ||
| 444 | hernanlopes <.d0t.> gmail | ||
| 445 | |||
| 446 | =head1 COPYRIGHT | ||
| 447 | |||
| 448 | This program is free software; you can redistribute | ||
| 449 | it and/or modify it under the same terms as Perl itself. | ||
| 450 | |||
| 451 | The full text of the license can be found in the | ||
| 452 | LICENSE file included with this module. | ||
| 453 | |||
| 454 | |||
| 455 | =head1 SEE ALSO | ||
| 456 | |||
| 457 | perl(1). | ||
| 458 | |||
| 459 | =cut | ||
| 460 | |||
| 461 | 1; | ||
| 462 | |||