| 1 | =head1 NAME | ||
| 2 | |||
| 3 | StreamFinder::Anystream - Fetch any raw streamable URLs from an HTML page. | ||
| 4 | |||
| 5 | =head1 AUTHOR | ||
| 6 | |||
| 7 | This module is Copyright (C) 2017-2026 by | ||
| 8 | |||
| 9 | Jim Turner, C<< <turnerjw784 at yahoo.com> >> | ||
| 10 | |||
| 11 | Email: turnerjw784@yahoo.com | ||
| 12 | |||
| 13 | All rights reserved. | ||
| 14 | |||
| 15 | You may distribute this module under the terms of either the GNU General | ||
| 16 | Public License or the Artistic License, as specified in the Perl README | ||
| 17 | file. | ||
| 18 | |||
| 19 | =head1 SYNOPSIS | ||
| 20 | |||
| 21 | #!/usr/bin/perl | ||
| 22 | |||
| 23 | use strict; | ||
| 24 | |||
| 25 | use StreamFinder::Anystream; | ||
| 26 | |||
| 27 | die "..usage: $0 ID|URL\n" unless ($ARGV[0]); | ||
| 28 | |||
| 29 | my $station = new StreamFinder::Anystream($ARGV[0]); | ||
| 30 | |||
| 31 | die "Invalid URL or no streams found!\n" unless ($station); | ||
| 32 | |||
| 33 | my $firstStream = $station->get(); | ||
| 34 | |||
| 35 | print "First Stream URL=$firstStream\n"; | ||
| 36 | |||
| 37 | my $url = $station->getURL(); | ||
| 38 | |||
| 39 | print "Stream URL=$url\n"; | ||
| 40 | |||
| 41 | my $stream_count = $station->count(); | ||
| 42 | |||
| 43 | print "--Stream count=$stream_count=\n"; | ||
| 44 | |||
| 45 | my @streams = $station->get(); | ||
| 46 | |||
| 47 | foreach my $s (@streams) { | ||
| 48 | |||
| 49 | print "------ stream URL=$s=\n"; | ||
| 50 | |||
| 51 | } | ||
| 52 | |||
| 53 | =head1 DESCRIPTION | ||
| 54 | |||
| 55 | StreamFinder::Anystream accepts any valid html URL | ||
| 56 | and returns any actual stream URL(s) found. StreamFinder::Anystream is intended | ||
| 57 | mainly as a "last resort" search of webpages that do not match any of the supported | ||
| 58 | websites handled (better) by the other site-specific StreamFinder modules. | ||
| 59 | The purpose is that one needs one of these URLs in order to have the option to | ||
| 60 | stream the station in one's own choice of media player software rather than | ||
| 61 | using their web browser and accepting any / all flash, ads, javascript, | ||
| 62 | cookies, trackers, web-bugs, and other crapware that can come with that method | ||
| 63 | of play. The author uses his own custom all-purpose media player called | ||
| 64 | "fauxdacious" (his custom hacked version of the open-source "audacious" | ||
| 65 | audio player). "fauxdacious" can incorporate this module to decode and play | ||
| 66 | streams. NOTE: It is recommended to use the main StreamFinder module or | ||
| 67 | one of the other StreamFinder submodules for searching supported sites, as this | ||
| 68 | module does not return "station" (website)-specific or stream-specific metadata | ||
| 69 | as those modules can and do for their supported sites! The main StreamFinder | ||
| 70 | module will first try to use the proper matching submodule supporting the website | ||
| 71 | (if one exists) before trying this one last. | ||
| 72 | |||
| 73 | One or more streams can be returned. | ||
| 74 | |||
| 75 | =head1 SUBROUTINES/METHODS | ||
| 76 | |||
| 77 | =over 4 | ||
| 78 | |||
| 79 | =item B<new>(I<ID>|I<url> [, I<-keep> => "type1,type2?..." | [type1,type2?...] ] | ||
| 80 | [, I<-maxstreams> [ => # ]] [, I<-secure> [ => 0|1 ]] [, I<-debug> [ => 0|1|2 ]]) | ||
| 81 | |||
| 82 | Accepts an HTML URL (webpage) and creates and returns a new station | ||
| 83 | object, or I<undef> if the URL contains any valid stream URLs (matching the | ||
| 84 | list of default extensions). The URL must be the full URL. | ||
| 85 | |||
| 86 | The optional I<-keep> argument can be either a comma-separated string or an | ||
| 87 | array reference ([...]) of stream types (extensions) to keep (include) and | ||
| 88 | returned in order specified (type1, type2...). Each "type" (extension) can be | ||
| 89 | one of: "mp3", "m4a", "mp4", "pls" (playlist), etc. NOTE: Since these are | ||
| 90 | actual extensions used to identify streams, there is NO | ||
| 91 | "any/all/stream/playlist" catch-all options as used by some of the other | ||
| 92 | (more specific) StreamFinder-supported sites! Streams will be returned sorted | ||
| 93 | by extension in the order specified in this list. | ||
| 94 | |||
| 95 | DEFAULT I<-keep> list is: "mp3,ogg,flac,mp4,m4a,mpd,aac,m3u8,m3u,pls", meaning | ||
| 96 | that all mp3 streams found (if any), followed by all "ogg" streams, etc. | ||
| 97 | |||
| 98 | The optional I<-maxstreams> argument specifies the maximum number of streams to | ||
| 99 | consider (some sites have tons of them). | ||
| 100 | |||
| 101 | DEFAULT I<-maxstreams> is 20 - return up to the first 20 streams found. | ||
| 102 | |||
| 103 | The optional I<-secure> argument can be either 0 or 1 (I<false> or I<true>). | ||
| 104 | If 1 then only secure ("https://") streams will be returned. | ||
| 105 | NOTE: If I<-secure> is specified, the number returned could be less as both | ||
| 106 | secure and unsecure streams are counted against the total as they are found. | ||
| 107 | |||
| 108 | DEFAULT I<-secure> is 0 (false) - return all streams (http and https). | ||
| 109 | |||
| 110 | Additional options: | ||
| 111 | |||
| 112 | I<-log> => "I<logfile>" | ||
| 113 | |||
| 114 | Specify path to a log file. If a valid and writable file is specified, A line | ||
| 115 | will be appended to this file every time one or more streams is successfully | ||
| 116 | fetched for a url. | ||
| 117 | |||
| 118 | DEFAULT I<-none-> (no logging). | ||
| 119 | |||
| 120 | I<-logfmt> specifies a format string for lines written to the log file. | ||
| 121 | |||
| 122 | DEFAULT "I<[time] [url] - [site]: [title] ([total])>". | ||
| 123 | |||
| 124 | The valid field I<[variables]> are: [stream]: The url of the first/best | ||
| 125 | stream found. [site]: The site name (Anystream). [url]: The url searched | ||
| 126 | for streams. [time]: Perl timestamp when the line was logged. [title], | ||
| 127 | [artist], [album], [description], [year], [genre], [total], [albumartist]: | ||
| 128 | The corresponding field data returned (or "I<-na->", if no value). | ||
| 129 | |||
| 130 | =item $station->B<get>(['playlist']) | ||
| 131 | |||
| 132 | Returns an array of strings representing all stream URLs found. | ||
| 133 | If I<"playlist"> is specified, then an extended m3u playlist is returned | ||
| 134 | instead of stream url(s). | ||
| 135 | |||
| 136 | =item $station->B<getURL>([I<options>]) | ||
| 137 | |||
| 138 | Similar to B<get>() except it only returns a single stream representing | ||
| 139 | the first valid stream found. | ||
| 140 | |||
| 141 | Current options are: I<"random">, I<"nopls">, and I<"noplaylists">. | ||
| 142 | By default, the first ("best"?) stream is returned. If I<"random"> is | ||
| 143 | specified, then a random one is selected from the list of streams found. | ||
| 144 | If I<"nopls"> is specified, and the stream to be returned is a ".pls" | ||
| 145 | playlist, it is first fetched and the first entry (or a random entry if | ||
| 146 | I<"random"> is specified) is returned. This is needed by Fauxdacious | ||
| 147 | Mediaplayer. If I<"noplaylists"> is specified, and the stream to be returned | ||
| 148 | is a "playlist" (either .pls or .m3u? extension), it is first fetched and the | ||
| 149 | first entry (or a random entry if I<"random"> is specified) in the playlist | ||
| 150 | is returned. | ||
| 151 | |||
| 152 | =item $station->B<count>() | ||
| 153 | |||
| 154 | Returns the number of streams found for the station. | ||
| 155 | |||
| 156 | =item $station->B<getID>() | ||
| 157 | |||
| 158 | Returns the station's ID (alphanumeric). | ||
| 159 | NOTE: Since this module only looks for any streams found on any | ||
| 160 | specifed website, this function always returns the base name of | ||
| 161 | the website being searched, plus the first word (subdirectory name) | ||
| 162 | following it (if any). | ||
| 163 | |||
| 164 | =item $station->B<getTitle>(['desc']) | ||
| 165 | |||
| 166 | Returns the station's title, or (long description). | ||
| 167 | NOTE: Since this module only looks for any streams found on any | ||
| 168 | specifed website, this function usually returns the base name of | ||
| 169 | the website being searched, or the full URL passed in, unless an | ||
| 170 | html <TITLE> tag is found, in which case that will be returned. | ||
| 171 | If 'I<desc>' is specified, then it will usually be full URL of the | ||
| 172 | website being searched, unless a standard "description" HTML metatag | ||
| 173 | is found, in which case that will be returned. | ||
| 174 | |||
| 175 | =item $station->B<getIconURL>() | ||
| 176 | |||
| 177 | Returns the URL for the station's "cover art" icon image, if any. | ||
| 178 | NOTE: For most websites, this will return an empty string, however, | ||
| 179 | a few have an image link contained in a standard HTML meta-tag which, | ||
| 180 | if found, will be returned. | ||
| 181 | |||
| 182 | =item $station->B<getIconData>() | ||
| 183 | |||
| 184 | Returns a two-element array consisting of the extension (ie. "png", | ||
| 185 | "gif", "jpeg", etc.) and the actual icon image (binary data), if any. | ||
| 186 | NOTE: For most websites, this will return an empty string, however, | ||
| 187 | a few have an image link contained in a standard HTML meta-tag which, | ||
| 188 | if found, will be fetched and returned as image data. | ||
| 189 | |||
| 190 | =item $station->B<getImageURL>() | ||
| 191 | |||
| 192 | Returns the URL for the station's "cover art" (usually larger) | ||
| 193 | banner image. | ||
| 194 | NOTE: Since this module only looks for any streams found on any | ||
| 195 | specifed website, this function always returns an empty string! | ||
| 196 | |||
| 197 | =item $station->B<getImageData>() | ||
| 198 | |||
| 199 | Returns a two-element array consisting of the extension (ie. "png", | ||
| 200 | "gif", "jpeg", etc.) and the actual station's banner image (binary data). | ||
| 201 | NOTE: Since this module only looks for any streams found on any | ||
| 202 | specifed website, this function always returns an empty array! | ||
| 203 | |||
| 204 | =item $station->B<getType>() | ||
| 205 | |||
| 206 | Returns the station's type ("Anystream"). | ||
| 207 | |||
| 208 | =back | ||
| 209 | |||
| 210 | =head1 CONFIGURATION FILES | ||
| 211 | |||
| 212 | The default root location directory for StreamFinder configuration files | ||
| 213 | is "~/.config/StreamFinder". To use an alternate location directory, | ||
| 214 | specify it in the "I<STREAMFINDER>" environment variable, ie.: | ||
| 215 | B<$ENV{STREAMFINDER} = "/etc/StreamFinder">. | ||
| 216 | |||
| 217 | =over 4 | ||
| 218 | |||
| 219 | =item ~/.config/StreamFinder/Anystream/config | ||
| 220 | |||
| 221 | Optional text file for specifying various configuration options | ||
| 222 | for a specific site (submodule). Each option is specified on a | ||
| 223 | separate line in the format below: | ||
| 224 | NOTE: Do not follow the lines with a semicolon, comma, or any other | ||
| 225 | separator. Non-numeric I<values> should be surrounded with quotes, either | ||
| 226 | single or double. Blank lines and lines beginning with a "#" sign as | ||
| 227 | their first non-blank character are ignored as comments. | ||
| 228 | |||
| 229 | 'option' => 'value' [,] | ||
| 230 | |||
| 231 | and the options are loaded into a hash used only by the specific | ||
| 232 | (submodule) specified. Valid options include | ||
| 233 | I<-debug> => [0|1|2] and most of the L<LWP::UserAgent> options. | ||
| 234 | |||
| 235 | Options specified here override any specified in | ||
| 236 | I<~/.config/StreamFinder/config>. | ||
| 237 | |||
| 238 | Among options valid for Anystream streams is the I<-keep> option | ||
| 239 | previously described in the B<new()> function. | ||
| 240 | |||
| 241 | =item ~/.config/StreamFinder/config | ||
| 242 | |||
| 243 | Optional text file for specifying various configuration options. | ||
| 244 | Each option is specified on a separate line in the format below: | ||
| 245 | |||
| 246 | 'option' => 'value' [,] | ||
| 247 | |||
| 248 | and the options are loaded into a hash used by all sites | ||
| 249 | (submodules) that support them. Valid options include | ||
| 250 | I<-debug> => [0|1|2] and most of the L<LWP::UserAgent> options. | ||
| 251 | |||
| 252 | =back | ||
| 253 | |||
| 254 | NOTE: Options specified in the options parameter list of the I<new()> | ||
| 255 | function will override those corresponding options specified in these files. | ||
| 256 | |||
| 257 | =head1 KEYWORDS | ||
| 258 | |||
| 259 | Anystream | ||
| 260 | |||
| 261 | =head1 DEPENDENCIES | ||
| 262 | |||
| 263 | L<URI::Escape>, L<HTML::Entities>, L<LWP::UserAgent> | ||
| 264 | |||
| 265 | =head1 RECCOMENDS | ||
| 266 | |||
| 267 | wget | ||
| 268 | |||
| 269 | =head1 BUGS | ||
| 270 | |||
| 271 | Please report any bugs or feature requests to C<bug-streamFinder-Anystream at rt.cpan.org>, or through | ||
| 272 | the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=StreamFinder-Anystream>. I will be notified, and then you'll | ||
| 273 | automatically be notified of progress on your bug as I make changes. | ||
| 274 | |||
| 275 | =head1 SUPPORT | ||
| 276 | |||
| 277 | You can find documentation for this module with the perldoc command. | ||
| 278 | |||
| 279 | perldoc StreamFinder::Anystream | ||
| 280 | |||
| 281 | You can also look for information at: | ||
| 282 | |||
| 283 | =over 4 | ||
| 284 | |||
| 285 | =item * RT: CPAN's request tracker (report bugs here) | ||
| 286 | |||
| 287 | L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=StreamFinder-Anystream> | ||
| 288 | |||
| 289 | =item * AnnoCPAN: Annotated CPAN documentation | ||
| 290 | |||
| 291 | L<http://annocpan.org/dist/StreamFinder-Anystream> | ||
| 292 | |||
| 293 | =item * Search CPAN | ||
| 294 | |||
| 295 | L<http://search.cpan.org/dist/StreamFinder-Anystream/> | ||
| 296 | |||
| 297 | =back | ||
| 298 | |||
| 299 | =head1 LICENSE AND COPYRIGHT | ||
| 300 | |||
| 301 | Copyright 2017-2026 Jim Turner. | ||
| 302 | |||
| 303 | This program is free software; you can redistribute it and/or modify it | ||
| 304 | under the terms of the the Artistic License (2.0). You may obtain a | ||
| 305 | copy of the full license at: | ||
| 306 | |||
| 307 | L<http://www.perlfoundation.org/artistic_license_2_0> | ||
| 308 | |||
| 309 | Any use, modification, and distribution of the Standard or Modified | ||
| 310 | Versions is governed by this Artistic License. By using, modifying or | ||
| 311 | distributing the Package, you accept this license. Do not use, modify, | ||
| 312 | or distribute the Package, if you do not accept this license. | ||
| 313 | |||
| 314 | If your Modified Version has been derived from a Modified Version made | ||
| 315 | by someone other than you, you are nevertheless required to ensure that | ||
| 316 | your Modified Version complies with the requirements of this license. | ||
| 317 | |||
| 318 | This license does not grant you the right to use any trademark, service | ||
| 319 | mark, tradename, or logo of the Copyright Holder. | ||
| 320 | |||
| 321 | This license includes the non-exclusive, worldwide, free-of-charge | ||
| 322 | patent license to make, have made, use, offer to sell, sell, import and | ||
| 323 | otherwise transfer the Package with respect to any patent claims | ||
| 324 | licensable by the Copyright Holder that are necessarily infringed by the | ||
| 325 | Package. If you institute patent litigation (including a cross-claim or | ||
| 326 | counterclaim) against any party alleging that the Package constitutes | ||
| 327 | direct or contributory patent infringement, then this Artistic License | ||
| 328 | to you shall terminate on the date that such litigation is filed. | ||
| 329 | |||
| 330 | Disclaimer of Warranty: THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER | ||
| 331 | AND CONTRIBUTORS "AS IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. | ||
| 332 | THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR | ||
| 333 | PURPOSE, OR NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY | ||
| 334 | YOUR LOCAL LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR | ||
| 335 | CONTRIBUTOR WILL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR | ||
| 336 | CONSEQUENTIAL DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, | ||
| 337 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 338 | |||
| 339 | =cut | ||
| 340 | |||
| 341 | package StreamFinder::Anystream; | ||
| 342 | |||
| 343 | #use warnings; | ||
| 344 | use URI::Escape; | ||
| 345 | use HTML::Entities (); | ||
| 346 | use LWP::UserAgent (); | ||
| 347 | use parent 'StreamFinder::_Class'; | ||
| 348 | |||
| 349 | my $DEBUG = 0; | ||
| 350 | |||
| 351 | sub new | ||
| 352 | { | ||
| 353 | my $class = shift; | ||
| 354 | my $url = shift; | ||
| 355 | |||
| 356 | return undef unless ($url); | ||
| 357 | |||
| 358 | my $self = $class->SUPER::new('Anystream', @_); | ||
| 359 | $DEBUG = $self->{'debug'} if (defined $self->{'debug'}); | ||
| 360 | |||
| 361 | my @okStreams; | ||
| 362 | $self->{'maxstreams'} = 20; | ||
| 363 | while (@_) { | ||
| 364 | if ($_[0] =~ /^\-?keep$/o) { | ||
| 365 | shift; | ||
| 366 | if (defined $_[0]) { | ||
| 367 | my $keeporder = shift; | ||
| 368 | @okStreams = (ref($keeporder) =~ /ARRAY/) ? @{$keeporder} : split(/\,\s*/, $keeporder); | ||
| 369 | } | ||
| 370 | } elsif ($_[0] =~ /^\-?maxstreams$/o) { | ||
| 371 | shift; | ||
| 372 | $self->{'maxstreams'} = (defined $_[0]) ? shift : 20; | ||
| 373 | } else { | ||
| 374 | shift; | ||
| 375 | } | ||
| 376 | } | ||
| 377 | if (!defined($okStreams[0]) && defined($self->{'keep'})) { | ||
| 378 | @okStreams = (ref($self->{'keep'}) =~ /ARRAY/) ? @{$self->{'keep'}} : split(/\,\s*/, $self->{'keep'}); | ||
| 379 | } | ||
| 380 | @okStreams = (qw(mp3 ogg flac mp4 m4a mpd aac m3u8 m3u pls)) unless (defined $okStreams[0]); | ||
| 381 | |||
| 382 | my $url2fetch = $url; | ||
| 383 | my $urlPrefix = ''; | ||
| 384 | my $domainName = ''; | ||
| 385 | my $baseURL = ''; | ||
| 386 | $url2fetch = 'https://' . $url unless ($url =~ m#^https?\:\/\/#); | ||
| 387 | my $t = $url2fetch; | ||
| 388 | if ($t =~ s#(https?\:\/\/)([^\/\?\&\#]+).?##) { | ||
| 389 | $urlPrefix = $1; | ||
| 390 | $domainName = $2; | ||
| 391 | $baseURL = $urlPrefix . $domainName; | ||
| 392 | $self->{'id'} = $domainName; | ||
| 393 | $self->{'id'} .= "/$1" if ($t =~ s#^([^\/\?\&\#\.\=]+)##); | ||
| 394 | } else { | ||
| 395 | print STDERR "u:Anystream - Invalid URL ($url2fetch) t=$t=\n" if ($DEBUG); | ||
| 396 | #---PUNT!--- | ||
| 397 | return undef; | ||
| 398 | } | ||
| 399 | $self->{'id'} =~ s#[^a-z0-9\-\_\.\/]##gi; | ||
| 400 | $url2fetch =~ s#\/$##; | ||
| 401 | print STDERR "-url2fetch=$url2fetch= ID=".$self->{'id'}."= PFX=$urlPrefix= DN=$domainName= BASE=$baseURL=\n" if ($DEBUG); | ||
| 402 | |||
| 403 | $self->{'albumartist'} = $url2fetch; | ||
| 404 | my $html = ''; | ||
| 405 | print STDERR "-0(Anystream): URL=$url2fetch=\n" if ($DEBUG); | ||
| 406 | if ($url2fetch =~ /\b(?:youtube\.|youtu.be|ytimg\.)\b/) { | ||
| 407 | #WE CAN'T EXTRACT STREAMS IN YOUTUBE PAGES, SO IF HERE, IT'S LIKELY | ||
| 408 | #BECAUSE StreamFinder::Youtube FAILED & IT FELL THROUGH TO US, SO | ||
| 409 | #JUST PUNT & DON'T WASTE ANY ORE TIME HERE ON IT!: | ||
| 410 | print STDERR "u:Anystream: We're a YOUTUBE page & StreamFinder::Youtube likely FAILED, so PUNT!\n" if ($DEBUG); | ||
| 411 | return undef; | ||
| 412 | } | ||
| 413 | |||
| 414 | my $ua = LWP::UserAgent->new(@{$self->{'_userAgentOps'}}); | ||
| 415 | $ua->timeout($self->{'timeout'}); | ||
| 416 | $ua->max_size(2048); #LIMIT FETCH-SIZE TO AVOID INFINITELY DOWNLOADING A STREAM! | ||
| 417 | $ua->cookie_jar({}); | ||
| 418 | $ua->env_proxy; | ||
| 419 | my $response = $ua->get($url2fetch); | ||
| 420 | if ($response->is_success) { | ||
| 421 | $html = $response->decoded_content; | ||
| 422 | } else { | ||
| 423 | print STDERR $response->status_line if ($DEBUG); | ||
| 424 | } | ||
| 425 | unless ($html) { #STEP 1 FAILED, INVALID URL, PUNT! | ||
| 426 | print STDERR "-!!!- COULD NOT FETCH URL=$url2fetch=\n" if ($DEBUG); | ||
| 427 | return undef; | ||
| 428 | } | ||
| 429 | my $isHLSpage = ($html =~ /#EXTM3U/) ? 1 : 0; #SEE AUDACIOUS ISSUE#1169 FOR EMBEDDED HLS IN HTML PAGE EXAMPLE USED FOR THIS CODE: | ||
| 430 | my $streamExts = join('|',@okStreams); | ||
| 431 | print STDERR "-1: HLS=$isHLSpage=\n----html=$html=\n" if ($DEBUG > 1); | ||
| 432 | return undef unless ($html =~ /\<\!DOCTYPE\s+(?:html|text)/i | ||
| 433 | || ($isHLSpage && $streamExts =~ /(?:m3u8|any|all)/i)); #STEP 1 FAILED, INVALID STATION URL, PUNT! | ||
| 434 | |||
| 435 | print STDERR "-1: GOT SOME(<=2048 BYTES) HTML!\n" if ($DEBUG); | ||
| 436 | $ua->requests_redirectable([]) if ($isHLSpage); | ||
| 437 | unless ($isHLSpage) { | ||
| 438 | print STDERR "-1a: NOW RE-FETCH FULL PAGE!\n" if ($DEBUG); | ||
| 439 | $ua->max_size(undef); #(NOW OK TO FETCH THE WHOLE DOCUMENT) | ||
| 440 | $response = $ua->get($url2fetch); | ||
| 441 | if ($response->is_success) { | ||
| 442 | $html = $response->decoded_content; | ||
| 443 | } else { | ||
| 444 | print STDERR $response->status_line if ($DEBUG); | ||
| 445 | } | ||
| 446 | unless ($html) { #STEP 1 FAILED, INVALID URL, PUNT! | ||
| 447 | print STDERR "-!!!- COULD NOT FETCH URL=$url2fetch=\n" if ($DEBUG); | ||
| 448 | return undef; | ||
| 449 | } | ||
| 450 | } | ||
| 451 | |||
| 452 | $self->{'_havehls'} = 0; | ||
| 453 | if ($isHLSpage) { #JWT:NOTE:MAY END UP NEEDING TO REMOVE TRAILING "$"!: | ||
| 454 | $self->{'title'} = $self->{'id'} || $url; | ||
| 455 | $self->{'description'} = $url2fetch; | ||
| 456 | if ($self->{'hls_bandwidth'} > 0 && $url2fetch =~ /\.m3u8$/) { #JWT:NOTE:MAY END UP NEEDING TO REMOVE TRAILING "$"!: | ||
| 457 | my $location = $response->header('location'); | ||
| 458 | ($baseURL = $url2fetch) =~ s#\/[^\/]+$##; | ||
| 459 | print STDERR "-!!!- WE'RE AN HLS PAGE!: HLS BASE URL=$baseURL= locn=$location\n" if ($DEBUG); | ||
| 460 | my @lines = split(/\r?\n/, $html); | ||
| 461 | my @plentries = (); | ||
| 462 | my $firstTitle = ''; | ||
| 463 | my $plidx = ($arglist =~ /\b\-?random\b/) ? 1 : 0; | ||
| 464 | my $line = 1; | ||
| 465 | (my $urlpath = $firstStream) =~ s#[^\/]+$##; | ||
| 466 | my $highestBW = 0; | ||
| 467 | my %streamsByBandwidth; | ||
| 468 | while ($line <= $#lines) { #FIND HIGHEST BANDWIDTH STREAM (WITHIN ANY USER-SET BANDWIDTH): | ||
| 469 | if ($lines[$line] =~ /\s*\#EXT\-X\-STREAM\-INF\:(?:.*?)BANDWIDTH\=(\d+)/o) { | ||
| 470 | my $bw = $1; | ||
| 471 | if ($lines[$line] =~ /\,\s*URI\=\"([^\"]+)/o) { #SOMETIMES THEY STICK IT ON THE END OF CURRENT LINE!: | ||
| 472 | $lines[$line] = $1; | ||
| 473 | } else { | ||
| 474 | $line++; | ||
| 475 | } | ||
| 476 | if ($line <= $#lines) { | ||
| 477 | $bw =~ s/^\d*x//o; | ||
| 478 | if ($lines[$line] =~ m#\.m3u8#o && ($bw <= $self->{'hls_bandwidth'})) { | ||
| 479 | my $url = $lines[$line]; | ||
| 480 | if ($lines[$line] =~ m#^https?\:\/\/#o) { | ||
| 481 | $streamsByBandwidth{$bw} = $lines[$line]; | ||
| 482 | } elsif ($lines[$line] =~ m#^\/\/#o) { | ||
| 483 | my $protocol = $1 if ($url2fetch =~ /^(https?)/); | ||
| 484 | $streamsByBandwidth{$bw} = $protocol . ':' . $lines[$line]; | ||
| 485 | } elsif ($lines[$line] =~ m#^\/#) { | ||
| 486 | $streamsByBandwidth{$bw} = $baseURL . $lines[$line]; | ||
| 487 | } else { | ||
| 488 | $streamsByBandwidth{$bw} = $baseURL . '/' . $lines[$line]; | ||
| 489 | } | ||
| 490 | $self->{'_havehls'} = 1; | ||
| 491 | print STDERR "----1($bw): found stream=$streamsByBandwidth{$bw}= bw=$bw=...\n" if ($DEBUG); | ||
| 492 | } | ||
| 493 | } | ||
| 494 | } | ||
| 495 | $line++; | ||
| 496 | } | ||
| 497 | foreach $bw (sort {$b <=> $a} keys %streamsByBandwidth) { | ||
| 498 | push (@{$self->{'streams'}}, $streamsByBandwidth{$bw}) | ||
| 499 | if (!$self->{'secure'} || $streamURL =~ /^https/o); | ||
| 500 | } | ||
| 501 | } else { | ||
| 502 | push (@{$self->{'streams'}}, $url2fetch); | ||
| 503 | } | ||
| 504 | } else { | ||
| 505 | print STDERR "--WE'RE A PAGE THAT MIGHT CONTAIN HLS:\n" if ($DEBUG); | ||
| 506 | $baseURL = $1 if ($html =~ m#\<BASE\s+HREF\=\"([^\"]+)#si); | ||
| 507 | $baseURL .= '/' unless ($baseURL =~ m#\/$#o); | ||
| 508 | $self->{'title'} = $self->{'id'} || $url; | ||
| 509 | $self->{'description'} = $url2fetch; | ||
| 510 | $self->{'title'} = $1 if ($html =~ /\<TITLE>([^\<]+)/si); | ||
| 511 | print STDERR "-2: title=".$self->{'title'}."=\n" if ($DEBUG); | ||
| 512 | $self->{'cnt'} = 0; #LIMIT HOW MANY STREAMS TO CONSIDER - SOME PAGES (ie. $self->{'maxstreams'} CAN HAVE BAGILLIONS: | ||
| 513 | my $streams = ''; | ||
| 514 | print STDERR "--EXTS=$streamExts=\n" if ($DEBUG); | ||
| 515 | my $streamLimiter = 0; | ||
| 516 | while ($html =~ s#\<meta\s+([^\>]+)\>##iso) { | ||
| 517 | my $metatag = $1; | ||
| 518 | if ($metatag =~ m#\scontent\=\"([^\"]+)#io) { | ||
| 519 | my $content = $1; | ||
| 520 | $self->{'iconurl'} ||= $content if ($metatag =~ /\:?image\"/o); | ||
| 521 | $self->{'title'} ||= $content if ($metatag =~ /\:?title\"/o); | ||
| 522 | $self->{'description'} ||= $content if ($metatag =~ /\:?description\"/o); | ||
| 523 | $self->{'artist'} ||= $content if ($metatag =~ /\:?site_name\"/o); | ||
| 524 | } | ||
| 525 | } | ||
| 526 | while ($html =~ s#(https?\:\/\/|\.\.\/)?([^\s\'\"\:\<\>\[\]\{\}]+)\.($streamExts)##s | ||
| 527 | && $streamLimiter <= $self->{'maxstreams'}) { | ||
| 528 | (my $one = $1 . $2) =~ s#\\\/#\/#gs; | ||
| 529 | my $ext = $3; | ||
| 530 | $one =~ s/\\u00([0-9A-Fa-f]{2})/chr(hex($1))/eg; | ||
| 531 | my $streamURL = $one.'.'.$ext; | ||
| 532 | print STDERR "--1: streamURL=$streamURL= baseURL=$baseURL=\n" if ($DEBUG); | ||
| 533 | my $tmpbase = $baseURL; | ||
| 534 | while ($streamURL =~ s#\.\.\/##o) { | ||
| 535 | $tmpbase =~ s#[^\/]+\/$##o; | ||
| 536 | } | ||
| 537 | if ($streamURL =~ m#^\/\/#o) { #STREAM URL STARTS WITH "//", ASSUME ABSOLUTE: | ||
| 538 | if ($baseURL =~ /http\:/io) { | ||
| 539 | $streamURL = 'http:' . $streamURL; | ||
| 540 | } else { | ||
| 541 | $streamURL = 'https:' . $streamURL; | ||
| 542 | } | ||
| 543 | print STDERR "--2a: stream=$streamURL=\n" if ($DEBUG); | ||
| 544 | } elsif ($streamURL =~ m#^\/#o) { #STREAM URL STARTS WITH "/", ASSUME ABSOLUTE TO BASE PAGE URL ("TITLE"): | ||
| 545 | $tmpbase =~ s#\/$##o; | ||
| 546 | $streamURL = $tmpbase . $streamURL; | ||
| 547 | print STDERR "--2b: baseURL=$tmpbase= stream=$streamURL=\n" if ($DEBUG); | ||
| 548 | } elsif ($streamURL !~ /^http/o) { #NO PREFIX, ASSUME RELATIVE TO THE FETCHED URL ("LONG DESC."): | ||
| 549 | if ($isHLSpage) { | ||
| 550 | $tmpbase .= '/' unless ($tmpbase =~ m#\/$#o); | ||
| 551 | $streamURL = $tmpbase . $streamURL; | ||
| 552 | } else { | ||
| 553 | $streamURL = $url2fetch . '/' . $streamURL; | ||
| 554 | } | ||
| 555 | print STDERR "--2c: baseURL=$tmpbase= stream=$streamURL=\n" if ($DEBUG); | ||
| 556 | } #OTHERWISE STREAM URL IS A FULL URL (NO CHANGE). | ||
| 557 | $streams .= "$ext=$streamURL|" unless ($self->{'secure'} && $streamURL !~ /^https/o); | ||
| 558 | $streamLimiter++; | ||
| 559 | } | ||
| 560 | print STDERR "==streams=$streams=\n" if ($DEBUG); | ||
| 561 | my $stindex = 0; | ||
| 562 | my $savestreams = $streams; | ||
| 563 | my %havestreams = (); | ||
| 564 | my ($one, $ext); | ||
| 565 | foreach my $streamtype (@okStreams) { | ||
| 566 | $streams = $savestreams; | ||
| 567 | while ($streams =~ /^([^\=]*)\=([^\|]+)/o) { | ||
| 568 | $ext = $1; $one = $2; | ||
| 569 | if ($ext =~ /^${streamtype}$/i && !defined($havestreams{"$ext|$one"}) | ||
| 570 | && (!$self->{'secure'} || $streamURL =~ /^https/o)) { | ||
| 571 | $self->{'streams'}->[$stindex++] = $one; | ||
| 572 | $havestreams{"$ext|$one"}++; | ||
| 573 | } | ||
| 574 | $streams =~ s/^[^\|]*\|//o; | ||
| 575 | } | ||
| 576 | } | ||
| 577 | } | ||
| 578 | $self->{'cnt'} = scalar @{$self->{'streams'}}; | ||
| 579 | unless ($self->{'cnt'} > 0) { | ||
| 580 | #SEE IF WE HAVE ANYTHING IN AN IFRAME THAT STREAMFINDER'S OTHER* MODULES MIGHT RECOGNIZE | ||
| 581 | # (* MODULES OTHER THAN THIS ONE OR YOUTUBE (WHICH IF HERE, WE'VE ALREADY CHECKED!) | ||
| 582 | # FOR TIME'S SAKE, WE ONLY LOOK AT THE 1ST IFRAME CONTAINING A URL: | ||
| 583 | my $embedded_video; | ||
| 584 | while ($html && $html =~ s#\<iframe([^\>]+)\>##so) { | ||
| 585 | my $one = $1; | ||
| 586 | my $embeddedURL = ($one =~ m#\"(https?\:\/\/[^\"]+)#s) ? $1 : ''; | ||
| 587 | if ($embeddedURL) { | ||
| 588 | $embeddedURL =~ s/[\?\&].*$// unless ($self->{'notrim'}); | ||
| 589 | print STDERR "--embedded IFRAME url=$embeddedURL=\n" if ($DEBUG); | ||
| 590 | my $haveStreamFinder = 0; | ||
| 591 | eval { require 'StreamFinder.pm'; $haveStreamFinder = 1; }; | ||
| 592 | if ($haveStreamFinder) { | ||
| 593 | my %globalArgs = (-omit => 'Youtube,Anystream', -noiframes => 1, | ||
| 594 | -debug => $DEBUG); | ||
| 595 | foreach my $arg (qw(log logfmt)) { | ||
| 596 | $globalArgs{$arg} = $self->{$arg} if (defined($self->{$arg}) && $self->{$arg}); | ||
| 597 | } | ||
| 598 | $embedded_video = new StreamFinder($embeddedURL, %globalArgs); | ||
| 599 | } | ||
| 600 | last; | ||
| 601 | } | ||
| 602 | } | ||
| 603 | return $embedded_video if (defined($embedded_video) && $embedded_video->count() > 0); | ||
| 604 | } | ||
| 605 | $self->{'total'} = $self->{'cnt'}; | ||
| 606 | $self->{'title'} = HTML::Entities::decode_entities($self->{'title'}); | ||
| 607 | $self->{'title'} = uri_unescape($self->{'title'}); | ||
| 608 | $self->{'title'} =~ s/^\s+//s; | ||
| 609 | $self->{'title'} =~ s/\s+$//s; | ||
| 610 | $self->{'description'} = HTML::Entities::decode_entities($self->{'description'}); | ||
| 611 | $self->{'description'} = uri_unescape($self->{'description'}); | ||
| 612 | if ($self->{'total'} > 0) { | ||
| 613 | $self->{'Url'} = $self->{'streams'}->[0]; | ||
| 614 | print STDERR "-SUCCESS: 1st stream=".$self->{'Url'}."= total=".$self->{'total'}."=\n" if ($DEBUG); | ||
| 615 | $self->{'playlist'} = "#EXTM3U\n"; | ||
| 616 | for (my $i=0;$i<=$#{$self->{'streams'}};$i++) { | ||
| 617 | $self->{'playlist'} .= "#EXTINF:-1, " . $self->{'title'} | ||
| 618 | . "\n" . ${$self->{'streams'}}[$i] . "\n"; | ||
| 619 | } | ||
| 620 | } | ||
| 621 | if ($DEBUG) { | ||
| 622 | foreach my $i (sort keys %{$self}) { | ||
| 623 | print STDERR "--KEY=$i= VAL=".$self->{$i}."=\n"; | ||
| 624 | } | ||
| 625 | print STDERR "--FIRST STREAM=".${$self->{'streams'}}[0]."=\n"; | ||
| 626 | } | ||
| 627 | $self->_log($url); | ||
| 628 | |||
| 629 | bless $self, $class; #BLESS IT! | ||
| 630 | |||
| 631 | return $self; | ||
| 632 | } | ||
| 633 | |||
| 634 | sub getURL #LIKE GET, BUT ONLY RETURN THE SINGLE ONE W/BEST BANDWIDTH AND RELIABILITY: | ||
| 635 | { | ||
| 636 | my $self = shift; | ||
| 637 | my $arglist = (defined $_[0]) ? join('|',@_) : ''; | ||
| 638 | my $idx = ($arglist =~ /\b\-?random\b/) ? int rand scalar @{$self->{'streams'}} : 0; | ||
| 639 | my $firstStream = ${$self->{'streams'}}[$idx]; | ||
| 640 | |||
| 641 | return '' unless (defined $firstStream); | ||
| 642 | |||
| 643 | return ($self->{'_havehls'} == 1) ? $firstStream | ||
| 644 | : $self->StreamFinder::_Class::getURL(@_); | ||
| 645 | } | ||
| 646 | |||
| 647 | 1 |