File Coverage

blib/lib/Data/Downloader.pm

Criterion	Covered	Total	%
statement	1	3	33.3
branch			n/a
condition			n/a
subroutine	1	1	100.0
pod			n/a
total	2	4	50.0

=head1 NAME

Data::Downloader -- Download and organize files using RSS feeds and templates.

=head1 SYNOPSIS

    use Data::Downloader -init_logging => "INFO";

    my $repo = Data::Downloader::Repository->new(name => "ozone");
    $repo->load(speculative => 1) or die "ozone repository not configured";

    for my $feed (@{ $dado->feeds }) {
        $feed->refresh;
    }

    for my $file (@{ $dado->files }) {
        $file->download;
    }

=head1 DESCRIPTION

Data::Downloader allows one to download and maintain local repositories
of files.  File metadata may be obtained from RSS or Atom feeds.  Files
are stored using MD5 sums, and symbolic links can be created based on
the metadata for the files.

A command line version of Data::Downloader, L, is also available.

Data::Downloader uses an SQLite L
to store both the L, as well
as information about the Ls and
L stored
in a repository.

Parameters may be sent when updating Ls;
parameters are replace tokens in URLs for the RSS/Atom feeds, just
as in an L URL template.

=head1 BACKGROUND

Several efforts are underway to extend Atom and RSS as a mechanism
for distribution of scientific data.  For example, L provides a versatile response format
as well as a client capable of graphically oriented searches for data.
ESIP Discovery Services (e.g. L)
are working on specifications for server-side filtering of data, based
on the L specification.

In addition, standards such as L and
L provide ways to represent structured
metadata for resources which are often downloaded and organized on a local disk.

In addition to reading feeds, Data::Downloader may be used as an
L L which may
be populated directly without querying Atom/RSS feeds.

=head1 EXAMPLE

This is an example of subscribing to flickr's mrss feed to download
images.  For more examples, please see L.

    my $images = Data::Downloader::Repository->new(
        name           => "images",
        storage_root   => "/usr/local/datastore/data",
        cache_strategy => "Keep",
        feeds          => [
            {
                name => "flickr",
                feed_template => 'http://api.flickr.com/services/feeds/photos_public.gne?tags=&lang=en-us&format=rss_200',
                file_source => {
                    url_xpath      => 'media:content/@url',
                    filename_xpath => 'media:content/@url',
                    filename_regex => '/([^/]*)$',
                },
                metadata_sources => [
                    { name => 'date_taken', xpath => 'dc:date.Taken' },
                    { name => 'tags',       xpath => 'media:category' },
                ],
            },
        ],
        metadata_transformations => [
            {
                input         => "tags",
                output        => "tag",
                function_name => "split",
            },
        ],
        linktrees => [
            {
                root          => "/usr/local/datastore/by_tag",
                condition     => undef,
                path_template => ""
            },
            {
                root          => "/usr/local/datastore/by_date",
                condition     => undef,
                path_template => ""
            },
        ],
    );

    $images->load(speculative => 1) or $images->save;

    for my $feed ($images->feeds) {
        $feed->refresh(tags => "apples");
    }

    $images->download_all;

=head1 SEE ALSO

L,
L,
L,
L,
L,
L,
L,
L

=cut

package Data::Downloader;
use Rose::DB::Object::Loader;
use Lingua::EN::Inflect qw/def_noun/;
use Log::Log4perl qw(:easy);

use Data::Downloader::DB;
use Data::Downloader::DB::Object;
use Data::Downloader::DB::Object::Cached;
use Data::Downloader::Config;
use Data::Downloader::MetadataPivot;
use Data::Downloader::FileMetadata;
use strict;

def_noun "metadatum" => "metadata";

our $VERSION = '0.9907';
our $db;
our $useProgressBars;   # set during import to turn on Smart::Comments
our $setupDone;

sub import {
    my ($class,@params) = @_;
    return if $setupDone;
    $setupDone = 1;
    $db = Data::Downloader::DB->new("main");
    if (@params && grep /-init_logging/, @params) {
        my ($level) = grep { /^(DEBUG|INFO|WARN|ERROR|FATAL)$/ } @params;
        $level ||= "INFO";
        my $init = <<"EOT";
           log4perl.rootLogger = $level, Screen
           log4perl.appender.Screen = Log::Log4perl::Appender::ScreenColoredLevels
           log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
           log4perl.appender.Screen.layout.ConversionPattern = @{[ $ENV{HARNESS_ACTIVE} ? '#' : '' ]} [%-5p] %d %F{1} (%L) %m %n
EOT
        Log::Log4perl::init(\$init);
    }
    if (@params && grep /-use_progress_bars/, @params) {
        $useProgressBars = 1;
    }
    if (! -s $db->database) {
       INFO "initializing database ".$db->database."\n";
       _initialize_db();
    }
   _setup_classes();
}

sub _initialize_db {
    local $/ = ';';
    while () {
        my $stmt = $_;
        if (/CREATE TRIGGER/) {
            do {
                $stmt .= ( $_ =  );
            } until /END/ || !defined($_);
        }
        TRACE "SQL:\n$stmt\n";
        next unless $stmt =~ /\w/;
        $db->dbh->do($stmt) or die "failed to execute '$stmt'\n\n".$db->dbh->errstr;
    }
}

sub _setup_classes {
    our $cm = Rose::DB::Object::ConventionManager->new();
    $cm->tables_are_singular(1);
    $cm->singular_to_plural_function(\&Lingua::EN::Inflect::PL);

    my $loader_dynamic = Rose::DB::Object::Loader->new(
        base_class         => "Data::Downloader::DB::Object",
        db                 => $db,
        db_class           => "Data::Downloader::DB",
        class_prefix       => "Data::Downloader",
        convention_manager => $cm,
        post_init_hook     => sub { shift->error_mode('return'); },
        # TODO use log4perl for error mode somehow
    );

    my $loader_cached = Rose::DB::Object::Loader->new(
        base_class         => "Data::Downloader::DB::Object::Cached",
        db                 => $db,
        db_class           => "Data::Downloader::DB",
        class_prefix       => "Data::Downloader",
        convention_manager => $cm,
        post_init_hook     => sub { shift->error_mode('return'); },
    );

    my @config_tables = qw/repository feed disk feed_parameter file_source metadata_source metadata_transformation/;
    my @classes = $loader_dynamic->make_classes(exclude_tables => \@config_tables)
        or LOGDIE "Error: unable to load classes from ".$db->dbh->database;

    push @classes, $loader_cached->make_classes(include_tables => \@config_tables);

    for (@classes) {
        eval "use $_";
        no strict 'refs';
        *{"${_}::init_db"} = sub { Data::Downloader::DB->new_or_cached("main") };
        die "Errors using $_ : $@" if $@ && $@ !~ /Can't locate/;
    }

    Data::Downloader::MetadataPivot->do_setup;
    Data::Downloader::FileMetadata->do_setup;
}

1;

<<"=head1 SCHEMA"; # my trick to put __DATA__ into pod

=head1 SCHEMA

__DATA__