File Coverage

blib/lib/Text/Mining.pm

Criterion	Covered	Total	%
statement	3	3	100.0
branch			n/a
condition			n/a
subroutine	1	1	100.0
pod			n/a
total	4	4	100.0

package Text::Mining;
use base qw(Text::Mining::Base);
use Class::Std;
use Class::Std::Utils;
use Text::Mining::Corpus;
use Text::Mining::Corpus::Document;
use Text::Mining::Shell;

use warnings;
use strict;
use Carp;

use version; our $VERSION = qv('0.0.8');

{
	my %attribute_of : ATTR( get => 'attribute', set => 'attribute' );

	sub BUILD {
		my ($self, $ident, $arg_ref) = @_;

	#	&DBConnect( %{ $self->_library_connect_parameters() } );

		return;
	}

	sub shell { my $shell = Text::Mining::Shell->new(); $shell->cmdloop(); }
	sub version { return "VERSION $VERSION"; }

	sub create_corpus             { my ( $self, $arg_ref ) = @_; return Text::Mining::Corpus->new( $arg_ref ); }
	sub get_corpus                { my ( $self, $arg_ref ) = @_; return Text::Mining::Corpus->new( $arg_ref ); }
	sub delete_corpus             { my ( $self, $arg_ref ) = @_; my $corpus = Text::Mining::Corpus->new(); return $corpus->delete( $arg_ref ); }

	sub get_root_dir              { my ( $self ) = @_; return $self->_get_root_dir(); }
	sub get_root_url              { my ( $self ) = @_; return $self->_get_root_url(); }
	sub get_data_dir              { my ( $self, $corpus_id ) = @_; return $self->_get_data_dir( $corpus_id ); }

	sub get_submitted_document    { my ( $self, $arg_ref ) = @_; return Text::Mining::Corpus::Document->new( $arg_ref ); }
	sub count_submitted_waiting   { my ( $self ) = @_; my ( $count ) = $self->library()->sqlexec( "select count(*) from submitted_documents where exit_date = '0000-00-00 00:00:00'", '@' ); return $count; }
	sub count_submitted_complete  { my ( $self ) = @_; my ( $count ) = $self->library()->sqlexec( "select count(*) from submitted_documents where exit_date != '0000-00-00 00:00:00'", '@' ); return $count; }

	sub parse_document {
		my ( $self, $arg_ref ) = @_;
		my $document  = defined $arg_ref->{document}  ? $arg_ref->{document}  : $self->_status( "No document to parse." );
		my $algorithm = defined $arg_ref->{algorithm} ? $arg_ref->{algorithm} : $self->_status( "No algorithm defined." );

		return $document;
	}

	sub get_all_corpuses          {
		my ( $self, @corpuses) = @_;
		my $corpuses = $self->library()->sqlexec( "select corpus_id from corpuses order by corpus_id asc", '\@@' );
		foreach my $corpus (@$corpuses) { push @corpuses, Text::Mining::Corpus->new({ corpus_id => $corpus->[0] }); }
		return \@corpuses;
	}

	sub get_corpus_id             {
		my ( $self, $arg_ref ) = @_;
		my   $corpus           = Text::Mining::Corpus->new();
		my ( $corpus_id )      = $self->library()->sqlexec( "select corpus_id from corpuses where corpus_name = '" . $arg_ref->{corpus_name} . "'", '@' );
		return $corpus_id;
	}

	sub process_urls {
		my ( $self ) = @_;
		my $corpuses = $self->get_all_corpuses();
		foreach my $corpus( @$corpuses ) {
			my $data_dir      = $self->get_data_dir( $corpus->get_id() );
			my $sql           = "select submitted_url_id, corpus_id, submitted_by_user_id, submitted_url from submitted_urls where exit_date = '0000-00-00 00:00:00' and file_not_found = 0";
			my $urls          = $self->library()->sqlexec( $sql, '\@@' );
			foreach my $url ( @$urls ) { $self->_download_url( $url, $data_dir ); }
		}
	}

	sub reprocess_urls {
		my ( $self ) = @_;
		my $corpuses = $self->get_all_corpuses();
		foreach my $corpus( @$corpuses ) {
			my ( $corpus_id ) = @$corpus;
			my $data_dir      = $self->get_data_dir( $corpus->get_id() );
			my $sql           = "select submitted_url_id, corpus_id, submitted_by_user_id, submitted_url from submitted_urls where file_not_found = 1";
			my $urls          = $self->library()->sqlexec( $sql, '\@@' );
			foreach my $url ( @$urls ) { $self->_download_url( $url, $data_dir ); }
		}
	}

	sub _download_url {
		my ( $self, $url_row, $data_dir )          = @_;
		my ( $id, $corpus_id, $user_id, $url ) = @$url_row;
		my $file_name = $self->_parse_file_name( $url );
		my $path      = $self->_build_directories( $url, $data_dir );
		my $bytes     = $self->_download_file({ target_dir => $data_dir . $path,
		                                        url        => $url,
						        file_name  => $file_name });
		if ( $bytes ) {
			my $sql  = "insert into submitted_documents (submitted_url_id, corpus_id, submitted_by_user_id, document_path, document_file_name, bytes ) ";
			   $sql .= "values ('$id', '$corpus_id', '$user_id', '$path', '$file_name', '$bytes' )";
			$self->library()->sqlexec( $sql );
			$self->library()->sqlexec( "update submitted_urls set exit_date = now(), file_found = 1, file_not_found = 0 where submitted_url_id = '$id'" ); }
		else 	      {
			$self->library()->sqlexec( "update submitted_urls set exit_date = now(), file_found = 0, file_not_found = 1 where submitted_url_id = '$id'" ); }
	}

	sub _build_directories {
		my ( $self, $url, $corpus_data_dir ) = @_;
		my @path     = split(/\//, $url); shift(@path); shift(@path); # Remove protocol
		my $file     = pop(@path);
		my $path     = '';

		foreach my $part (@path) {
			$path .= '/' . $part;
			if (! -e $corpus_data_dir . $path) { mkdir $corpus_data_dir . $path; }
		}
		return $path;
	}

}

1; # Magic true value required at end of module
__END__