File Coverage

blib/lib/MediaWiki/DumpFile/Pages.pm

Criterion	Covered	Total	%
statement	16	18	88.8
branch			n/a
condition			n/a
subroutine	6	6	100.0
pod			n/a
total	22	24	91.6

package MediaWiki::DumpFile::Pages;

our $VERSION = '0.2.2';
our $TESTED_SCHEMA_VERSION = 0.5;

use strict;
use warnings;
use Scalar::Util qw(reftype);
use Carp qw(croak);
use Data::Dumper;

use XML::TreePuller;
use XML::LibXML::Reader;
use IO::Uncompress::AnyUncompress qw($AnyUncompressError);

sub new {
	my ($class, @args) = @_;
	my $self = {};
	my $reftype;
	my $xml;
	my $input;
	my %conf;
	my $io;

	bless($self, $class);

	$self->{siteinfo} = undef;
	$self->{version} = undef;
	$self->{fast_mode} = undef;
	$self->{version_ignore} = 1;

	if (scalar(@args) == 0) {
		croak "must specify a file path or open file handle object or a hash of options";
	} elsif (scalar(@args) == 1) {
		$input = $args[0];
	} elsif (! scalar(@args) % 2) {
		croak "must specify a hash as an argument";
	} else {
		%conf = @args;

		if (! defined($input = $conf{input})) {
			croak "input is a required option";
		}

		if (defined($conf{fast_mode})) {
			$self->{fast_mode} = $conf{fast_mode};
		}

		if (defined($conf{strict})) {
			$self->{version_ignore} = $conf{version_ignore};
		}
	}

	$reftype = reftype($input);

	if (! defined($reftype)) {
		if (! -e $input) {
			croak("$input is not a file");
		}

	} elsif ($reftype ne 'GLOB') {
		croak('must provide a GLOB reference');
	}

	$self->{input} = $input;
	$io = IO::Uncompress::AnyUncompress->new($input);
	$xml = $self->_new_puller(IO => $io);

	if (exists($ENV{MEDIAWIKI_DUMPFILE_VERSION_IGNORE})) {
		$self->{version_ignore} = $ENV{MEDIAWIKI_DUMPFILE_VERSION_IGNORE};
	}

	if (exists($ENV{MEDIAWIKI_DUMPFILE_FAST_MODE})) {
		$self->{fast_mode} = $ENV{MEDIAWIKI_DUMPFILE_FAST_MODE};
	}

	$self->{xml} = $xml;
	$self->{reader} = $xml->reader;
	$self->{input} = $input;
	$self->{io} = $io;

	$self->_init_xml;

	return $self;
}

sub next {
	my ($self, $fast) = @_;
	my $version;
	my $new;

	if ($fast || $self->{fast_mode}) {
		my ($title, $text);

		if ($self->{finished}) {
			return ();
		}

		eval { ($title, $text) = $self->_fast_next; };

		if ($@) {
			chomp($_);
			croak("E_XML_PARSE_FAILED \"$@\" see the ERRORS section of the MediaWiki::DumpFile::Pages Perl module documentation for what to do");
		}

		unless (defined($title)) {
			$self->{finished} = 1;
			return ();
		}

		return MediaWiki::DumpFile::Pages::FastPage->new($title, $text);
	}

	if ($self->{finished}) {
		return undef;
	}

	$version = $self->{version};
	eval { $new  = $self->{xml}->next; };

	if ($@) {
		chomp($_);
		croak("E_XML_PARSE_FAILED \"$@\" see the ERRORS section of the MediaWiki::DumpFile::Pages Perl module documentation for what to do");
	}

	unless (defined($new)) {
		$self->{finished} = 1;
		return undef;
	}

	return MediaWiki::DumpFile::Pages::Page->new($new, $version);
}

sub size {
	my $source = $_[0]->{input};

	unless(defined($source) && ref($source) eq '') {
		return undef;
	}

	#if we are decompressing a file on the fly then don't report the size
	#of the file because we don't actually know the uncompressed size,
	#only the compressed size
	if (defined($_[0]->{io}->getHeaderInfo)) {
		return undef;
	}

	my @stat = stat($source);
	return $stat[7];
}

sub current_byte {
	return $_[0]->{xml}->reader->byteConsumed;
}

sub completed {
	my ($self) = @_;
	my $size = $self->size;
	my $current = $self->current_byte;

	return -1 unless (defined($size) && defined($current));

	return int($current / $size * 100);
}

sub version {
	return $_[0]->{version};
}

#private methods

sub _init_xml {
	my ($self) = @_;
	my $xml = $self->{xml};
	my $version;

	$xml->iterate_at('/mediawiki', 'short');
	$xml->iterate_at('/mediawiki/siteinfo', 'subtree');
	$xml->iterate_at('/mediawiki/page', 'subtree');

	$version = $self->{version} = $xml->next->attribute('version');

	unless ($self->{version_ignore}) {
		$self->_version_enforce($version);
	}

	if ($version > 0.2) {
		$self->{siteinfo} = $xml->next;

		bless($self, 'MediaWiki::DumpFile::PagesSiteinfo');
	}

	return undef;
}

sub _version_enforce {
	my ($self, $version) = @_;

	if ($version > $TESTED_SCHEMA_VERSION) {
		my $filename;
		my $msg;

		if (ref($self->{input}) eq '') {
			$filename = $self->{input};
		} else {
			$filename = ref($self->{input});
		}

		$msg = "E_UNTESTED_DUMP_VERSION Version $version dump file \"$filename\" has not been tested with ";
		$msg .= __PACKAGE__ . " version $VERSION; see the ERRORS section of the MediaWiki::DumpFile::Pages Perl module documentation for what to do";

		die $msg;
	}

}

sub _new_puller {
	my ($self, @args) = @_;
	my $ret;

	eval { $ret = XML::TreePuller->new(@args) };

	if ($@) {
		chomp($@);
		croak("E_XML_CREATE_FAILED \"$@\" see the ERRORS section of the MediaWiki::DumpFile::Pages Perl module documentation for what to do")
	}

	return $ret;
}

sub _get_text {
	my ($self) = @_;
	my $r = $self->{reader};
	my @buffer;
	my $type;

	while($r->nodeType != XML_READER_TYPE_TEXT && $r->nodeType != XML_READER_TYPE_END_ELEMENT) {
		$r->read or die "could not read";
	}

	while($r->nodeType != XML_READER_TYPE_END_ELEMENT) {
		if ($r->nodeType == XML_READER_TYPE_TEXT) {
			push(@buffer, $r->value);
		}

		$r->read or die "could not read";
	}

	return join('', @buffer);
}

sub _fast_next {
	my ($self) = @_;
	my $reader = $self->{reader};
	my ($title, $text);

	if ($self->{finished}) {
		return ();
	}

	while(1) {
		my $type = $reader->nodeType;

		if ($type == XML_READER_TYPE_ELEMENT) {
			if ($reader->name eq 'title') {
				$title = $self->_get_text();
				last unless $reader->nextElement('text') == 1;
				next;
			} elsif ($reader->name eq 'text') {
				$text = $self->_get_text();
				$reader->nextElement('page');
				last;
			}
		}

		last unless $reader->nextElement == 1;
	}

	if (! defined($title) || ! defined($text)) {
		$self->{finished} = 1;
		return ();
	}

	return($title, $text);
}

package MediaWiki::DumpFile::PagesSiteinfo;

use base qw(MediaWiki::DumpFile::Pages);

use Data::Dumper;

use MediaWiki::DumpFile::Pages::Lib qw(_safe_text);

sub _site_info {
	my ($self, $name) = @_;
	my $siteinfo = $self->{siteinfo};

	return _safe_text($siteinfo, $name);
}

sub sitename {
	return $_[0]->_site_info('sitename');
}

sub base {
	return $_[0]->_site_info('base');
}

sub generator {
	return $_[0]->_site_info('generator');
}

sub case {
	return $_[0]->_site_info('case');
}

sub namespaces {
	my ($self) = @_;
	my @e = $self->{siteinfo}->get_elements('namespaces/namespace');
	my %ns;

	map({ $ns{ $_->attribute('key') } = $_->text } @e);

	return %ns;
}

package MediaWiki::DumpFile::Pages::Page;

use strict;
use warnings;
use Data::Dumper;

use MediaWiki::DumpFile::Pages::Lib qw(_safe_text);

sub new {
	my ($class, $element, $version) = @_;
	my $self = { tree => $element };

	bless($self, $class);

	if ($version >= 0.4) {
		bless ($self, 'MediaWiki::DumpFile::Pages::Page000004000');
	}

	return $self;
}

sub title {
	return _safe_text($_[0]->{tree}, 'title');
}

sub id {
	return _safe_text($_[0]->{tree}, 'id');
}

sub revision {
	my ($self) = @_;
	my @revisions;

	foreach ($self->{tree}->get_elements('revision')) {
		push(@revisions, MediaWiki::DumpFile::Pages::Page::Revision->new($_));
	}

	if (wantarray()) {
		return (@revisions);
	}

	return pop(@revisions);
}

package MediaWiki::DumpFile::Pages::Page000004000;

use base qw(MediaWiki::DumpFile::Pages::Page);

use strict;
use warnings;

sub redirect {
	return 1 if defined $_[0]->{tree}->get_elements('redirect');
	return 0;
}


package MediaWiki::DumpFile::Pages::Page::Revision;

use strict;
use warnings;

use MediaWiki::DumpFile::Pages::Lib qw(_safe_text);

sub new {
	my ($class, $tree) = @_;
	my $self = { tree => $tree };

	return bless($self, $class);
}

sub text {
	return _safe_text($_[0]->{tree}, 'text');
}

sub id {
	return _safe_text($_[0]->{tree}, 'id');
}

sub timestamp {
	return _safe_text($_[0]->{tree}, 'timestamp');
}

sub comment {
	return _safe_text($_[0]->{tree}, 'comment');
}

sub minor {
	return 1 if defined $_[0]->{tree}->get_elements('minor');
	return 0;
}

sub contributor {
	return MediaWiki::DumpFile::Pages::Page::Revision::Contributor->new(
		$_[0]->{tree}->get_elements('contributor') );
}

package MediaWiki::DumpFile::Pages::Page::Revision::Contributor;

use strict;
use warnings;

use Carp qw(croak);

use overload
	'""' => 'astext',
	fallback => 'TRUE';

sub new {
	my ($class, $tree) = @_;
	my $self = { tree => $tree };

	return bless($self, $class);
}

sub astext {
	my ($self) = @_;

	if (defined($self->ip)) {
		return $self->ip;
	}

	return $self->username;
}

sub username {
	my $user = $_[0]->{tree}->get_elements('username');

	return undef unless defined $user;

	return $user->text;
}

sub id {
	my $id = $_[0]->{tree}->get_elements('id');

	return undef unless defined $id;

	return $id->text;
}

sub ip {
	my $ip = $_[0]->{tree}->get_elements('ip');

	return undef unless defined $ip;

	return $ip->text;
}

package MediaWiki::DumpFile::Pages::FastPage;

sub new {
	my ($class, $title, $text) = @_;
	my $self = { title => $title, text => $text };

	bless($self, $class);

	return $self;
}

sub title {
	return $_[0]->{title};
}

sub text {
	return $_[0]->{text};
}

sub revision {
	return $_[0];
}

1;

__END__