MediaWikiDump/Revisions.pm untested

filter: Statementstmt 0.0% 0 / 179 Branchbran 0.0% 0 / 28 Conditioncond 0.0% 0 / 3 MC/DCmcdc 0.0% 0 / 2 Subroutinesub 0.0% 0 / 28 Podpod 0.0% 0 / 21 totaltotal 0.0% 0 / 261 CC 15 SCAR 20.3
CC
15
Cov
0%
new
20.0
parse_more
20.0
next
12.0
CRAP 7.6
1 package Parse::MediaWikiDump::Revisions;
2  
3 our $VERSION = '1.0.4';
4  
5 use 5.8.0;
6  
7 use strict;
8 use warnings;
9 use Carp;
10  
11 use List::Util;
12 use Scalar::Util qw(weaken reftype);
13 use Data::Dumper;
14  
15 sub DESTROY {
16 my ($self) = @_;
17
18 if (! $self->{FINISHED}) {
19 $self->{EXPAT}->parse_done;
20 }
21 }
22  
23 #public methods
24 sub new {
25 my ($class, @args) = @_;
26 my $self = {};
27 my $source;
28
29 if (scalar(@args) == 0) {
30 die "you must specify an argument to new()";
31 } elsif (scalar(@args) == 1) {
32 $source = $args[0];
33 } else {
34 my %conf = @args;
35
36 if (! defined($conf{input})) {
37 die "input is a required parameter to new()";
38 }
39
40 $source = $conf{input};
41 }
42
43 bless($self, $class);
44  
45 $$self{XML} = undef; #holder for XML::Accumulator
46 $$self{EXPAT} = undef; #holder for expat under XML::Accumulator
47 $$self{SITEINFO} = {}; #holder for the data from siteinfo
48 $$self{PAGE_LIST} = []; #place to store articles as they come out of XML::Accumulator
49 $$self{BYTE} = 0;
50 $$self{CHUNK_SIZE} = 32768;
51 $$self{FINISHED} = 0;
52  
53 $self->open($source);
54 $self->init;
55
56 return $self;
57 }
58  
59 sub next {
60 my ($self) = @_;
61 my $case = $self->{SITEINFO}->{CASE};
62 my $namespaces = $self->{SITEINFO}->{namespaces};
63
64 my $page;
65
66 #look for an available page and if one isn't
67 #there then parse more XML
68 while(1) {
69 $page = shift(@{ $self->{PAGE_LIST} } );
70
71 if (defined($page)) {
72
73 return Parse::MediaWikiDump::page->new($page, $self->get_category_anchor, $case, $namespaces);
74 }
75
76 return undef unless $self->parse_more;
77 }
78
79 die "should not get here";
80 }
81  
82 sub version {
83 my ($self) = @_;
84 return $self->{SITEINFO}{version};
85 }
86  
87 sub sitename {
88 my ($self) = @_;
89 return $$self{SITEINFO}{sitename};
90 }
91  
92 sub base {
93 my ($self) = @_;
94 return $$self{SITEINFO}{base};
95 }
96  
97 sub generator {
98 my ($self) = @_;
99 return $$self{SITEINFO}{generator};
100 }
101  
102 sub case {
103 my ($self) = @_;
104 return $$self{SITEINFO}{case};
105 }
106  
107 sub namespaces {
108 my ($self) = @_;
109 return $$self{SITEINFO}{namespaces};
110 }
111  
112 sub namespaces_names {
113 my $self = shift;
114 my @result;
115
116 foreach (@{ $$self{SITEINFO}{namespaces} }) {
117 push(@result, $_->[1]);
118 }
119
120 return \@result;
121 }
122  
123 sub current_byte {
124 my ($self) = @_;
125 return $$self{BYTE};
126 }
127  
128 sub size {
129 my ($self) = @_;
130
131 return undef unless defined $$self{SOURCE_FILE};
132  
133 my @stat = stat($$self{SOURCE_FILE});
134  
135 return $stat[7];
136 }
137  
138 #private functions with OO interface
139  
140 sub open {
141 my ($self, $source) = @_;
142  
143 if (defined(reftype($source)) && reftype($source) eq 'GLOB') {
144 $$self{SOURCE} = $source;
145 } else {
146 if (! open($$self{SOURCE}, $source)) {
147 die "could not open $source: $!";
148 }
149  
150 $$self{SOURCE_FILE} = $source;
151 }
152  
153 binmode($$self{SOURCE}, ':utf8');
154  
155 return 1;
156 }
157  
158 sub init {
159 my ($self) = @_;
160
161 $self->{XML} = $self->new_accumulator_engine;
162 my $expat_bb = $$self{XML}->parser->parse_start();
163 $$self{EXPAT} = $expat_bb;
164
165 #load the information from the siteinfo section so it is available before
166 #someone calls ->next
167 while(scalar(@{$self->{PAGE_LIST}}) < 1) {
168 die "hit end of document" unless $self->parse_more;
169 }
170 }
171  
172 sub new_accumulator_engine {
173 my ($self) = @_;
174 my $f = Parse::MediaWikiDump::XML::Accumulator->new;
175 my $store_siteinfo = $self->{SITEINFO};
176 my $store_page = $self->{PAGE_LIST};
177
178 my $root = $f->root;
179 my $mediawiki = $f->node('mediawiki', Start => \&handle_mediawiki_node);
180
181 #stuff for siteinfo
182 my $siteinfo = $f->node('siteinfo', End => sub { %$store_siteinfo = %{ $_[1] } } );
183 my $sitename = $f->textcapture('sitename');
184 my $base = $f->textcapture('base');
185 my $generator = $f->textcapture('generator');
186 my $case = $f->textcapture('case');
187 my $namespaces = $f->node('namespaces', Start => sub { $_[1]->{namespaces} = []; } );
188 my $namespace = $f->node('namespace', Character => \&save_namespace_node);
189
190 #stuff for page entries
191 my $page = $f->node('page', Start => sub { $_[0]->accumulator( {} ) } );
192 my $title = $f->textcapture('title');
193 my $id = $f->textcapture('id');
194 my $revision = $f->node('revision',
195 Start => \&handle_revision_node_start, End => sub { push(@$store_page, { %{ $_[1] } } ) } );
196 my $rev_id = $f->textcapture('id', 'revision_id');
197 my $minor = $f->node('minor', Start => sub { $_[1]->{minor} = 1 } );
198 my $time = $f->textcapture('timestamp');
199 my $contributor = $f->node('contributor');
200 my $username = $f->textcapture('username');
201 my $ip = $f->textcapture('ip', 'userip');
202 my $contrib_id = $f->textcapture('id', 'userid');
203 my $comment = $f->textcapture('comment');
204 my $text = $f->textcapture('text');
205 my $restr = $f->textcapture('restrictions');
206
207 #put together the tree
208 $siteinfo->add_child($sitename, $base, $generator, $case, $namespaces);
209 $namespaces->add_child($namespace);
210
211 $page->add_child($title, $id, $revision, $restr);
212 $revision->add_child($rev_id, $time, $contributor, $minor, $comment, $text);
213 $contributor->add_child($username, $ip, $contrib_id);
214
215 $mediawiki->add_child($siteinfo, $page);
216 $root->add_child($mediawiki);
217
218 my $engine = $f->engine($root, {});
219  
220 return $engine;
221 }
222  
223 sub parse_more {
224 my ($self) = @_;
225 my $buf;
226  
227 my $read = read($$self{SOURCE}, $buf, $$self{CHUNK_SIZE});
228  
229 if (! defined($read)) {
230 die "error during read: $!";
231 } elsif ($read == 0) {
232 $$self{FINISHED} = 1;
233 $$self{EXPAT}->parse_done;
234
235 return 0;
236 }
237  
238 #expat has a bug where the current_byte
239 #value overflows around 2 gigabytes
240 #so we track how much data has been
241 #processed ourselves
242 $$self{BYTE} += $read;
243 $$self{EXPAT}->parse_more($buf);
244
245 if ($self->{DIE_REQUESTED}) {
246 die "$self->{DIE_REQUESTED}\n";
247 }
248
249 return 1;
250 }
251  
252 sub get_category_anchor {
253 my ($self) = @_;
254 my $namespaces = $self->{SITEINFO}->{namespaces};
255  
256 foreach (@$namespaces) {
257 my ($id, $name) = @$_;
258 if ($id == 14) {
259 return $name;
260 }
261 }
262
263 return undef;
264 }
265  
266 #helper functions that the xml accumulator uses
267  
268 sub save_namespace_node {
269 my ($parser, $accum, $text, $element, $attrs) = @_;
270 my $key = $attrs->{key};
271 my $namespaces = $accum->{namespaces};
272
273 push(@{ $accum->{namespaces} }, [$key, $text] );
274 }
275  
276 sub handle_mediawiki_node {
277 my ($engine, $a, $element, $attrs) = @_;
278 my $version = $attrs->{version};
279  
280 #checking versions of the dump file removed in 1.0.6
281 #see the migration notes for why
282 # if ($version ne '0.3' && $version ne '0.4') {
283 # die "Only version 0.3 and 0.4 dump files are supported";
284 # }
285
286 $a->{version} = $version;
287 }
288  
289 sub handle_revision_node_start {
290 my (undef, $a) = @_;
291
292 $a->{minor} = 0;
293 delete($a->{username});
294 delete($a->{userid});
295 delete($a->{userip});
296 }
297  
298 sub save_siteinfo {
299 my ($self, $info) = @_;
300 my %info = %$info;
301
302 $self->{SITEINFO} = \%info;
303 }
304  
305 1;
306  
307 __END__