| 1 | package RDF::Trine::Store::File; | ||
| 2 | |||
| 3 | 1 | ▶ | use 5.006; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 4 | 1 | ▶ | use strict; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 5 | 1 | ▶ | use warnings; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 6 | 1 | ▶ | use base qw(RDF::Trine::Store); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 7 | 1 | ▶ | use RDF::Trine::Error qw(:try); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 8 | 1 | ▶ | use RDF::Trine::Serializer::NTriples; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 9 | 1 | ▶ | use RDF::Trine::Parser; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 10 | 0 | ▶ | use File::Data; |
|
Subroutine
BEGIN: called CC=1 SCAR=2.6
| |||
| 11 | use File::Util; | ||
| 12 | use Scalar::Util qw(blessed); | ||
| 13 | use File::Temp qw/tempfile/; | ||
| 14 | use Carp qw/croak/; | ||
| 15 | use Log::Log4perl; | ||
| 16 | use Digest::MD5 ('md5_hex'); | ||
| 17 | |||
| 18 | |||
| 19 | =head1 NAME | ||
| 20 | |||
| 21 | RDF::Trine::Store::File - Using a file with N-Triples as triplestore | ||
| 22 | |||
| 23 | =head1 VERSION | ||
| 24 | |||
| 25 | Version 0.1 | ||
| 26 | |||
| 27 | =cut | ||
| 28 | |||
| 29 | our $VERSION = '0.1'; | ||
| 30 | |||
| 31 | |||
| 32 | =head1 SYNOPSIS | ||
| 33 | |||
| 34 | To be used as a normal triple store. | ||
| 35 | |||
| 36 | my $store = RDF::Trine::Store::File->new($filename); | ||
| 37 | $store->add_statement(RDF::Trine::Statement->new( | ||
| 38 | RDF::Trine::Node::Resource->new('http://example.org/a'), | ||
| 39 | RDF::Trine::Node::Resource->new('http://example.org/b'), | ||
| 40 | RDF::Trine::Node::Resource->new('http://example.org/c') | ||
| 41 | )); | ||
| 42 | ... | ||
| 43 | |||
| 44 | =head1 METHODS | ||
| 45 | |||
| 46 | =head2 new($filename) | ||
| 47 | |||
| 48 | A constructor, takes a filename as parameter. If the file doesn't | ||
| 49 | exist, it will be created. | ||
| 50 | |||
| 51 | =cut | ||
| 52 | |||
| 53 | sub new { | ||
| 54 | my $class = shift; | ||
| 55 | my $file = shift; | ||
| 56 | my($fu) = File::Util->new(); | ||
| 57 | unless (-e $file) { | ||
| 58 | $fu->touch($file); | ||
| 59 | } | ||
| 60 | my $self = bless( | ||
| 61 | { | ||
| 62 | file => $file, | ||
| 63 | fu => $fu, | ||
| 64 | nser => RDF::Trine::Serializer::NTriples->new, | ||
| 65 | log => Log::Log4perl->get_logger("rdf.trine.store.file") | ||
| 66 | }, $class); | ||
| 67 | return $self; | ||
| 68 | } | ||
| 69 | |||
| 70 | =head2 new_with_string('File;'.$filename) | ||
| 71 | |||
| 72 | A constructor, takes a string config as parameter. If the file doesn't | ||
| 73 | exist, it will be created. The string will typically begin with C<File;>, e.g. | ||
| 74 | |||
| 75 | my $store = RDF::Trine::Store::File->new_with_string('File;/path/to/file.nt'); | ||
| 76 | |||
| 77 | =cut | ||
| 78 | |||
| 79 | sub _new_with_string { | ||
| 80 | my ($class, $filename) = @_; | ||
| 81 | return $class->new($filename); | ||
| 82 | } | ||
| 83 | |||
| 84 | =head2 new_with_config({ storetype => 'File', file => $filename}); | ||
| 85 | |||
| 86 | A constructor, takes a hashref config as parameter. If the file doesn't | ||
| 87 | exist, it will be created. It needs to have a C<storetype> key with C<File> as the value, e.g. | ||
| 88 | |||
| 89 | my $store = RDF::Trine::Store::File->new_with_config({ storetype => 'File', file => $filename}); | ||
| 90 | |||
| 91 | |||
| 92 | =cut | ||
| 93 | |||
| 94 | sub _new_with_config { | ||
| 95 | my $class = shift; | ||
| 96 | my $config = shift; | ||
| 97 | return $class->new($config->{file}); | ||
| 98 | } | ||
| 99 | |||
| 100 | =head2 temporary_store | ||
| 101 | |||
| 102 | Constructor that creates a temporary file to work on. | ||
| 103 | |||
| 104 | =cut | ||
| 105 | |||
| 106 | sub temporary_store { | ||
| 107 | my $class = shift; | ||
| 108 | my ($fh, $filename) = tempfile(EXLOCK => 0); | ||
| 109 | return $class->new($filename); | ||
| 110 | } | ||
| 111 | |||
| 112 | |||
| 113 | =head2 add_statement($statement) | ||
| 114 | |||
| 115 | Adds the specified C<$statement> to the underlying model. | ||
| 116 | |||
| 117 | =cut | ||
| 118 | |||
| 119 | sub add_statement { | ||
| 120 | my ($self, $st) = @_; | ||
| 121 | unless (blessed($st) and $st->isa('RDF::Trine::Statement')) { | ||
| 122 | throw RDF::Trine::Error::MethodInvocationError -text => "Not a valid statement object passed to add_statement"; | ||
| 123 | } | ||
| 124 | my $mm = RDF::Trine::Model->temporary_model; | ||
| 125 | $mm->add_statement($st); | ||
| 126 | $self->{log}->debug("Attempting addition of statement"); | ||
| 127 | my $fd = File::Data->new($self->{file}); | ||
| 128 | $fd->append($self->{nser}->serialize_model_to_string($mm)); | ||
| 129 | return; | ||
| 130 | } | ||
| 131 | |||
| 132 | =head2 get_statements($subject, $predicate, $object) | ||
| 133 | |||
| 134 | Returns a stream object of all statements matching the specified subject, | ||
| 135 | predicate and objects. Any of the arguments may be undef to match any value. | ||
| 136 | |||
| 137 | =cut | ||
| 138 | |||
| 139 | sub get_statements { | ||
| 140 | my $self = shift; | ||
| 141 | my @lines = $self->_search_statements(@_); | ||
| 142 | my $parser = RDF::Trine::Parser->new( 'ntriples' ); | ||
| 143 | my $mm = RDF::Trine::Model->temporary_model; | ||
| 144 | $parser->parse_into_model( '', join('', @lines), $mm ); | ||
| 145 | return $mm->get_statements(undef, undef, undef, undef); | ||
| 146 | } | ||
| 147 | |||
| 148 | =head2 count_statements($subject, $predicate, $object) | ||
| 149 | |||
| 150 | Returns a count of all the statements matching the specified subject, | ||
| 151 | predicate and objects. Any of the arguments may be undef to match any value. | ||
| 152 | |||
| 153 | =cut | ||
| 154 | |||
| 155 | sub count_statements { | ||
| 156 | my $self = shift; | ||
| 157 | my @lines = $self->_search_statements(@_); | ||
| 158 | return scalar @lines; | ||
| 159 | } | ||
| 160 | |||
| 161 | # Private method to actually search for statements based on a regexp. | ||
| 162 | |||
| 163 | sub _search_statements { | ||
| 164 | my $self = shift; | ||
| 165 | my $regexp = $self->_search_regexp(@_); | ||
| 166 | my $fd = File::Data->new($self->{file}); | ||
| 167 | $self->{log}->debug("Searching with regexp $regexp"); | ||
| 168 | return $fd->SEARCH($regexp); | ||
| 169 | } | ||
| 170 | |||
| 171 | |||
| 172 | |||
| 173 | =head2 remove_statement($statement) | ||
| 174 | |||
| 175 | Removes the specified C<$statement> from the underlying model. | ||
| 176 | |||
| 177 | =cut | ||
| 178 | |||
| 179 | sub remove_statement { | ||
| 180 | my ($self, $st) = @_; | ||
| 181 | unless (blessed($st) and $st->isa('RDF::Trine::Statement')) { | ||
| 182 | throw RDF::Trine::Error::MethodInvocationError -text => "Not a valid statement object passed to remove_statement"; | ||
| 183 | } | ||
| 184 | my $mm = RDF::Trine::Model->temporary_model; | ||
| 185 | $mm->add_statement($st); | ||
| 186 | $self->{log}->debug("Attempting removal of statement"); | ||
| 187 | my $fd = File::Data->new($self->{file}); | ||
| 188 | my $triple = $self->{nser}->serialize_model_to_string($mm); | ||
| 189 | $triple =~ s/\^/\\^/g; | ||
| 190 | $triple =~ s/\\u/\\\\u/g; | ||
| 191 | $fd->REPLACE($triple, ''); | ||
| 192 | return; | ||
| 193 | } | ||
| 194 | |||
| 195 | |||
| 196 | =head2 remove_statements($subject, $predicate, $object) | ||
| 197 | |||
| 198 | Removes all statement matching the specified subject, predicate and | ||
| 199 | objects. Any of the arguments may be undef to match any value. | ||
| 200 | |||
| 201 | =cut | ||
| 202 | |||
| 203 | sub remove_statements { | ||
| 204 | my $self = shift; | ||
| 205 | my $regexp = $self->_search_regexp(@_); | ||
| 206 | my $fd = File::Data->new($self->{file}); | ||
| 207 | $self->{log}->debug("Removing with regexp $regexp"); | ||
| 208 | my @lines = $fd->REPLACE($regexp, ''); | ||
| 209 | $self->{log}->info("Removed " . scalar @lines . " statements."); | ||
| 210 | return; | ||
| 211 | } | ||
| 212 | |||
| 213 | |||
| 214 | |||
| 215 | =head2 get_contexts | ||
| 216 | |||
| 217 | Contexts are not supported for this store. | ||
| 218 | |||
| 219 | =cut | ||
| 220 | |||
| 221 | |||
| 222 | sub get_contexts { | ||
| 223 | croak "Contexts not supported for the File store"; | ||
| 224 | } | ||
| 225 | |||
| 226 | =head2 size | ||
| 227 | |||
| 228 | Returns the number of statements in the store. Breaks if there are | ||
| 229 | comments or empty lines in the file. | ||
| 230 | |||
| 231 | =cut | ||
| 232 | |||
| 233 | sub size { | ||
| 234 | my $self = shift; | ||
| 235 | return $self->{fu}->line_count($self->{file}); | ||
| 236 | } | ||
| 237 | |||
| 238 | =head2 etag | ||
| 239 | |||
| 240 | Returns an etag based on the last modification time of the file. Note: | ||
| 241 | This has resolution of one second, so it cannot be relied on for data | ||
| 242 | that changes fast. | ||
| 243 | |||
| 244 | =cut | ||
| 245 | |||
| 246 | sub etag { | ||
| 247 | my $self = shift; | ||
| 248 | return md5_hex($self->{fu}->last_modified($self->{file})); | ||
| 249 | } | ||
| 250 | |||
| 251 | =head2 nuke | ||
| 252 | |||
| 253 | Permanently removes the store file and its data. | ||
| 254 | |||
| 255 | =cut | ||
| 256 | |||
| 257 | sub nuke { | ||
| 258 | my $self = shift; | ||
| 259 | unlink $self->{file}; | ||
| 260 | return $self; | ||
| 261 | } | ||
| 262 | |||
| 263 | # Private method to create a regexp to be used in all kind of searching | ||
| 264 | |||
| 265 | sub _search_regexp { | ||
| 266 | my $self = shift; | ||
| 267 | my $i = 1; | ||
| 268 | my @stmt; | ||
| 269 | foreach my $term (@_[0..2]) { # Create an array of RDF terms for later replacing for variables, discard context | ||
| 270 | my $outterm = $term || RDF::Trine::Node::Resource->new("urn:rdf-trine-store-file-$i"); | ||
| 271 | $outterm = RDF::Trine::Node::Resource->new("urn:rdf-trine-store-file-$i") if ($outterm->isa('RDF::Trine::Node::Variable')); | ||
| 272 | push(@stmt, $outterm); | ||
| 273 | $i++; | ||
| 274 | } | ||
| 275 | my $mm = RDF::Trine::Model->temporary_model; | ||
| 276 | $mm->add_statement(RDF::Trine::Statement->new(@stmt)); | ||
| 277 | my $triple_resources = $self->{nser}->serialize_model_to_string($mm); | ||
| 278 | chomp($triple_resources); | ||
| 279 | $triple_resources =~ s/\.\s*$/\\./; | ||
| 280 | $triple_resources =~ s/urn:rdf-trine-store-file-(1|2)/.*?/g; | ||
| 281 | $triple_resources =~ s/<urn:rdf-trine-store-file-3>/.*/; | ||
| 282 | $triple_resources =~ s/\^/\\^/g; | ||
| 283 | $triple_resources =~ s/\\u/\\\\u/g; | ||
| 284 | my $out = '(' . $triple_resources . '\n)'; | ||
| 285 | $self->{log}->debug("Search regexp: $out"); | ||
| 286 | return $out; | ||
| 287 | } | ||
| 288 | |||
| 289 | =head1 DISCUSSION | ||
| 290 | |||
| 291 | This module is intended mostly as a simple backend to dump data to a | ||
| 292 | file and do as little as possible in memory. Thus, it is mostly | ||
| 293 | suitable in cases where a lot of data is written to file. It should be | ||
| 294 | possible to use it as a SPARQL store with L<RDF::Query>, but the | ||
| 295 | performance is likely to be somewhere between terrible and abyssmal, | ||
| 296 | so don't do that unless you are prepared to be waiting around. | ||
| 297 | |||
| 298 | On the good side, adding statements should be pretty fast, as it just | ||
| 299 | appends to a file. The C<size> method should be pretty fast too, as it | ||
| 300 | just counts the lines in that file. Finally, it supports the C<etag> | ||
| 301 | method, not perfectly, but that's still pretty good! | ||
| 302 | |||
| 303 | It uses a lot of heuristics tied to the format chosen, i.e. | ||
| 304 | N-Triples. That's a line-based format, with predictable amounts of | ||
| 305 | whitespace, allowing us to create relatively simple regular | ||
| 306 | expressions as search patterns in the file. This is likely to be | ||
| 307 | somewhat fragile (it is making assumptions about the file that is true | ||
| 308 | in the L<RDF::Trine::Serializer::Ntriples> case, but not in the | ||
| 309 | format), but it kinda works. | ||
| 310 | |||
| 311 | It is important to note that this module does nothing to prevent you | ||
| 312 | from adding duplicate statements like other stores should do. This is | ||
| 313 | because it would dramatically reduce C<add_statement> performance and | ||
| 314 | thus kill the main use case for this module. Perhaps it could be made | ||
| 315 | optional at some point, but for now, just be aware that this may not | ||
| 316 | always return the right counts if two identical statements are | ||
| 317 | inserted and possibly produce unexpected results if you against the | ||
| 318 | advice above should attempt to use this a store for SPARQL. | ||
| 319 | |||
| 320 | |||
| 321 | I've decided to use L<File::Data> to actually do the work with the | ||
| 322 | file. Locking and that kind of stuff is done there and is thus Not My | ||
| 323 | Problem. If it is yours, then L<File::Data> is probably the right | ||
| 324 | place to go and fix it. | ||
| 325 | |||
| 326 | |||
| 327 | =head1 TODO | ||
| 328 | |||
| 329 | This is beta-quality software but it has a pretty comprehensive test | ||
| 330 | suite. These are some things that could be done: | ||
| 331 | |||
| 332 | =over | ||
| 333 | |||
| 334 | =item * Support bulk operations (somewhat less important) | ||
| 335 | |||
| 336 | =item * Find a way to do duplicate checking efficiently | ||
| 337 | |||
| 338 | =back | ||
| 339 | |||
| 340 | =head1 AUTHOR | ||
| 341 | |||
| 342 | Kjetil Kjernsmo, C<< <kjetilk at cpan.org> >> | ||
| 343 | |||
| 344 | =head1 BUGS | ||
| 345 | |||
| 346 | Please report any bugs or feature requests to | ||
| 347 | C<bug-rdf-trine-store-file at rt.cpan.org>, or through the web | ||
| 348 | interface at | ||
| 349 | L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=RDF-Trine-Store-File>. | ||
| 350 | I will be notified, and then you'll automatically be notified of | ||
| 351 | progress on your bug as I make changes. | ||
| 352 | |||
| 353 | |||
| 354 | |||
| 355 | |||
| 356 | =head1 SUPPORT | ||
| 357 | |||
| 358 | You can find documentation for this module with the perldoc command. | ||
| 359 | |||
| 360 | perldoc RDF::Trine::Store::File | ||
| 361 | |||
| 362 | The perlrdf mailing list is the right place to seek help and discuss this module: | ||
| 363 | |||
| 364 | L<http://lists.perlrdf.org/listinfo/dev> | ||
| 365 | |||
| 366 | You can also look for information at: | ||
| 367 | |||
| 368 | =over 4 | ||
| 369 | |||
| 370 | =item * RT: CPAN's request tracker (report bugs here) | ||
| 371 | |||
| 372 | L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=RDF-Trine-Store-File> | ||
| 373 | |||
| 374 | =item * AnnoCPAN: Annotated CPAN documentation | ||
| 375 | |||
| 376 | L<http://annocpan.org/dist/RDF-Trine-Store-File> | ||
| 377 | |||
| 378 | =item * CPAN Ratings | ||
| 379 | |||
| 380 | L<http://cpanratings.perl.org/d/RDF-Trine-Store-File> | ||
| 381 | |||
| 382 | =item * Search CPAN | ||
| 383 | |||
| 384 | L<http://search.cpan.org/dist/RDF-Trine-Store-File/> | ||
| 385 | |||
| 386 | =back | ||
| 387 | |||
| 388 | |||
| 389 | =head1 LICENSE AND COPYRIGHT | ||
| 390 | |||
| 391 | Copyright 2011 Kjetil Kjernsmo. | ||
| 392 | |||
| 393 | This program is free software; you can redistribute it and/or modify it | ||
| 394 | under the terms of either: the GNU General Public License as published | ||
| 395 | by the Free Software Foundation; or the Artistic License. | ||
| 396 | |||
| 397 | See http://dev.perl.org/licenses/ for more information. | ||
| 398 | |||
| 399 | |||
| 400 | =cut | ||
| 401 | |||
| 402 | 1; # End of RDF::Trine::Store::File | ||