File Coverage

blib/lib/File/Catalog.pm

Criterion	Covered	Total	%
statement	41	89	46.0
branch	0	34	0.0
condition	0	6	0.0
subroutine	14	17	82.3
pod			n/a
total	55	146	37.6

package File::Catalog;

use 5.010;
use strict;
use warnings FATAL => 'all';
use Env;
use Cwd qw(abs_path);
use File::Basename qw(dirname basename);
use Fcntl ':mode';
use Digest::MD5::File qw(file_md5_hex);
use Log::Log4perl qw(:easy);
use Term::ProgressBar;
use UI::Dialog;
use Data::Dumper qw(Dumper);

use File::Catalog::Env qw(nom_local);
use File::Catalog::DB;

=head1 NAME

File::Catalog - The great new File::Catalog!

=head1 VERSION

Version 0.003

=cut

our $VERSION = '0.003';

# timeout pour operations systeme
#my $timeout_court = 30;     # secondes
#my $timeout_long  = 300;    # secondes

# repertoires ignores
my @exclusions_rep = qw(.git .svn CVS _no_ktl RECYCLER $RECYCLE.BIN);
my @exclusions_fic = qw(Thumbs.db sync.ffs_db);

=head1 SYNOPSIS

Quick summary of what the module does.

Perhaps a little code snippet.

    use File::Catalog;

    my $foo = File::Catalog->new();
    ...

=head1 EXPORT

A list of functions that can be exported.  You can delete this section
if you don't export anything, such as for a purely object-oriented module.

=head1 SUBROUTINES/METHODS

=head2 initialiser

=cut

sub initialiser {
    my ($class, %options) = @_;
    my $self = {};
    bless $self, $class;
    $self->{i}   = 0;
    $self->{moi} = basename($0);

    # init log
    my $opt_debug = (exists $options{debug}) ? $options{debug} : undef;
    Log::Log4perl->easy_init(($opt_debug) ? $DEBUG : $INFO);
    DEBUG "debug: ON";

    # taille max en kio pour calcul md5
    # (juste plus grand qu'une photo)
    $self->{no_md5} = (exists $options{no_md5}) ? $options{no_md5} : 0;
    $self->{taille_limite_md5} =
      (exists $options{taille_limite_md5})
      ? $options{taille_limite_md5}
      : 1024 * 10;

    # rep tmp
    $self->{tmp} =
      (exists $options{tmp} and defined $options{tmp})
      ? $options{tmp}
      : File::Catalog::Env::tmp();
    DEBUG "tmp: $self->{tmp}";

    # archives
    $self->{archives} = (exists $options{archives}) ? $options{archives} : 0;
    DEBUG "archives: $self->{archives}";
    if ($self->{archives}) {

        # saisie du mot de passe si besoin
        my $opt_pwd =
          (exists $options{password}) ? $options{password} : undef;
        if (defined($opt_pwd) and !$opt_pwd) {
            my $d = new UI::Dialog(
                backtitle => $self->{moi},
                title     => 'Mot de passe'
            );
            $opt_pwd = $d->password(text => 'Tu peux taper ton mot de passe. Je ne regarde pas...');
        }
        $self->{password} = $opt_pwd;
        DEBUG "password: $self->{password}" if $opt_pwd;

        # 7zip
        $self->{exe7z} = File::Catalog::Env::exe7z();
        $self->{opt7z} = (defined $opt_pwd) ? "-p$opt_pwd" : "";
        DEBUG "exe7z: $self->{exe7z}" if $opt_pwd;
    }

    # bd par defaut
    #my $serveur = File::Catalog::Env::serveur();
    my $fic_bd = (exists $options{bd}) ? $options{bd} : undef;
    $fic_bd = ".ktl.sqlite" unless defined $fic_bd;
    $self->{fic_bd} = $fic_bd;
    DEBUG "fic_bd: $self->{fic_bd}";
    $self->{db} =
      File::Catalog::DB->connect($self->{fic_bd}, $options{extension});

    # contexte archive
    $self->{ctx_arch} = 0;

    # retour
    return $self;
}

=head2 terminer

=cut

# fermeture
sub terminer {
    my ($self) = @_;

    $self->{db}->disconnect();
}

=head2 parcourir_repertoire

=cut

# parcours
sub parcourir_repertoire {
    my ($self, $repertoire, $volume) = @_;
    my $err = 0;

    # volume courant
    if (!defined $volume) {
        chomp($volume = `df -P $repertoire | tail -n 1`);
        $volume =~ s|.+%\s+||;

        # (re)initialisation de la liste des fichiers catalogues
        $self->{repficcat} = [];
    }
    DEBUG "$volume: $repertoire";

    # listage repertoire
    if (opendir my $fh_rep, $repertoire) {
        my @lst_fic = grep { !/^\.\.?$/ } readdir $fh_rep;
        closedir $fh_rep or die "Pb fermeture repertoire $repertoire !\n";

        # analyse
        foreach my $entree (@lst_fic) {
            $err = $self->analyser_entree("$repertoire/$entree", $volume);
        }
    }
    else {
        WARN "Echec ouverture repertoire $repertoire !\n";
    }

    return $err;
}

=head2 analyser_entree

=cut

# analyse
sub analyser_entree {
    my ($self, $entree, $volume, $force_archive) = @_;
    my $rep_tmp = $self->{tmp};
    (my $repfic = $entree) =~ s|^$rep_tmp/||;
    my $rep = dirname $repfic;
    my $fic = basename $repfic;
    return 0 if $fic ~~ @exclusions_fic;
    my $archive  = ($fic =~ /\.(gz|tar|tgz|zip|bz2|tbz2|7z)$/);
    my @stat     = lstat $entree;
    my $inode    = $stat[1];
    my $volinode = "$volume/$inode";
    my $rows;
    my $err = 0;

    #--- table Fichier
    $rows =
      $self->{db}->execute('lister_volinodes', $repfic)->fetchall_arrayref();
    if (my $row = shift @$rows) {

        # si maj volinode
        if ($row->[0] ne $volinode) {

            # maj table Fichier
            $self->{db}->execute('updateF', $volinode, $repfic);
            push @{ $self->{repficcat} }, $repfic;
        }
        else {
            DEBUG "$repfic deja vu";
        }
    }
    else {

        # insertion table Fichier
        DEBUG "insertF: " . $repfic;
        my @autres_infos = $self->{db}->{trigger}->($fic);
        my $ok_trigger   = shift @autres_infos;
        if ($ok_trigger) {
            $self->{db}->execute('insertF', $repfic, $rep, $fic, $volinode, @autres_infos);
            push @{ $self->{repficcat} }, $repfic;
        }
        else {
            $volinode = undef;
        }
    }

    #--- table Inode
    my $md5;
    my $majmd5 = 0;
    if ($volinode) {
        $rows = $self->{db}->execute('lire_volinode', $volinode)->fetchall_arrayref();
        if (my $row = shift @$rows) {

            # si maj infos
            if (
                $row->[3] != $stat[2]       # mode
                or $row->[4] != $stat[7]    # taille

                #or $row->[5] != $stat[8]  # acces
                or $row->[6] != $stat[9]     # modif
                or $row->[7] != $stat[10]    # creation
              )
            {
                my $old_md5 = $row->[8];

                # maj table Inode
                $md5 = $self->md5sum($entree, \@stat) unless $self->{no_md5};
                $self->{db}
                  ->execute("updateI", $stat[2], $stat[7], $stat[8], $stat[9], $stat[10], $archive, $md5, $volinode);
                $majmd5 = (defined $md5 and ($old_md5 ne $md5));
                DEBUG "majmd5: " . $majmd5;
            }
            else {
                DEBUG "inode $inode deja vu";
            }
        }
        else {

            # insertion table Inode
            $md5 = $self->md5sum($entree, \@stat) unless $self->{no_md5};
            DEBUG "insertI: " . $volinode;
            $self->{db}->execute(
                'insertI', $volinode, $volume,   $inode,   $stat[2], $stat[7],
                $stat[8],  $stat[9],  $stat[10], $archive, $md5
            );
            $majmd5 = 1;
        }
    }

    #--- progression
    if (    S_ISREG($stat[2])
        and defined($self->{barre})
        and !$self->{ctx_arch})
    {
        $self->{barre}->update(++$self->{nbfic});
    }

    #--- archive
    if ($md5 and ($self->{archives} or $force_archive) and $archive) {

        DEBUG "archive: " . $md5;

        # nb occurrences md5sum
        my $count = $self->{db}->nb_occurrences_md5sum($md5);
        DEBUG "count = $count";
        if ($count > 1 or !$majmd5) {
            DEBUG "archive $fic deja analysee";
        }
        else {

            #INFO "analyse archive $fic";
            #DEBUG "timeout: $timeout_court";

            #ZZZ essayer de trouver la taille necessaire
            #ZZZ pour eviter de saturer l'espace temporaire
            #my $err = timeout $timeout_court => sub {
            my $entree7z = nom_local($entree);
            my $cmd      = "$self->{exe7z} $self->{opt7z} t \"$entree7z\" > $rep_tmp/$md5.log 2>&1";
            DEBUG $cmd;
            $err = system($cmd) >> 8;

            #};
            #if ($@) {
            #WARN "[PB] test timed-out : $entree";
            #$err = 5;
            #}
            if ($err) {
                WARN "[PB] archive non reconnue ($err) : $entree";
                my @log = `cat $rep_tmp/$md5.log`;
                foreach (@log) {
                    ERROR $_;
                }
            }
            else {
                $err = $self->analyser_archive($entree, $md5);
            }
        }
    }

    # fin si repertoire exclu
    return $err if $fic ~~ @exclusions_rep;

    # repertoire
    $err = $self->parcourir_repertoire($entree, $volume) if S_ISDIR $stat[2];

    return $err;
}

=head2 lister_resultat

=cut

# liste des fichiers catalogues
# par la commande precedente
sub lister_resultat {
    my ($self) = @_;

    return $self->{repficcat};
}

=head2 analyser_archive

=cut

sub analyser_archive {
    my ($self, $archive, $md5) = @_;
    my $err = 0;

    # debut contexte archive
    $self->{ctx_arch}++;

    # md5
    if (!$md5) {
        my @stat = lstat $archive;
        $md5 = $self->md5sum($archive, \@stat);
    }

    # tar, zip, bz2, 7z : 7za x -o{Directory} + rm -rf
    # tgz, tar.gz : gunzip -c + 7za x -o{Directory} + rm -rf
    # tbz2, tar.bz2 : bunzip2 -c + 7za x -o{Directory} + rm -rf
    my $archive7z = nom_local($archive);
    my $rep_tmp   = $self->{tmp};
    my $reptmp    = "$rep_tmp/$md5";
    my $reptmp7z  = nom_local($reptmp);

    #INFO "analyse sous $reptmp";

    # extraction
    if ($archive =~ /\.(tar\.gz|tgz|tar\.bz2|tbz2)$/) {

        # cas d'une archive (b|g)zippee
        my $dec = ($archive =~ /\.t?gz$/) ? "gunzip" : "bunzip2";
        my $cmd = "$dec -c \"$archive\" > $reptmp.tar 2>$reptmp.log";
        DEBUG $cmd;
        $err = system($cmd) >> 8;
        if ($err) {
            ERROR "[PB] gunzip ($err) : $archive";
        }
        else {

            # extraction avec tar
            mkdir $reptmp;
            my $cmd = "tar xf $reptmp7z.tar -C $reptmp7z > $reptmp.log 2>&1";
            DEBUG $cmd;
            $err = system($cmd) >> 8;
        }
    }
    else {

        # extraction avec 7zip
        my $cmd = "$self->{exe7z} $self->{opt7z} x -y -o\"$reptmp7z\" \"$archive7z\" > $reptmp.log 2>&1";
        DEBUG $cmd;
        $err = system($cmd) >> 8;
    }

    if (!-d $reptmp) {

        # cas reptmp inexistant
        ERROR "[PB] repertoire $reptmp inexistant ($archive) !";
    }
    if ($err or !-d $reptmp) {

        # affichage du log en cas d'erreur
        ERROR "[PB] extraction archive ($err) : $archive";
        my @log = `cat $reptmp.log`;
        foreach (@log) {
            ERROR $_;
        }
    }
    else {

        # parcours de l'arborescence temporaire
        $self->parcourir_repertoire($reptmp, $md5);
    }

    # nettoyage
    my $cmd = "rm -rf $reptmp*";
    DEBUG $cmd;
    my $cr = system($cmd) >> 8;

    # fin contexte archive
    $self->{ctx_arch}--;

    # retour
    return $err || $cr;
}

=head2 purger_entree

=cut

# purge d'une entree du catalogue
# les fichiers et leur inode sont supprimes du catalogue
# les fichiers ne sont pas supprimes du disque
# on n'accede a aucun moment au disque
sub purger_entree {
    my ($self, $entree) = @_;

    DEBUG "purge: $entree";

    # recherche de l'entree dans le catalogue
    my $lst = $self->{db}->lister_fichiers($entree);

    foreach my $elt (@$lst) {
        my ($md5sum, $type, $repfic) = @{$elt};

        # volinode de ce repfic
        my ($volinode, $archive) =
          @{ $self->{db}->lire_volinode_archive($repfic) };

        # nombre de fichiers supplementaires pour ce volinode
        my $nbfic = $self->{db}->nb_occurrences_volinode($volinode) - 1;
        DEBUG sprintf "autres occurrences de ce fichier = %d", $nbfic;

        # si nbfic
        if ($nbfic) {
            INFO sprintf("%s ! reste %d occurrence%s de ce fichier", $md5sum, $nbfic, ($nbfic > 1) ? "s" : "");
        }
        else {

            # alors supprimer enregistrement table Inode
            DEBUG "deleteI: " . $volinode;
            $self->{db}->execute('deleteI', $volinode);
        }

        # delete repfic
        # supprimer enregistrement table Fichier
        DEBUG "deleteF: " . $repfic;
        $self->{db}->execute('deleteF', $repfic);

        # progression
        if (defined $self->{barre}) {
            $self->{barre}->update(++$self->{nbfic});
        }

        # archive : TODO ...

    }

}

=head2 md5sum

=cut

# calcul du md5
sub md5sum {
    my ($self, $fic, $refStat) = @_;

    if (!$refStat) {
        my @stat = lstat($fic);
        $refStat = \@stat;
    }
    my $md5;
    if (S_ISREG $refStat->[2]) {
        if ($refStat->[7] < $self->{taille_limite_md5} * 1024) {

            # taille inferieure a la limite
            $md5 = file_md5_hex($fic);
        }
        else {

            # taille bornee a la limite
            my $tmp =
              $self->{tmp} . "/" . $self->{moi} . "_" . $$ . "_" . $self->{i}++;
            my $cmd = "dd if=\"$fic\" of=$tmp bs=1024 count=$self->{taille_limite_md5} 2>/dev/null";
            DEBUG $cmd;
            my $err = system($cmd) >> 8;
            $md5 = file_md5_hex($tmp);
            unlink $tmp;
        }
    }
    return $md5;
}

=head1 AUTHOR

Patrick Hingrez, C<<  >>

=head1 BUGS

Please report any bugs or feature requests to C, or through
the web interface at L.  I will be notified, and then you'll
automatically be notified of progress on your bug as I make changes.




=head1 SUPPORT

You can find documentation for this module with the perldoc command.

    perldoc File::Catalog


You can also look for information at:

=over 4

=item * RT: CPAN's request tracker (report bugs here)

L

=item * AnnoCPAN: Annotated CPAN documentation

L

=item * CPAN Ratings

L

=item * Search CPAN

L

=back


=head1 ACKNOWLEDGEMENTS


=head1 LICENSE AND COPYRIGHT

Copyright 2015 Patrick Hingrez.

This program is free software; you can redistribute it and/or modify it
under the terms of the the Artistic License (2.0). You may obtain a
copy of the full license at:

L

Any use, modification, and distribution of the Standard or Modified
Versions is governed by this Artistic License. By using, modifying or
distributing the Package, you accept this license. Do not use, modify,
or distribute the Package, if you do not accept this license.

If your Modified Version has been derived from a Modified Version made
by someone other than you, you are nevertheless required to ensure that
your Modified Version complies with the requirements of this license.

This license does not grant you the right to use any trademark, service
mark, tradename, or logo of the Copyright Holder.

This license includes the non-exclusive, worldwide, free-of-charge
patent license to make, have made, use, offer to sell, sell, import and
otherwise transfer the Package with respect to any patent claims
licensable by the Copyright Holder that are necessarily infringed by the
Package. If you institute patent litigation (including a cross-claim or
counterclaim) against any party alleging that the Package constitutes
direct or contributory patent infringement, then this Artistic License
to you shall terminate on the date that such litigation is filed.

Disclaimer of Warranty: THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER
AND CONTRIBUTORS "AS IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE, OR NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY
YOUR LOCAL LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR
CONTRIBUTOR WILL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR
CONSEQUENTIAL DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


=cut

1;    # End of File::Catalog