File Coverage

blib/lib/Text/Fingerprint.pm

Criterion	Covered	Total	%
statement	26	26	100.0
branch			n/a
condition			n/a
subroutine	8	8	100.0
pod	2	2	100.0
total	36	36	100.0

package Text::Fingerprint;
# ABSTRACT: perform simple text clustering by key collision


use strict;
use utf8;
use warnings qw(all);

use base q(Exporter);

our %EXPORT_TAGS    = (all => [qw(fingerprint fingerprint_ngram)]);
our @EXPORT_OK      = (@{$EXPORT_TAGS{all}});
our @EXPORT         = qw();

use List::MoreUtils qw(uniq);
use Text::Unidecode;

our $VERSION = '0.006'; # VERSION


my $NON_WORD = qr{ [\W_]+ }x;

sub fingerprint ($) {
    my ($string) = @_;

    $string = lc unidecode $string;
    $string =~ s{^ $NON_WORD | $NON_WORD $}{}gosx;

    return join q( ) =>
        sort(
            uniq(
                split(
                    m{ $NON_WORD }ox,
                    $string
                )
            )
        );
}


sub fingerprint_ngram ($;$) {
    my ($string, $n) = (@_, 2);

    $string = lc unidecode $string;
    $string =~ s{ $NON_WORD }{}gosx;

    return join '' =>
        sort(
            uniq(
                $string =~ m{
                    (?=
                        (.{$n})
                    )
                }gx
            )
        );
}


1;

__END__