File Coverage

blib/lib/Text/TFIDF.pm

Criterion	Covered	Total	%
statement	68	68	100.0
branch	11	14	78.5
condition	1	2	50.0
subroutine	11	11	100.0
pod	5	5	100.0
total	96	100	96.0

package Text::TFIDF;

use 5.012003;
use strict;
use warnings;
use Carp;
require Encode;

our $VERSION = '0.04';


sub new {
	my $class = shift;
	my $self = {};

	bless $self, $class;

	my %args = @_;

	if ($args{file}) {
		$self->process_files(@{$args{file}});
	}

	return $self;
}

sub TFIDF {
	my $self = shift;
	my $file = shift;
	my $word = shift;

	if (!defined $file) {
		carp("You must give a filename for the TFIDF measure.\n");
		return undef;
	}

	if (!defined $word) {
		carp("You must give a word for the TFIDF measure.\n");
		return undef;
	}

#	$word =~  s/[?;:!,.'"]//g;
        $word =~ s/[?;:!,."\(\)]//g;
	return undef if (!defined $self->{file}->{$file});
	return $self->TF($file,$word)*$self->IDF($word);
}


sub TF {
	my $self = shift;
	my $file = shift;
	my $word = shift;

	return $self->{file}->{$file}->{$word};
}


#IDF = log(number of documents/(number of documents containing the word))
sub IDF {

	my $self = shift;
	my $word = shift;

	my $count = 0;

	foreach my $el (keys %{$self->{file}}) {
		$count++ if (defined $self->{file}->{$el}->{$word});
	}

	return log(scalar(keys %{$self->{file}})/($count))/log(10);
}

sub process_files {

	my $self = shift;
        my @documents = @_;

        foreach my $el (@documents) {
                $self->_process_file($el);
        }

	return 1;
}

sub _process_file {
	my $self = shift;
        my $file = shift;

        my $hash;
	return undef if (!-r $file);
	open my $handle, '<:encoding(UTF-8)', $file || die $file," ",$!;
        while (<$handle>) {
                chop;
                my $line = lc($_);
                my @words = split(/\s+/,$line);
                foreach my $el (@words) {
                        $el =~ s/[?;:!,."\(\)]//g;
			my $word = Encode::encode("utf8",$el);
                        if (defined $hash->{$word}) {
                                $hash->{$word}++;
                        }
                        else {
                                $hash->{$word} = 1;
                        }
                }
        }
	close($handle);

	$self->{file}->{$file} = $hash;

}

1;
__END__