File Coverage

blib/lib/Text/NGrammer.pm

Criterion	Covered	Total	%
statement	58	67	86.5
branch	8	16	50.0
condition			n/a
subroutine	9	11	81.8
pod	7	7	100.0
total	82	101	81.1

# Copyright 2018 Francesco Nidito. All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.

package Text::NGrammer;

use strict;
use Carp;
use Lingua::Sentence;

use vars qw($VERSION);
$VERSION = '0.04';

sub new {
  my $class = shift;
  my %config = (lang => 'en', # used by the sentencer
               );

  my %param = @_;

  for my $opt (keys %param) {
    croak "option $opt unsupported by version $VERSION of Text::NGrammer" unless exists $config{$opt};
    $config{$opt} = $param{$opt};
  }
  $config{version} = $VERSION;
  return bless \%config, $class;
}

##
# Skip-grams

sub skipgrams_array {
  my $self   = shift;
  my $n      = shift;
  my $k      = shift;
  my $length = scalar(@_);

  croak "the n-gram length cannot be lesser than 1" if $n < 1;
  croak "the tokens to be skipped cannot be lesser than 0" if $k < 0;

  my @ngrams = ();
  my $step = $k + 1;
  for (my $i = 0; $i <= ($length - ($n+($k*($n-1)))); $i += 1) {
    my @tokens = ();
    my $at = $i;
    while (@tokens < $n) {
      push @tokens, $_[$at];
      $at += $step;
    }
    push @ngrams, \@tokens;
  }

  return @ngrams;
}

sub skipgrams_sentence {
  my $self     = shift;
  my $n        = shift;
  my $k        = shift;
  my $sentence = shift;

  croak "the n-gram length cannot be lesser than 1" if $n < 1;
  croak "the tokens to be skipped cannot be lesser than 0" if $k < 0;

  # splits a string -- assumed to be a sentence -- according to spaces, control chars, etc.
  my @tokens = grep /\S+/, split(/(?:\p{C}|\p{M}|\p{P}|\p{S}|\p{Z})+/, $sentence);
  return () if @tokens < $n;
  return $self->skipgrams_array($n, $k, @tokens);
}

sub skipgrams_text {
  my $self = shift;
  my $n    = shift;
  my $k    = shift;
  my $text = shift;

  croak "the n-gram length cannot be lesser than 1" if $n < 1;
  croak "the tokens to be skipped cannot be lesser than 0" if $k < 0;

  my @ngrams = ();

  my $splitter = Lingua::Sentence->new($self->{lang});
  for my $sentence ($splitter->split_array($text)) {
    push @ngrams, $self->skipgrams_sentence($n, $k, $sentence);
  }

  return @ngrams;
}

##
# N-Grams

sub ngrams_array {
  my $self   = shift;
  my $n      = shift;

  return $self->skipgrams_array($n, 0, @_);
}

sub ngrams_sentence {
  my $self     = shift;
  my $n        = shift;
  my $sentence = shift;

  return $self->skipgrams_sentencey($n, 0, $sentence);
}

sub ngrams_text {
  my $self = shift;
  my $n    = shift;
  my $text = shift;

  return $self->skipgrams_text($n, 0, $text);
}


1;

__END__