File Coverage

blib/lib/Search/Tools/TokenListUtils.pm

Criterion	Covered	Total	%
statement	55	76	72.3
branch	15	26	57.6
condition	9	14	64.2
subroutine	5	6	83.3
pod	4	4	100.0
total	88	126	69.8

package Search::Tools::TokenListUtils;
use Moo::Role;
use Carp;

our $VERSION = '1.004';

=head1 NAME

Search::Tools::TokenListUtils - mixin methods for TokenList and TokenListPP

=head1 SYNOPSIS

 my $tokens = $tokenizer->tokenize( $string );
 if ( $tokens->str eq $string) {
    print "string is same, before and after tokenize()\n";
 }
 else {
    warn "I'm filing a bug report against Search::Tools right away!\n";
 }

 my ($start_pos, $end_pos) = $tokens->get_window( 5, 20 );
 # $start_pos probably == 0
 # $end_pos probably   == 25

 my $slice = $tokens->get_window_pos( 5, 20 );
 for my $token (@$slice) {
    print "token = $token\n";
 }

=head1 DESCRIPTION

Search::Tools::TokenListUtils contains pure-Perl methods inhertited
by both Search::Tools::TokenList and Search::Tools::TokenListPP.

=head1 METHODS

=head2 str

Returns a serialized version of the TokenList. If you haven't
altered the TokenList since you got it from tokenize(),
then str() returns a scalar string identical to (but not the same as)
the string you passed to tokenize().

Both Search::Tools::TokenList and TokenListPP are overloaded
to stringify to the str() value.

=cut

sub str {
    my $self   = shift;
    my $joiner = shift(@_);
    if ( !defined $joiner ) {
        $joiner = '';
    }
    return join( $joiner, map {"$_"} @{ $self->as_array } );
}

=head2 get_window( I [, I, I] )

Returns array with two values: I and I positions
for the array of length I on either side of I.
Like taking a slice of the TokenList.

Note that I is the number of B not B.
So if you're looking for the number of "words", think about
I*2.

Note too that I is the number of B on B
side of I. So the entire window width (length of the returned
slice) is I*2 +/-1. The window is guaranteed to be bounded
by B.

If I is true, the window is shifted to try and match
the first token prior to I that returns true for is_sentence_start().

=cut

sub get_window {
    my $self = shift;
    my $pos  = shift;
    if ( !defined $pos ) {
        croak "pos required";
    }

    my $size        = int(shift) || 20;
    my $as_sentence = shift      || 0;
    my $max_index   = $self->len - 1;

    if ( $pos > $max_index or $pos < 0 ) {
        croak "illegal pos value: no such index in TokenList";
    }

    #warn "window size $size for pos $pos";

    # get the $size tokens on either side of $tok
    my ( $start, $end );

    # is token too close to the top of the stack?
    if ( $pos > $size ) {
        $start = $pos - $size;
    }

    # is token too close to the bottom of the stack?
    if ( $pos < ( $max_index - $size ) ) {
        $end = $pos + $size;
    }
    $start ||= 0;
    $end   ||= $max_index;

    if ($as_sentence) {
        my $sentence_starts = $self->get_sentence_starts;

        # default to what we have.
        my $start_for_pos = $start;
        my $i             = 0;

        #warn "looking for sentence_start for start = $start end = $end\n";
        for (@$sentence_starts) {

            #warn " $_ [$i]\n";
            if ( $_ >= $pos ) {
                $start_for_pos = $sentence_starts->[$i];
                last;
            }
            $i++;
        }

        #warn "found $start_for_pos (start = $start end = $end)\n";
        if ( $start_for_pos != $start ) {
            if ( $start_for_pos < $start ) {
                $end -= ( $start - $start_for_pos );
            }
            else {
                $end += ( $start_for_pos - $start );
            }
            $start = $start_for_pos;
        }

        #warn "now $start_for_pos (start = $start end = $end)\n";
    }
    else {

        # make sure window starts and ends with is_match
        while ( !$self->get_token($start)->is_match ) {
            $start++;
        }
        while ( !$self->get_token($end)->is_match ) {
            $end--;
        }
    }

    #warn "return $start .. $end";
    #warn "$size ~~ " . ( $end - $start );

    return ( $start, $end );
}

=head2 get_window_tokens( I [, I] )

Like get_window() but returns an array ref of a slice
of the TokenList containing Tokens.

=cut

sub get_window_tokens {
    my $self = shift;
    my ( $start, $end ) = $self->get_window(@_);
    my @slice = ();
    for ( $start .. $end ) {
        push( @slice, $self->get_token($_) );
    }
    return \@slice;
}

=head2 as_sentences([I])

Returns a reference to an array of arrays,
where each child array is a "sentence" worth of Token objects.
You can stringify each sentence array like:

 my $sentences = $tokenlist->as_sentences;
 for my $s (@$sentences) {
     printf("sentence: %s\n", join("", map {"$_"} @$s));
 }

If you pass a single true value to as_sentences(),
then the array returned will consist of plain scalar strings
with whitespace normalized.

=cut

sub as_sentences {
    my $self = shift;
    my $stringed = shift || 0;
    my @sents;
    my @s;

    # use array method since we do not know the iterator position
    for my $t ( @{ $self->as_array } ) {
        if ( $t->is_sentence_start ) {

            # if has any, add anonymous copy to master
            if (@s) {
                push @sents, [@s];
            }

            # reset
            @s = ();
        }

        # add
        push @s, $t;
    }
    if (@s) {
        push @sents, [@s];
    }
    if ($stringed) {
        my @stringed;
        for my $s (@sents) {
            my $str = join( "", map {"$_"} @$s );
            $str =~ s/\s\s+/\ /g;
            $str =~ s/\s+$//;
            push @stringed, $str;
        }
        return \@stringed;
    }

    return \@sents;
}

1;

__END__