File Coverage

blib/lib/Lingua/TermWeight/WordSegmenter/SplitBySpace.pm

Criterion	Covered	Total	%
statement	31	31	100.0
branch	10	14	71.4
condition			n/a
subroutine	8	8	100.0
pod	4	4	100.0
total	53	57	92.9

# SPDX-FileCopyrightText: 2014 Koichi SATOH
# SPDX-FileCopyrightText: 2026 Wesley Schwengle
#
# SPDX-License-Identifier: MIT

package Lingua::TermWeight::WordSegmenter::SplitBySpace;
our $VERSION = '0.01';
# ABSTRACT: Simple word segmenter suitable for most european languages

use v5.20;
use warnings;
use Object::Pad;

class Lingua::TermWeight::WordSegmenter::SplitBySpace {

    field $lower_case :accessor :param = 0;
    field $remove_punctuations :accessor :param = 0;
    field $stop_words :accessor :param = [];

    method segment ($document) {
        my @words = split /\s+/, ref $document ? $$document : $document;

        @words = map lc, @words if $self->lower_case;

        if ($self->remove_punctuations) {
            s/^\W+|\W+$//g for @words;
        }

        if (@{ $self->stop_words } != 0) {
            my %stop_words = map { ($_ => 1) } @{ $self->stop_words };
            @words = grep { not exists $stop_words{$_} } @words;
        }

        return sub { shift @words };
    }
}

1;

__END__