File Coverage

blib/lib/Dancer/SearchApp.pm

Criterion	Covered	Total	%
statement	30	55	54.5
branch	0	8	0.0
condition	0	3	0.0
subroutine	10	13	76.9
pod	0	3	0.0
total	40	82	48.7

package Dancer::SearchApp;
use strict;
use File::Basename 'basename';
use Dancer;
use Search::Elasticsearch::Async;
use URI::Escape 'uri_unescape';
use URI::file;
#use Search::Elasticsearch::TestServer;

use Dancer::SearchApp::Defaults 'get_defaults';

use Dancer::SearchApp::Entry;

use vars qw($VERSION $es %indices);
$VERSION = '0.05';

=head1 NAME

Dancer::SearchApp - A simple local search engine

=head1 SYNOPSIS

=head1 QUICKSTART

Also see L.

  cpanm --look Dancer::SearchApp

  # Install prerequisites
  cpanm --installdeps .

  # Install Elasticsearch https://www.elastic.co/downloads/elasticsearch
  # Start Elasticsearch
  # Install Apache Tika from https://tika.apache.org/download.html into jar/

  # Launch the web frontend
  plackup --host 127.0.0.1 -p 8080 -Ilib -a bin\app.pl

  # Edit filesystem configuration
  cat >>fs-import.yml
  fs:
    directories:
        - folder: "C:\\Users\\Corion\\Projekte\\App-StarTraders"
          recurse: true
          exclude:
             - ".git"
        - folder: "t\\documents"
          recurse: true

  # Collect some content
  perl -Ilib -w bin/index-filesystem.pl -f

  # Search in your browser

=head1 CONFIGURATION

Configuration happens through config.yml

  elastic_search:
    home: "./elasticsearch-2.1.1/"
    index: "dancer-searchapp"

The Elasticsearch instance to used can also be passed in C<%ENV>
as C.

=cut

use Data::Dumper;
$Data::Dumper::Sortkeys = 1;

my $config = get_defaults(
    env      => \%ENV,
    config   => config(),
    #defaults => \%
    names => [
        ['elastic_search/index' => 'elastic_search/index' => 'SEARCHAPP_ES_INDEX', 'searchapp'],
        ['elastic_search/nodes' => 'elastic_search/nodes' => 'SEARCHAPP_ES_NODES', 'localhost:9200'],
    ],
);

sub search {
    if( ! $es ) {
        my $nodes = config->{elastic_search}->{nodes};
        $es = Search::Elasticsearch->new(
            nodes => $nodes,
        );
    };

    $es
};

$Template::Stash::PRIVATE = $Template::Stash::PRIVATE = 1;

get '/' => sub {
    # Later, separate out the code paths between
    # search and index page only, to serve the index
    # page as a static file

    my $statistics;
    my $results;

    my $from = params->{'from'} || 0;
    $from =~ s!\D!!g;
    my $size = params->{'size'} || 25;
    $size =~ s!\D!!g;
    my $search_term = params->{'q'};

    if( defined $search_term) {

        #warning "Reading ES indices\n";
        %indices = %{ search->indices->get({index => ['*']}) };
        #warning $_ for sort keys %indices;

        my @restrict_type;
        my $type;
        if( $type = params->{'type'} and $type =~ m!([a-z0-9+-]+)/[a-z0-9+-]+!i) {
            #warn "Filtering for '$type'";
            @restrict_type = (filter => { term => { mime_type => $type }});
        };

        # Move this to an async query, later
        my $index = config->{elastic_search}->{index};
        $results = search->search(
            # Wir suchen in allen Sprachindices
            index => [ grep { /^\Q$index\E/ } sort keys %indices ],
            body => {
                from => $from,
                size => $size,
                query => {
                    filtered => {
                        query => {
                            query_string => {
                                query => $search_term,
                                fields => ['title','folder','content', 'author'] #'creation_date']
                            },
                        },
                        @restrict_type,
                    },
                },
                sort => {
                    _score => { order => 'desc' },
                },
               "highlight" => {
                    "pre_tags" => '',
                    "post_tags" => '',
                    "fields" => {
                        "content" => {}
                    }
                }
            }
        );

        #warn Dumper $results->{hits};
    } else {
        # Update the statistics
        #$statistics = search->search(
        #    search_type => 'count',
        #    index => config->{index},
        #    body        => {
        #        query       => {
        #            match_all => {}
        #        }
        #    }
        #);
        #warn Dumper $statistics;
    };

    if( $results ) {
        for( @{ $results->{ hits }->{hits} } ) {
            $_->{source} = Dancer::SearchApp::Entry->from_es( $_ );
            for my $key ( qw( id index type )) {
                $_->{$key} = $_->{"_$key"}; # thanks, Template::Toolkit
            };

        };
    };

    # Output the search results
    template 'index', {
            results => ($results ? $results->{hits} : undef ),
            params => {
                q    => $search_term,
                from => $from,
                size => $size,
            },
    };
};

# Show (cached) elements
get '/cache/:index/:type/:id' => sub {
    my $index = params->{index};
    my $type = params->{type};
    my $id = uri_unescape( params->{id} );
    my $document = retrieve($index,$type,$id);
    #warn $document->basic_mime_type;

    $document->{type} = $type;
    $document->{index} = $index;

    if( $document ) {
        return template 'view_document', {
            result => $document,
            backlink => scalar( request->referer ),
        }
    } else {
        status 404;
        return <
        That file does (not) exist anymore in the index.
SORRY
        # We could delete that item from the index here...
        # Or schedule reindexing of the resource?
    }
};

# Reproxy elements from disk
sub reproxy {
    my( $document, $local, $disposition, %options ) = @_;

    # Now, if the file exists both in the index and locally, let's reproxy the content
    if( $document and -f $local) {
        status 200;
        content_type( $document->mime_type );
        header( "Content-Disposition" => sprintf '%s; filename="%s"', $disposition, basename $local);
        my $abs = File::Spec->rel2abs( $local, '.' );
        open my $fh, '<', $local
            or die "Couldn't read local file '$local': $!";
        binmode $fh;
        local $/;
        <$fh>

    } else {
        status 404; # sorry
        return <
        That file does (not) exist anymore or is currently unreachable
        for this webserver. We'll need to implement
        cleaning up the index from dead items.
SORRY
        # We could delete that item from the index here...
        # Or schedule reindexing of the resource?
    }
};

sub retrieve {
    my( $index, $type, $id ) = @_;
    my $document;
    if( eval {
        $document = search->get(index => $index, type => $type, id => $id);
        1
    }) {
        my $res = Dancer::SearchApp::Entry->from_es($document);
        return $res
    } else {
        warn "$@";
    };
    # Not found in the Elasticsearch index
    return undef
}

get '/open/:index/:type/:id' => sub {
    my $index = params->{index};
    my $type = params->{type};
    my $id = uri_unescape params->{id};
    my $document = retrieve($index,$type,$id);
    if( $type eq 'http' ) {
        return
            redirect $id
    } else {
        my $local = URI::file->new( $id )->file;
        return
        reproxy( $document, $local, 'Attachment',
            index => $index,
            type => $type,
        );
    }
};

get '/inline/:index/:type/:id' => sub {
    my $index = params->{index};
    my $type = params->{type};
    my $id = uri_unescape params->{id};
    my $document = retrieve($index,$type,$id);

    my $local;
    if( 'http' eq $type ) {
        $document->content
    } else {
        $local = URI::file->new( $id )->file;
    };

    reproxy( $document, $local, 'Inline',
        index => $index,
        type => $type,
    );

};

true;

__END__