File Coverage

lib/Ubic/Watchdog.pm

Criterion	Covered	Total	%
statement	27	116	23.2
branch	0	36	0.0
condition			n/a
subroutine	9	18	50.0
pod	5	5	100.0
total	41	175	23.4

package Ubic::Watchdog;
$Ubic::Watchdog::VERSION = '1.59';
use strict;
use warnings;

# ABSTRACT: watchdog code


use POSIX;
use IO::Handle;
use Params::Validate qw(:all);
use Try::Tiny;
use List::MoreUtils qw(any);
use Ubic;

use Ubic::Logger;

sub run {
    my $class = shift;
    my $options = validate(@_, {
        glob_filter => { type => ARRAYREF, default => [] },
        compile_timeout => { type => SCALAR, regex => qr/^\d+$/ },
        verbose => { type => SCALAR|UNDEF },
    });

    my @filter;
    {
        for my $arg (@{ $options->{glob_filter} }) {
            $arg =~ /^[*\w.-]+$/ or die "Invalid argument '$arg', expected service name or shell-style glob";
            $arg =~ s/\./\\./g;
            $arg =~ s/\*/.*/g;
            push @filter, qr/^$arg$/;
        }
    }
    $options->{filter} = \@filter if @filter;
    delete $options->{glob_filter};

    my $self = bless $options => $class;

    my @services = $self->load_services(Ubic->root_service);
    $self->check_all(@services);
}

sub match($$) {
    my ($name, $filter) = @_;
    do {
        return 1 if $name =~ $filter;
    } while ($name =~ s/\.[^.]+$//);
    return;
}

sub load_services {
    my $self = shift;
    my ($parent) = @_;
    alarm($self->{compile_timeout});
    $SIG{ALRM} = sub {
        die "Couldn't compile $parent services in $self->{compile_timeout} seconds";
    };
    my @services = $parent->services;
    alarm(0);
    return @services;
}

sub check_all {
    my $self = shift;
    my @services = @_;
    for my $service (@services) {
        my $name = $service->full_name;
        if ($service->isa('Ubic::Multiservice')) {
            INFO("$name is multiservice, checking subservices") if $self->{verbose};
            $self->check_all($self->load_services($service));
            next;
        }
        if ($self->{filter}) {
            next unless any { match($name, $_) } @{ $self->{filter} };
        }

        # trying to get logs a little bit more ordered
        STDOUT->flush;
        STDERR->flush;

        my $child = fork;
        unless (defined $child) {
            die "fork failed";
        }
        unless ($child) {
            POSIX::setsid; # so we could kill this watchdog and its children safely later
            $self->check($service);
            exit;
        }
    }
    1 while wait() > 0;
    return;
}

sub check($) {
    my $self = shift;
    my $service = shift;
    my $name = $service->full_name;
    if ($self->{verbose}) {
        INFO("Checking $name");
    }
    $0 = "ubic-watchdog $name";

    try {
        alarm($service->check_timeout);

        # TODO - do additional fork, so that if service code overrides SIG{ALRM} or resets alarm(), watchdog still will finish in time
        $SIG{ALRM} = sub {
            ERROR("$name check_timeout exceeded");
            STDOUT->flush;
            STDERR->flush;
            kill -9 => $$; # suicide
            ERROR "kill sent, still alive"; # should never happen, we called setsid earlier
        };

        # permanently use service credentials
        # this line optimizes the number of fork calls - future status/restart calls would perform forked_call() otherwise
        Ubic::Credentials->new( service => $service )->set;

        # so we don't need access guard for this lock
        my $watchdog_lock = Ubic::SingletonLock->new(Ubic->get_data_dir()."/watchdog/lock/".$name, { blocking => 0 });

        unless ($watchdog_lock) {
            if ($self->{verbose}) {
                INFO "$name is locked by another watchdog process";
            }
            return;
        }

        my $lock = Ubic->lock($name);
        unless (Ubic->is_enabled($name)) {
            INFO("$name disabled") if $self->{verbose};
            return;
        }

        my $cached_status = Ubic->cached_status($name);
        my $status = Ubic->status($name);
        unless ($status->status eq 'running') {
            # following code can throw an exception, so we want to cache invalid status immediately
            Ubic->set_cached_status($name, $status);

            if ($cached_status eq "autostarting") {
                INFO("$name is autostarting");
            }
            else {
                ERROR("$name status is '$status', restarting");
            }

            Ubic->restart($name);

            # This is a precaution against services with wrong start/status logic.
            $status = Ubic->status($name);
            if ($status->status ne 'running') {
                INFO("$name started, but status is still '$status'");
            }
        }

        alarm(0);
        Ubic->set_cached_status($name, $status); # if service's start implementation is invalid, ubic-watchdog will restart it every minute, so be careful
    }
    catch {
        ERROR("Failed to revive $name: $_");
    };

    INFO("$name checked") if $self->{verbose};
}


1;

__END__