| 1 | # Object to handle subscriber records (Scheduler.pm) | ||
| 2 | |||
| 3 | # Multiple processes may be run locally or on multiple hosts. | ||
| 4 | |||
| 5 | # Upon startup, does sanity checks on the tables to ensure everything | ||
| 6 | # is as expected. | ||
| 7 | |||
| 8 | # On SIGTERM signal, kills off all of its own children. Upon restart, | ||
| 9 | # these will be resumed. SIGUSR1 turns on debug messages to syslog. | ||
| 10 | |||
| 11 | package RWDE::Scheduler::Scheduler; | ||
| 12 | |||
| 13 | use strict; | ||
| 14 | use warnings; | ||
| 15 | |||
| 16 | use Error qw(:try); | ||
| 17 | |||
| 18 | use POSIX qw(:sys_wait_h setsid ceil); | ||
| 19 | |||
| 20 | use RWDE::DB::DbRegistry; | ||
| 21 | use RWDE::DB::Record; | ||
| 22 | use RWDE::Exceptions; | ||
| 23 | use RWDE::Time; | ||
| 24 | |||
| 25 | use RWDE::Scheduler::SchedulerWorker; | ||
| 26 | use RWDE::Scheduler::Pending_action; | ||
| 27 | |||
| 28 | use base qw(RWDE::DB::DefaultDB RWDE::DB::Deletable RWDE::DB::Record RWDE::Runnable); | ||
| 29 | |||
| 30 | our ($db, $table, $index, $id, @fieldnames, $ccrcontext, %fields, %static_fields, %modifiable_fields, @static_fieldnames, @modifiable_fieldnames); | ||
| 31 | |||
| 32 | use vars qw($VERSION); | ||
| 33 | $VERSION = sprintf "%d", q$Revision: 535 $ =~ /(\d+)/; | ||
| 34 | |||
| 35 | BEGIN { | ||
| 36 | $table = 'schedulers'; | ||
| 37 | $id = 'scheduler_id'; | ||
| 38 | $index = 'schedulers_scheduler_id_seq'; | ||
| 39 | |||
| 40 | #all of the static fields present in the scheduler table | ||
| 41 | %static_fields = ( | ||
| 42 | |||
| 43 | # Field => [Type, Descr] | ||
| 44 | scheduler_id => [ 'int', 'Scheduler ID' ], | ||
| 45 | scheduler_created => [ 'timestamp', 'Scheduler created on' ], # datetime | ||
| 46 | lastmod => [ 'timestamp', 'last seen' ], | ||
| 47 | ); | ||
| 48 | |||
| 49 | #all of the fields allowed to be modified in the scheduler table | ||
| 50 | %modifiable_fields = ( | ||
| 51 | |||
| 52 | # Field => [Type, Descr] | ||
| 53 | scheduler_terminated => [ 'timestamp', 'Scheduler created on' ], # datetime | ||
| 54 | scheduler_name => [ 'char', 'Scheduler name' ], | ||
| 55 | scheduler_pid => [ 'char', 'Scheduler process id' ], | ||
| 56 | scheduler_status => [ 'char', 'Scheduler status' ], | ||
| 57 | ); | ||
| 58 | |||
| 59 | %fields = (%static_fields, %modifiable_fields); | ||
| 60 | |||
| 61 | @static_fieldnames = sort keys %static_fields; | ||
| 62 | @modifiable_fieldnames = sort keys %modifiable_fields; | ||
| 63 | @fieldnames = sort keys %fields; | ||
| 64 | |||
| 65 | } | ||
| 66 | |||
| 67 | sub fetch_by_name { | ||
| 68 | my ($self, $params) = @_; | ||
| 69 | |||
| 70 | my $scheduler; | ||
| 71 | if (exists($$params{scheduler_name})) { | ||
| 72 | $scheduler = $self->fetch_one( | ||
| 73 | { | ||
| 74 | query => 'scheduler_name = ?', | ||
| 75 | query_params => [ $$params{scheduler_name} ] | ||
| 76 | } | ||
| 77 | ); | ||
| 78 | } | ||
| 79 | else { | ||
| 80 | throw RWDE::DevelException({ info => 'Scheduler::Inappropriate parameters passed to fetch_by_name' }); | ||
| 81 | } | ||
| 82 | |||
| 83 | return $scheduler; | ||
| 84 | } | ||
| 85 | |||
| 86 | sub get_workers { | ||
| 87 | my ($self, $params) = @_; | ||
| 88 | |||
| 89 | return RWDE::Scheduler::SchedulerWorker->fetch({ query => 'scheduler_id = ?', query_params => [ $self->scheduler_id ] }); | ||
| 90 | } | ||
| 91 | |||
| 92 | sub get_pending_actions { | ||
| 93 | my ($self, $params) = @_; | ||
| 94 | |||
| 95 | if (!$self->{Pending_actions}) { | ||
| 96 | $self->{Pending_actions} = RWDE::Scheduler::Pending_action->fetch({ query => 'scheduler_id = ?', query_params => [ $self->scheduler_id ] }); | ||
| 97 | } | ||
| 98 | return $self->{Pending_actions}; | ||
| 99 | } | ||
| 100 | |||
| 101 | # Get a job for a worker, if available, and this scheduler is the one with most free schedulers, take the job | ||
| 102 | # Mark the job as taken inside of the database so nobody else takes it | ||
| 103 | # This has the side-effect of temporarily locking the db, but hopefully for a very short time | ||
| 104 | sub get_next_job { | ||
| 105 | my ($self, $params) = @_; | ||
| 106 | |||
| 107 | $self->debug_info('devel', 'Scheduler::get_next_job'); | ||
| 108 | my $action; | ||
| 109 | |||
| 110 | try { | ||
| 111 | |||
| 112 | #begin a transaction | ||
| 113 | transaction { | ||
| 114 | |||
| 115 | #election query | ||
| 116 | $action = RWDE::Scheduler::Pending_action->fetch_one( | ||
| 117 | { | ||
| 118 | query => "action_due < NOW() AND scheduler_id IS NULL AND (action_scheduler_name IS NULL or action_scheduler_name = ?) ORDER BY action_priority,action_due", | ||
| 119 | query_params => [ $self->scheduler_name ], | ||
| 120 | } | ||
| 121 | ); | ||
| 122 | |||
| 123 | $action->scheduler_id($self->scheduler_id); | ||
| 124 | $action->update_record(); | ||
| 125 | }; | ||
| 126 | } | ||
| 127 | |||
| 128 | catch RWDE::DataNotFoundException with { | ||
| 129 | my $ex = shift; | ||
| 130 | |||
| 131 | #there was no work found | ||
| 132 | }; | ||
| 133 | |||
| 134 | return $action; | ||
| 135 | } | ||
| 136 | |||
| 137 | sub cleanup { | ||
| 138 | my ($self, $params) = @_; | ||
| 139 | |||
| 140 | # get all the outstanding workers | ||
| 141 | foreach my $worker (@{ $self->get_workers }) { | ||
| 142 | $worker->cleanup; | ||
| 143 | } | ||
| 144 | |||
| 145 | #get all the reserved jobs (might not be assigned to workers | ||
| 146 | foreach my $job (@{ $self->get_pending_actions }) { | ||
| 147 | $self->debug_info('devel', 'Cleaning_up for job: ' . $job->action_id); | ||
| 148 | $job->cleanup; | ||
| 149 | } | ||
| 150 | |||
| 151 | $self->delete_record; | ||
| 152 | |||
| 153 | return (); | ||
| 154 | } | ||
| 155 | |||
| 156 | sub terminate_scheduler { | ||
| 157 | my ($self, $params) = @_; | ||
| 158 | |||
| 159 | $self->syslog_msg('info', 'Terminating scheduler'); | ||
| 160 | |||
| 161 | $self->scheduler_status('terminated'); | ||
| 162 | $self->scheduler_terminated(RWDE::Time->now()); | ||
| 163 | $self->scheduler_pid(0); | ||
| 164 | $self->update_record; | ||
| 165 | |||
| 166 | return; | ||
| 167 | } | ||
| 168 | |||
| 169 | sub fork_off { | ||
| 170 | my ($self, $params) = @_; | ||
| 171 | |||
| 172 | my $job = $$params{job}; | ||
| 173 | |||
| 174 | # fork a child to run the job | ||
| 175 | my $cpid = fork; | ||
| 176 | |||
| 177 | if (not defined $cpid or $cpid) { | ||
| 178 | |||
| 179 | #parent | ||
| 180 | $self->add_job($cpid, $job); | ||
| 181 | return (); | ||
| 182 | } | ||
| 183 | |||
| 184 | else { | ||
| 185 | |||
| 186 | #child | ||
| 187 | my $result = RWDE::Scheduler::SchedulerWorker->run({ job => $job, scheduler_id => $self->scheduler_id }); | ||
| 188 | exit($result); | ||
| 189 | } | ||
| 190 | |||
| 191 | } | ||
| 192 | |||
| 193 | # cleanup: After child terminates, perform housekeeping | ||
| 194 | sub child_cleanup { | ||
| 195 | my ($self, $cpid, $status) = @_; | ||
| 196 | |||
| 197 | $self->debug_info('devel', "Cleanup for child $cpid, exited with $status"); | ||
| 198 | $self->remove_job($cpid); | ||
| 199 | |||
| 200 | return; | ||
| 201 | } | ||
| 202 | |||
| 203 | sub print_status { | ||
| 204 | my ($self) = @_; | ||
| 205 | |||
| 206 | $self->debug_info('info', sprintf("%-10s %-10s %10s\n", 'PID', 'MSGID', 'RUNNING')); | ||
| 207 | |||
| 208 | my $active_jobs_ref = $self->{active_jobs}; | ||
| 209 | |||
| 210 | foreach my $cpid (keys %{$active_jobs_ref}) { | ||
| 211 | my $alive = kill 0, $cpid; | ||
| 212 | my $job = $active_jobs_ref->{$cpid}; | ||
| 213 | $self->debug_info('info', sprintf("%-10d %-10d %10s\n", $cpid, $job->action_id, $alive ? 'yes' : 'no')); | ||
| 214 | } | ||
| 215 | |||
| 216 | return; | ||
| 217 | } | ||
| 218 | |||
| 219 | sub add_job { | ||
| 220 | my ($self, $cpid, $job) = @_; | ||
| 221 | |||
| 222 | my $active_jobs_ref = $self->{active_jobs}; | ||
| 223 | $active_jobs_ref->{$cpid} = $job; | ||
| 224 | |||
| 225 | $self->{workers_free} = $self->{workers_free} - 1; | ||
| 226 | |||
| 227 | return; | ||
| 228 | } | ||
| 229 | |||
| 230 | sub remove_job { | ||
| 231 | my ($self, $cpid) = @_; | ||
| 232 | |||
| 233 | my $active_jobs_ref = $self->{active_jobs}; | ||
| 234 | |||
| 235 | if (exists $active_jobs_ref->{$cpid}) { | ||
| 236 | |||
| 237 | #remove the job from the active jobs | ||
| 238 | delete $active_jobs_ref->{$cpid}; | ||
| 239 | |||
| 240 | #we are going back into the worker pool now | ||
| 241 | $self->{workers_free} = $self->{workers_free} + 1; | ||
| 242 | } | ||
| 243 | else { | ||
| 244 | $self->syslog_msg('warning', 'waitpid got unknown pid: ' . $cpid); | ||
| 245 | } | ||
| 246 | |||
| 247 | return; | ||
| 248 | } | ||
| 249 | |||
| 250 | sub get_active_jobs { | ||
| 251 | my ($self) = @_; | ||
| 252 | |||
| 253 | my $active_jobs = $self->{active_jobs}; | ||
| 254 | |||
| 255 | return keys(%{$active_jobs}); | ||
| 256 | } | ||
| 257 | |||
| 258 | sub setup { | ||
| 259 | my ($self, $params) = @_; | ||
| 260 | |||
| 261 | # Process command line options: | ||
| 262 | # -s SECONDS seconds to sleep between scans. Default 180. | ||
| 263 | # -m PROCS Max number of workers. Default 5. | ||
| 264 | # -n NAME The name for the scheduler | ||
| 265 | ################################################################################################# | ||
| 266 | |||
| 267 | my $sleeptime = $$params{s} || 30; | ||
| 268 | my $workers = $$params{m} || 5; | ||
| 269 | my $name = $$params{n} || "temp"; | ||
| 270 | |||
| 271 | # Before we start, make sure we tidy up the remnants of my previous run(s) | ||
| 272 | try { | ||
| 273 | my $previous_run = $self->fetch_by_name({ scheduler_name => $name }); | ||
| 274 | $previous_run->cleanup(); | ||
| 275 | } | ||
| 276 | |||
| 277 | catch RWDE::DataNotFoundException with { | ||
| 278 | my $ex = shift; | ||
| 279 | |||
| 280 | # there are no schedulers with the same name | ||
| 281 | }; | ||
| 282 | |||
| 283 | # non-persistent fields | ||
| 284 | $self->{workers_free} = $self->{workers} = $workers; | ||
| 285 | $self->{sleeptime} = $sleeptime; | ||
| 286 | $self->{cleanup} = $self->{terminating} = $self->{printstatus} = 0; # initialize for sig handler safety. | ||
| 287 | $self->{active_jobs} = {}; | ||
| 288 | |||
| 289 | # setup signal handlers | ||
| 290 | $SIG{'TERM'} = $SIG{'INT'} = sub { $self->{terminating} = 1; }; | ||
| 291 | $SIG{'INFO'} = sub { $self->{printstatus} = 1; }; | ||
| 292 | $SIG{'CHLD'} = sub { $self->{cleanup} = 1; }; | ||
| 293 | |||
| 294 | # persistent fields | ||
| 295 | $self->scheduler_name($name); | ||
| 296 | $self->scheduler_status('running'); | ||
| 297 | $self->scheduler_pid($$); | ||
| 298 | |||
| 299 | # Register ourselves, so other schedulers know we are running | ||
| 300 | $self->create_record; | ||
| 301 | |||
| 302 | return (); | ||
| 303 | } | ||
| 304 | |||
| 305 | sub start { | ||
| 306 | my ($self, $params) = @_; | ||
| 307 | |||
| 308 | # set a listener for DB notifications | ||
| 309 | RWDE::DB::DbRegistry->add_db_settings({ db => $self->get_db, db_settings => ['LISTEN pending'], }); | ||
| 310 | |||
| 311 | $self->syslog_msg('info', "Starting Scheduler:: name: " . $self->scheduler_name . " sleeptime: " . $self->{sleeptime} . " workers: " . $self->{workers}); | ||
| 312 | |||
| 313 | while (1) { | ||
| 314 | |||
| 315 | # check for any child processes that have terminated | ||
| 316 | if ($self->{cleanup}) { # set by SIGCHLD handler | ||
| 317 | $self->debug_info('devel', 'reaping old children'); | ||
| 318 | while ((my $cpid = waitpid(-1, WNOHANG)) > 0) { | ||
| 319 | $self->child_cleanup($cpid, $?); | ||
| 320 | } | ||
| 321 | $self->{cleanup} = 0; | ||
| 322 | } | ||
| 323 | |||
| 324 | if ($self->{printstatus}) { # set by SIGUSR1 handler | ||
| 325 | $self->print_status(); | ||
| 326 | $self->{printstatus} = 0; | ||
| 327 | } | ||
| 328 | |||
| 329 | if ($self->{terminating} and ($self->get_active_jobs == 0)) { | ||
| 330 | $self->terminate_scheduler(); | ||
| 331 | exit(0); | ||
| 332 | } | ||
| 333 | |||
| 334 | if ($self->{workers_free} > 0 && !($self->{terminating})) { | ||
| 335 | $self->debug_info('devel', 'Number of workers: ' . $self->{workers_free}); | ||
| 336 | my $action = $self->get_next_job(); | ||
| 337 | if (defined $action) { | ||
| 338 | $self->fork_off({ job => $action }); | ||
| 339 | } | ||
| 340 | } | ||
| 341 | |||
| 342 | else { | ||
| 343 | $self->debug_info('devel', 'No free workers'); | ||
| 344 | } | ||
| 345 | |||
| 346 | } | ||
| 347 | continue { | ||
| 348 | my $notifications_ref = RWDE::DB::DbRegistry->get_db_notifications({ sleeptime => $self->{sleeptime}, db => $self->get_db() }); | ||
| 349 | } | ||
| 350 | |||
| 351 | return; | ||
| 352 | } | ||
| 353 | |||
| 354 | 1; | ||
| 355 |