| 1 | # -*- perl -*- | ||
| 2 | |||
| 3 | # Copyright (c) 2010 AdCopy | ||
| 4 | # Author: Jeff Weisberg | ||
| 5 | # Created: 2010-Jan-14 17:04 (EST) | ||
| 6 | # Function: get list of files to map | ||
| 7 | # | ||
| 8 | # $Id: FileList.pm,v 1.1 2010/11/01 18:41:42 jaw Exp $ | ||
| 9 | |||
| 10 | package AC::MrGamoo::FileList; | ||
| 11 | use AC::MrGamoo::Customize; | ||
| 12 | use AC::Import; | ||
| 13 | use strict; | ||
| 14 | |||
| 15 | our @ISA = 'AC::MrGamoo::Customize'; | ||
| 16 | our @EXPORT = qw(get_file_list); | ||
| 17 | our @CUSTOM = @EXPORT; | ||
| 18 | |||
| 19 | 1; | ||
| 20 | |||
| 21 | =head1 NAME | ||
| 22 | |||
| 23 | AC::MrGamoo::FileList - get list of files | ||
| 24 | |||
| 25 | =head1 SYNOPSIS | ||
| 26 | |||
| 27 | emacs /myperldir/Local/MrGamoo/FileList.pm | ||
| 28 | copy. paste. edit. | ||
| 29 | |||
| 30 | use lib '/myperldir'; | ||
| 31 | my $m = AC::MrGamoo::D->new( | ||
| 32 | class_filelist => 'Local::MrGamoo::FileList', | ||
| 33 | ); | ||
| 34 | |||
| 35 | =head1 IMPORTANT | ||
| 36 | |||
| 37 | You can fire up the system, and get the servers talking to each other, and | ||
| 38 | perform some limited tests without this file. | ||
| 39 | |||
| 40 | But you must provide this file in order to actually run map/reduce jobs. | ||
| 41 | |||
| 42 | =head1 DESCRIPTION | ||
| 43 | |||
| 44 | MrGamoo only runs map/reduce jobs. | ||
| 45 | It is up to you to get the files on to the servers | ||
| 46 | and keep track of where they are. And to tell MrGamoo. | ||
| 47 | |||
| 48 | Some people keep the file meta-information in a sql database. | ||
| 49 | Some people keep the file meta-information in a yenta map. | ||
| 50 | Some people keep the file meta-information in the filesystem. | ||
| 51 | |||
| 52 | When a new job starts, your C<get_file_list> function will be | ||
| 53 | called with the job config, and should return an arrayref | ||
| 54 | of matching files along with meta-info. | ||
| 55 | |||
| 56 | Each element of the returned arrayref should be a hashref | ||
| 57 | containing at least the following fields: | ||
| 58 | |||
| 59 | =head2 filename | ||
| 60 | |||
| 61 | the name of the file, relative to the C<basedir> in your config file. | ||
| 62 | |||
| 63 | filename => 'www/2010/01/17/23/5943_prod_5x2N5qyerdeddsNi' | ||
| 64 | |||
| 65 | =head2 location | ||
| 66 | |||
| 67 | an arrayref of servers where this file is located. the locations | ||
| 68 | should be the persistent-ids of the servers (see MySelf). | ||
| 69 | |||
| 70 | if the same file is replicated on multiple servers, mrgamoo will | ||
| 71 | be able to both intelligently determine which servers will process | ||
| 72 | which files, as well as recover from failures. | ||
| 73 | |||
| 74 | location => [ 'mrm@athena.example.com', 'mrm@zeus.example.com' ] | ||
| 75 | |||
| 76 | =head2 size | ||
| 77 | |||
| 78 | this should be the size of the file, in bytes. mrgamoo will consider | ||
| 79 | the sizes of files in determining which servers will process which files. | ||
| 80 | |||
| 81 | size => 10843 | ||
| 82 | |||
| 83 | =head1 BUGS | ||
| 84 | |||
| 85 | none. you write this yourself. | ||
| 86 | |||
| 87 | =head1 SEE ALSO | ||
| 88 | |||
| 89 | AC::MrGamoo | ||
| 90 | |||
| 91 | =head1 AUTHOR | ||
| 92 | |||
| 93 | You! | ||
| 94 | |||
| 95 | =cut |