2 # Batch EXecutor -- Show Queued Jobs
3 # (c) 2011-2015 Martin Mares <mj@ucw.cz>
26 Usage: bex queue [<options and actions>] [[!]<machine-or-group> ...]
29 --by-job Show jobs sorted by job ID (default)
30 -h, --by-host Show jobs sorted by host
31 --rm Remove jobs from the queue
32 --move-to=<queue> Move jobs to a different queue
35 -a, --attn Show only jobs needing attention (e.g., failures)
36 -f, --filenames Show filenames of jobs and log files
37 -j, --job=<id> Act on the specified job (default: on all)
38 -q, --queue=<name> Act on the given queue
39 -s, --summary Show only a summary
40 -S, --state=<regex> Act only on jobs whose state matches the given regex
41 -w, --why[=<lines>] In case of failed jobs, display last few lines of output
46 Getopt::Long::Configure("bundling");
48 "by-job!" => \$op_by_job,
49 "h|by-host!" => \$op_by_host,
51 "move-to=s" => \$op_move_to,
52 "a|attn!" => \$attention,
53 "f|filenames!" => \$filenames,
54 "j|job=s" => \$given_job,
55 "q|queue=s" => \$queue_name,
56 "s|summary!" => \$summary,
57 "S|state=s" => \$state_regex,
60 ) or die "Try `bex queue --help' for more information.\n";
62 my @machines = BEX::Config::parse_machine_list(@ARGV ? @ARGV : '*');
63 my $queue = BEX::Queue->new($queue_name);
64 $given_job = $queue->resolve_job_id($given_job) if defined $given_job;
67 my %status_cache = ();
70 $status_cache{$m}{$j} or $status_cache{$m}{$j} = $queue->read_job_status($m, $j);
71 return $status_cache{$m}{$j};
75 return get_status($m, $j)->{'Status'} // 'UNKNOWN';
81 for my $m (@machines) {
82 for my $j ($queue->scan($m)) {
83 if (defined $given_job) {
84 next if $j ne $given_job;
86 if (defined $attention) {
87 next if get_stat($m, $j) =~ m{^(NEW|NOPING)$};
89 if (defined $state_regex) {
90 next unless get_stat($m, $j) =~ m{^($state_regex)$};
92 push @{$jobs{$j}}, $m;
93 push @{$machs{$m}}, $j;
101 my $ops = 0 + defined($op_by_host) + defined($op_by_job) + defined($op_rm) + defined($op_move_to);
102 if ($ops > 1) { die "Multiple actions are not allowed\n"; }
104 if ($op_rm) { do_rm(); }
105 elsif (defined $op_move_to) { do_move_to(); }
112 my %mach_locked = ();
114 my %cnt_by_mach = ();
116 for my $m (keys %machs) {
117 $mach_locked{$m} = $queue->is_locked($m, undef);
118 for my $j (@{$machs{$m}}) {
119 my $st = get_status($m, $j);
120 my $s = get_stat($m, $j);
121 my $adir = $queue->attachment_dir($j);
122 my $has_adir = -d $adir;
123 $cnt_by_job{$j}{$s}++;
124 $cnt_by_mach{$m}{$s}++;
127 $why{$m}{$j} .= "\t\t== Job file: " . $queue->queue_file($m, $j) . "\n";
128 $why{$m}{$j} .= "\t\t== Attachments: $adir\n" if $has_adir;
130 if (defined($st->{'Time'}) && defined($st->{'Status'})) {
131 $stat{$m}{$j} = ' [' . $st->{'Status'} . ' on ' .
132 POSIX::strftime('%Y-%m-%d', localtime $st->{'Time'}) . ']';
133 if (defined($why) && $st->{'Status'} eq 'FAILED') {
134 my $lines = $why ? $why : 3;
135 my $log = $queue->log_file($m, $j);
137 $why{$m}{$j} .= join("", map { "\t\t>> $_" } `tail -n$lines $log`);
143 if ($mach_locked{$m} || $queue->is_locked($m, $j)) {
144 $stat{$m}{$j} .= ' [LOCKED]';
146 $stat{$m}{$j} .= ' +ATT' if $has_adir;
150 if ($queue->is_locked(undef, undef)) {
151 print "### Queue lock present\n\n";
156 for my $m (sort keys %cnt_by_mach) {
157 print "$m: ", join(" ", map { "$_:" . $cnt_by_mach{$m}{$_} } sort keys %{$cnt_by_mach{$m}}), "\n";
160 for my $j (sort keys %cnt_by_job) {
161 print $queue->job_name($j), ": ", join(" ", map { "$_:" . $cnt_by_job{$j}{$_} } sort keys %{$cnt_by_job{$j}}), "\n";
166 for my $m (sort keys %machs) {
167 print "$m", ($mach_locked{$m} ? ' [LOCKED]' : ''), "\n";
168 for my $j (@{$machs{$m}}) {
169 print "\t" . $queue->job_name($j) . $stat{$m}{$j}, "\n";
174 for my $j (sort keys %jobs) {
175 print $queue->job_name($j), "\n";
176 for my $m (sort @{$jobs{$j}}) {
177 print "\t$m", $stat{$m}{$j}, "\n";
188 for my $m (sort keys %machs) {
189 for my $j (sort @{$machs{$m}}) {
190 if (!$queue->lock($m, $j)) {
191 print STDERR "Cannot remove $m:", $queue->job_name($j), ", it is locked\n";
194 $queue->update_job_status($m, $j, 'REMOVED');
195 $queue->remove($m, $j);
196 print "Removed $m:", $queue->job_name($j), "\n";
207 my $dest = BEX::Queue->new($op_move_to);
208 $dest->{'Name'} ne $queue->{'Name'} or die "Moving to the same queue is not permitted\n";
209 for my $j (sort keys %jobs) {
210 my $job = BEX::Job->new_from_file($queue->job_file($j));
211 for my $m (sort @{$jobs{$j}}) {
212 if (!$queue->lock($m, $j)) {
213 print STDERR "Cannot move $m:", $queue->job_name($j), ", it is locked\n";
216 my $enq = $dest->enqueue($m, $job);
218 $dest->update_job_status($m, $job->id, 'NEW', 'Moved to this queue');
220 $dest->log($m, $job->id, 'REQUEUE', 'Moved to this queue');
222 $queue->update_job_status($m, $job->id, 'REMOVED', 'Moved from this queue');
223 $queue->remove($m, $j);
224 print "Moved $m:", $dest->job_name($j);
225 print " (already queued)" if !$enq;