source: subversion/applications/utils/tirex/bin/tirex-batch @ 20987

Revision 20987, 12.0 KB checked in by jochen, 4 years ago (diff)

fixed older() filter in tirex-batch (logic was inverted)
added older(filename) filter to tirex-batch

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2#-----------------------------------------------------------------------------
3#
4#  Tirex Tile Rendering System
5#
6#  tirex-batch
7#
8#-----------------------------------------------------------------------------
9#  See end of this file for documentation.
10#-----------------------------------------------------------------------------
11#
12#  Copyright (C) 2010  Frederik Ramm <frederik.ramm@geofabrik.de> and
13#                      Jochen Topf <jochen.topf@geofabrik.de>
14
15#  This program is free software; you can redistribute it and/or
16#  modify it under the terms of the GNU General Public License
17#  as published by the Free Software Foundation; either version 2
18#  of the License, or (at your option) any later version.
19
20#  This program is distributed in the hope that it will be useful,
21#  but WITHOUT ANY WARRANTY; without even the implied warranty of
22#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23#  GNU General Public License for more details.
24
25#  You should have received a copy of the GNU General Public License
26#  along with this program; If not, see <http://www.gnu.org/licenses/>.
27#
28#-----------------------------------------------------------------------------
29
30use strict;
31use warnings;
32
33use Getopt::Long qw( :config gnu_getopt );
34use IO::Socket;
35use Socket;
36use JSON;
37use Time::HiRes;
38use Pod::Usage;
39use File::stat;
40
41use Tirex;
42use Tirex::Status;
43
44#-----------------------------------------------------------------------------
45# Reading command line and config
46#-----------------------------------------------------------------------------
47
48my %opts = ();
49GetOptions( \%opts, 'help|h', 'debug|d', 'config|c=s', 'num|n=i', 'prio|p=i', 'expire|e=s', 'filter|f=s', 'remove', 'count-only' ) or exit(2);
50
51if ($opts{'help'})
52{
53    pod2usage(
54        -verbose => 1,
55        -msg     => "tirex-batch - send rendering requests to tirex master\n",
56        -exitval => 0
57    );
58}
59
60if ($opts{'debug'})
61{
62    $Tirex::DEBUG = $opts{'debug'};
63}
64
65my $prio = $opts{'prio'} || 99;          # default batch prio is 99
66my $num  = $opts{'num'}  || 999_999_999; # huge queue size as default max size
67
68my @filters;
69@filters = split(qr{\s*;\s*}, $opts{'filter'}) if (defined $opts{'filter'});
70
71foreach my $filter (@filters)
72{
73    if ($filter !~ qr{^exists|not-exists|older\(([^)]+)\)|multi\(2,[01]\)$})
74    {
75        print STDERR "unknown filter: $filter\n";
76        exit(2);
77    }
78}
79
80if ($Tirex::DEBUG)
81{
82    print STDERR "Using prio: $prio\n";
83    print STDERR "Using expire: $opts{'expire'}\n" if (exists $opts{'expire'});
84    print STDERR "Using filters: ", join(', ', @filters) ,"\n";
85}
86
87my $config_file = $opts{'config'} || $Tirex::TIREX_CONFIGFILE;
88Tirex::Config::init($config_file);
89
90#-----------------------------------------------------------------------------
91
92my $socket = IO::Socket::INET->new( LocalAddr => 'localhost', Proto => 'udp' );
93$socket->connect( Socket::pack_sockaddr_in($Tirex::MASTER_UDP_PORT, Socket::inet_aton('localhost')) );
94
95my $status;
96if ($opts{'num'})
97{
98    $status = eval { Tirex::Status->new(); };
99    die("Can't connect to shared memory. Is the tirex-master running?\n") if ($@);
100}
101
102#-----------------------------------------------------------------------------
103
104my $mx = Tirex::Config::get('metatile_columns') || $Tirex::METATILE_COLUMNS;
105my $my = Tirex::Config::get('metatile_rows')    || $Tirex::METATILE_ROWS;
106
107#-----------------------------------------------------------------------------
108
109my $count = 0;
110
111# if there are still command line args, use those as init string
112if (scalar(@ARGV) > 0)
113{
114    $count += handle_init(join(' ', @ARGV));
115}
116# else read init strings from STDIN
117else
118{
119    while (<STDIN>)
120    {
121        chomp;
122        $count += handle_init($_);
123    }
124}
125
126print "metatiles: $count\n" if ($opts{'count-only'});
127
128exit(0);
129
130#-----------------------------------------------------------------------------
131# get queue size of given priority from master
132#-----------------------------------------------------------------------------
133sub queue_size
134{
135    my $prio = shift;
136
137    if (defined $status)
138    {
139        my $s = $status->read();
140        if (defined $s)
141        {
142            my $queues = JSON::from_json($s)->{'queue'}->{'prioqueues'};
143            foreach my $q (@$queues)
144            {
145                return $q->{'size'} if ($q->{'prio'} == $prio);
146            }
147            return 0;
148        }
149        else
150        {
151            die("can't read status\n");
152        }
153    }
154    else
155    {
156        die("can't get status\n");
157    }
158}
159
160#-----------------------------------------------------------------------------
161# handle one init string, ie. decode and send job requests to server
162#-----------------------------------------------------------------------------
163sub handle_init
164{
165    my $init = shift;
166
167    my $count_metatiles = 0;
168
169    # if this looks like a metatile path name, decode it
170    if ($init =~ qr{^\.?/})
171    {
172        my $metatile = Tirex::Metatile->new_from_filename($init);
173        $init = $metatile->to_s();
174    }
175
176    my $range = eval { Tirex::Metatiles::Range->new(init => $init); };
177    if ($@)
178    {
179        print STDERR "Error parsing init string: $@";
180        exit(2);
181    }
182    print STDERR "Range: ", $range->to_s(), "\n" if ($Tirex::DEBUG);
183
184    while (1)
185    {
186        my $queue_size = 0;
187
188        if (!$opts{'count-only'} && $opts{'num'})
189        {
190            # wait for queue to have some space
191            while (($queue_size = queue_size($prio)) >= $opts{'num'})
192            {
193                print STDERR " queue size $queue_size >= max queue size $opts{'num'}. waiting...\n" if ($Tirex::DEBUG);
194                sleep(1);
195            }
196            print STDERR " queue size $queue_size, can send up to ", $opts{'num'}-$queue_size ," jobs\n" if ($Tirex::DEBUG);
197        }
198
199        # send as many jobs as will fit into queue
200        METATILE:
201        while ($queue_size <= $num)
202        {
203            my $metatile = $range->next();
204
205            # if there are no more jobs, we are done
206            return $count_metatiles unless (defined $metatile);
207
208            print STDERR "Considering ", $metatile->to_s(), "\n" if ($Tirex::DEBUG);
209            foreach my $filter (@filters)
210            {
211                if    ($filter eq 'exists')                         { next METATILE unless ($metatile->exists()); }
212                elsif ($filter eq 'not-exists')                     { next METATILE if     ($metatile->exists()); }
213                elsif ($filter =~ qr{^older\(([0-9]+)\)$})          { next METATILE unless ($metatile->older($1)); } # seconds since epoch
214                elsif ($filter =~ qr{^older\(([^)]+)\)$})           { next METATILE unless ($metatile->older(get_mtime($1))); } # filename
215                elsif ($filter =~ qr{^multi\(([0-9]+),([0-9]+)\)$}) { next METATILE if     (($metatile->get_x()/$mx + $metatile->get_y()/$my) % $1 != $2); }
216            }
217
218            $count_metatiles++;
219            if (!$opts{'count-only'})
220            {
221                $queue_size++;
222
223                my %jobparams = ( metatile => $metatile, prio => $prio );
224                if (defined $opts{'expire'})
225                {
226                    if ($opts{'expire'} =~ /^\+/)
227                    {
228                        $jobparams{'expire'} = time() + $opts{'expire'};
229                    }
230                    else
231                    {
232                        $jobparams{'expire'} = $opts{'expire'};
233                    }
234                }
235                my $job = Tirex::Job->new(%jobparams);
236
237                my $request = $job->to_msg( type => $opts{'remove'} ? 'metatile_remove_request' : 'metatile_enqueue_request' );
238                print STDERR " sending: ", $request->to_s(), "\n" if ($Tirex::DEBUG);
239                my $ret = $request->send($socket);
240                if (! defined $ret)
241                {
242                    print STDERR "Can't send request. Is the master server running?\n";
243                    exit(1);
244                }
245                Time::HiRes::usleep(1000);    # don't send more than 1000 requests/s to not overwhelm the UDP receive buffer or the master
246            }
247        }
248        sleep(1) unless ($opts{'count-only'});
249    }
250}
251
252sub get_mtime
253{
254    my $filename = shift;
255    my $st = File::stat::stat($filename);
256
257    if (! $st) {
258        print "Can't stat $filename: $!\n";
259        exit(2);
260    }
261
262    return $st->mtime;
263}
264
265
266__END__
267
268=head1 NAME
269
270tirex-batch - send rendering requests to tirex master
271
272=head1 SYNOPSIS
273
274tirex-batch [OPTIONS] [INIT]
275
276=head1 OPTIONS
277
278=over 8
279
280=item B<-h>, B<--help>
281
282Display help message.
283
284=item B<-d>, B<--debug>
285
286Run in debug mode. You'll see the actual messages sent and received.
287
288=item B<-c>, B<--config=FILE>
289
290Use the config file FILE instead of /etc/tirex/tirex.conf.
291
292=item B<-n>, B<--num=NUM>
293
294Try to keep the number of jobs in the queue below this number (Only checked
295once per second). Disable with NUM=0.
296
297=item B<-p>, B<--prio=PRIO>
298
299Priority for requests.
300
301=item B<-e>, B<--expire=TIME>
302
303Expire time (seconds since epoch) for jobs. If it starts with '+', number of
304seconds added to current time.
305
306=item B<-f>, B<--filter>
307
308Add filters to metatile selection, see section FILTERS.
309
310=item B<--remove>
311
312Send remove request instead of rendering request. Jobs will be removed from
313the queue.
314
315=item B<--count-only>
316
317Only count how many metatiles would be rendered, do not actually send the
318requests. This will take the filters into account, so it will possibly
319check the disk for thousands of files!
320
321=back
322
323=head1 DESCRIPTION
324
325INIT is a string describing a range of tiles. If no INIT string is given on the
326command line, tirex-batch reads init strings from STDIN, one per line.
327
328Several different ways of describing the tiles are allowed:
329
330Single metatile:
331 map=foo x=4 y=7 z=15 (coordinates will be rounded to metatile numbers)
332
333Multiple metatiles:
334 map=foo x=0-32 y=16-32 z=15
335
336Using bounding box (8 to 9 degrees longitude, 48 to 49 degrees latitude)
337 map=foo lon=8,9 lat=48,49 z=15-17
338
339Multiple maps are allowed, too:
340 map=foo,bar
341
342You can use a z range (z=10-15). This does not work together with x
343and y ranges, but works with lon/lat ranges.
344
345Ranges of x, y, and z numbers are written as "MIN,MAX" or "MIN-MAX".
346Ranges of lon and lat are written as "MIN,MAX" (lon and lat can be
347negative, so range with "-" is problematic).
348
349You can also just give a pathname of a metatile file as INIT string. It
350has to start with './' or '/'. The first directory component must be
351the name of the map.
352
353=head1 FILTERS
354
355FILTER is a ;-separated list of filter options. Metatiles not matching
356the filter are skipped.
357
358Filter Options:
359
360=over 8
361
362=item B<exists>
363
364Matches if the meta tile exists on disk.
365
366=item B<not-exists>
367
368Matches if the meta tile does not exist on disk.
369
370=item B<older(time)>
371
372Matches if the meta tile's last modification time is before the given Unix time
373stamp (seconds since the epoch, 1970-01-01 00:00:00). Also matches if the meta
374tile does not exist on disk. If you want to match only files older than the
375given date which do actually exist, add the I<exists> filter.
376
377=item B<older(filename)>
378
379Instead of the time in seconds since the epoch you can also enter a filename
380here. The mtime (last modified) of this file will be used. tirex-batch will
381exit with return code 2 if the file does not exist.
382
383=item B<multi(count,num)>
384
385A magic filter that divides all meta tiles up in I<count> classes, and
386matches only if the current meta tile is in class I<num> of these. Hence
387the allowed range for I<num> is always 0..I<count>-1. Currently only
388I<count>=2 is supported. This filter can be used to distribute rendering
389requests among different tile servers (which may or may not then use
390F<tirex-syncd> to share resulting tiles).
391
392=back
393
394=head1 FILES
395
396=over 8
397
398=item F</etc/tirex/tirex.conf>
399
400The configuration file. See tirex.conf(5) for further details.
401
402=back
403
404=head1 DIAGNOSTICS
405
406Returns 0 on success, 1 if there was a problem sending the request and 2 if there was
407a problem parsing the command line or init string.
408
409=head1 AUTHORS
410
411Frederik Ramm <frederik.ramm@geofabrik.de>, Jochen Topf
412<jochen.topf@geofabrik.de> and possibly others.
413
414=cut
415
416
417#-- THE END ----------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.