source: subversion/applications/utils/tirex/bin/tirex-tiledir-check @ 21037

Last change on this file since 21037 was 20619, checked in by jochen, 10 years ago

tirex doc improvements

  • Property svn:executable set to *
File size: 8.6 KB
Line 
1#!/usr/bin/perl
2#-----------------------------------------------------------------------------
3#
4#  Tirex Tile Rendering System
5#
6#  tirex-tiledir-check
7#
8#-----------------------------------------------------------------------------
9#  See end of this file for documentation.
10#-----------------------------------------------------------------------------
11#
12#  Copyright (C) 2010  Frederik Ramm <frederik.ramm@geofabrik.de> and
13#                      Jochen Topf <jochen.topf@geofabrik.de>
14
15#  This program is free software; you can redistribute it and/or
16#  modify it under the terms of the GNU General Public License
17#  as published by the Free Software Foundation; either version 2
18#  of the License, or (at your option) any later version.
19
20#  This program is distributed in the hope that it will be useful,
21#  but WITHOUT ANY WARRANTY; without even the implied warranty of
22#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23#  GNU General Public License for more details.
24
25#  You should have received a copy of the GNU General Public License
26#  along with this program; If not, see <http://www.gnu.org/licenses/>.
27#
28#-----------------------------------------------------------------------------
29
30use strict;
31use warnings;
32
33use Getopt::Long qw( :config gnu_getopt );
34use File::Find;
35use File::stat;
36use Pod::Usage;
37use List::Util;
38use IO::File;
39use IO::Handle;
40use JSON;
41
42use Tirex;
43
44#-----------------------------------------------------------------------------
45
46my $EMPTY_TILE_SIZE = 7124;
47
48my %opts = ( list => '', stats => '' );
49GetOptions( \%opts, 'help|h', 'config|c=s', 'list|l=s', 'stats|s=s' ) or exit(2);
50
51if ($opts{'help'})
52{
53    pod2usage(
54        -verbose => 1,
55        -msg     => "tirex-tiledir-check - check tile dir\n",
56        -exitval => 0
57    );
58}
59
60die("--list and --stats can't both be -\n") if ($opts{'list'} eq '-' && $opts{'stats'} eq '-');
61
62my $fh_list;
63my $fh_stats;
64
65if ($opts{'list'} eq '-')
66{
67    $fh_list = IO::Handle->new();
68    $fh_list->fdopen(fileno(STDOUT), 'w');
69}
70elsif ($opts{'list'})
71{
72    $fh_list = IO::File->new($opts{'list'}, 'w') or die("Can't open list file '$opts{'list'}': $!\n");
73}
74
75if ($opts{'stats'} eq '-')
76{
77    $fh_stats = IO::Handle->new();
78    $fh_stats->fdopen(fileno(STDOUT), 'w');
79}
80elsif ($opts{'stats'})
81{
82    $fh_stats = IO::File->new($opts{'stats'}, 'w') or die("Can't open stats file '$opts{'stats'}': $!\n");
83}
84
85my $config_file = $opts{'config'} || $Tirex::TIREX_CONFIGFILE;
86Tirex::Config::init($config_file);
87
88#-----------------------------------------------------------------------------
89
90my $REGEX_DIR  = qr{^/[a-zA-Z0-9_-]+(/[0-9]+){0,5}$};
91my $REGEX_FILE = qr{^/[a-zA-Z0-9_-]+(/[0-9]+){0,5}/[0-9]+\.meta$};
92my $REGEX_TMP  = qr{^/[a-zA-Z0-9_-]+(/[0-9]+){0,5}/[0-9]+\.meta\.[0-9]+\.tmp$};
93
94my $BLOCKSIZE = 512;
95
96my $mtdir = Tirex::Config::get('metatile_dir');
97chop $mtdir if ($mtdir =~ qr{/$});
98
99my %stats;
100my $exitcode = 0;
101my $list = '';
102
103find({ wanted => \&process, follow_fast => 1, no_chdir => 1 }, $mtdir);
104
105if ($fh_list)
106{
107    $fh_list->print($list);
108}
109if ($fh_stats)
110{
111    my $json = JSON::to_json(\%stats, { pretty => 1 });
112    $fh_stats->print("$json\n");
113}
114
115exit($exitcode);
116
117#-----------------------------------------------------------------------------
118
119sub error
120{
121    my $errmsg = shift;
122
123    print STDERR "$errmsg\n";
124    $exitcode = 1;
125}
126
127sub process
128{
129    (my $path = $_) =~ s/^$mtdir//;
130
131    # ignore lost+found directories
132    if ($path =~ /lost\+found/)
133    {
134        $File::Find::prune = 1;
135        return;
136    }
137
138    # ignore /planet-import-complete (used by mod_tile)
139    if ($path eq '/planet-import-complete')
140    {
141        return;
142    }
143
144    # directories
145    if (-d $_)
146    {
147        return if ($path eq '');
148
149        if ($path !~ $REGEX_DIR)
150        {
151            $File::Find::prune = 1;
152            error("Unknown directory format: $path");
153        }
154
155        return;
156    }
157
158    my $stat = stat($_);
159
160    # tmp files
161    if ($path =~ $REGEX_TMP)
162    {
163        #  older than 1 minute
164        error("Old tmp file: $path") if ($stat->mtime() < time() - 60);
165        return;
166    }
167
168    # metatile files
169    if ($path !~ $REGEX_FILE)
170    {
171        error("Unknown file format: $path");
172        return;
173    }
174
175    my $metatile = Tirex::Metatile->new_from_filename($path);
176
177    if (!$stat)
178    {
179        error("Can't stat file: $path");
180        return;
181    }
182
183    my $age    = time() - $stat->mtime();
184    my $size   = $stat->size();
185    my $blocks = $stat->blocks();
186
187    $list .= join(',', $age, $size, $blocks, $metatile->to_s()) . "\n" if ($fh_list);
188
189    return unless ($fh_stats);
190
191    if (! defined $stats{$metatile->get_map()}->[$metatile->get_z()])
192    {
193        $stats{$metatile->get_map()}->[$metatile->get_z()] = {
194            'minage'     => 999_999_999,
195            'maxage'     => 0,
196            'sumage'     => 0,
197            'minsize'    => 999_999_999,
198            'maxsize'    => 0,
199            'sumsize'    => 0,
200            'minblocks'  => 999_999_999,
201            'maxblocks'  => 0,
202            'sumblocks'  => 0,
203            'count'      => 0,
204            'countempty' => 0,
205        };
206    }
207
208    my $s = $stats{$metatile->get_map()}->[$metatile->get_z()];
209
210    $s->{'minage'}    = 0 + List::Util::min($s->{'minage'}   , $age);
211    $s->{'maxage'}    = 0 + List::Util::max($s->{'maxage'}   , $age);
212    $s->{'minsize'}   = 0 + List::Util::min($s->{'minsize'}  , $size);
213    $s->{'maxsize'}   = 0 + List::Util::max($s->{'maxsize'}  , $size);
214    $s->{'minblocks'} = 0 + List::Util::min($s->{'minblocks'}, $blocks);
215    $s->{'maxblocks'} = 0 + List::Util::max($s->{'maxblocks'}, $blocks);
216    $s->{'sumage'}    += $age;
217    $s->{'sumsize'}   += $size;
218    $s->{'sumblocks'} += $blocks;
219    $s->{'count'}++;
220    $s->{'countempty'}++ if ($size == $EMPTY_TILE_SIZE);
221}
222
223__END__
224
225=head1 NAME
226
227tirex-tiledir-check - check tile dir
228
229=head1 SYNOPSIS
230
231tirex-tiledir-check [OPTIONS]
232
233=head1 OPTIONS
234
235=over 8
236
237=item B<-h>, B<--help>
238
239Display help message.
240
241=item B<-c>, B<--config=FILE>
242
243Use the config file FILE instead of /etc/tirex/tirex.conf.
244
245=item B<-l>, B<--list=FILE>
246
247Write list of tiles to FILE. See below for format.
248
249=item B<-s>, B<--stats=FILE>
250
251Write stats about tiles to FILE. See below for format.
252
253=back
254
255=head1 DESCRIPTION
256
257Walks recursively through the tiles directory and checks for wrong directory
258or file names or files that can't be accessed. If problems are found, messages
259are written to STDERR.
260
261When the --list and/or --stats options are given, it outputs information about
262each tile and/or generates statistics, respectively. The list and stats are
263written to the filenames given, if you give '-' as the filename STDOUT is used.
264You can only use STDOUT for either the list file or the stats file.
265
266Caution: This command will go through all metatiles on your disk and stat each
267file. On a lightly loaded machine with lots of RAM this can go pretty quick,
268but under high IO loads it might take a long time. Take this into account if
269you want to run it regularly from cron or similar.
270
271=head1 LIST FILE FORMAT
272
273The list file is in CSV format with one line per metatile. The
274fields are:
275
276=over 8
277
278=item mtime
279
280Last modified time in seconds since epoch.
281
282=item size
283
284Size in bytes of the metatile.
285
286=item blocks
287
288Number of blocks used for this metatile.
289
290=item metatile
291
292Description of metatile. Format: map=foo x=8 y=0 z=10
293
294=back
295
296This data can be read with other programs to create statistics etc.
297
298=head1 STATS FILE FORMAT
299
300The stats file is in JSON format. It contains a hash with the names
301of all maps as keys and a list of zoom levels as their values. For
302each zoom level there is a nested hash containing the statistics:
303
304=over 8
305
306=item count - Number of tiles
307
308=item maxage - Maximum age in seconds
309
310=item maxblocks - Maximum number of blocks
311
312=item maxsize - Maximum file size in bytes
313
314=item minage - Minimum age in seconds
315
316=item minblocks - Minimum number of blocks
317
318=item minsize - Minimum file size in bytes
319
320=item sumage - Sum of ages in seconds
321
322=item sumblocks - Sum of number of blocks
323
324=item sumsize - Sum of file size in bytes
325
326=back
327
328The sumage, sumblocks, and sumsize values can be divided by count to get
329the average.
330
331This file can be displayed on a human readable format with the Program
332tirex-tiledir-stat. It is also read by several Munin plugins.
333
334=head1 FILES
335
336=over 8
337
338=item F</etc/tirex/tirex.conf>
339
340The configuration file. See tirex.conf(5) for further details.
341
342=back
343
344=head1 DIAGNOSTICS
345
346Returns 0 if no errors were found or 1 if there were errors. If there were
347errors parsing the command line 2 is returned.
348
349=head1 AUTHORS
350
351Frederik Ramm <frederik.ramm@geofabrik.de>, Jochen Topf
352<jochen.topf@geofabrik.de> and possibly others.
353
354=cut
355
356
357#-- THE END ------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.