source: subversion/applications/rendering/tilesAtHome/lib/Compress.pm @ 11132

Last change on this file since 11132 was 11131, checked in by Dirk Stoecker, 12 years ago

readded message to fix unfinished line ending

  • Property svn:executable set to *
File size: 14.4 KB
Line 
1package Compress;
2
3use warnings;
4use strict;
5use File::Copy;
6use File::Path;
7use Fcntl ':flock'; #import LOCK_* constants
8use English '-no_match_vars';
9use Error qw(:try);
10use tahlib;
11use TahConf;
12
13#-----------------------------------------------------------------------------
14# OpenStreetMap tiles@home, compress module
15# Takes any tiles generated and adds them into ZIP files
16#
17# Contact OJW on the Openstreetmap wiki for help using this program
18#-----------------------------------------------------------------------------
19# Copyright 2006, Oliver White, Dirk-Lueder Kreie
20#
21# This program is free software; you can redistribute it and/or
22# modify it under the terms of the GNU General Public License
23# as published by the Free Software Foundation; either version 2
24# of the License, or (at your option) any later version.
25#
26# This program is distributed in the hope that it will be useful,
27# but WITHOUT ANY WARRANTY; without even the implied warranty of
28# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
29# GNU General Public License for more details.
30#
31# You should have received a copy of the GNU General Public License
32# along with this program; if not, write to the Free Software
33# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
34#-----------------------------------------------------------------------------
35
36#-------------------------------------------------------------------
37# Create a new Compress instance
38#-------------------------------------------------------------------
39sub new
40{
41    my $class = shift;
42    my $self  = {};
43
44    $self = {
45        Config  => TahConf->getConfig(),
46    };
47    $self->{TileDir} = $self->{Config}->get("WorkingDirectory"),
48    $self->{UploadDir}  = File::Spec->catdir($self->{Config}->get("WorkingDirectory"), "/uploadable"),
49
50    bless ($self, $class);
51
52    #set global progressbar task
53    $::currentSubTask ='compress';
54    return $self;
55}
56
57#-------------------------------------------------------------------
58# main function. Compresses all tileset.dir's in $self->{TileDir}.
59#-------------------------------------------------------------------
60sub compressAll
61{
62    my $self = shift;
63    my $Config = $self->{Config};
64
65    my $progress = 0;
66    $::progressPercent = 0;
67    $::currentSubTask = "compress";
68    my $LOCKFILE;
69
70    if ($Config->get("LocalSlippymap"))
71    {
72       throw CompressError "No compressing - LocalSlippymap set in config file", "LocalSlippymap";
73    }
74
75    my (@prefixes,$allowedPrefixes);
76   
77    ::statusMessage("Searching for tilesets to be compressed",0,3);
78
79    # compile a list of the "Prefix" values of all configured layers,
80    # separated by |
81    foreach my $layer(split(/,/, $Config->get("LayersCapability")))
82    {
83        push(@prefixes, $Config->get($layer."_Prefix"));
84    }
85    $allowedPrefixes = join('|', @prefixes);
86
87    # Go through all files in TileDir and grep the right directories
88    opendir(my $dp, $self->{TileDir}) or throw CompressError "Can't open directory " . $self->{TileDir};
89    my @dir = readdir($dp);
90    my @tilesets = grep { /($allowedPrefixes)_\d+_\d+_\d+\.dir$/ } @dir;
91    closedir($dp);
92
93    if(!@tilesets)
94    {
95        ::statusMessage("Nothing found to compress",1,3);
96    }
97
98    foreach my $File(@tilesets)
99    {   # go through all complete tilesets ie "*.dir" firectories
100        $File =~ m{^([^_]+)_};
101        my $layer = $1;
102        my $FullTilesetPath = File::Spec->join($self->{TileDir}, $File);
103
104        # get a file handle, then try to lock the file exclusively.
105        # if flock fails it is being handled by a different upload process
106        # also check if the file still exists when we get to it
107        open ($LOCKFILE, '>', $FullTilesetPath."lock");
108        my $flocked = !$Config->get('flock_available')
109                      || ($LOCKFILE && flock($LOCKFILE, LOCK_EX|LOCK_NB));
110        if ($flocked && -d $FullTilesetPath )
111        {   # got exclusive lock, now compress
112            $::currentSubTask ='optimize';
113            ::statusMessage("optimizing PNG files",0,3);
114            $self->optimizePNGs($FullTilesetPath, $layer);
115            $::currentSubTask ='compress';
116            $::progressPercent = 0;
117            ::statusMessage("compressing $File",0,3);
118            if ($Config->get("CreateTilesetFile")) {
119                $self->createTilesetFile($FullTilesetPath);
120            }
121            else {
122                $self->compress($FullTilesetPath);
123            }
124            # TODO: We always kill the tileset.dir independent of success and never return a success value!
125            rmtree $FullTilesetPath;    # should be empty now
126        }
127        else
128        {   # could not get exclusive lock, this is being handled elsewhere now
129            ::statusMessage("$File compressed by different process. skipping",1,3);
130        }
131        # finally unlock zipfile and release handle
132        if ($LOCKFILE)
133        {
134            flock ($LOCKFILE, LOCK_UN);
135            close ($LOCKFILE);
136            unlink($FullTilesetPath."lock") if $flocked;
137        }
138    }
139}
140
141#-----------------------------------------------------------------------------
142# Compress all PNG files from one directory, creating a .zip file.
143# Parameters:  FullTilesetPath
144#
145# It will never delete the source files, so the caller has to delete them
146#-----------------------------------------------------------------------------
147sub compress
148{
149    my $self = shift;
150    my $Config = $self->{Config};
151
152    my ($FullTilesetPathDir) = @_;
153 
154    my $Filename;
155    my $Tempfile;
156
157    $FullTilesetPathDir =~ m{([^_\/\\]+)_(\d+)_(\d+)_(\d+).dir$};
158    my ($layer, $Z, $X, $Y) = ($1, $2, $3, $4);
159
160    # Create the output directory if it doesn't exist...
161    if( ! -d $self->{UploadDir} )
162    {
163        mkpath $self->{UploadDir};# TODO: Error handling
164    }
165
166    $Filename = File::Spec->join($self->{UploadDir},
167                                sprintf("%s_%d_%d_%d_%d.zip",
168                                $layer, $Z, $X, $Y, ::GetClientId()));
169   
170    $Tempfile = File::Spec->join($self->{TileDir},
171                                sprintf("%s_%d_%d_%d_%d.zip",
172                                $layer, $Z, $X, $Y, ::GetClientId()));
173    # ZIP all the tiles into a single file
174    # First zip into "$Filename.part" and move to "$Filename" when finished
175    my $stdOut = File::Spec->join($self->{TileDir}, "zip.stdout");
176    my $zipCmd;
177    if ($Config->get("7zipWin"))
178    {
179        $zipCmd = sprintf('"%s" %s "%s" "%s"',
180          $Config->get("Zip"),
181          "a -tzip",
182          $Tempfile,
183          File::Spec->join($FullTilesetPathDir,"*.png"));
184    }
185    else
186    {
187        $zipCmd = sprintf('"%s" -r -j "%s" "%s" > "%s"',
188          $Config->get("Zip"),
189          $Tempfile,
190          $FullTilesetPathDir,
191          $stdOut);
192    }
193
194    if (::dirEmpty($FullTilesetPathDir))
195    {
196        ::statusMessage("Skipping emtpy tileset directory: $FullTilesetPathDir",1,0);
197        return 0;
198    }
199    # Run the zip command
200    ::runCommand($zipCmd, $PID) or throw CompressError "Error running $zipCmd";
201
202    # stdOut is currently never used, so delete it unconditionally   
203    unlink($stdOut);
204   
205    # rename to final name so any uploader could pick it up now
206    move ($Tempfile, $Filename); # TODO: Error handling
207   
208    return 1;
209}
210
211#-----------------------------------------------------------------------------
212# Pack all PNG files from one directory into a Tileset file.
213# Parameters:  FullTilesetPath
214#
215# It will never delete the source files, so the caller has to delete them
216#-----------------------------------------------------------------------------
217sub createTilesetFile
218{
219    my $self = shift;
220    my $Config = $self->{Config};
221
222    my ($FullTilesetPathDir) = @_;
223 
224    my $Filename;
225    my $Tempfile;
226
227    $FullTilesetPathDir =~ m{([^_\/\\]+)_(\d+)_(\d+)_(\d+).dir$};
228    my ($layer, $Z, $X, $Y) = ($1, $2, $3, $4);
229
230    my @offsets;
231    my $levels = 6;                       # number of layers in a tileset file, currently 6
232    my $tiles = ((4 ** $levels) - 1) / 3; # 1365 for 6 zoom levels
233    my $currpos = 8 + (4 * ($tiles + 1)); # 5472 for 6 zoom levels
234
235    my $userid = 0; # the server will fill this in
236
237    $Filename = File::Spec->join($self->{UploadDir},
238                                 sprintf("%s_%d_%d_%d_%d.tileset",
239                                         $layer, $Z, $X, $Y, ::GetClientId()));
240   
241    $Tempfile = File::Spec->join($self->{TileDir},
242                                 sprintf("%s_%d_%d_%d_%d.tileset",
243                                         $layer, $Z, $X, $Y, ::GetClientId()));
244
245    open my $fh, ">$Tempfile" or throw CompressError "Couldn't open '$Tempfile' ($!)";
246    seek $fh, $currpos, 0 or throw CompressError "Couldn't seek.";
247
248    for my $iz (0 .. $levels - 1) {
249        my $width = 2**$iz;
250        for my $iy (0 .. $width-1) {
251            for my $ix (0 .. $width-1) {
252                my $Pngname = File::Spec->join($FullTilesetPathDir,
253                                               sprintf("%s_%d_%d_%d.png",
254                                                       $layer, $Z+$iz, $X*$width+$ix, $Y*$width+$iy));
255                my $length = -s $Pngname;
256                if (! -e $Pngname) {
257                    push(@offsets, 0);
258                }
259                elsif ($length == 67) {
260                    push(@offsets, 2);
261                }
262                elsif ($length == 69) {
263                    push(@offsets, 1);
264                }
265                else {
266                    open my $png, "<$Pngname" or throw CompressError "Couldn't open file $Pngname ($!)";
267                    my $buffer;
268                    if( read($png, $buffer, $length) != $length ) {
269                        throw CompressError "Read failed from $Pngname ($!)"
270                    }
271                    close $png;
272                    print $fh $buffer or throw CompressError "Write failed on output to $Tempfile ($!)";
273                    push @offsets, $currpos;
274                    $currpos += $length;
275                }
276            }
277        }
278    }
279    push @offsets, $currpos;
280
281    if( scalar( @offsets ) != $tiles + 1 ) {
282        throw CompressError "Bad number of offsets: " . scalar( @offsets );
283    }
284
285    seek $fh, 0, 0;
286    print $fh pack("CxxxVV*", 1, $userid, @offsets) or throw CompressError "Write failed to $Tempfile ($!)";
287    close $fh;
288
289    move($Tempfile, $Filename) or throw CompressError "Could not move tileset file $Tempfile to $Filename ($!)";
290}
291
292
293#-----------------------------------------------------------------------------
294# Run pngcrush on each split tile, then delete the temporary cut file
295# parameter (FullPathToPNGDir, $layer)
296#-----------------------------------------------------------------------------
297sub optimizePNGs
298{
299    my $self = shift;
300    my $Config = $self->{Config};
301    my $PNGDir = shift;
302    my $layer  = shift;
303    my $transparent_layer = int($Config->get($layer."_Transparent"));
304
305    $::progressPercent = 0;
306    my $TmpFilename_suffix = ".cut";
307    my $Redirect = ">/dev/null";
308    my $Cmd;
309
310    if ($^O eq "MSWin32")
311    {
312        $Redirect = "";
313    }
314
315    # read in all the PNG files in the Dir
316    my @pngfiles;
317    if (opendir(PNGDIR, $PNGDir))
318    {
319        @pngfiles = grep { /\.png$/ } readdir(PNGDIR);
320        closedir PNGDIR;
321    }
322    else 
323    {
324       throw CompressError "could not read $PNGDir";
325    }
326
327    my $NumPNG = scalar(@pngfiles);
328    my $progress = 0;
329    ::statusMessage("Optimizing $NumPNG images", 0, 3);
330
331    foreach my $PngFileName(@pngfiles)
332    {  # go through all PNG files
333       $progress ++;
334       $::progressPercent = 100 * $progress / $NumPNG;
335
336       my $PngFullFileName = File::Spec->join($PNGDir, $PngFileName);
337       # don't optimize empty sea or empty land tiles (file size 67 and 69)
338       next if ((-s $PngFullFileName) =~ /67|69/);
339
340       # Temporary filename between quantizing and optimizing
341       my $TmpFullFileName = $PngFullFileName.$TmpFilename_suffix;
342
343       if ($transparent_layer)
344       {    # Don't quantize if it's transparent
345            rename($PngFullFileName, $TmpFullFileName);
346       }
347       elsif (($Config->get("PngQuantizer")||'') eq "pngnq") 
348       {
349           $Cmd = sprintf("\"%s\" -e .png%s -s1 -n256 %s %s",
350                                   $Config->get("pngnq"),
351                                   $TmpFilename_suffix,
352                                   $PngFullFileName,
353                                   $Redirect);
354
355           ::statusMessage("ColorQuantizing $PngFileName",0,6); 
356           if(::runCommand($Cmd,$PID))
357           {   # Color quantizing successful
358               unlink($PngFullFileName);
359           }
360           else
361           {   # Color quantizing failed
362               ::statusMessage("ColorQuantizing $PngFileName with ".$Config->get("PngQuantizer")." failed",1,0);
363               rename($PngFullFileName, $TmpFullFileName);
364            }
365       }
366       else
367       {
368           ::statusMessage("Not Color Quantizing $PngFileName, pngnq not installed?",0,6);
369           rename($PngFullFileName, $TmpFullFileName);
370       }
371
372       # Finished quantizing. The file is in TmpFullFileName now.
373
374       if ($Config->get("PngOptimizer") eq "pngcrush")
375       {
376           $Cmd = sprintf("\"%s\" -q %s %s %s",
377                  $Config->get("Pngcrush"),
378                  $TmpFullFileName,
379                  $PngFullFileName,
380                  $Redirect);
381       }
382       elsif ($Config->get("PngOptimizer") eq "optipng")
383       {
384           $Cmd = sprintf("\"%s\" %s -out %s %s", #no quiet, because it even suppresses error output
385                  $Config->get("Optipng"),
386                  $TmpFullFileName,
387                  $PngFullFileName,
388                  $Redirect);
389       }
390       else
391       {
392           ::statusMessage("SplitImageX:PngOptimizer not configured (should not happen, update from svn, and check config file)",1,0);
393           ::talkInSleep("Install a PNG optimizer and configure it.",15);
394       }
395
396       ::statusMessage("Optimizing $PngFileName",0,6);
397       if(::runCommand($Cmd,$PID))
398       {
399           unlink($TmpFullFileName);
400       }
401       else
402       {
403           ::statusMessage("Optimizing $PngFileName with ".$Config->get("PngOptimizer")." failed",1,0);
404           rename($TmpFullFileName, $PngFullFileName);
405       }
406
407       # Assign the job time to this file
408       # TODO:
409       #utime $JobTime, $JobTime, $PngFullFileName;
410
411    } # foreach my $PngFileName
412}
413
414
415#-----------------------------------------------------------------------------------------------------------------
416# class CompressError
417#
418# Exception to be thrown by Compress methods
419
420package CompressError;
421use base 'Error::Simple';
422
4231;
Note: See TracBrowser for help on using the repository browser.