source: subversion/applications/rendering/tilesAtHome/compress.pl @ 9930

Last change on this file since 9930 was 9930, checked in by spaetz, 10 years ago

remove last traces of MultipleClient? setting which is superceded by UploadToDirectory?. It was causing error messages.

  • Property svn:executable set to *
File size: 9.2 KB
Line 
1#!/usr/bin/perl
2use strict;
3use File::Copy;
4use File::Path;
5use Fcntl ':flock'; #import LOCK_* constants
6use English '-no_match_vars';
7use tahconfig;
8use tahlib;
9use AppConfig qw(:argcount);
10
11#-----------------------------------------------------------------------------
12# OpenStreetMap tiles@home, compress module
13# Takes any tiles generated and adds them into ZIP files
14#
15# Contact OJW on the Openstreetmap wiki for help using this program
16#-----------------------------------------------------------------------------
17# Copyright 2006, Oliver White, Dirk-Lueder Kreie
18#
19# This program is free software; you can redistribute it and/or
20# modify it under the terms of the GNU General Public License
21# as published by the Free Software Foundation; either version 2
22# of the License, or (at your option) any later version.
23#
24# This program is distributed in the hope that it will be useful,
25# but WITHOUT ANY WARRANTY; without even the implied warranty of
26# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
27# GNU General Public License for more details.
28#
29# You should have received a copy of the GNU General Public License
30# along with this program; if not, write to the Free Software
31# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
32#-----------------------------------------------------------------------------
33
34# conf file, will contain username/password and environment info
35our $Config = AppConfig->new({
36                CREATE => 1,                      # Autocreate unknown config variables
37                GLOBAL => {
38                  DEFAULT  => undef,    # Create undefined Variables by default
39                  ARGCOUNT => ARGCOUNT_ONE, # Simple Values (no arrays, no hashmaps)
40                }
41              });
42
43$Config->define("help|usage!");
44$Config->define("nodownload=s");
45$Config->set("nodownload",0);
46$Config->file("config.defaults", "layers.conf", "tilesAtHome.conf", "authentication.conf"); #first read configs in order, each (possibly) overwriting settings from the previous
47$Config->args();              # overwrite config options with command line options
48$Config->file("general.conf");  # overwrite with hardcoded values that must not be changed
49
50if ($Config->get("LocalSlippymap"))
51{
52    print "No upload - LocalSlippymap set in config file\n";
53    exit 1;
54}
55
56my $ZipFileCount = 0;
57
58my $ZipDir = $Config->get("WorkingDirectory") . "/uploadable";
59
60my @sorted;
61
62# when called from tilesGen, use these for nice display
63my $progress = 0;
64our $progressPercent = 0;
65our $progressJobs = $ARGV[0] or 1;
66our $currentSubTask = "zipping";
67
68
69my $tileCount;
70
71my @tiles;
72
73    my $allowedPrefixes;
74   
75    my $TileDir = $Config->get("WorkingDirectory");
76   
77    # Group and upload the tiles
78    statusMessage("Searching for tiles in $TileDir",0,3);
79    # compile a list of the "Prefix" values of all configured layers,
80    #     # separated by |
81   
82    foreach my $UploadLayer (split(/,/, $Config->get("LayersCapability")))
83    {
84        $allowedPrefixes = $Config->get($UploadLayer."_Prefix"); #just select the current layer for compressing
85        ## DEBUG print "\n.$allowedPrefixes.\n";
86        opendir(my $dp, $TileDir) or die("Can't open directory $TileDir\n");
87        my @dir = readdir($dp);
88        @tiles = grep { /($allowedPrefixes)_\d+_\d+_\d+\.png$/ } @dir;
89        my @tilesets = grep { /($allowedPrefixes)_\d+_\d+_\d+\.dir$/ } @dir;
90        closedir($dp);
91       
92        foreach my $File(@tilesets)
93        {   # go through all complete tilesets ie "*.dir" firectories
94            my $FullTileDirPath = "$TileDir/$File";
95
96            # get a file handle, then try to lock the file exclusively.
97            # if open fails (file has been uploaded and removed by other process)
98            # the subsequent flock will also fail and skip the file.
99            # if just flock fails it is being handled by a different upload process
100            open (ZIPDIR, $FullTileDirPath);
101            if (flock(ZIPDIR, LOCK_EX|LOCK_NB))
102            {   # got exclusive lock, now compress
103                compress($FullTileDirPath, $ZipDir, 'yes', $allowedPrefixes);
104                # TODO: We always kill the tileset.dir independent of success and never return a success value!
105                rmtree $FullTileDirPath;    # should be empty now
106            }
107            else
108            {   # could not get exclusive lock, this is being handled elsewhere now
109                statusMessage("$File compressed by different process. skipping",0,3);
110            }
111            # finally unlock zipfile and release handle
112            flock (ZIPDIR, LOCK_UN);
113            close (ZIPDIR);
114        }
115
116        ### NOTE: The following code block deals with single tiles in the TileDir(=WorkingDir)
117        ### as we don't produce single tiles currently, this is commented out for now.
118        #$tileCount = scalar(@tiles);       
119        #if ($tileCount)
120        #{
121        #    while (processTileBatch(
122        #      $TileDir,
123        #      $TileDir . "/gather", ## FIXME: this is one of the things that make compress.pl not multithread safe
124        #      $ZipDir,
125        #      $allowedPrefixes))
126        #    {};
127        #}
128        ## TODO: fix progress display
129    }
130    statusMessage("done",0,3); 
131
132## end of main
133
134#-----------------------------------------------------------------------------
135# Moves tiles into a "gather" directory until a certain size is reached,
136# then compress and upload those files
137#-----------------------------------------------------------------------------
138sub processTileBatch
139{
140    my ($TileDir, $TempDir, $OutputDir, $allowedPrefixes) = @_;
141    my ($Size,$Count) = (0,0);
142    my $MB = 1024*1024;
143
144    mkdir $TempDir if ! -d $TempDir;
145    mkdir $OutputDir if ! -d $OutputDir;
146
147    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount;
148    statusMessage(scalar(@tiles)." tiles to process for ".$allowedPrefixes,0,3);
149
150    while(my $file = shift @tiles)
151    {
152        my $Filename1 = "$TileDir/$file";
153        my $Filename2 = "$TempDir/$file";
154        if($file =~ /($allowedPrefixes)_\d+_\d+_\d+\.png$/i)
155        {
156            $Size += -s $Filename1;
157            $Count++;
158
159            rename($Filename1, $Filename2);
160        }
161    }
162
163    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount; 
164   
165    if($Count)
166    {
167        statusMessage(sprintf("Got %d files (%d bytes), compressing", $Count, $Size),0,3);
168        my $zip_result = compress($TempDir, $OutputDir, 'no', $allowedPrefixes);
169        rmtree $TempDir;
170        return $zip_result;
171    }
172    else
173    {   # No tiles in directory, leave with success
174        statusMessage("compress finished",0,3);
175        return 0;
176    }
177}
178
179#-----------------------------------------------------------------------------
180# Compress all PNG files from one directory, creating a .zip file.
181# Parameters:  Dir, OutputDir, SingleTileset, Layer
182# Dir: directory where the *.png files reside in.
183# OutputDir: directory where the resulting .zip file will be stored
184# SingleTileset: 'yes'/'no' whether it's a single tileset, affects .zip file name
185# Layer: layer name will be incorporated in .zip file name
186# It will never delete the source files, so the caller has to delete them after success
187# (usually by removing the temporary direcory they reside in)
188# returns 1, if the zip command succeeded and 0 otherwise
189#-----------------------------------------------------------------------------
190sub compress
191{
192    my ($Dir, $OutputDir, $SingleTileset, $Layer) = @_;
193    $SingleTileset = ($SingleTileset eq 'yes' ? '_tileset' : '');
194 
195    my $Filename;
196
197    my $epochtime = time;
198 
199    # Create the output directory if it doesn't exist...
200    if( ! -e $OutputDir )
201    {
202        mkdir $OutputDir;
203    }
204   
205    if($Config->get("UseHostnameInZipname"))
206    {
207        my $hostname = `hostname`;
208        chomp $hostname;
209        $hostname .= "XXXXX";
210        $Filename = sprintf("%s/%d_%s_%d_%d_%d_%s%s.zip", $OutputDir, $epochtime,
211          substr($hostname,0,5), GetClientId(), $$, $ZipFileCount++, $Layer, $SingleTileset);
212    }
213    else 
214    {
215        $Filename = sprintf("%s/%d_%d_%d_%d_%s%s.zip", $OutputDir,
216          $epochtime, GetClientId(), $$, $ZipFileCount++, $Layer, $SingleTileset);
217    }
218   
219    # ZIP all the tiles into a single file
220    my $stdOut = $Config->get("WorkingDirectory")."/".$PID.".stdout";
221    my $Command1;
222    if ($Config->get("7zipWin"))
223    {
224        $Command1 = sprintf("\"%s\" %s %s %s",
225          $Config->get("Zip"),
226          "a -tzip",
227          $Filename,
228          "$Dir/*.png");
229    }
230    else
231    {
232        $Command1 = sprintf("\"%s\" -r -j %s %s > %s",
233          $Config->get("Zip"),
234          $Filename,
235          "$Dir",
236          $stdOut);
237    }
238   
239    # Run the zip command
240    my $zip_result = runCommand($Command1,$PID);
241
242    # stdOut is currently never used, so delete it unconditionally   
243    unlink($stdOut);
244
245    # if collecting single tiles (incomplete tileset and zipping failed,
246    # move all the .png back into $WorkingDirectory
247    if ($SingleTileset eq '' and $zip_result == 0)
248    {
249        opendir (GATHERDIR, $Dir);
250        my @zippedFiles = grep { /.png$/ } readdir(GATHERDIR);
251        closedir (GATHERDIR);
252        while(my $File = shift @zippedFiles)
253        {
254            rename($Dir . "/" . $File, $Config->get("WorkingDirectory") . $File);
255        }
256    }
257   
258    return $zip_result;
259}
Note: See TracBrowser for help on using the repository browser.