source: subversion/applications/rendering/tilesAtHome/compress.pl @ 9930

Revision 9930, 9.2 KB checked in by spaetz, 6 years ago (diff)

remove last traces of MultipleClient? setting which is superceded by UploadToDirectory?. It was causing error messages.

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2use strict;
3use File::Copy;
4use File::Path;
5use Fcntl ':flock'; #import LOCK_* constants
6use English '-no_match_vars';
7use tahconfig;
8use tahlib;
9use AppConfig qw(:argcount);
10
11#-----------------------------------------------------------------------------
12# OpenStreetMap tiles@home, compress module
13# Takes any tiles generated and adds them into ZIP files
14#
15# Contact OJW on the Openstreetmap wiki for help using this program
16#-----------------------------------------------------------------------------
17# Copyright 2006, Oliver White, Dirk-Lueder Kreie
18#
19# This program is free software; you can redistribute it and/or
20# modify it under the terms of the GNU General Public License
21# as published by the Free Software Foundation; either version 2
22# of the License, or (at your option) any later version.
23#
24# This program is distributed in the hope that it will be useful,
25# but WITHOUT ANY WARRANTY; without even the implied warranty of
26# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
27# GNU General Public License for more details.
28#
29# You should have received a copy of the GNU General Public License
30# along with this program; if not, write to the Free Software
31# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
32#-----------------------------------------------------------------------------
33
34# conf file, will contain username/password and environment info
35our $Config = AppConfig->new({
36                CREATE => 1,                      # Autocreate unknown config variables
37                GLOBAL => {
38                  DEFAULT  => undef,    # Create undefined Variables by default
39                  ARGCOUNT => ARGCOUNT_ONE, # Simple Values (no arrays, no hashmaps)
40                }
41              });
42
43$Config->define("help|usage!");
44$Config->define("nodownload=s");
45$Config->set("nodownload",0);
46$Config->file("config.defaults", "layers.conf", "tilesAtHome.conf", "authentication.conf"); #first read configs in order, each (possibly) overwriting settings from the previous
47$Config->args();              # overwrite config options with command line options
48$Config->file("general.conf");  # overwrite with hardcoded values that must not be changed
49
50if ($Config->get("LocalSlippymap"))
51{
52    print "No upload - LocalSlippymap set in config file\n";
53    exit 1;
54}
55
56my $ZipFileCount = 0;
57
58my $ZipDir = $Config->get("WorkingDirectory") . "/uploadable";
59
60my @sorted;
61
62# when called from tilesGen, use these for nice display
63my $progress = 0;
64our $progressPercent = 0;
65our $progressJobs = $ARGV[0] or 1;
66our $currentSubTask = "zipping";
67
68
69my $tileCount;
70
71my @tiles;
72
73    my $allowedPrefixes;
74   
75    my $TileDir = $Config->get("WorkingDirectory");
76   
77    # Group and upload the tiles
78    statusMessage("Searching for tiles in $TileDir",0,3);
79    # compile a list of the "Prefix" values of all configured layers,
80    #     # separated by |
81   
82    foreach my $UploadLayer (split(/,/, $Config->get("LayersCapability")))
83    {
84        $allowedPrefixes = $Config->get($UploadLayer."_Prefix"); #just select the current layer for compressing
85        ## DEBUG print "\n.$allowedPrefixes.\n";
86        opendir(my $dp, $TileDir) or die("Can't open directory $TileDir\n");
87        my @dir = readdir($dp);
88        @tiles = grep { /($allowedPrefixes)_\d+_\d+_\d+\.png$/ } @dir;
89        my @tilesets = grep { /($allowedPrefixes)_\d+_\d+_\d+\.dir$/ } @dir;
90        closedir($dp);
91       
92        foreach my $File(@tilesets)
93        {   # go through all complete tilesets ie "*.dir" firectories
94            my $FullTileDirPath = "$TileDir/$File";
95
96            # get a file handle, then try to lock the file exclusively.
97            # if open fails (file has been uploaded and removed by other process)
98            # the subsequent flock will also fail and skip the file.
99            # if just flock fails it is being handled by a different upload process
100            open (ZIPDIR, $FullTileDirPath);
101            if (flock(ZIPDIR, LOCK_EX|LOCK_NB))
102            {   # got exclusive lock, now compress
103                compress($FullTileDirPath, $ZipDir, 'yes', $allowedPrefixes);
104                # TODO: We always kill the tileset.dir independent of success and never return a success value!
105                rmtree $FullTileDirPath;    # should be empty now
106            }
107            else
108            {   # could not get exclusive lock, this is being handled elsewhere now
109                statusMessage("$File compressed by different process. skipping",0,3);
110            }
111            # finally unlock zipfile and release handle
112            flock (ZIPDIR, LOCK_UN);
113            close (ZIPDIR);
114        }
115
116        ### NOTE: The following code block deals with single tiles in the TileDir(=WorkingDir)
117        ### as we don't produce single tiles currently, this is commented out for now.
118        #$tileCount = scalar(@tiles);       
119        #if ($tileCount)
120        #{
121        #    while (processTileBatch(
122        #      $TileDir,
123        #      $TileDir . "/gather", ## FIXME: this is one of the things that make compress.pl not multithread safe
124        #      $ZipDir,
125        #      $allowedPrefixes))
126        #    {};
127        #}
128        ## TODO: fix progress display
129    }
130    statusMessage("done",0,3); 
131
132## end of main
133
134#-----------------------------------------------------------------------------
135# Moves tiles into a "gather" directory until a certain size is reached,
136# then compress and upload those files
137#-----------------------------------------------------------------------------
138sub processTileBatch
139{
140    my ($TileDir, $TempDir, $OutputDir, $allowedPrefixes) = @_;
141    my ($Size,$Count) = (0,0);
142    my $MB = 1024*1024;
143
144    mkdir $TempDir if ! -d $TempDir;
145    mkdir $OutputDir if ! -d $OutputDir;
146
147    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount;
148    statusMessage(scalar(@tiles)." tiles to process for ".$allowedPrefixes,0,3);
149
150    while(my $file = shift @tiles)
151    {
152        my $Filename1 = "$TileDir/$file";
153        my $Filename2 = "$TempDir/$file";
154        if($file =~ /($allowedPrefixes)_\d+_\d+_\d+\.png$/i)
155        {
156            $Size += -s $Filename1;
157            $Count++;
158
159            rename($Filename1, $Filename2);
160        }
161    }
162
163    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount; 
164   
165    if($Count)
166    {
167        statusMessage(sprintf("Got %d files (%d bytes), compressing", $Count, $Size),0,3);
168        my $zip_result = compress($TempDir, $OutputDir, 'no', $allowedPrefixes);
169        rmtree $TempDir;
170        return $zip_result;
171    }
172    else
173    {   # No tiles in directory, leave with success
174        statusMessage("compress finished",0,3);
175        return 0;
176    }
177}
178
179#-----------------------------------------------------------------------------
180# Compress all PNG files from one directory, creating a .zip file.
181# Parameters:  Dir, OutputDir, SingleTileset, Layer
182# Dir: directory where the *.png files reside in.
183# OutputDir: directory where the resulting .zip file will be stored
184# SingleTileset: 'yes'/'no' whether it's a single tileset, affects .zip file name
185# Layer: layer name will be incorporated in .zip file name
186# It will never delete the source files, so the caller has to delete them after success
187# (usually by removing the temporary direcory they reside in)
188# returns 1, if the zip command succeeded and 0 otherwise
189#-----------------------------------------------------------------------------
190sub compress
191{
192    my ($Dir, $OutputDir, $SingleTileset, $Layer) = @_;
193    $SingleTileset = ($SingleTileset eq 'yes' ? '_tileset' : '');
194 
195    my $Filename;
196
197    my $epochtime = time;
198 
199    # Create the output directory if it doesn't exist...
200    if( ! -e $OutputDir )
201    {
202        mkdir $OutputDir;
203    }
204   
205    if($Config->get("UseHostnameInZipname"))
206    {
207        my $hostname = `hostname`;
208        chomp $hostname;
209        $hostname .= "XXXXX";
210        $Filename = sprintf("%s/%d_%s_%d_%d_%d_%s%s.zip", $OutputDir, $epochtime,
211          substr($hostname,0,5), GetClientId(), $$, $ZipFileCount++, $Layer, $SingleTileset);
212    }
213    else 
214    {
215        $Filename = sprintf("%s/%d_%d_%d_%d_%s%s.zip", $OutputDir,
216          $epochtime, GetClientId(), $$, $ZipFileCount++, $Layer, $SingleTileset);
217    }
218   
219    # ZIP all the tiles into a single file
220    my $stdOut = $Config->get("WorkingDirectory")."/".$PID.".stdout";
221    my $Command1;
222    if ($Config->get("7zipWin"))
223    {
224        $Command1 = sprintf("\"%s\" %s %s %s",
225          $Config->get("Zip"),
226          "a -tzip",
227          $Filename,
228          "$Dir/*.png");
229    }
230    else
231    {
232        $Command1 = sprintf("\"%s\" -r -j %s %s > %s",
233          $Config->get("Zip"),
234          $Filename,
235          "$Dir",
236          $stdOut);
237    }
238   
239    # Run the zip command
240    my $zip_result = runCommand($Command1,$PID);
241
242    # stdOut is currently never used, so delete it unconditionally   
243    unlink($stdOut);
244
245    # if collecting single tiles (incomplete tileset and zipping failed,
246    # move all the .png back into $WorkingDirectory
247    if ($SingleTileset eq '' and $zip_result == 0)
248    {
249        opendir (GATHERDIR, $Dir);
250        my @zippedFiles = grep { /.png$/ } readdir(GATHERDIR);
251        closedir (GATHERDIR);
252        while(my $File = shift @zippedFiles)
253        {
254            rename($Dir . "/" . $File, $Config->get("WorkingDirectory") . $File);
255        }
256    }
257   
258    return $zip_result;
259}
Note: See TracBrowser for help on using the repository browser.