source: subversion/applications/rendering/tilesAtHome/compress.pl @ 6008

Last change on this file since 6008 was 6002, checked in by deelkar, 13 years ago

enable uploading to one central upload directory

  • Property svn:executable set to *
File size: 9.9 KB
Line 
1#!/usr/bin/perl
2use strict;
3use File::Copy;
4use English '-no_match_vars';
5use tahconfig;
6use tahlib;
7#-----------------------------------------------------------------------------
8# OpenStreetMap tiles@home, compress module
9# Takes any tiles generated and adds them into ZIP files
10#
11# Contact OJW on the Openstreetmap wiki for help using this program
12#-----------------------------------------------------------------------------
13# Copyright 2006, Oliver White, Dirk-Lueder Kreie
14#
15# This program is free software; you can redistribute it and/or
16# modify it under the terms of the GNU General Public License
17# as published by the Free Software Foundation; either version 2
18# of the License, or (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program; if not, write to the Free Software
27# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
28#-----------------------------------------------------------------------------
29
30# conf file, will contain username/password and environment info
31my %Config = ReadConfig("tilesAtHome.conf", "general.conf", "authentication.conf", "layers.conf");
32
33if ($Config{"LocalSlippymap"})
34{
35    print "No upload - LocalSlippymap set in config file\n";
36    exit 1;
37}
38
39my $ZipFileCount = 0;
40
41## FIXME: this is one of the things that make upload.pl not multithread safe
42my $ZipDir = $Config{WorkingDirectory} . "/uploadable";
43
44my @sorted;
45
46# when called from tilesGen, use these for nice display
47my $progress = 0;
48my $progressPercent = 0;
49my $progressJobs = $ARGV[1] or 1;
50my $currentSubTask = "ziprun" . $ARGV[0] or " ";
51
52
53my $lastmsglen;
54
55### TODO: implement locking, this is one of the things that make compress.pl not multithread-safe.
56
57my $tileCount;
58
59my @tiles;
60
61## TODO: re-implement MultipleClients so it shoves the completed zips onto the upload computer instead of the single tiles.
62
63if($Config{MultipleClients}) #Trigger the _other_ codepath...
64# move the finished tiles to a subfolder of UploadDirectory
65# First name the folder timestamp_hostname_inprogress
66# then rename the folder to timestamp_hostname
67{
68    print "MultipleClients config option is deprecated, use UploadToDirectory instead.\n\n";
69    my $epochtime = time;
70    my $hostname = `hostname`;
71    chomp $hostname;
72    $hostname.="XXXXXXXX";
73    my $UploadDir = $Config{UploadDirectory};
74    my $WorkDir = $Config{WorkingDirectory};
75    my $folder = sprintf("%s/%s_%s_%d", $UploadDir, $epochtime, substr($hostname,0,8),$$);
76    while(-e $folder)  # avoid the improbable... the folder exists.
77    {
78        $folder .= "x";
79    }
80    my $inprogress = $folder."_inprogress";
81    print "Making dir\n";
82    mkdir($inprogress);
83    print "Moving to progress\n";
84    for my $tilefile ( glob "$WorkDir/*" ) 
85    {
86         next if -d $tilefile; # skip folders
87         print "Moving $tilefile to $inprogress\n";
88         move($tilefile,$inprogress) or die "$!\n";
89    } 
90    print "Rename progress dir\n";
91    move("$folder"."_inprogress",$folder) or die "$!\n"; 
92 
93    # the files should be on the upload computer now!!!
94}
95else
96{
97
98   
99    # We calculate % differently this time so we don't need "progress"
100    # $progress = 0;
101   
102    $progressPercent = 0;
103   
104    my $allowedPrefixes;
105   
106    my $TileDir = $Config{WorkingDirectory};
107   
108    # Group and upload the tiles
109    statusMessage("Searching for tiles in $TileDir", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
110    # compile a list of the "Prefix" values of all configured layers,
111    #     # separated by |
112   
113    foreach my $UploadLayer (split(/,/, $Config{"Layers"}))
114    {
115        $allowedPrefixes = $Config{"Layer.$UploadLayer.Prefix"}; #just select the current layer for compressing
116        ## DEBUG print "\n.$allowedPrefixes.\n";
117        opendir(my $dp, $TileDir) or die("Can't open directory $TileDir\n");
118        my @dir = readdir($dp);
119        @tiles = grep { /($allowedPrefixes)_\d+_\d+_\d+\.png$/ } @dir;
120        my @tilesets = grep { /($allowedPrefixes)_\d+_\d+_\d+\.dir$/ } @dir;
121        closedir($dp);
122       
123        foreach (@tilesets)        # not split into chunks
124        {
125            my $set = "$TileDir/$_";
126            $set =~ s|\.dir$||;
127            if (rename "$set.dir", "$set.upload") 
128            {
129                compress("$set.upload", $ZipDir, 'yes', $allowedPrefixes);
130                rmdir "$set.upload";    # should be empty now
131            }
132            else
133            {
134                print STDERR "ERROR\n  Failed to rename $set.dir to $set.upload --tileset not uploaded\n";
135            }
136        }
137
138        ## look again in the workdir, there might be new files from split tilesets:
139       
140        opendir($dp, $TileDir) or die("Can't open directory $TileDir\n");
141        @dir = readdir($dp);
142        @tiles = grep { /($allowedPrefixes)_\d+_\d+_\d+\.png$/ } @dir;
143        closedir($dp);
144       
145        $tileCount = scalar(@tiles);
146       
147        if ($tileCount) 
148        {
149            while (processTileBatch(
150              $TileDir, 
151              $TileDir . "/gather", ## FIXME: this is one of the things that make compress.pl not multithread safe
152              $ZipDir, 
153              $allowedPrefixes)) 
154            {};
155        }
156        statusMessage("done", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent, 0); 
157        ## TODO: fix progress display
158    }
159} #done main/else.
160
161#-----------------------------------------------------------------------------
162# Moves tiles into a "gather" directory until a certain size is reached,
163# then compress and upload those files
164#-----------------------------------------------------------------------------
165sub processTileBatch
166{
167    my ($TileDir, $TempDir, $OutputDir, $allowedPrefixes) = @_;
168    my ($Size,$Count) = (0,0);
169    my $MB = 1024*1024;
170    my $SizeLimit = $Config{"UploadChunkSize"} * $MB;
171    my $CountLimit = $Config{"UploadChunkCount"};
172
173    #prevent too small zips, 683=half a tileset
174    $CountLimit = 683 if ($CountLimit < 100);
175
176    mkdir $TempDir if ! -d $TempDir;
177    mkdir $OutputDir if ! -d $OutputDir;
178
179    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount;
180    statusMessage(scalar(@tiles)." tiles to process for ".$allowedPrefixes, $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
181
182    while(($Size < $SizeLimit) && ($Count < $CountLimit) && (my $file = shift @tiles))
183    {
184        my $Filename1 = "$TileDir/$file";
185        my $Filename2 = "$TempDir/$file";
186        if($file =~ /($allowedPrefixes)_\d+_\d+_\d+\.png$/i)
187        {
188            $Size += -s $Filename1;
189            $Count++;
190
191            rename($Filename1, $Filename2);
192        }
193    }
194
195    $progressPercent = ( $tileCount - scalar(@tiles) ) * 100 / $tileCount; 
196   
197    if($Count)
198    {
199        statusMessage(sprintf("Got %d files (%d bytes), compressing", $Count, $Size), $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
200        return compress($TempDir, $OutputDir, 'no', $allowedPrefixes);
201    }
202    else
203    {
204
205        statusMessage("compress finished", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
206        return 0;
207    }
208}
209
210#-----------------------------------------------------------------------------
211# Compress all PNG files from one directory, creating
212#-----------------------------------------------------------------------------
213sub compress
214{
215    my ($Dir, $OutputDir, $SingleTileset, $Layer) = @_;
216    $SingleTileset = ($SingleTileset eq 'yes' ? '_tileset' : '');
217 
218    my $Filename;
219
220    my $epochtime = time;
221 
222    # Create the output directory if it doesn't exist...
223    if( ! -e $OutputDir )
224    {
225        mkdir $OutputDir;
226    }
227   
228    if($Config{UseHostnameInZipname})
229    {
230        my $hostname = `hostname`."XXXXX";
231        $Filename = sprintf("%s/%d_%s_%d_%d_%s%s.zip", $OutputDir, $epochtime,
232          substr($hostname,0,5), $$, $ZipFileCount++, $Layer, $SingleTileset);
233    }
234    else 
235    {
236        $Filename = sprintf("%s/%d_%d_%d_%s%s.zip", $OutputDir,
237          $epochtime, $$, $ZipFileCount++, $Layer, $SingleTileset);
238    }
239   
240    # ZIP all the tiles into a single file
241    my $stdOut = $Config{WorkingDirectory}."/".$PID.".stdout";
242    my $Command1;
243    if ($Config{"7zipWin"})
244    {
245        $Command1 = sprintf("\"%s\" %s %s %s",
246          $Config{"Zip"},
247          "a -tzip",
248          $Filename,
249          "$Dir/*.png");
250    }
251    else
252    {
253        $Command1 = sprintf("\"%s\" -r -j %s %s > %s",
254          $Config{"Zip"},
255          $Filename,
256          "$Dir",
257          $stdOut);
258    }
259   
260    ## FIXME: this is one of the things that make compress.pl not multithread safe
261    # Delete files in the gather directory
262    opendir (GATHERDIR, $Dir);
263    my @zippedFiles = grep { /.png$/ } readdir(GATHERDIR);
264    closedir (GATHERDIR);
265   
266    # Run the two commands
267    if (runCommand($Command1,$PID)) 
268    {
269        killafile($stdOut);
270        while(my $File = shift @zippedFiles)
271        {
272            killafile ($Dir . "/" . $File);
273        }
274    }
275    else
276    {
277        while(my $File = shift @zippedFiles)
278        {
279            rename($Dir . "/" . $File, $Config{WorkingDirectory} . $File);
280        }
281    }
282   
283    my $ZipSize += -s $Filename; ## Add the 10 MB check here.
284    if($ZipSize > $Config{ZipHardLimit} * 1000 * 1000) 
285    {
286        statusMessage($Filename." is larger than ".$Config{ZipHardLimit}." MB, retrying as split tileset.", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
287        runCommand("unzip -qj $Filename -d $Config{WorkingDirectory}",$PID);
288
289        if($Config{DeleteZipFilesAfterUpload})
290        {
291            unlink($Filename);
292        }
293        else
294        {
295            rename($Filename, $Filename."_oversized"); 
296        }
297
298    }
299
300    return 1;
301}
Note: See TracBrowser for help on using the repository browser.