source: subversion/applications/rendering/tilesAtHome/upload.pl @ 3643

Revision 3643, 8.5 KB checked in by deelkar, 7 years ago (diff)

move ZipHardLimit? (server limit) to general.conf to have in one central place

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2use strict;
3use FindBin qw($Bin);
4use LWP::UserAgent;
5use File::Copy;
6use English '-no_match_vars';
7use tahconfig;
8use tahlib;
9#-----------------------------------------------------------------------------
10# OpenStreetMap tiles@home, upload module
11# Takes any tiles generated, adds them into ZIP files, and uploads them
12#
13# Contact OJW on the Openstreetmap wiki for help using this program
14#-----------------------------------------------------------------------------
15# Copyright 2006, Oliver White
16#
17# This program is free software; you can redistribute it and/or
18# modify it under the terms of the GNU General Public License
19# as published by the Free Software Foundation; either version 2
20# of the License, or (at your option) any later version.
21#
22# This program is distributed in the hope that it will be useful,
23# but WITHOUT ANY WARRANTY; without even the implied warranty of
24# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
25# GNU General Public License for more details.
26#
27# You should have received a copy of the GNU General Public License
28# along with this program; if not, write to the Free Software
29# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
30#-----------------------------------------------------------------------------
31
32# conf file, will contain username/password and environment info
33my %Config = ReadConfig("tilesAtHome.conf", "general.conf", "authentication.conf", "layers.conf");
34
35if ($Config{"LocalSlippymap"})
36{
37    print "No upload - LocalSlippymap set in config file\n";
38    exit 1;
39}
40
41my $ZipFileCount = 0;
42
43## FIXME: this is one of the things that make upload.pl not multithread safe
44my $ZipDir = $Config{WorkingDirectory} . "/uploadable";
45
46my @sorted;
47
48# when called from tilesGen, use these for nice display
49my $progress = 0;
50my $progressPercent = 0;
51my $progressJobs = $ARGV[0] or 1;
52my $currentSubTask;
53 
54my $lastmsglen;
55
56### TODO: implement locking, this is one of the things that make upload not multithread-safe.
57my $sleepdelay;
58my $failFile = $Config{WorkingDirectory} . "/failurecount.txt";
59if (open(FAILFILE, "<", $failFile))
60{
61    $sleepdelay = <FAILFILE>;
62    chomp $sleepdelay;
63    close FAILFILE;
64}
65elsif (open(FAILFILE, ">", $failFile))
66{
67    $sleepdelay = 0; 
68    print FAILFILE $sleepdelay;
69    close FAILFILE;
70}
71
72compress(1); ## first run
73
74# Upload any ZIP files which are still waiting to go
75if(opendir(ZIPDIR, $ZipDir))
76{
77    processOldZips(1);
78}
79
80# We might have created lots of single tiles if some tileset zips were larger than 10 MB, so re-check here
81
82compress(2); ## second (and last) run.
83
84# Do we have new zips? (try to) upload them all!
85
86if(opendir(ZIPDIR, $ZipDir))
87{
88    processOldZips(2);
89}
90
91## update the failFile with current failure count from processOldZips
92
93if (open(FAILFILE, ">", $failFile))
94{
95    print FAILFILE $sleepdelay;
96    close FAILFILE;
97}
98
99## end main
100
101sub processOldZips
102{
103    my ($runNumber) = @_;
104    $currentSubTask = "upload" . $runNumber;
105    $progress = 0;
106    my @zipfiles = grep { /\.zip$/ } readdir(ZIPDIR);
107    close ZIPDIR;
108    @sorted = sort { $a cmp $b } @zipfiles; # sort by ASCII value (i.e. upload oldest first if timestamps used)
109    my $zipCount = scalar(@sorted);
110    statusMessage(scalar(@sorted)." zip files to upload", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
111    my $Reason = "queue full";
112    while(my $File = shift @sorted)
113    {
114        if($File =~ /\.zip$/i)
115        {
116           
117            my $FailureMode = 0; # 0 ->hard failure (i.e. Err503 on upload),
118                                 # 1 ->no failure,
119                                 # 10..1000 ->soft failure (with load% * 10)
120            while ($FailureMode != 1) # while not upload success or complete failure
121            {
122                $FailureMode = upload("$ZipDir/$File");
123
124                if ($FailureMode >= 10)
125                {
126                    $sleepdelay = 1.25 * $sleepdelay * (1.25 * ($FailureMode/1000)); ## 1.25 * 0.8 = 1 -> try to keep the queue at 80% full, if more increase sleepdelay by 25% plus the amount the queue is too full.
127                    $Reason = "queue full";
128                }
129                elsif ($FailureMode == 1) ## success
130                {
131                    $sleepdelay = 0.75 * $sleepdelay; # reduce sleepdelay by 25%
132                    $Reason = "uploaded ".$File;
133                }
134                elsif ($FailureMode == 0) ## hard fail
135                {
136                    last;
137                }
138                $sleepdelay = int($sleepdelay) + 1; 
139                if ($sleepdelay > 600)  ## needs adjusting based on real-world experience, if this check is true the above load adapting failed and the server is too overloaded to reasonably process the queue relative to the rendering speed
140                {
141                    $sleepdelay = 600; ## FIXME: since the checking of the queue is much less costly than trying to upload, need to further adapt the max delay.
142                }
143
144                statusMessage($Reason.", sleeping for " . $sleepdelay . " seconds", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
145                sleep ($sleepdelay);
146            }
147
148        }
149        $progress++;
150        $progressPercent = $progress * 100 / $zipCount;
151        statusMessage(scalar(@sorted)." zip files left to upload", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
152       
153    }
154}
155
156#-----------------------------------------------------------------------------
157# Upload a ZIP file
158#-----------------------------------------------------------------------------
159sub upload
160{
161    my ($File) = @_;
162    my $ZipSize += -s $File;
163    if($ZipSize > $Config{ZipHardLimit} * 1000 * 1000) 
164    {
165        statusMessage("zip is larger than ".$Config{ZipHardLimit}." MB, retrying as split tileset.", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,1);
166        runCommand("unzip -qj $File -d $Config{WorkingDirectory}",$PID);
167
168        if($Config{DeleteZipFilesAfterUpload})
169        {
170            unlink($File);
171        }
172        else
173        {
174            rename($File, $File."_oversized"); 
175        }
176
177        return 0;
178    }
179    my $SingleTileset = ($File =~ /_tileset\.zip/) ? 'yes' : 'no';
180   
181    my $ua = LWP::UserAgent->new(keep_alive => 1, timeout => 360);
182
183    $ua->protocols_allowed( ['http'] );
184    $ua->agent("tilesAtHomeZip");
185    $ua->env_proxy();
186   
187    my $Password = join("|", ($Config{UploadUsername}, $Config{UploadPassword}));
188    my $URL = $Config{"UploadURL2"};
189   
190    my ($UploadToken,$Load) = UploadOkOrNot();
191   
192    if ($UploadToken) 
193    {
194        statusMessage("Uploading $File", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
195        my $res = $ua->post($URL,
196          Content_Type => 'form-data',
197          Content => [ file => [$File], 
198          mp => $Password,
199          version => $Config{ClientVersion},
200          single_tileset => $SingleTileset,
201          token => $UploadToken ]);
202     
203        if(!$res->is_success())
204        {
205            print STDERR "ERROR\n";
206            print STDERR "  Error uploading $File to $URL:\n";
207            print STDERR "  ".$res->status_line."\n";
208            return 0; # hard fail
209        } 
210   
211        if($Config{DeleteZipFilesAfterUpload})
212        {
213            unlink($File);
214        }
215        else
216        {
217            rename($File, $File."_uploaded");
218        }
219    }
220    else
221    {
222        return $Load; #soft fail
223    }
224   
225    return 1;
226}
227
228sub compress
229{
230    ## Run compress directly because it uses same messaging as tilesGen.pl and upload.pl
231    ## no need to hide output at all.
232
233    my ($runNumber) = @_;
234
235    my $UploadScript = "$Bin/compress.pl $runNumber $progressJobs";
236    my $retval = system($UploadScript);
237    return $retval;
238}
239
240
241sub UploadOkOrNot
242{
243    my $LocalFilename = $Config{WorkingDirectory} . "/go-nogo-".$PID.".tmp";
244    statusMessage("Checking server queue", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
245    DownloadFile($Config{GoNogoURL}, $LocalFilename, 1);
246    open(my $fp, "<", $LocalFilename) || return;
247    my $Load = <$fp>; ##read first line from file
248    my $Token = <$fp>; ##read another line from file
249    chomp $Load;
250    chomp $Token;
251    close $fp;
252    killafile($LocalFilename);
253    $Load=1-$Load;
254    ##DEBUG print STDERR "\nLoad: $Load \n";
255    # $Token=1 if (! $Token);
256    if ($Load > 0.8) 
257    {
258        statusMessage("Not uploading, server queue full", $Config{Verbose}, $currentSubTask, $progressJobs, $progressPercent,0);
259        sleep(1);
260        return (0,$Load*1000);
261    }
262    else
263    {
264        #DEBUG: print STDERR "\n $Token\n";
265        return ($Token,$Load*1000);
266    }
267}
Note: See TracBrowser for help on using the repository browser.