source: subversion/applications/rendering/tilesAtHome/lib/Upload.pm @ 12207

Last change on this file since 12207 was 11950, checked in by matthiasj, 12 years ago

merge r11949 from _unstable

  • Property svn:executable set to *
File size: 10.3 KB
Line 
1package Upload;
2
3use warnings;
4use strict;
5use LWP::UserAgent;
6use File::Copy;
7use File::Spec;
8use Fcntl ':flock'; #import LOCK_* constants
9use English '-no_match_vars';
10use Error qw(:try);
11use tahlib;
12use TahConf;
13
14#-----------------------------------------------------------------------------
15# OpenStreetMap tiles@home, upload module
16# Takes any tiles generated, adds them into ZIP files, and uploads them
17#
18# Contact OJW on the Openstreetmap wiki for help using this program
19#-----------------------------------------------------------------------------
20# Copyright 2006, Oliver White, Dirk-Lueder Kreie, Sebastian Spaeth
21#
22# This program is free software; you can redistribute it and/or
23# modify it under the terms of the GNU General Public License
24# as published by the Free Software Foundation; either version 2
25# of the License, or (at your option) any later version.
26#
27# This program is distributed in the hope that it will be useful,
28# but WITHOUT ANY WARRANTY; without even the implied warranty of
29# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
30# GNU General Public License for more details.
31#
32# You should have received a copy of the GNU General Public License
33# along with this program; if not, write to the Free Software
34# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
35#-----------------------------------------------------------------------------
36
37
38#-------------------------------------------------------------------
39# Create a new Upload instance
40#-------------------------------------------------------------------
41sub new
42{
43    my $class = shift;
44    my $self  = {};
45
46    $self = {
47        Config  => TahConf->getConfig(),
48    };
49    $self->{TileDir} = $self->{Config}->get("WorkingDirectory"),
50    $self->{ZipDir}  = File::Spec->catdir($self->{Config}->get("WorkingDirectory"), "/uploadable"),
51
52    bless ($self, $class);
53
54    $self->{sleepdelay} = ::getFault('upload') * 10;
55    # uploading to a local directory is less costly
56    $self->{MaxSleep} = $self->{Config}->get("UploadToDirectory") ? 30 : 600;
57
58    #set global progressbar task
59    $::currentSubTask ='upload';
60    return $self;
61}
62
63
64
65#-------------------------------------------------------------------
66# Returns number of uploaded files.
67#-------------------------------------------------------------------
68sub uploadAllZips
69{
70    my $self = shift;
71    my $Config = $self->{Config};
72    $::progressPercent = 0;
73    my $LOCKFILE;
74
75    if ($Config->get("LocalSlippymap")) {
76        throw UploadError "No upload - LocalSlippymap set in config file";
77    }
78
79    # read in all the zip files in ZipDir
80    my @files;
81    if (opendir(ZIPDIR, $self->{ZipDir})) {
82        @files = grep { /\.(zip|tileset)$/ } readdir(ZIPDIR);
83        closedir ZIPDIR;
84    }
85    else {
86        return 0; # do nothing if ZipDir doesn't exist, just assume there is nothing to upload
87    }
88    my $file_count = scalar(@files);
89
90    my $files_uploaded = 0; # num handled files
91    foreach my $file (@files) {
92        ::statusMessage(($file_count - $files_uploaded) . " files to upload", 0, 0);
93        # get a file handle, then try to lock the file exclusively.
94        # if flock fails it is being handled by a different upload process
95        # also check if the file still exists when we get to it
96        open ($LOCKFILE, '>', File::Spec->join($self->{ZipDir},$file . ".lock"));
97        my $flocked = !$Config->get('flock_available')
98                      || ($LOCKFILE && flock($LOCKFILE, LOCK_EX|LOCK_NB));
99        if ($flocked && -e File::Spec->join($self->{ZipDir}, $file)) {
100            # got exclusive lock, now upload
101
102            my $UploadFailedHardOrDone=0;
103            # while not upload success or complete failure
104            while ($UploadFailedHardOrDone != 1) {
105                my $res_str; #stores success or error msg for status line
106                try {
107                    $self->upload($file);
108
109                    # reset sleepdelay
110                    $self->{sleepdelay} = 0;
111                    $res_str = "uploaded " . $file;
112                    $files_uploaded++;
113                    $::progressPercent = $files_uploaded * 100 / $file_count;
114                    $UploadFailedHardOrDone = 1;
115                }
116                catch UploadError with {
117                    my $err = shift();
118                    if (my $queue_length = $err->value()) { 
119                        # set sleepdelay to 30 seconds
120                        $self->{sleepdelay} = 30;
121                        $res_str = $err->text();
122                    }
123                    else {
124                        $err->throw();
125                    }
126                };
127
128                # Finally wait sleepdelay seconds until next upload
129                ::talkInSleep($res_str, int($self->{sleepdelay}));
130            }
131
132        }
133        else {
134            # could not get exclusive lock, this is being handled elsewhere now
135            ::statusMessage("$file uploaded by different process. skipping", 0, 3);
136        }
137        # finally unlock zipfile and release handle
138        if ($LOCKFILE) {
139            flock ($LOCKFILE, LOCK_UN);
140            close ($LOCKFILE);
141            unlink(File::Spec->join($self->{ZipDir}, $file . ".lock")) if $flocked;
142        }
143    }
144    if($files_uploaded)
145    {
146      ::statusMessage("uploaded $files_uploaded ". ($files_uploaded == 1 ? "file" :"files"),1,3);
147    }
148    elsif($file_count) # we need to print something to get a line end
149    {
150      ::statusMessage("Nothing uploaded",1,3);
151    }
152
153    return $files_uploaded;
154}
155
156#-----------------------------------------------------------------------------
157# Upload a ZIP or tileset file
158# Parameter (filename) is the name of a file in ZipDir
159#-----------------------------------------------------------------------------
160sub upload
161{
162    my $self = shift;
163    my $file_name = shift;
164    my $file      = File::Spec->join($self->{ZipDir}, $file_name);
165    my $file_size = -s $file;   # zip file size
166    my $file_age  = -M $file;   # days since last modified
167    my $Config = $self->{Config};
168
169    # delete zips that are already older than 2 days.
170    if ($file_age > 2) {
171        if ($Config->get("DeleteZipFilesAfterUpload")) {
172            unlink($file);
173        }
174        else {
175            rename($file, $file . "_overage"); 
176        }
177
178        throw UploadError "File $file too old", "overage";
179    }
180
181    $file_name =~ m{^([^_]+)_(\d+)_(\d+)_(\d+)_(\d+)\.(zip|tileset)$};
182    my $layer = $1;
183    my $zoom = $2;
184    my $x = $3;
185    my $y = $4;
186    my $client_id = $5;
187
188    if (! $Config->get("UploadToDirectory")) {
189        my $ua = LWP::UserAgent->new(keep_alive => 1, timeout => 360);
190       
191        $ua->protocols_allowed( ['http'] );
192        $ua->agent("tilesAtHomeZip");
193        $ua->env_proxy();
194        push @{ $ua->requests_redirectable }, 'POST';
195       
196        my $URL = $Config->get("UploadURL");
197       
198        ::statusMessage("Uploading $file_name", 0, 3);
199        my $res = $ua->post($URL,
200                            Content_Type => 'form-data',
201                            Content => [ file => [$file],
202                                         user => $Config->get("UploadUsername"),
203                                         passwd => $Config->get("UploadPassword"),
204                                         version => $Config->get("ClientVersion"),
205                                         layer => $layer,
206                                         z => $zoom,
207                                         x => $x,
208                                         y => $y,
209                                         client_uuid => ($::Mode eq "upload_loop") ? $client_id : ::GetClientId() ]);
210
211        if (!$res->is_success()) {
212            throw UploadError "Error uploading $file_name to $URL: " . $res->status_line, "ServerError"; # hard fail
213        }
214        else {
215            print $res->content if ($Config->get("Debug"));
216        }
217    }
218    else {
219        #Upload To Directory rather than server
220        ## Check "queue" length
221        my @queue_files;
222        if (opendir(UPDIR, $Config->get("UploadTargetDirectory"))) {
223            @queue_files = grep { /\.(zip|tileset)$/ } readdir(UPDIR);
224            closedir UPDIR;
225        }
226        else {
227            throw UploadError "Can not open target directory";
228        }
229
230        my $queue_length = scalar(@queue_files);
231        my $max_queue = $Config->get("UploadToDirectoryMaxQueue");
232        if ($queue_length >= $max_queue) {
233            ::statusMessage("Not uploading, upload directory full",0,0);
234            sleep(1);
235            throw UploadError "Not uploading, upload directory full", $queue_length;
236        }
237        else {
238            my $tmpfile = File::Spec->join($Config->get("UploadTargetDirectory"), $file_name . "_part");
239
240            # copy the file over using a temporary name
241            copy($file, $tmpfile) or throw UploadError "Failed to copy file to Upload Directory: $!";
242            # rename so it can be picked up by central uploading client.
243            move($tmpfile, File::Spec->join($Config->get("UploadTargetDirectory"), $file_name)) 
244                or throw UploadError "Failed to rename file in Upload Directory: $!";
245        }
246    }
247
248    # if we didn't encounter any errors error we get here
249    if($Config->get("DeleteZipFilesAfterUpload")) {
250        unlink($file);
251    }
252    else {
253        rename($file, $file . "_uploaded");
254    }
255}
256
257
258#-----------------------------------------------------------
259# check the go_nogo URL and retrieve the server upload queue
260# returns the status between [0,1000] (0:empty, 1000:full)
261# returns 1000 if an error occured while fetching the load
262#-----------------------------------------------------------
263sub UploadOkOrNot
264{
265    my $self = shift;
266    my $Config = $self->{Config};
267    ::statusMessage("Checking server queue",0,3);
268    my $ua = LWP::UserAgent->new('agent' =>'tilesAtHome');
269    $ua->env_proxy();
270    my $res = $ua->get($Config->get("GoNogoURL"));
271
272    if (! $res->is_success)
273    {    # Failed to retrieve server load
274         # $res->status_line; contains result here.
275         ::statusMessage("Failed to retrieve server queue load. Assuming full queue.",1,0);
276         return 1000;
277   }
278    # Load is a float value between [0,1]
279    my $Load = $res->content;
280    chomp $Load;
281    return ($Load*1000);
282}
283
284#-----------------------------------------------------------------------------------------------------------------
285# class UploadError
286#
287# Exception to be thrown by Upload methods
288
289package UploadError;
290use base 'Error::Simple';
291
2921;
Note: See TracBrowser for help on using the repository browser.