source: subversion/utils/tilesAtHome/upload.pl @ 2061

Revision 2061, 5.0 KB checked in by dirkl, 7 years ago (diff)

added patch by mcn to loop upload.pl until all tiles are up or an error occurs

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2use strict;
3use LWP::UserAgent;
4use File::Copy;
5#-----------------------------------------------------------------------------
6# OpenStreetMap tiles@home, upload module
7# Takes any tiles generated, adds them into ZIP files, and uploads them
8#
9# Contact OJW on the Openstreetmap wiki for help using this program
10#-----------------------------------------------------------------------------
11# Copyright 2006, Oliver White
12#
13# This program is free software; you can redistribute it and/or
14# modify it under the terms of the GNU General Public License
15# as published by the Free Software Foundation; either version 2
16# of the License, or (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
26#-----------------------------------------------------------------------------
27
28# conf file, will contain username/password and environment info
29my %Config;
30my $ZipFileCount = 0;
31ReadConfigFile("tilesAtHome.conf");
32
33my $ZipDir = $Config{WorkingDirectory} . "/uploadable";
34
35# Upload any ZIP files which are still waiting to go
36if(opendir(ZIPDIR, $ZipDir)){
37  while(my $File = readdir(ZIPDIR)){
38    if($File =~ /\.zip$/i){
39      upload("$ZipDir/$File");
40    }
41  }
42  close ZIPDIR;
43}
44
45# Group and upload the tiles
46while (uploadTileBatch(
47  $Config{WorkingDirectory}, 
48  $Config{WorkingDirectory} . "/gather", 
49  $ZipDir)) {};
50
51#-----------------------------------------------------------------------------
52# Moves tiles into a "gather" directory until a certain size is reached,
53# then compress and upload those files
54#-----------------------------------------------------------------------------
55sub uploadTileBatch(){
56  my ($TileDir, $TempDir, $OutputDir) = @_;
57  my ($Size,$Count) = (0,0);
58  my $MB = 1024*1024;
59  my $SizeLimit = $Config{"UploadChunkSize"} * $MB;
60  my $CountLimit = $Config{"UploadChunkCount"};
61
62  #prevent too small zips, 683=half a tileset
63  $CountLimit = 683 if ($CountLimit < 100);
64
65  mkdir $TempDir if ! -d $TempDir;
66  mkdir $OutputDir if ! -d $OutputDir;
67 
68  print "Searching for tiles in $TileDir\n";
69  opendir(my $dp, $TileDir) || return;
70  while((my $file = readdir($dp)) && ($Size < $SizeLimit) && ($Count < $CountLimit)){
71    my $Filename1 = "$TileDir/$file";
72    my $Filename2 = "$TempDir/$file";
73    if($file =~ /tile_\d+_\d+_\d+\.png$/i){
74      $Size += -s $Filename1;
75      $Count++;
76     
77      rename($Filename1, $Filename2);
78    }
79  }
80  closedir($dp);
81 
82  if($Count){
83    printf("Got %d files (%d bytes), compressing...", $Count, $Size);
84    return compressTiles($TempDir, $OutputDir);
85  }
86  else
87  {
88    print "Finished.\n";
89    return 0;
90  }
91}
92
93#-----------------------------------------------------------------------------
94# Compress all PNG files from one directory, creating
95#-----------------------------------------------------------------------------
96sub compressTiles(){
97  my ($Dir, $OutputDir) = @_;
98 
99  my $Filename;
100
101  my $epochtime = time;
102 
103  if($Config{UseHostnameInZipname}){
104      my $hostname = `hostname`."XXX";
105      $Filename = sprintf("%s/%s_%d_%d.zip", $OutputDir, substr($hostname,0,3), $$, $ZipFileCount++);
106  } else {
107      $Filename = sprintf("%s/%d_%d_%d.zip", $OutputDir, $epochtime, $$, $ZipFileCount++);
108  }
109 
110  # ZIP all the tiles into a single file
111  my $Command1 = sprintf("%s %s %s",
112    "zip",
113    $Filename,
114    "$Dir/*");
115  # ZIP filename is currently our process ID - change this if one process
116  # becomes capable of generating multiple zip files
117 
118  # Delete files in the gather directory
119  my $Command2 = sprintf("%s %s",
120    "rm",
121    "$Dir/*.png");
122   
123  # Run the two commands
124  `$Command1`;
125  `$Command2`;
126 
127  return upload($Filename);
128}
129
130#-----------------------------------------------------------------------------
131# Upload a ZIP file
132#-----------------------------------------------------------------------------
133sub upload(){
134  my ($File) = @_;
135 
136  my $ua = LWP::UserAgent->new(keep_alive => 1, timeout => 120);
137
138  $ua->protocols_allowed( ['http'] );
139  $ua->agent("tilesAtHomeZip");
140 
141  my $Password = join("|", ($Config{UploadUsername}, $Config{UploadPassword}));
142  my $URL = $Config{"UploadURL2"};
143 
144  print "Uploading \n  $File\n  to $URL\n";
145  my $res = $ua->post($URL,
146    Content_Type => 'form-data',
147    Content => [ file => [$File], mp => $Password]);
148   
149  if(!$res->is_success()){
150    print("Error uploading file");
151    return 0;
152  } 
153 
154  if($Config{DeleteZipFilesAfterUpload}){
155    unlink($File);
156  }
157  else
158  {
159    rename($File, $File."_uploaded");
160  }
161 
162  return 1;
163}
164
165sub ReadConfigFile(){
166  open(my $fp, "<", shift()) || die("Can't open config file\n");
167  foreach my $Line(<$fp>){
168    $Line =~ s/\s*#.*$//; # Strip out comments
169    if($Line =~ /(\w+)=(.*)/){
170      $Config{$1} = $2;
171    }
172  }
173}
174
Note: See TracBrowser for help on using the repository browser.