source: subversion/utils/tilesAtHome/upload.pl @ 2061

Last change on this file since 2061 was 2061, checked in by dirkl, 10 years ago

added patch by mcn to loop upload.pl until all tiles are up or an error occurs

  • Property svn:executable set to *
File size: 5.0 KB
Line 
1#!/usr/bin/perl
2use strict;
3use LWP::UserAgent;
4use File::Copy;
5#-----------------------------------------------------------------------------
6# OpenStreetMap tiles@home, upload module
7# Takes any tiles generated, adds them into ZIP files, and uploads them
8#
9# Contact OJW on the Openstreetmap wiki for help using this program
10#-----------------------------------------------------------------------------
11# Copyright 2006, Oliver White
12#
13# This program is free software; you can redistribute it and/or
14# modify it under the terms of the GNU General Public License
15# as published by the Free Software Foundation; either version 2
16# of the License, or (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
26#-----------------------------------------------------------------------------
27
28# conf file, will contain username/password and environment info
29my %Config;
30my $ZipFileCount = 0;
31ReadConfigFile("tilesAtHome.conf");
32
33my $ZipDir = $Config{WorkingDirectory} . "/uploadable";
34
35# Upload any ZIP files which are still waiting to go
36if(opendir(ZIPDIR, $ZipDir)){
37  while(my $File = readdir(ZIPDIR)){
38    if($File =~ /\.zip$/i){
39      upload("$ZipDir/$File");
40    }
41  }
42  close ZIPDIR;
43}
44
45# Group and upload the tiles
46while (uploadTileBatch(
47  $Config{WorkingDirectory}, 
48  $Config{WorkingDirectory} . "/gather", 
49  $ZipDir)) {};
50
51#-----------------------------------------------------------------------------
52# Moves tiles into a "gather" directory until a certain size is reached,
53# then compress and upload those files
54#-----------------------------------------------------------------------------
55sub uploadTileBatch(){
56  my ($TileDir, $TempDir, $OutputDir) = @_;
57  my ($Size,$Count) = (0,0);
58  my $MB = 1024*1024;
59  my $SizeLimit = $Config{"UploadChunkSize"} * $MB;
60  my $CountLimit = $Config{"UploadChunkCount"};
61
62  #prevent too small zips, 683=half a tileset
63  $CountLimit = 683 if ($CountLimit < 100);
64
65  mkdir $TempDir if ! -d $TempDir;
66  mkdir $OutputDir if ! -d $OutputDir;
67 
68  print "Searching for tiles in $TileDir\n";
69  opendir(my $dp, $TileDir) || return;
70  while((my $file = readdir($dp)) && ($Size < $SizeLimit) && ($Count < $CountLimit)){
71    my $Filename1 = "$TileDir/$file";
72    my $Filename2 = "$TempDir/$file";
73    if($file =~ /tile_\d+_\d+_\d+\.png$/i){
74      $Size += -s $Filename1;
75      $Count++;
76     
77      rename($Filename1, $Filename2);
78    }
79  }
80  closedir($dp);
81 
82  if($Count){
83    printf("Got %d files (%d bytes), compressing...", $Count, $Size);
84    return compressTiles($TempDir, $OutputDir);
85  }
86  else
87  {
88    print "Finished.\n";
89    return 0;
90  }
91}
92
93#-----------------------------------------------------------------------------
94# Compress all PNG files from one directory, creating
95#-----------------------------------------------------------------------------
96sub compressTiles(){
97  my ($Dir, $OutputDir) = @_;
98 
99  my $Filename;
100
101  my $epochtime = time;
102 
103  if($Config{UseHostnameInZipname}){
104      my $hostname = `hostname`."XXX";
105      $Filename = sprintf("%s/%s_%d_%d.zip", $OutputDir, substr($hostname,0,3), $$, $ZipFileCount++);
106  } else {
107      $Filename = sprintf("%s/%d_%d_%d.zip", $OutputDir, $epochtime, $$, $ZipFileCount++);
108  }
109 
110  # ZIP all the tiles into a single file
111  my $Command1 = sprintf("%s %s %s",
112    "zip",
113    $Filename,
114    "$Dir/*");
115  # ZIP filename is currently our process ID - change this if one process
116  # becomes capable of generating multiple zip files
117 
118  # Delete files in the gather directory
119  my $Command2 = sprintf("%s %s",
120    "rm",
121    "$Dir/*.png");
122   
123  # Run the two commands
124  `$Command1`;
125  `$Command2`;
126 
127  return upload($Filename);
128}
129
130#-----------------------------------------------------------------------------
131# Upload a ZIP file
132#-----------------------------------------------------------------------------
133sub upload(){
134  my ($File) = @_;
135 
136  my $ua = LWP::UserAgent->new(keep_alive => 1, timeout => 120);
137
138  $ua->protocols_allowed( ['http'] );
139  $ua->agent("tilesAtHomeZip");
140 
141  my $Password = join("|", ($Config{UploadUsername}, $Config{UploadPassword}));
142  my $URL = $Config{"UploadURL2"};
143 
144  print "Uploading \n  $File\n  to $URL\n";
145  my $res = $ua->post($URL,
146    Content_Type => 'form-data',
147    Content => [ file => [$File], mp => $Password]);
148   
149  if(!$res->is_success()){
150    print("Error uploading file");
151    return 0;
152  } 
153 
154  if($Config{DeleteZipFilesAfterUpload}){
155    unlink($File);
156  }
157  else
158  {
159    rename($File, $File."_uploaded");
160  }
161 
162  return 1;
163}
164
165sub ReadConfigFile(){
166  open(my $fp, "<", shift()) || die("Can't open config file\n");
167  foreach my $Line(<$fp>){
168    $Line =~ s/\s*#.*$//; # Strip out comments
169    if($Line =~ /(\w+)=(.*)/){
170      $Config{$1} = $2;
171    }
172  }
173}
174
Note: See TracBrowser for help on using the repository browser.