source: subversion/sites/other/tilesAtHome/Upload/Run/index.php @ 3399

Revision 3399, 13.3 KB checked in by ojw, 7 years ago (diff)

Fix bug - mysql_close assumes only one upload per script. plus add
logging

Line 
1<?php
2# Tiles queue handler, accepts ZIP files, saves to database
3# OJW 2007, GNU GPL v2 or later
4
5# All error-messages etc are plain text for use by clients
6header("Content-type:text/plain");
7
8if(0){ // Option to turn off uploads
9  AbortWithError("Disabled");
10}
11
12include("../../lib/log.inc");
13include("../../lib/tilenames.inc");
14include("../../lib/users.inc");
15include("../../lib/versions.inc");
16include("../../lib/layers.inc");
17include("../../lib/requests.inc");
18include("../../lib/checkupload.inc");
19include("../../lib/cpu.inc");
20include("../../connect/connect.php");
21
22if(1){
23  $Load = GetLoadAvg();
24  //logMsg("$Load load", 4);
25  if($Load < 0){
26    logMsg("Load average failed", 4);
27  }
28  elseif($Load > 2.6){
29    AbortWithError("Too busy...");
30  }
31}
32
33
34$QueueDir = "/home/ojw/tiles-ojw2/Queue/";
35list($Uploads, $Tiles) = HandleNextFilesFromQueue($QueueDir, 1);
36
37logMsg(sprintf("Queue runner - done %d uploads with %d tiles", $Uploads, $Tiles), 2);
38
39
40function HandleNextFilesFromQueue($Dir, $NumToProcess){
41  $CountUploads = 0;
42  $CountTiles = 0;
43  $fp = opendir($Dir);
44  if(!$fp)
45      return;
46  while($File = readdir($fp)){
47    if($Count < $NumToProcess){
48      if(preg_match("{(\w+)\.txt}", $File, $Matches)){
49        $Name = $Matches[1];
50        printf( "\n\n===%s===\n\n", htmlentities($Name));
51        $CountTiles += HandleQueueItem($Name, $Dir);
52        $CountUploads++;
53      }
54    }
55  }
56  closedir($fp);
57  return(array($CountUploads, $CountTiles));
58}
59
60function HandleQueueItem($Name, $Dir){
61    $MetaFile = $Dir . $Name . ".txt";
62    $ZipFile = $Dir . $Name . ".zip";
63    if(!file_exists($MetaFile)){
64        print "No meta file\n";
65        return;
66    }
67    if(!file_exists($ZipFile)){
68        print "No zip file\n";
69        return;
70    }
71   
72    $Meta = MetaFileInfo($MetaFile);
73   
74    logMsg(sprintf("Doing %s by user %d version %d", $ZipFile, $Meta["user"], $Meta["version"]), 3);
75   
76    $Count = HandleUpload($ZipFile, $Meta["user"], $Meta["version"]);
77   
78    unlink($MetaFile);
79    unlink($ZipFile);
80   
81    return($Count);
82}
83function MetaFileInfo($File){
84    $fp = fopen($File, "r");
85    if(!$fp)
86      return(array("valid"=>0));
87    $Return = array();
88    while($Line = fgets($fp, 200)){
89        if(preg_match('{^(\w+)\s*=\s*(.*)$}', $Line, $Matches)){
90            $Return[$Matches[1]] = $Matches[2];
91        }
92    }
93    fclose($fp);
94    return($Return);
95   
96}
97function AbortWithError($Message){
98  logMsg("Aborted with $Message", 5);
99  printf("\n\nError: %s\n", $Message);
100  exit;
101}
102
103
104function HandleUpload($File, $UserID, $VersionID){
105  # Decide on the name of a Temporary directory
106  $Dir = TempDir();
107 
108  # Check the uploaded ZIP file
109  $Size = filesize($File);
110
111  logMsg("Handling $File ($Size bytes) by $UserID (version $VersionID)", 4);
112 
113  if($Size <= 0){
114    AbortWithError("No file uploaded or file too large");
115  }
116
117  # Keep going if the user presses stop, to ensure temporary directories get erased
118  # see also register_shutdown_function() for another option
119  ignore_user_abort();
120   
121  # Create temporary directory
122  if(!mkdir($Dir)){
123    AbortWithError("Can't create temporary directory");
124  }
125     
126  # Uncompress the uploaded tiles
127  # -j means to ignore any pathnames in the ZIP file
128  # -d $Dir specifies the directory to unzip to
129  # $Filename is the zip file
130  $Command = sprintf("unzip -j -d %s %s", $Dir, $File);
131  logMsg("Running '$Command'", 3);
132  system($Command);
133 
134  logMsg("Handling directory $Dir", 3);
135 
136  # Process all the tiles (return number of tiles done)
137  $Count = HandleDir($Dir, $UserID, $VersionID);
138       
139  # Delete the temporary directory and everything inside
140  DelDir($Dir);
141 
142  logMsg("OK, $Count tiles in upload", 3);
143  return($Count);
144}
145
146#----------------------------------------------------------------------
147# Delete the temporary directory and everything inside
148#----------------------------------------------------------------------
149function DelDir($Dir){
150  $dp = opendir($Dir);
151  while(($file = readdir($dp)) !== false){
152    if($file != "." && $file != ".."){
153      $Filename = "$Dir/$file";
154      unlink($Filename);
155    }
156  }
157  closedir($dp);
158  rmdir($Dir);
159}
160
161#----------------------------------------------------------------------
162# Processes tiles that are currently sitting in a temp directory
163#----------------------------------------------------------------------
164function HandleDir($Dir, $UserID, $VersionID){
165  $Count = 0;
166  $TileList = array();
167  $BlankTileList = array();
168
169  list($ValidTileset, $TilesetX, $TilesetY, $TilesetLayer) = CheckUploadDir($Dir);
170 
171  $dp = opendir($Dir);
172  while(($file = readdir($dp)) !== false){
173    $Filename = "$Dir/$file";
174    logMsg("Handling file $file", 6);
175    $Count += HandleFile($Filename, $VersionID, $TileList, $BlankTileList);
176  }
177  closedir($dp);
178
179  if($ValidTileset)
180    SaveTilesetMetadata($TilesetX,$TilesetY,$TilesetLayer, $UserID, $VersionID);
181  else
182    SaveMetadata($TileList, $UserID, $VersionID);
183
184  SaveBlankTiles($BlankTileList, $UserID);
185
186  return($Count);
187}
188
189#-----------------------------------------------------------------------------------
190# Save metadata for each tile in the upload
191#-----------------------------------------------------------------------------------
192function SaveMetadata($TileList, $UserID, $VersionID){
193 
194  SaveUserStats($UserID, $VersionID, count($TileList));
195 
196  RemoveFromQueue($TileList);
197 
198  # Each element in TileList is a snippet of values (x,y,z,type,size) for each tile
199  foreach($TileList as $SqlSnippet){
200   
201    // Use this line if you need access to separate fields
202    // list($X, $Y, $Z, $Layer, $Size) = explode(",", $CSV);
203
204    $Fields = "x, y, z, type, size, date, user, version, tileset";
205    $Values = sprintf("%s, now(), %d, %d, 0", $SqlSnippet, $UserID, $VersionID);
206 
207    $SQL = sprintf("replace into `tiles_meta` (%s) values (%s);", $Fields, $Values);
208    mysql_query($SQL);
209  }
210}
211
212#------------------------------------------------------------------------------------
213# Save uploaded blank tiles in the database
214#------------------------------------------------------------------------------------
215function SaveBlankTiles($BlankTileList, $UserID){
216  # Each element in BlankTileList is a snippet of values (x,y,z,type,size) for each tile
217  foreach($BlankTileList as $SqlSnippet){
218
219    // TODO: blank tiles can be z-12, which means they can fulfil a request
220    list($X, $Y, $Z, $Layer, $Type) = explode(",", $SqlSnippet);
221    if($Z == 12){
222      moveRequest($X, $Y, REQUEST_ACTIVE, REQUEST_DONE, 0);
223    }
224   
225    # Make a blank tile
226    if( $Type >= 0 )
227    {
228      $Fields = "x, y, z, layer, type, date, user";
229      $Values = sprintf("%s, now(), %d", $SqlSnippet, $UserID);
230
231      $SQL = sprintf("replace into `tiles_blank` (%s) values (%s);", $Fields, $Values);
232    }
233    else
234    {
235      # Delete a blank tile
236      $SQL = sprintf("delete from `tiles_blank` where `x`=%d AND `y`=%s AND `z`=%s AND `layer`=%d", $X, $Y, $Z, $Layer);
237    }
238    DeleteRealTile($X,$Y,$Z,$Layer);
239
240    mysql_query($SQL);
241   
242    logSqlError();
243  }
244}
245
246#------------------------------------------------------------------------------------
247# Delete a tile and its metadata (usually when a blank tile is uploaded in its place)
248#------------------------------------------------------------------------------------
249function DeleteRealTile($X,$Y,$Z,$LayerID){
250 
251  # Delete the meta database entry
252  $SQL = sprintf(
253    "DELETE FROM `tiles_meta` WHERE `x`=%d AND `y`=%d AND `z`=%d AND `type`=%d;",
254      $X,$Y,$Z,$LayerID);
255  mysql_query($SQL);
256  logSqlError();
257 
258  # Delete the image, if exists
259  $NewFilename = TileName($X,$Y,$Z, layerDir($LayerID));
260  if($NewFilename){
261    if(file_exists($NewFilename)){
262      unlink($NewFilename);
263    }
264  }
265}
266
267#-----------------------------------------------------------------------------
268# Save metadata when an entire tileset is uploaded at once
269#-----------------------------------------------------------------------------
270function SaveTilesetMetadata($X,$Y,$Layer,$UserID, $VersionID){
271  SaveUserStats($UserID, $VersionID, 1365);
272 
273  moveRequest($X, $Y, REQUEST_ACTIVE, REQUEST_DONE, 0);
274 
275  $LayerID = checkLayer($Layer);
276
277  $Fields = "x, y, z, type, size, date, user, version, tileset";
278  $Values = sprintf("%d,%d,%d,%d,%d,now(), %d, %d, 1", $X,$Y,12, $LayerID, 0, $UserID, $VersionID);
279 
280  $SQL = sprintf("replace into `tiles_meta` (%s) values (%s);", $Fields, $Values);
281  mysql_query($SQL);
282 
283  logSqlError();
284
285}
286
287#-----------------------------------------------------------------------------
288# Removes completed* tilesets from queue
289# * where completed means "z12 was uploaded"
290#-----------------------------------------------------------------------------
291function RemoveFromQueue($TileList){
292  foreach($TileList as $CSV){
293    list($X, $Y, $Z, $Layer, $Size) = explode(",", $CSV);
294    if($Z == 12){
295   
296      moveRequest($X, $Y, REQUEST_ACTIVE, REQUEST_DONE, 0);
297       
298      #logMsg(sprintf("Moving tile %d, %d from %d to %d", $X, $Y, REQUEST_ACTIVE, REQUEST_DONE), 4);
299      logSqlError();
300    }
301  }
302}
303
304#---------------------------------------------------------------------------
305# Update user info with their latest upload
306#---------------------------------------------------------------------------
307function SaveUserStats($UserID, $VersionID, $NumTiles){
308  $SQL = 
309    "update `tiles_users` set ".
310      "`uploads` = `uploads` + 1, ".
311      sprintf("`tiles` = `tiles` + %d, ", $NumTiles).
312      sprintf("`version` = %d, ", $VersionID).
313      "`last_upload` = now() ".
314    " where ".
315    sprintf("`id`=%d;", $UserID);
316   
317  mysql_query($SQL);
318}
319
320#----------------------------------------------------------------------
321# Processes tile PNG images
322#----------------------------------------------------------------------
323function HandleFile($Filename, $VersionID, &$TileList, &$BlankTileList){
324  if(preg_match("/([a-z]+)_(\d+)_(\d+)_(\d+)\.png/", $Filename, $Matches)){
325    $Layername = $Matches[1];
326    $Z = $Matches[2];
327    $X = $Matches[3];
328    $Y = $Matches[4];
329    $Valid = TileValid($X,$Y,$Z);
330    if($Valid){
331     
332      $Layer = checkLayer($Layername);
333      if($Layer > 0){
334        InsertTile($X,$Y,$Z,$Layer,$Filename, $VersionID, $TileList, $BlankTileList);
335        return(1);
336      }
337      else{
338        logMsg("Invalid layer $Layer from $UserID ($Layername)", 2);
339      }
340    }
341    else{
342      logMsg("Invalid tile $Filename from $UserID", 3);
343    }
344  }
345  else{
346    # logMsg("$Filename doesn't match regular expression", 2);
347  }
348  return(0);
349}
350
351function InsertTile($X,$Y,$Z,$Layer,$OldFilename, $VersionID, &$TileList, &$BlankTileList){
352  if(!TileValid($X,$Y,$Z)){
353    printf("INVALID %d,%d,%d\n", $X,$Y,$Z);
354    return;
355  }
356 
357  $Size = filesize($OldFilename);
358
359 
360  # Decide on a filename
361  $NewFilename = TileName($X,$Y,$Z, layerDir($Layer));
362  if(!$NewFilename){
363    logMsg("Invalid filename created for $X,$Y,$Z,$Layer",2);
364    return;
365  }
366 
367  if($VersionID < 5){ // Prior to "cambridge", no blank-tile detection
368    # Don't store blank tiles
369    if($Size < 1000){
370      printf("%s -> blank, not saved\n", $OldFilename);
371      return;
372    }
373  }
374 
375  if($Size == 67){
376    # This is a request to delete existing tiles and create a "blank land" tile
377    # TODO: make an enumeration for blank land/sea
378    $SqlSnippet = sprintf("%d,%d,%d,%d,%d", $X, $Y, $Z, $Layer, 2);
379    array_push($BlankTileList, $SqlSnippet);
380    return;
381  }
382  if($Size == 69){
383    # This is a request to create a sea tile
384    $SqlSnippet = sprintf("%d,%d,%d,%d,%d", $X, $Y, $Z, $Layer, 1);
385    array_push($BlankTileList, $SqlSnippet);
386    return;
387  }
388  if($Size == 0){
389    # This is a request to delete a tile (both real and blank)
390    $SqlSnippet = sprintf("%d,%d,%d,%d,%d", $X, $Y, $Z, $Layer, -1);
391    array_push($BlankTileList, $SqlSnippet);
392    return;
393  }
394
395  if($Size < 100){
396    # TODO: WTF is this tile
397    return;
398  }
399 
400  # Remember tile details, in a form that can be added to SQL easily
401  $SqlSnippet = sprintf("%d,%d,%d,%d,%d", $X, $Y, $Z, $Layer, $Size);
402  array_push($TileList, $SqlSnippet);
403
404 
405  # Check directory exists
406  CreateDirectoryToHold($NewFilename);
407 
408  # Move the file to its new home
409  rename($OldFilename, $NewFilename);
410  printf("%s -> %s\n", $OldFilename, $NewFilename);
411
412  # Make world-writeable, so that it's easier to move files using shell
413  # (note: anyone with shell account on dev can access these files anyway
414  #  through their website running as htuser)
415  chmod($NewFilename, 0666);
416}
417
418function CreateDirectoryToHold($Filename){
419  # Get the components of the directory structure
420  $Parts = explode("/", $Filename);
421 
422  # Remove the last element, which is the filename
423  array_pop($Parts);
424  # and the first element, which is a zero-length string
425  array_shift($Parts);
426 
427  $AssumedToExist = 4; // var/www/ojw/Tiles don't get created
428 
429  # For each part...
430  $Dir = "";
431  $Count = 0;
432  foreach($Parts as $Part){
433    $Dir .= "/".$Part;
434    $Count++;
435   
436    if($Count > $AssumedToExist){ 
437      CreateDir($Dir);
438    }
439  }
440}
441
442function CreateDir($Dir){
443  if(file_exists($Dir)){
444    #printf("Directory exists: %s\n", $Dir);
445    return(1);
446  }
447 
448  if(!mkdir($Dir, 0777)){
449    printf("Failed to create directory %s\n", $Dir);
450    return(0);
451  }
452 
453  #printf("Creating dir \"%s\"\n", $Dir);
454  return(1);
455}
456
457#----------------------------------------------------------------------
458# Chooses the name for a temporary directory
459#
460# * everything under one temp dir
461# * md5 gives alphanumeric filename
462# * uniqid means multiple threads are unlikely to conflict
463#----------------------------------------------------------------------
464function TempDir(){
465  return(sprintf("/home/ojw/tiles-ojw2/temp/%s", md5(uniqid(rand(), 1))));
466}
467
468?>
Note: See TracBrowser for help on using the repository browser.