source: subversion/sites/other/tilesAtHome_tahngo/tah_intern/process_uploads.py @ 9723

Revision 9723, 8.9 KB checked in by spaetz, 6 years ago (diff)

empty queue returns IndexError? now, so check for that instead

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2
3import os, sys, logging, zipfile, random, re, stat, signal
4# we need to insert the basedir to the python path (strip 2 path components) if we want to directly execute this file
5sys.path.insert(0, os.path.dirname(os.path.dirname(sys.path[0])))
6os.environ['DJANGO_SETTINGS_MODULE'] = "tah.settings"
7import shutil
8from datetime import datetime
9from time import sleep,clock,time
10from django.conf import settings
11from tah.tah_intern.models import Settings, Layer
12from tah.tah_intern.Tileset import Tileset
13from tah.tah_intern.Tile import Tile
14from django.contrib.auth.models import User
15from tah.requests.models import Request,Upload
16
17### TileUpload returns 0 on success and >0 otherwise
18class TileUpload:
19  unzip_path=None #path to be used for unzipping (temporary files)
20  base_tilepath=None # base tile directory
21  fname=None      #complete path of uploaded tileset file
22  uid=None        #random unique id which is used for pathnames
23  upload=None     #current handled upload object
24  tmptiledir=None #usually unzip_path+uid contains the unzipped files
25
26  def __init__(self,config):
27    self.unzip_path = config.getSetting(name='unzipPath')
28    self.base_tilepath = settings.TILES_ROOT
29    if self.unzip_path == None or self.base_tilepath == None:
30      sys.exit("Failed to get required settings.")
31
32  def process(self):
33   try:
34    while True:
35      #find the oldest unlocked upload file
36      self.upload = None
37      while not self. upload:
38        # repeat fetching until there is one
39        try:
40          self.upload = Upload.objects.filter(is_locked=False)[0]
41        except IndexError:
42          #logging.debug('No uploaded request. Sleeping 10 sec.')
43          sleep(10)
44      starttime = (time(),clock()) # start timing tileset handling now
45      self.fname = self.upload.get_file_filename()
46      if os.path.isfile(self.fname):
47        #logging.debug('Handling next tileset: ' + self.upload.file)
48        self.uid = str(random.randint(0,9999999999999999999))
49        if self.unzip():
50          tset = self.movetiles()
51          if tset.layer and tset.base_z and tset.x and tset.y:
52          #It's a valid tileset. Save the tileset at it's place
53            time_save = [time()]
54            logging.debug("Saving tileset at (%s,%d,%d,%d) from user %s (client id %d)" % (tset.layer,tset.base_z,tset.x,tset.y,self.upload.user_id,self.upload.client_uuid))
55            (retval,unknown_tiles) = tset.save(self.base_tilepath, self.upload.user_id.id)
56            time_save.append(time())
57            if retval:
58              # everything went fine. Add to user statistics
59              self.add_user_stats(1365-unknown_tiles)
60              # now match up the upload with a request and mark the request as finished
61              reqs = Request.objects.filter(min_z = tset.base_z, x = tset.x ,y = tset.y, status__lt=2)
62              for req in reqs:
63                # remove corresponding layer from request and set it to status=2 when all layers are done
64                req.layers.remove(tset.layer)
65                if req.layers.count() == 0:
66                  req.status=2
67                  req.clientping_time=datetime.now()
68                  req.save()
69              logging.debug('Finished "%s,%d,%d,%d" in %.1f sec (CPU %.1f). Saving took %.1f sec. %d unknown tiles.' % (tset.layer,tset.base_z,tset.x,tset.y,time()-starttime[0],clock()-starttime[1], time_save[1] - time_save[0], unknown_tiles))
70            else:
71              # saving the tileset went wrong
72              logging.error('Saving tileset "%s,%d,%d,%d" failed. Aborting tileset. Took %.1f sec (CPU %.1f). %d unknown tiles.' % (tset.layer,tset.base_z,tset.x,tset.y,time()-starttime[0],clock()-starttime[1], unknown_tiles))
73          else:
74            # movetiles did not return a valid tileset
75            logging.error('Unzipped file was no valid tileset. Took %.1f sec (CPU %.1f).' % (time()-starttime[0],clock()-starttime[1]))
76        self.cleanup(True)
77
78      else:
79        logging.info('uploaded file not found, deleting upload.')
80        self.upload.delete()
81   except KeyboardInterrupt:
82     if self.upload: self.cleanup(False)
83     logging.info('Ctrl-C pressed. Shutdown gracefully.')
84     sys.exit("Ctrl-C pressed. Shutdown gracefully. Upload was: %s" % self.upload)
85  #-----------------------------------------------------------------
86  def unzip(self):
87    now = clock()
88    outfile = None
89    self.tmptiledir = dir = os.path.join(self.unzip_path,self.uid)
90    os.mkdir(dir, 0777)
91    try:
92      zfobj = zipfile.ZipFile(self.fname)
93
94      for name in zfobj.namelist():
95        if name.endswith('/'):
96          os.mkdir(os.path.join(dir, name))
97        else:
98          outfile = open(os.path.join(dir, name), 'wb')
99          outfile.write(zfobj.read(name))
100          outfile.close()
101    except zipfile.BadZipfile:
102      logging.warning('found bad zip file %s uploaded by user %s', (self.uid, self.upload.user_id))
103      if outfile: outfile.close()
104      return 0
105    except:
106      logging.warning('unknown zip file error in file uploaded by user %s' % self.upload.user_id)
107      if outfile: outfile.close()
108      return(0)
109
110    logging.debug('Unzipped tileset in %.1f sec.' % ((clock()-now)))
111    return(1)
112
113  #-----------------------------------------------------------------
114  def movetiles(self):
115    # 67 byte file => mark blank land;69 byte => mark blank sea; ignore every thing else below 100 bytes
116    smalltiles = 0
117    layer = None
118    r = re.compile('^([a-zA-Z]+)_(\d+)_(\d+)_(\d+).png$')
119    tset = Tileset()
120
121    for f in os.listdir(self.tmptiledir):
122      ignore_file = False
123      full_filename = os.path.join(self.tmptiledir,f)
124      m = r.match(f)
125      if not m:
126        logging.info('found weird file '+f+' in zip from user %s. Ignoring.' % self.upload.user_id)
127        ignore_file = True
128
129      if not ignore_file:
130        # get the layer if it's any different
131        if not (layer and layer.name == m.group(1)):
132          try: layer = Layer.objects.get(name=m.group(1))
133          except Layer.DoesNotExist:
134            logging.info("unknown layer '%s' in upload by user %s" % (m.group(1), self.upload.user_id))
135            return 0
136        t = Tile(layer=layer,z=m.group(2),x=m.group(3),y=m.group(4))
137        #print "found layer:"+m.group(1)+'z: '+m.group(2)+'x: '+m.group(3)+'y: '+m.group(4)
138
139        #check the size to catch special meaning. This check can be removed once blankness
140        #information is conveyed by other means than file size...
141        fsize = os.stat(full_filename)[stat.ST_SIZE]
142        if fsize < 100:
143          if fsize == 67:
144            #mark blank land
145            t.set_blank_land()
146            tset.add_tile(t,None)
147          elif fsize == 69:
148            #mark blank sea
149            t.set_blank_sea()
150            tset.add_tile(t,None)
151          else:
152            # ignore unknown small png files
153            smalltiles += 1
154        else:
155          #png has regular filesize
156          tset.add_tile(t,full_filename)
157
158    if smalltiles: logging.debug('Ignored %d too small png files' % smalltiles)
159    return tset
160
161  #-----------------------------------------------------------------
162  def add_user_stats(self, uploaded_tiles):
163    """ Update the tah user statistics after a successfull upload """
164    tahuser = self.upload.user_id.tahuser_set.get()
165    try: tahuser.kb_upload += os.stat(self.upload.get_file_filename())[stat.ST_SIZE] // 1024
166    except OSError: pass
167    tahuser.renderedTiles += uploaded_tiles
168    tahuser.save()
169
170  #-----------------------------------------------------------------
171  def sigterm(self, signum, frame):
172    """ This is called when a SIGTERM signal is issued """
173    print "Received SIGTERM signal. Shutdown gracefully."
174    self.cleanup(None, False)
175    sys.exit(0)
176
177  #-----------------------------------------------------------------
178  def cleanup(self, del_upload = True):
179    """ Removes all temporary files and removes the upload object
180        (and the uploaded file if 'del_upload' is True.
181    """
182    # Delete the unzipped files directory
183    shutil.rmtree(self.tmptiledir, True)
184    self.uid=None
185    self.fname=None
186    self.tmptiledir=None
187    if del_upload:
188      # delete the uploaded file itself
189      try: os.unlink(self.upload.get_file_filename())
190      except: pass
191      # delete the upload db entry
192      self.upload.delete()
193
194#---------------------------------------------------------------------
195
196if __name__ == '__main__':
197  config = Settings()
198  logging.basicConfig(level=logging.DEBUG,
199                    format='%(asctime)s %(levelname)s %(message)s',
200                    datefmt = "%y-%m-%d-%H:%M:%S", 
201                    filename= config.getSetting(name='logFile'),
202                    ) 
203
204  logging.info('Starting tile upload processor')
205  u = TileUpload(config)
206  signal.signal(signal.SIGTERM,u.sigterm)
207  if not u.process():
208      logging.critical('Upload handling returned with error. Aborting.')
209      sys.stderr.write('Upload handling returned with error')
210      sys.exit(1)
211else:
212  sys.stderr.write('You need to run this as the main program.')
213  sys.exit(1)
Note: See TracBrowser for help on using the repository browser.