source: subversion/sites/other/tilesAtHome_tahngo/requests/views.py @ 10639

Revision 10639, 21.0 KB checked in by spaetz, 6 years ago (diff)

put fetching request into separate function. perhaps we can lock this

Line 
1import logging, os
2from django.db import transaction   #, connection
3from django.shortcuts import render_to_response
4import django.views.generic.list_detail
5from django.http import HttpResponse, HttpResponseNotFound, HttpResponseForbidden
6from tah.requests.models import Request,Upload
7from tah.requests.forms import *
8from django.conf import settings
9from django.forms import widgets
10from datetime import datetime, timedelta
11import urllib
12import xml.dom.minidom
13from django.contrib.auth import authenticate
14from tah.tah_intern.models import Layer
15from tah.tah_intern.Tile import Tile
16from tah.tah_intern.Tileset import Tileset
17from django.views.decorators.cache import cache_control,  cache_page
18from tah.tah_intern.models import Settings
19
20###############################################
21# requests.views.py does all the action with regard to creating requests,
22# taking error feedback and receiving uploads
23###############################################
24
25#-------------------------------------------------------
26# Show the base request homepage
27
28def index(request):
29  return render_to_response('base_requests.html');
30
31#-------------------------------------------------------
32# Show the recent uploads in flashy ajax
33
34def show_ajax(request):
35  return render_to_response('requests_show_ajax.html');
36
37#-------------------------------------------------------
38# shortcut to see the first page of the show_request page
39
40def show_first_page(request):
41  return show_requests(request,1)
42
43
44#-------------------------------------------------------
45#### show_requests (request,page) ####
46# show the list of recently created requests and the list
47# of recently taken requests
48
49@cache_control(max_age=10)
50def show_requests(request,page):
51  if page: pagination=30 
52  else: pagination=0
53  return django.views.generic.list_detail.object_list(request, \
54    queryset=Request.objects.filter(status=0).order_by('-request_time'), \
55    template_name='requests_show.html', allow_empty=True, paginate_by=pagination, \
56    page=page, template_object_name='new_reqs', extra_context = {
57    'active_reqs_list':Request.objects.filter(status=1).order_by('-clientping_time')[:30]
58    });
59
60#-------------------------------------------------------
61# Show a list of recently uploaded requests
62
63@cache_control(max_age = 10)
64def show_uploads_page(request):
65  return django.views.generic.list_detail.object_list(request, \
66        queryset=Request.objects.filter(status=2).order_by('-clientping_time')[:30], \
67        template_name='requests_show_uploads.html',allow_empty=True, \
68        template_object_name='reqs');
69
70#-------------------------------------------------------
71# not a public view, but a helper function that creates a new request based on data in
72# 'form' and performs sanity checks etc. The form must have been validated previously.
73
74#@transaction.commit_on_success
75def saveCreateRequestForm(request, form):
76    """ Returns (Request, reason), with Request being 'None' on failure and
77        the saved request object on success.
78        Reason is a string that describes the error in case of failure.
79    """
80    formdata = form.cleaned_data.copy()
81
82    # delete layer here. We use the layers from ^'form' later.
83    del formdata['layers']
84
85    #sanity changes and default values on some attributes
86    if not formdata['min_z'] in ['6','12']: formdata['min_z'] = 12
87    if not formdata['priority'] or \
88      formdata['priority'] > 4 or formdata['priority'] < 1: 
89          formdata['priority'] = 3
90
91    if not formdata['src']: formdata['src'] = '' # requester did not supply a 'src' string
92    formdata['ipaddress'] = request.META['REMOTE_ADDR']
93
94    # catch invalid x,y and return if found
95    if not Tile(None,formdata['min_z'],formdata['x'],formdata['y']).is_valid():
96        return (None, 'Invalid tile coordinates')
97
98    # Deny request if the same is already out rendering
99    if Request.objects.filter(status=1, min_z=formdata['min_z'], x=form.data['x'], \
100      y=form.data['y']).count():
101          return (None, 'Request currently rendering')
102
103    # Create a new request, or get the existing one
104    newRequest, created_new = Request.objects.get_or_create(status=0, \
105        min_z=formdata['min_z'], x=form.data['x'], y=form.data['y'],defaults=formdata)
106
107    # set more values in the new request item
108    newRequest.ipaddress = request.META['REMOTE_ADDR']
109    newRequest.max_z = {0:5,6:11,12:17}[formdata['min_z']]
110
111    if not created_new:
112        # increase priority if we update existing request (if needed)
113        newRequest.priority = min(formdata['priority'], newRequest.priority)
114
115    ##check if the IP has already lot's of high priority requests going and auto-bump down
116    if formdata['priority'] == 1:
117        ip_requested = Request.objects.filter(status__lt= 2, \
118          ipaddress= request.META['REMOTE_ADDR']).count()
119
120        if ip_requested > 15:
121            # auto bump down to a minimum of 2
122            newRequest.priority = max(2,newRequest.priority)
123
124    # finally save the updated request, the required layers are still missing then.
125    newRequest.save()
126
127    # select layers for the request
128    if form.data.has_key('layers'):
129        # save the chosen layers
130        layers = Layer.objects.filter(pk__in=form['layers'].data)
131    else:
132        # no layers selected -> save default layers
133        layers = Layer.objects.filter(default=True)
134
135
136    # only add layers if they make sense
137    numLayers=0
138    for l in layers: 
139        if l.max_z >= newRequest.max_z and l.min_z <= newRequest.min_z:
140            newRequest.layers.add(l)
141            numLayers += 1
142
143    if numLayers == 0:
144        # could not add ANY sensible layer, return error
145        #transaction.rollback()
146        newRequest.delete()
147        return (None, 'no sensible layer selected for request')
148
149    # Finally return the request. Success!
150    return (newRequest,'')
151
152
153#-------------------------------------------------------
154# view that creates a new request
155
156def create(request):
157    html="XX|unknown error"
158    req = None
159    CreateFormClass = CreateForm
160    CreateFormClass.base_fields['priority'].required = False
161    CreateFormClass.base_fields['min_z'].required = False 
162    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
163         choices=CreateFormClass.base_fields['layers'].choices)
164    CreateFormClass.base_fields['layers'].required = False
165    form = CreateFormClass()
166
167    if request.method == 'POST':
168      form = CreateForm(request.POST)
169      if form.is_valid():
170        req, reason = saveCreateRequestForm(request, form)
171      else:
172        html="form is not valid. "+str(form.errors)
173        return HttpResponse(html)
174    else:
175      #Create request using GET"
176      form = CreateForm(request.GET)
177      if form.is_valid():
178        req, reason = saveCreateRequestForm(request, form)
179      else:
180         # view the plain form webpage with default values filled in
181         return render_to_response('requests_create.html', \
182                {'createform': form, 'host':request.META['HTTP_HOST']})
183    if req: html = "Render '%s' (%s,%s,%s) at priority %d" % \
184                    (req.layers_str,req.min_z,req.x,req.y,req.priority)
185    else:   html = "Request failed (%s,%s,%s): %s" \
186                    % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'],reason)
187    return HttpResponse(html)
188
189def feedback(request):
190    html="XX|unknown error"
191    CreateFormClass = CreateForm
192    CreateFormClass.base_fields['priority'].required = False
193    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
194         choices=CreateFormClass.base_fields['layers'].choices)
195    CreateFormClass.base_fields['layers'].required = False
196    form = CreateFormClass()
197    if request.method == 'POST':
198      authform = ClientAuthForm(request.POST)
199      if authform.is_valid():
200        name     = authform.cleaned_data['user']
201        passwd   = authform.cleaned_data['passwd']
202        user = authenticate(username=name, password=passwd)
203        if user is not None:
204          #"You provided a correct username and password!"
205
206          form = CreateForm(request.POST)
207          if form.is_valid():
208            formdata = form.cleaned_data
209            try:
210                # get the current render job and reset it
211                req = Request.objects.get(status=1,x = formdata['x'],y=formdata['y'],min_z = formdata['min_z'], client=user)
212                req.status=0
213                req.request_time = req.request_time + timedelta(hours=2)
214                req.save()
215                html = "Reset tileset (%d,%d,%d)" % \
216                        (formdata['min_z'],formdata['x'],formdata['y'])
217                logging.info("%s by %s (uuid: %s). Cause: %s" %(html,user,request.POST.get('client_uuid','0'),request.POST.get('cause','unknown')))
218            except Request.DoesNotExist:
219                # tried to reset request that is not assigned to us
220                html = "tileset (%d,%d,%d) is not assigned to us. Not resetting" % \
221                        (formdata['min_z'],formdata['x'],formdata['y'])
222                logging.info("%s. %s (%s)" %(html,user,request.POST.get('client_uuid','0')))
223          else:
224            html="XX|4|form is not valid. "+str(form.errors)
225      else:
226        # authentication failed here, or authform failed to validate
227        html="XX|4|Invalid username. Your username and password were incorrect or the user has been disabled."
228    elif request.method == 'GET':
229         # view the plain form webpage with default values filled in
230         return render_to_response('requests_create.html', \
231                {'createform': form, 'host':request.META['HTTP_HOST']})
232    return HttpResponse(html)
233
234
235#-------------------------------------------------------
236# Upload a finished tileset
237
238def upload_request(request):
239    html='XX|Unknown error.'
240
241    if request.method == 'POST':
242      authform = ClientAuthForm(request.POST)
243      if authform.is_valid():
244        name     = authform.cleaned_data['user']
245        passwd   = authform.cleaned_data['passwd']
246        user = authenticate(username=name, password=passwd)
247        if user is not None:
248          #"You provided a correct username and password!"
249          formdata = request.POST.copy()
250          formdata['user_id'] = user.id # set the user to the correct value
251          # we had huge client_uuids in the beginnning, shorten if necessary
252          if formdata.has_key('client_uuid') and int(formdata['client_uuid']) > 65535:
253            formdata['client_uuid'] = formdata['client_uuid'][-4:]
254          #t@h client send layername, rather than layer number,
255          #find the right one if that is the case
256          if formdata.has_key('layer'):
257            try: int(formdata['layer'])
258            except ValueError:
259              # look up the layer id
260              try: formdata['layer'] = Layer.objects.get(name=formdata['layer']).id
261              except Layer.DoesNotExist: del formdata['layer']
262          form = UploadForm(formdata, request.FILES)
263
264          if form.is_valid():
265            file = request.FILES['file']
266            try:
267              newUpload= form.save(commit=False)
268              #             #   filetype = file['content-type']   
269              newUpload.ipaddress = request.META['REMOTE_ADDR']
270              # low priority upload by default
271              if not newUpload.priority: newUpload.priority = 3
272              if not newUpload.client_uuid: newUpload.client_uuid = 0
273              newUpload.is_locked = False
274              #newUpload.save_file_file(file['filename'],file['content'])
275              newUpload.save()
276              html="OK|4|"
277            except:
278              #TODO: delete possibly uploaded file, in case of exception
279              import sys
280              html ="XX| Exception thrown."+str(user.id)+str(sys.exc_info()[1])
281          else:
282            html="XX|4|form is not valid. "+str(form.errors)
283        else:
284            # authentication failed here, or authform failed to validate.
285            html="XX|4|Invalid username. Your username and password were " \
286                 "incorrect or the user has been disabled."
287      else:
288          # authform failed to validate.
289          html="XX|4|Invalid username. Please specify a username and password."
290
291    else:
292        # request.method==GET here. View the plain form webpage with
293        # default values filled in
294        authform = ClientAuthForm()
295        CreateFormClass = UploadForm
296        del CreateFormClass.base_fields['user_id']
297        form = CreateFormClass()
298        return render_to_response('requests_upload.html',{'uploadform': form, 
299                                  'authform': authform, 'host':request.META['HTTP_HOST']})
300    return HttpResponse(html);
301
302
303#-------------------------------------------------------
304# return upload queue load
305
306def upload_gonogo(request):
307    # return 'fullness' of the server queue between [0,1]
308    load = min(Upload.objects.all().count()/1500.0, 1)
309    return HttpResponse(str(load));
310
311
312#-------------------------------------------------------
313# helper function that fetched the next request for taking
314# it locks the table and commit on success
315# returns a request on success or an Request.DoesNotExist execption
316
317@transaction.commit_on_success
318def fetch_next_request( user, client_uuid):
319
320    # returns request or DoesNotExist exception
321    request = Request.objects.get_next_and_lock();
322
323    # if it's a low prio request, make sure the upload queue is not too full
324    upload_queue = Upload.objects.all().count() # [0...1500]
325    if upload_queue > {1:1500,2:1200,3:1100,4:900}[request.priority]:
326        # bomb out with a "No request in queue for you"
327        raise Request.DoesNotExist
328    request.status=1
329    request.client = user
330    request.client_uuid = client_uuid
331    request.clientping_time=datetime.now()
332    request.save()
333
334    return request
335
336
337#-------------------------------------------------------
338#### take(request) ####
339# retrieve a new request from the server
340# TODO, quite large by now. Split?
341
342def take(request):
343    html='XX|5|unknown error'
344    if request.method == 'POST':
345      # we had huge client_uuids in the beginnning, shorten if necessary
346      if request.POST.has_key('client_uuid') and \
347          int(request.POST['client_uuid']) > 65535:
348             request.POST['client_uuid'] = request.POST['client_uuid'][-4:]
349
350      # perform user authentication
351      authform = ClientAuthForm(request.POST)
352      form     = TakeRequestForm(request.POST)
353      form.is_valid() #clean data
354      if authform.is_valid():
355        name     = authform.cleaned_data['user']
356        passwd   = authform.cleaned_data['passwd']
357        user = authenticate(username=name, password=passwd)
358        if user is not None:
359          #"You provided a correct username and password!"
360          # next, check for a valid client version
361          if form.cleaned_data['version'] in ['Rapperswil', 'Saurimo']:
362            #next 2 lines are for limiting max #of active requests per usr
363            active_user_reqs = Request.objects.filter(status=1,client=user.id).count()
364
365            if active_user_reqs <= 150:
366              try: 
367                  # get the next request from the queue (or return DoesNotExist exception)
368                  #req = Request.objects.select_for_update().filter(status=0).order_by('priority','request_time')[0];
369                  client_uuid = form.cleaned_data.get('client_uuid', 0)
370                  req = fetch_next_request( user, client_uuid);
371
372                  # find out tileset filesize and age
373                  # always hardcode 'tile' layer for now.
374                  # need to find something better, like layerid=1 for default
375                  tilelayer = Layer.objects.get(name='tile')
376                  (tilepath, tilefile) = Tileset(tilelayer, req.min_z, \
377                              req.x, req.y).get_filename(settings.TILES_ROOT)
378                  tilefile = os.path.join(tilepath, tilefile)
379                  try: 
380                    fstat = os.stat(tilefile)
381                    (fsize,mtime) = (fstat[6], fstat[8])
382                  except OSError: 
383                    (fsize,mtime) = 0,0
384
385                  # next line is actually the successful request string!
386                  html="OK|5|%s|%d|%d" % (req,mtime,fsize)
387
388              except (IndexError, Request.DoesNotExist), e:
389                  # could not get_next_and_lock, queue empty
390                  html ="XX|5|No requests in queue for you. (%s)" % (e)
391            else:
392                  #  active_user_reqs > 150
393                  html ='XX|5|You have more than 150 active requests. '\
394                        'Check your client.'
395
396          else:
397            # client version not in whitelist
398            logging.info("User %s connects with disallowed client '%s'." %\
399                         (user,form.cleaned_data['version']))
400            html="XX|5|Invalid client version."
401        else:
402            # user is None, auth failed
403            html='XX|5|Invalid username. Your username and password were '\
404                 'incorrect or the user has been disabled.'
405      else: #form was not valid
406        html = "XX|5|Form invalid. "+str(form.errors)
407
408    else: #request.method != POST, show the web form
409        authform, form = ClientAuthForm(), TakeRequestForm()
410        return render_to_response('requests_take.html', \
411                              {'clientauthform': authform, 'takeform': form})
412
413    # Finally return html
414    return HttpResponse(html) #+"\n"+str(connection.queries))
415
416
417
418#-------------------------------------------------------
419
420@cache_control(no_cache=True)
421def request_changedTiles(request):
422    #http://www.openstreetmap.org/api/0.5/changes?start=2008-08-18-18:40&end=now
423    #<osm version="0.5" generator="OpenStreetMap server">
424    #<changes starttime="2008-08-18T18:40:00+01:00" endtime="2008-08-18T20:50:29+01:00">
425    setting = Settings()
426    #check if we access this page from the whitelisted ip address
427    allowed_ip = setting.getSetting('changed_tiles_allowed_IP')
428    if not allowed_ip: allowed_ip = setting.setSetting('changed_tiles_allowed_IP',request.META['REMOTE_ADDR'])
429    if allowed_ip != request.META['REMOTE_ADDR']:
430      return HttpResponseForbidden('Access not allowed from this IP address.')
431    #fetch the url that is called to retrieve the changed tiles
432    url = setting.getSetting('changed_tiles_api_url')
433    if not url: url = setting.setSetting('changed_tiles_api_url','http://www.openstreetmap.org/api/0.5/changes?hours=6')
434
435    html="Requested tiles:\n"
436    xml_dom = xml.dom.minidom.parse(urllib.urlopen(url))
437    tiles = xml_dom.getElementsByTagName("tile")
438    for tile in tiles:
439      (z,x,y) = tile.getAttribute('z'),tile.getAttribute('x'),tile.getAttribute('y')
440      CreateFormClass = CreateForm
441      CreateFormClass.base_fields['layers'].required = False 
442      form = CreateFormClass({'min_z': z, 'x': x, 'y': y, 'priority': 2, 'src':'ChangedTileAutoRequest'})
443      if form.is_valid():
444        req, reason = saveCreateRequestForm(request, form)
445        if req:
446          html += "Render '%s' (%s,%s,%s)\n" % (req.layers_str,form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'])
447        else: html +="Renderrequest failed (%s,%s,%s): %s\n" % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'], reason)
448      else:
449        html+="form is not valid. %s\n" % form.errors
450    xml_dom.unlink()
451    logging.info("Requested %d changes Tilesets." % len(tiles))
452    return HttpResponse(html,mimetype='text/plain')
453
454@cache_control(no_cache=True)
455def expire_tiles(request):
456    """ Rerequest old active request that haven't been pinged by the client
457        for a while. Also delete old finished requests that we don't need
458        anymore.
459    """
460    rerequested = Request.objects.filter(status=1,clientping_time__lt=datetime.now()-timedelta(0,0,0,0,0,6))
461    for r in rerequested:
462      #next django version will be able to just update() this
463      r.status=0
464      r.save()
465
466    expired = Request.objects.filter(status=2, clientping_time__lt=datetime.now()-timedelta(2,0,0,0,0,0))
467    expired.delete()
468
469    html="Reset %d requests to unfinished status. Expired %d old finished requests." % (len(rerequested),len(expired))
470    logging.debug(html)
471    return HttpResponse(html,mimetype='text/plain')
472
473@cache_control(no_cache=True)
474def stats_munin_requests(request,status):
475    reply=''
476    states={'pending':0,'active':1,'done':2}
477    if states.has_key(status): state = states[status]
478    else: return HttpResponseNotFound('Unknown request state')
479    reqs = Request.objects.filter(status=state)
480    if state < 2:
481      #output low/medium/high requests for unfinished ones
482      for val, state in enumerate(['high','medium','low']):
483        c = reqs.filter(priority=val+1).count()
484        reply += "%s.value %d\n" % (state,c)
485    else:
486      #output requests finished per last hour and 48 moving avg
487      reply += "_req_processed_last_hour.value %d\n" %  (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,1)).count())
488      reply += 'done.value %d' % (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,48)).count() // 48)
489    return HttpResponse(reply,mimetype='text/plain')
490
491
492#----------------------------------------------------------------------
493# Simply return the latest client version (cached for 15 Minutes)
494#@cache_page(60*15)
495def show_latest_client_version(request):
496    html = Settings().getSetting("latestClientVersion")
497    return HttpResponse(html,mimetype='text/plain')
Note: See TracBrowser for help on using the repository browser.