source: subversion/sites/other/tilesAtHome_tahngo/requests/views.py @ 9728

Last change on this file since 9728 was 9710, checked in by spaetz, 12 years ago

increase uploaded tileset queue to 1500

File size: 18.3 KB
Line 
1import logging, os
2from django.utils.encoding import force_unicode
3from django.shortcuts import render_to_response
4from django.contrib.auth.models import User
5import django.views.generic.list_detail
6from django.http import HttpResponse, HttpResponseNotFound, HttpResponseForbidden
7from tah.requests.models import Request,Upload
8from tah.requests.forms import *
9from django.conf import settings
10from django.forms import form_for_model,widgets
11from datetime import datetime, timedelta
12import urllib
13import xml.dom.minidom
14from django.contrib.auth import authenticate
15#from tah.user.auth import OSMBackend
16from tah.tah_intern.models import Layer
17from tah.tah_intern.Tile import Tile
18from tah.tah_intern.Tileset import Tileset
19from django.views.decorators.cache import cache_control
20from tah.tah_intern.models import Settings
21
22
23def index(request):
24  return render_to_response('base_requests.html');
25
26#shortcut view to see all requests
27def show_first_page(request):
28  return show_requests(request,1)
29
30@cache_control(must_revalidate=True, max_age=30)
31def show_requests(request,page):
32  if page: pagination=30 
33  else: pagination=0
34  return django.views.generic.list_detail.object_list(request, queryset=Request.objects.filter(status=0).order_by('-request_time'),template_name='requests_show.html',allow_empty=True,paginate_by=pagination,page=page,template_object_name='new_reqs',extra_context={'active_reqs_list':Request.objects.filter(status=1).order_by('-clientping_time')[:30]});
35
36
37@cache_control(must_revalidate=True, max_age=30)
38def show_uploads_page(request):
39  return django.views.generic.list_detail.object_list(request, queryset=Request.objects.filter(status=2).order_by('-clientping_time')[:30],template_name='requests_show_uploads.html',allow_empty=True,template_object_name='reqs');
40
41
42def saveCreateRequestForm(request, form):
43    """ Returns (Request, reason), with Request being 'None' on failure and
44        the saved request object on success.
45        Reason is a string that describes the error in case of failure.
46    """
47    formdata = form.cleaned_data.copy()
48    # delete entries that are not needed as default value or won't work
49    del formdata['layers']
50    del formdata['status']
51    if not formdata['min_z'] in ['6','12']: formdata['min_z'] = 12
52    if not formdata['max_z']: formdata['max_z'] = {0:5,6:11,12:17}[formdata['min_z']]
53    if not formdata['priority'] or \
54      formdata['priority']>4 or formdata['priority']<1: 
55        formdata['priority'] = 3
56    formdata['clientping_time'] = force_unicode(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
57
58    # catch invalid x,y
59    if not Tile(None,formdata['min_z'],formdata['x'],formdata['y']).is_valid():
60      return (None, 'Invalid tile coordinates')
61
62    # Deny request if the same is already out rendering
63    if Request.objects.filter(status=1, min_z=formdata['min_z'], x=form.data['x'], y=form.data['y']).count():
64      return (None, 'Request currently rendering')
65
66    # Create a new request, or get the existing one
67    newRequest, created_new = Request.objects.get_or_create(status=0,min_z=formdata['min_z'], x=form.data['x'], y=form.data['y'],defaults=formdata)
68
69    newRequest.ipaddress = request.META['REMOTE_ADDR']
70
71    if not created_new:
72      #update existing request with new request data
73      newRequest.max_z = max(formdata['max_z'],newRequest.max_z)
74      newRequest.priority = min(formdata['priority'],newRequest.priority)
75
76    ##check if the IP has already lot's of high priority requests going and auto-bump down
77    if formdata['priority'] == 1:
78      ip_requested = Request.objects.filter(status__lt= 2, ipaddress= request.META['REMOTE_ADDR']).count()
79      if ip_requested > 15:
80        newRequest.priority = max(2,newRequest.priority)
81
82    # finally save the updated request
83    newRequest.save()
84    if form.data.has_key('layers'):
85      # save the chosen layers
86      layers = Layer.objects.filter(pk__in=form['layers'].data)
87    else:
88      # no layers selected -> save default layers
89      layers = Layer.objects.filter(default=True)
90    for l in layers: newRequest.layers.add(l)
91    return (newRequest,'')
92
93def create(request):
94    html="XX|unknown error"
95    CreateFormClass = CreateForm
96    CreateFormClass.base_fields['ipaddress'].required = False
97    CreateFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
98    CreateFormClass.base_fields['priority'].required = False
99    CreateFormClass.base_fields['status'].required = False 
100    CreateFormClass.base_fields['status'].widget = widgets.HiddenInput()
101    CreateFormClass.base_fields['min_z'].required = False 
102    CreateFormClass.base_fields['max_z'].required = False 
103    CreateFormClass.base_fields['max_z'].widget = widgets.HiddenInput()
104    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
105         choices=CreateFormClass.base_fields['layers'].choices)
106    CreateFormClass.base_fields['layers'].required = False
107    CreateFormClass.base_fields['clientping_time'].required = False
108    CreateFormClass.base_fields['clientping_time'].widget = widgets.HiddenInput()
109    CreateFormClass.base_fields['client'].required = False
110    CreateFormClass.base_fields['client'].widget = widgets.HiddenInput()
111    form = CreateFormClass()
112
113    if request.method == 'POST':
114      form = CreateForm(request.POST)
115      if form.is_valid():
116        req, reason = saveCreateRequestForm(request, form)
117      else:
118        html="form is not valid. "+str(form.errors)
119    else:
120      #Create request using GET"
121      form = CreateForm(request.GET)
122      if form.is_valid():
123        req, reason = saveCreateRequestForm(request, form)
124      else:
125         # view the plain form webpage with default values filled in
126         return render_to_response('requests_create.html', \
127                {'createform': form, 'host':request.META['HTTP_HOST']})
128    if req: html = "Render '%s' (%s,%s,%s) at priority %d" % \
129                    (req.layers_str,req.min_z,req.x,req.y,req.priority)
130    else:   html = "Request failed (%s,%s,%s): %s" \
131                    % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'],reason)
132    return HttpResponse(html)
133
134def feedback(request):
135    html="XX|unknown error"
136    CreateFormClass = CreateForm
137    CreateFormClass.base_fields['ipaddress'].required = False
138    CreateFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
139    CreateFormClass.base_fields['priority'].required = False
140    CreateFormClass.base_fields['status'].required = False 
141    CreateFormClass.base_fields['status'].widget = widgets.HiddenInput()
142    #CreateFormClass.base_fields['client_uuid'].required = False
143    #CreateFormClass.base_fields['client_uuid'].widget = widgets.HiddenInput()
144    CreateFormClass.base_fields['max_z'].required = False 
145    CreateFormClass.base_fields['max_z'].widget = widgets.HiddenInput()
146    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
147         choices=CreateFormClass.base_fields['layers'].choices)
148    CreateFormClass.base_fields['layers'].required = False
149    CreateFormClass.base_fields['clientping_time'].required = False
150    CreateFormClass.base_fields['clientping_time'].widget = widgets.HiddenInput()
151    CreateFormClass.base_fields['client'].required = False
152    CreateFormClass.base_fields['client'].widget = widgets.HiddenInput()
153    form = CreateFormClass()
154    if request.method == 'POST':
155      authform = ClientAuthForm(request.POST)
156      if authform.is_valid():
157        name     = authform.cleaned_data['user']
158        passwd   = authform.cleaned_data['passwd']
159        user = authenticate(username=name, password=passwd)
160        if user is not None:
161          #"You provided a correct username and password!"
162
163          form = CreateForm(request.POST)
164          if form.is_valid():
165            formdata = form.cleaned_data
166            reqs = Request.objects.filter(status=1,x = formdata['x'],y=formdata['y'],min_z = formdata['min_z'])
167            num = reqs.count()
168            reqs.update(status=0)
169            html = "Reset %d tileset (%d,%d,%d)" % \
170                      (num,formdata['min_z'],formdata['x'],formdata['y'])
171            logging.info("%s by user %s (uuid: %s). Cause: %s" %(html,user,request.POST.get('client_uuid','0'),request.POST.get('cause','unknown')))
172          else:
173            html="XX|4|form is not valid. "+str(form.errors)
174      else:
175        # authentication failed here, or authform failed to validate
176        html="XX|4|Invalid username. Your username and password were incorrect or the user has been disabled."
177    elif request.method == 'GET':
178         # view the plain form webpage with default values filled in
179         return render_to_response('requests_create.html', \
180                {'createform': form, 'host':request.META['HTTP_HOST']})
181    return HttpResponse(html)
182
183def upload_request(request):
184    html='XX|Unknown error.'
185    UploadFormClass = UploadForm
186    UploadFormClass.base_fields['ipaddress'].required = False
187    UploadFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
188    UploadFormClass.base_fields['priority'].required = False
189    UploadFormClass.base_fields['file'].required = False
190    UploadFormClass.base_fields['client_uuid'].required = False
191    UploadFormClass.base_fields['client_uuid'].widget = widgets.HiddenInput()
192    UploadFormClass.base_fields['is_locked'].required = False
193    UploadFormClass.base_fields['is_locked'].widget = widgets.HiddenInput()
194    UploadFormClass.base_fields['user_id'].required = False
195    UploadFormClass.base_fields['user_id'].widget = widgets.HiddenInput()
196    form = UploadFormClass()
197
198    if request.method == 'POST':
199      authform = ClientAuthForm(request.POST)
200      if authform.is_valid():
201        name     = authform.cleaned_data['user']
202        passwd   = authform.cleaned_data['passwd']
203        user = authenticate(username=name, password=passwd)
204        if user is not None:
205          #"You provided a correct username and password!"
206          formdata = request.POST.copy()
207          formdata['user_id'] = user.id # set the user to the correct value
208          # we had huge client_uuids in the beginnning, shorten if necessary
209          if formdata.has_key('client_uuid') and int(formdata['client_uuid']) > 65535:
210            formdata['client_uuid'] = formdata['client_uuid'][-4:]
211          #t@h client send layername, rather than layer number,
212          #find the right one if that is the case
213          if formdata.has_key('layer'):
214            try: int(formdata['layer'])
215            except ValueError:
216              # look up the layer id
217              try: formdata['layer'] = Layer.objects.get(name=formdata['layer']).id
218              except Layer.DoesNotExist: del formdata['layer']
219          form = UploadFormClass(formdata, request.FILES)
220
221          if form.is_valid():
222            file = request.FILES['file']
223            try:
224              newUpload= form.save(commit=False)
225              #             #   filetype = file['content-type']   
226              newUpload.ipaddress = request.META['REMOTE_ADDR']
227              # low priority upload by default
228              if not newUpload.priority: newUpload.priority = 3
229              if not newUpload.client_uuid: newUpload.client_uuid = 0
230              newUpload.is_locked = False
231              #newUpload.save_file_file(file['filename'],file['content'])
232              newUpload.save()
233              html="OK|4|"
234            except:
235              #TODO: delete possibly uploaded file, in case of exception
236              import sys
237              html ="XX| Exception thrown."+str(user.id)+str(sys.exc_info()[1])
238          else:
239            html="XX|4|form is not valid. "+str(form.errors)
240        else:
241          # authentication failed here, or authform failed to validate.
242          html="XX|4|Invalid username. Your username and password were incorrect or the user has been disabled."
243    else:
244      # request.method==GET here. View the plain form webpage with default values filled in
245      authform = ClientAuthForm()
246      return render_to_response('requests_upload.html',{'uploadform': form, 'authform': authform, 'host':request.META['HTTP_HOST']})
247    return HttpResponse(html);
248
249@cache_control(max_age=30)
250def upload_gonogo(request):
251    # dummy, always return 1 for full speed ahead
252    load = 1 - Upload.objects.all().count()/1500.0
253    return HttpResponse(str(load));
254
255def take(request):
256    html='XX|5|unknown error'
257    if request.method == 'POST':
258      authform = ClientAuthForm(request.POST)
259      form     = TakeRequestForm(request.POST)
260      form.is_valid() #clean data
261      if authform.is_valid():
262        name     = authform.cleaned_data['user']
263        passwd   = authform.cleaned_data['passwd']
264        user = authenticate(username=name, password=passwd)
265        if user is not None:
266          #"You provided a correct username and password!"
267          # next, check for a valid client version
268          if form.cleaned_data['version'] in ['Quickborn', 'Rapperswil']:
269            try: 
270                #next 2 lines are for limiting max #of active requests per usr
271                active_user_reqs = Request.objects.filter(status=1,client=user.id).count()
272                if active_user_reqs <= 50:
273                  req = Request.objects.filter(status=0).order_by('priority','request_time')[0]
274                  req.status=1
275                  req.client = user
276                  req.clientping_time=datetime.now()
277                  req.save()
278                  # find out tileset filesize and age
279                  # (always 'tile' layer for now. need to find something better)
280                  tilelayer = Layer.objects.get(name='tile')
281                  (tilepath, tilefile) = Tileset(tilelayer, req.min_z, req.x, req.y).get_filename(settings.TILES_ROOT)
282                  tilefile = os.path.join(tilepath, tilefile)
283                  try: 
284                    fstat = os.stat(tilefile)
285                    (fsize,mtime) = (fstat[6], fstat[8])
286                  except OSError: 
287                    (fsize,mtime) = 0,0
288                  html="OK|5|%s|%d|%d" % (req,fsize,mtime)
289                else:
290                  html ="XX|5|You have more than 50 active requests. Check your client."
291            except IndexError:
292                html ="XX|5|No requests in queue"
293          else:
294            # client version no in whitelist
295            logging.info("User %s connects with disallowed client '%s'." %(user,form.cleaned_data['version']))
296            html="XX|5|Invalid client version."
297        else:
298            # user is None, auth failed
299            html="XX|5|Invalid username. Your username and password were incorrect or the user has been disabled."
300      else: #form was not valid
301        html = "XX|5|Form invalid. "+str(form.errors)
302      return HttpResponse(html);
303    else: #request.method != POST, show the web form
304      authform, form = ClientAuthForm(), TakeRequestForm()
305      return render_to_response('requests_take.html',{'clientauthform': authform, 'takeform': form})
306    return HttpResponse(html)
307
308@cache_control(no_cache=True)
309def request_changedTiles(request):
310    setting = Settings()
311    #check if we access this page from the whitelisted ip address
312    allowed_ip = setting.getSetting('changed_tiles_allowed_IP')
313    if not allowed_ip: allowed_ip = setting.setSetting('changed_tiles_allowed_IP',request.META['REMOTE_ADDR'])
314    if allowed_ip != request.META['REMOTE_ADDR']:
315      return HttpResponseForbidden('Access not allowed from this IP address.')
316    #fetch the url that is called to retrieve the changed tiles
317    url = setting.getSetting('changed_tiles_api_url')
318    if not url: url = setting.setSetting('changed_tiles_api_url','http://www.openstreetmap.org/api/0.5/changes?hours=6')
319
320    html="Requested tiles:\n"
321    xml_dom = xml.dom.minidom.parse(urllib.urlopen(url))
322    tiles = xml_dom.getElementsByTagName("tile")
323    for tile in tiles:
324      (z,x,y) = tile.getAttribute('z'),tile.getAttribute('x'),tile.getAttribute('y')
325      CreateFormClass = CreateForm
326      CreateFormClass.base_fields['max_z'].required = False 
327      CreateFormClass.base_fields['layers'].required = False 
328      CreateFormClass.base_fields['status'].required = False 
329      form = CreateFormClass({'min_z': z, 'x': x, 'y': y, 'priority': 2})
330      if form.is_valid():
331        req, reason = saveCreateRequestForm(request, form)
332        if req:
333          html += "Render '%s' (%s,%s,%s)\n" % (req.layers_str,form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'])
334        else: html +="Renderrequest failed (%s,%s,%s): %s\n" % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'], reason)
335      else:
336        html+="form is not valid. %s\n" % form.errors
337    xml_dom.unlink()
338    logging.info("Requested %d changes Tilesets." % len(tiles))
339    return HttpResponse(html,mimetype='text/plain')
340
341@cache_control(no_cache=True)
342def expire_tiles(request):
343    """ Rerequest old active request that haven't been pinged by the client
344        for a while. Also delete old finished requests that we don't need
345        anymore.
346    """
347    rerequested = Request.objects.filter(status=1,clientping_time__lt=datetime.now()-timedelta(0,0,0,0,0,6))
348    for r in rerequested:
349      #next django version will be able to just update() this
350      r.status=0
351      r.save()
352
353    expired = Request.objects.filter(status=2, clientping_time__lt=datetime.now()-timedelta(2,0,0,0,0,0))
354    expired.delete()
355
356    html="Reset %d requests to unfinished status. Expired %d old finished requests." % (len(rerequested),len(expired))
357    logging.debug(html)
358    return HttpResponse(html,mimetype='text/plain')
359
360@cache_control(no_cache=True)
361def stats_munin_requests(request,status):
362    reply=''
363    states={'pending':0,'active':1,'done':2}
364    if states.has_key(status): state = states[status]
365    else: return HttpResponseNotFound('Unknown request state')
366    reqs = Request.objects.filter(status=state)
367    if state < 2:
368      #output low/medium/high requests for unfinished ones
369      for val, state in enumerate(['high','medium','low']):
370        c = reqs.filter(priority=val+1).count()
371        reply += "%s.value %d\n" % (state,c)
372    else:
373      #output requests finished per last hour and 48 moving avg
374      reply += "_req_processed_last_hour.value %d\n" %  (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,1)).count())
375      reply += 'done.value %d' % (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,48)).count() // 48)
376    return HttpResponse(reply,mimetype='text/plain')
377
378@cache_control(must_revalidate=True, max_age=3600)
379def show_latest_client_version(request):
380    html = Settings().getSetting(name='latest_client_version')
381    return HttpResponse(html,mimetype='text/plain')
Note: See TracBrowser for help on using the repository browser.