source: subversion/sites/other/tilesAtHome_tahngo/requests/views.py @ 9729

Revision 9729, 18.2 KB checked in by spaetz, 6 years ago (diff)

clean up unused imports

Line 
1import logging, os
2from django.utils.encoding import force_unicode
3from django.shortcuts import render_to_response
4import django.views.generic.list_detail
5from django.http import HttpResponse, HttpResponseNotFound, HttpResponseForbidden
6from tah.requests.models import Request,Upload
7from tah.requests.forms import *
8from django.conf import settings
9from django.forms import widgets
10from datetime import datetime, timedelta
11import urllib
12import xml.dom.minidom
13from django.contrib.auth import authenticate
14#from tah.user.auth import OSMBackend
15from tah.tah_intern.models import Layer
16from tah.tah_intern.Tile import Tile
17from tah.tah_intern.Tileset import Tileset
18from django.views.decorators.cache import cache_control
19from tah.tah_intern.models import Settings
20
21
22def index(request):
23  return render_to_response('base_requests.html');
24
25#shortcut view to see all requests
26def show_first_page(request):
27  return show_requests(request,1)
28
29@cache_control(must_revalidate=True, max_age=30)
30def show_requests(request,page):
31  if page: pagination=30 
32  else: pagination=0
33  return django.views.generic.list_detail.object_list(request, queryset=Request.objects.filter(status=0).order_by('-request_time'),template_name='requests_show.html',allow_empty=True,paginate_by=pagination,page=page,template_object_name='new_reqs',extra_context={'active_reqs_list':Request.objects.filter(status=1).order_by('-clientping_time')[:30]});
34
35
36@cache_control(must_revalidate=True, max_age=30)
37def show_uploads_page(request):
38  return django.views.generic.list_detail.object_list(request, queryset=Request.objects.filter(status=2).order_by('-clientping_time')[:30],template_name='requests_show_uploads.html',allow_empty=True,template_object_name='reqs');
39
40
41def saveCreateRequestForm(request, form):
42    """ Returns (Request, reason), with Request being 'None' on failure and
43        the saved request object on success.
44        Reason is a string that describes the error in case of failure.
45    """
46    formdata = form.cleaned_data.copy()
47    # delete entries that are not needed as default value or won't work
48    del formdata['layers']
49    del formdata['status']
50    if not formdata['min_z'] in ['6','12']: formdata['min_z'] = 12
51    if not formdata['max_z']: formdata['max_z'] = {0:5,6:11,12:17}[formdata['min_z']]
52    if not formdata['priority'] or \
53      formdata['priority']>4 or formdata['priority']<1: 
54        formdata['priority'] = 3
55    formdata['clientping_time'] = force_unicode(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
56
57    # catch invalid x,y
58    if not Tile(None,formdata['min_z'],formdata['x'],formdata['y']).is_valid():
59      return (None, 'Invalid tile coordinates')
60
61    # Deny request if the same is already out rendering
62    if Request.objects.filter(status=1, min_z=formdata['min_z'], x=form.data['x'], y=form.data['y']).count():
63      return (None, 'Request currently rendering')
64
65    # Create a new request, or get the existing one
66    newRequest, created_new = Request.objects.get_or_create(status=0,min_z=formdata['min_z'], x=form.data['x'], y=form.data['y'],defaults=formdata)
67
68    newRequest.ipaddress = request.META['REMOTE_ADDR']
69
70    if not created_new:
71      #update existing request with new request data
72      newRequest.max_z = max(formdata['max_z'],newRequest.max_z)
73      newRequest.priority = min(formdata['priority'],newRequest.priority)
74
75    ##check if the IP has already lot's of high priority requests going and auto-bump down
76    if formdata['priority'] == 1:
77      ip_requested = Request.objects.filter(status__lt= 2, ipaddress= request.META['REMOTE_ADDR']).count()
78      if ip_requested > 15:
79        newRequest.priority = max(2,newRequest.priority)
80
81    # finally save the updated request
82    newRequest.save()
83    if form.data.has_key('layers'):
84      # save the chosen layers
85      layers = Layer.objects.filter(pk__in=form['layers'].data)
86    else:
87      # no layers selected -> save default layers
88      layers = Layer.objects.filter(default=True)
89    for l in layers: newRequest.layers.add(l)
90    return (newRequest,'')
91
92def create(request):
93    html="XX|unknown error"
94    CreateFormClass = CreateForm
95    CreateFormClass.base_fields['ipaddress'].required = False
96    CreateFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
97    CreateFormClass.base_fields['priority'].required = False
98    CreateFormClass.base_fields['status'].required = False 
99    CreateFormClass.base_fields['status'].widget = widgets.HiddenInput()
100    CreateFormClass.base_fields['min_z'].required = False 
101    CreateFormClass.base_fields['max_z'].required = False 
102    CreateFormClass.base_fields['max_z'].widget = widgets.HiddenInput()
103    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
104         choices=CreateFormClass.base_fields['layers'].choices)
105    CreateFormClass.base_fields['layers'].required = False
106    CreateFormClass.base_fields['clientping_time'].required = False
107    CreateFormClass.base_fields['clientping_time'].widget = widgets.HiddenInput()
108    CreateFormClass.base_fields['client'].required = False
109    CreateFormClass.base_fields['client'].widget = widgets.HiddenInput()
110    form = CreateFormClass()
111
112    if request.method == 'POST':
113      form = CreateForm(request.POST)
114      if form.is_valid():
115        req, reason = saveCreateRequestForm(request, form)
116      else:
117        html="form is not valid. "+str(form.errors)
118    else:
119      #Create request using GET"
120      form = CreateForm(request.GET)
121      if form.is_valid():
122        req, reason = saveCreateRequestForm(request, form)
123      else:
124         # view the plain form webpage with default values filled in
125         return render_to_response('requests_create.html', \
126                {'createform': form, 'host':request.META['HTTP_HOST']})
127    if req: html = "Render '%s' (%s,%s,%s) at priority %d" % \
128                    (req.layers_str,req.min_z,req.x,req.y,req.priority)
129    else:   html = "Request failed (%s,%s,%s): %s" \
130                    % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'],reason)
131    return HttpResponse(html)
132
133def feedback(request):
134    html="XX|unknown error"
135    CreateFormClass = CreateForm
136    CreateFormClass.base_fields['ipaddress'].required = False
137    CreateFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
138    CreateFormClass.base_fields['priority'].required = False
139    CreateFormClass.base_fields['status'].required = False 
140    CreateFormClass.base_fields['status'].widget = widgets.HiddenInput()
141    #CreateFormClass.base_fields['client_uuid'].required = False
142    #CreateFormClass.base_fields['client_uuid'].widget = widgets.HiddenInput()
143    CreateFormClass.base_fields['max_z'].required = False 
144    CreateFormClass.base_fields['max_z'].widget = widgets.HiddenInput()
145    CreateFormClass.base_fields['layers'].widget = widgets.CheckboxSelectMultiple( 
146         choices=CreateFormClass.base_fields['layers'].choices)
147    CreateFormClass.base_fields['layers'].required = False
148    CreateFormClass.base_fields['clientping_time'].required = False
149    CreateFormClass.base_fields['clientping_time'].widget = widgets.HiddenInput()
150    CreateFormClass.base_fields['client'].required = False
151    CreateFormClass.base_fields['client'].widget = widgets.HiddenInput()
152    form = CreateFormClass()
153    if request.method == 'POST':
154      authform = ClientAuthForm(request.POST)
155      if authform.is_valid():
156        name     = authform.cleaned_data['user']
157        passwd   = authform.cleaned_data['passwd']
158        user = authenticate(username=name, password=passwd)
159        if user is not None:
160          #"You provided a correct username and password!"
161
162          form = CreateForm(request.POST)
163          if form.is_valid():
164            formdata = form.cleaned_data
165            reqs = Request.objects.filter(status=1,x = formdata['x'],y=formdata['y'],min_z = formdata['min_z'])
166            num = reqs.count()
167            reqs.update(status=0)
168            html = "Reset %d tileset (%d,%d,%d)" % \
169                      (num,formdata['min_z'],formdata['x'],formdata['y'])
170            logging.info("%s by user %s (uuid: %s). Cause: %s" %(html,user,request.POST.get('client_uuid','0'),request.POST.get('cause','unknown')))
171          else:
172            html="XX|4|form is not valid. "+str(form.errors)
173      else:
174        # authentication failed here, or authform failed to validate
175        html="XX|4|Invalid username. Your username and password were incorrect or the user has been disabled."
176    elif request.method == 'GET':
177         # view the plain form webpage with default values filled in
178         return render_to_response('requests_create.html', \
179                {'createform': form, 'host':request.META['HTTP_HOST']})
180    return HttpResponse(html)
181
182def upload_request(request):
183    html='XX|Unknown error.'
184    UploadFormClass = UploadForm
185    UploadFormClass.base_fields['ipaddress'].required = False
186    UploadFormClass.base_fields['ipaddress'].widget = widgets.HiddenInput()
187    UploadFormClass.base_fields['priority'].required = False
188    UploadFormClass.base_fields['file'].required = False
189    UploadFormClass.base_fields['client_uuid'].required = False
190    UploadFormClass.base_fields['client_uuid'].widget = widgets.HiddenInput()
191    UploadFormClass.base_fields['is_locked'].required = False
192    UploadFormClass.base_fields['is_locked'].widget = widgets.HiddenInput()
193    UploadFormClass.base_fields['user_id'].required = False
194    UploadFormClass.base_fields['user_id'].widget = widgets.HiddenInput()
195    form = UploadFormClass()
196
197    if request.method == 'POST':
198      authform = ClientAuthForm(request.POST)
199      if authform.is_valid():
200        name     = authform.cleaned_data['user']
201        passwd   = authform.cleaned_data['passwd']
202        user = authenticate(username=name, password=passwd)
203        if user is not None:
204          #"You provided a correct username and password!"
205          formdata = request.POST.copy()
206          formdata['user_id'] = user.id # set the user to the correct value
207          # we had huge client_uuids in the beginnning, shorten if necessary
208          if formdata.has_key('client_uuid') and int(formdata['client_uuid']) > 65535:
209            formdata['client_uuid'] = formdata['client_uuid'][-4:]
210          #t@h client send layername, rather than layer number,
211          #find the right one if that is the case
212          if formdata.has_key('layer'):
213            try: int(formdata['layer'])
214            except ValueError:
215              # look up the layer id
216              try: formdata['layer'] = Layer.objects.get(name=formdata['layer']).id
217              except Layer.DoesNotExist: del formdata['layer']
218          form = UploadFormClass(formdata, request.FILES)
219
220          if form.is_valid():
221            file = request.FILES['file']
222            try:
223              newUpload= form.save(commit=False)
224              #             #   filetype = file['content-type']   
225              newUpload.ipaddress = request.META['REMOTE_ADDR']
226              # low priority upload by default
227              if not newUpload.priority: newUpload.priority = 3
228              if not newUpload.client_uuid: newUpload.client_uuid = 0
229              newUpload.is_locked = False
230              #newUpload.save_file_file(file['filename'],file['content'])
231              newUpload.save()
232              html="OK|4|"
233            except:
234              #TODO: delete possibly uploaded file, in case of exception
235              import sys
236              html ="XX| Exception thrown."+str(user.id)+str(sys.exc_info()[1])
237          else:
238            html="XX|4|form is not valid. "+str(form.errors)
239        else:
240          # authentication failed here, or authform failed to validate.
241          html="XX|4|Invalid username. Your username and password were incorrect or the user has been disabled."
242    else:
243      # request.method==GET here. View the plain form webpage with default values filled in
244      authform = ClientAuthForm()
245      return render_to_response('requests_upload.html',{'uploadform': form, 'authform': authform, 'host':request.META['HTTP_HOST']})
246    return HttpResponse(html);
247
248@cache_control(max_age=30)
249def upload_gonogo(request):
250    # dummy, always return 1 for full speed ahead
251    load = 1 - Upload.objects.all().count()/1500.0
252    return HttpResponse(str(load));
253
254def take(request):
255    html='XX|5|unknown error'
256    if request.method == 'POST':
257      authform = ClientAuthForm(request.POST)
258      form     = TakeRequestForm(request.POST)
259      form.is_valid() #clean data
260      if authform.is_valid():
261        name     = authform.cleaned_data['user']
262        passwd   = authform.cleaned_data['passwd']
263        user = authenticate(username=name, password=passwd)
264        if user is not None:
265          #"You provided a correct username and password!"
266          # next, check for a valid client version
267          if form.cleaned_data['version'] in ['Quickborn', 'Rapperswil']:
268            try: 
269                #next 2 lines are for limiting max #of active requests per usr
270                active_user_reqs = Request.objects.filter(status=1,client=user.id).count()
271                if active_user_reqs <= 50:
272                  req = Request.objects.filter(status=0).order_by('priority','request_time')[0]
273                  req.status=1
274                  req.client = user
275                  req.clientping_time=datetime.now()
276                  req.save()
277                  # find out tileset filesize and age
278                  # (always 'tile' layer for now. need to find something better)
279                  tilelayer = Layer.objects.get(name='tile')
280                  (tilepath, tilefile) = Tileset(tilelayer, req.min_z, req.x, req.y).get_filename(settings.TILES_ROOT)
281                  tilefile = os.path.join(tilepath, tilefile)
282                  try: 
283                    fstat = os.stat(tilefile)
284                    (fsize,mtime) = (fstat[6], fstat[8])
285                  except OSError: 
286                    (fsize,mtime) = 0,0
287                  html="OK|5|%s|%d|%d" % (req,fsize,mtime)
288                else:
289                  html ="XX|5|You have more than 50 active requests. Check your client."
290            except IndexError:
291                html ="XX|5|No requests in queue"
292          else:
293            # client version no in whitelist
294            logging.info("User %s connects with disallowed client '%s'." %(user,form.cleaned_data['version']))
295            html="XX|5|Invalid client version."
296        else:
297            # user is None, auth failed
298            html="XX|5|Invalid username. Your username and password were incorrect or the user has been disabled."
299      else: #form was not valid
300        html = "XX|5|Form invalid. "+str(form.errors)
301      return HttpResponse(html);
302    else: #request.method != POST, show the web form
303      authform, form = ClientAuthForm(), TakeRequestForm()
304      return render_to_response('requests_take.html',{'clientauthform': authform, 'takeform': form})
305    return HttpResponse(html)
306
307@cache_control(no_cache=True)
308def request_changedTiles(request):
309    setting = Settings()
310    #check if we access this page from the whitelisted ip address
311    allowed_ip = setting.getSetting('changed_tiles_allowed_IP')
312    if not allowed_ip: allowed_ip = setting.setSetting('changed_tiles_allowed_IP',request.META['REMOTE_ADDR'])
313    if allowed_ip != request.META['REMOTE_ADDR']:
314      return HttpResponseForbidden('Access not allowed from this IP address.')
315    #fetch the url that is called to retrieve the changed tiles
316    url = setting.getSetting('changed_tiles_api_url')
317    if not url: url = setting.setSetting('changed_tiles_api_url','http://www.openstreetmap.org/api/0.5/changes?hours=6')
318
319    html="Requested tiles:\n"
320    xml_dom = xml.dom.minidom.parse(urllib.urlopen(url))
321    tiles = xml_dom.getElementsByTagName("tile")
322    for tile in tiles:
323      (z,x,y) = tile.getAttribute('z'),tile.getAttribute('x'),tile.getAttribute('y')
324      CreateFormClass = CreateForm
325      CreateFormClass.base_fields['max_z'].required = False 
326      CreateFormClass.base_fields['layers'].required = False 
327      CreateFormClass.base_fields['status'].required = False 
328      form = CreateFormClass({'min_z': z, 'x': x, 'y': y, 'priority': 2})
329      if form.is_valid():
330        req, reason = saveCreateRequestForm(request, form)
331        if req:
332          html += "Render '%s' (%s,%s,%s)\n" % (req.layers_str,form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'])
333        else: html +="Renderrequest failed (%s,%s,%s): %s\n" % (form.cleaned_data['min_z'],form.cleaned_data['x'],form.cleaned_data['y'], reason)
334      else:
335        html+="form is not valid. %s\n" % form.errors
336    xml_dom.unlink()
337    logging.info("Requested %d changes Tilesets." % len(tiles))
338    return HttpResponse(html,mimetype='text/plain')
339
340@cache_control(no_cache=True)
341def expire_tiles(request):
342    """ Rerequest old active request that haven't been pinged by the client
343        for a while. Also delete old finished requests that we don't need
344        anymore.
345    """
346    rerequested = Request.objects.filter(status=1,clientping_time__lt=datetime.now()-timedelta(0,0,0,0,0,6))
347    for r in rerequested:
348      #next django version will be able to just update() this
349      r.status=0
350      r.save()
351
352    expired = Request.objects.filter(status=2, clientping_time__lt=datetime.now()-timedelta(2,0,0,0,0,0))
353    expired.delete()
354
355    html="Reset %d requests to unfinished status. Expired %d old finished requests." % (len(rerequested),len(expired))
356    logging.debug(html)
357    return HttpResponse(html,mimetype='text/plain')
358
359@cache_control(no_cache=True)
360def stats_munin_requests(request,status):
361    reply=''
362    states={'pending':0,'active':1,'done':2}
363    if states.has_key(status): state = states[status]
364    else: return HttpResponseNotFound('Unknown request state')
365    reqs = Request.objects.filter(status=state)
366    if state < 2:
367      #output low/medium/high requests for unfinished ones
368      for val, state in enumerate(['high','medium','low']):
369        c = reqs.filter(priority=val+1).count()
370        reply += "%s.value %d\n" % (state,c)
371    else:
372      #output requests finished per last hour and 48 moving avg
373      reply += "_req_processed_last_hour.value %d\n" %  (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,1)).count())
374      reply += 'done.value %d' % (reqs.filter(clientping_time__gt=datetime.now()-timedelta(0,0,0,0,0,48)).count() // 48)
375    return HttpResponse(reply,mimetype='text/plain')
376
377@cache_control(must_revalidate=True, max_age=3600)
378def show_latest_client_version(request):
379    html = Settings().getSetting(name='latest_client_version')
380    return HttpResponse(html,mimetype='text/plain')
Note: See TracBrowser for help on using the repository browser.