source: subversion/applications/rendering/tilesAtHome/tilesGen.pl @ 13851

Last change on this file since 13851 was 13755, checked in by deelkar, 11 years ago

don't render 0-complexity tiles if running a "weak" client

  • Property svn:executable set to *
  • Property svn:keywords set to Revision
File size: 37.4 KB
Line 
1#!/usr/bin/perl
2#-------------------------------------------------------------
3# OpenStreetMap tiles@home
4#
5# Contact Deelkar or OJW on the Openstreetmap wiki for help using this program
6#-----------------------------------------------------------------------------
7# Copyright 2006, Oliver White, Etienne Cherdlu, Dirk-Lueder Kreie,
8# Sebastian Spaeth and others
9#
10# This program is free software; you can redistribute it and/or
11# modify it under the terms of the GNU General Public License
12# as published by the Free Software Foundation; either version 2
13# of the License, or (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
23#-----------------------------------------------------------------------------
24
25use warnings;
26use strict;
27use lib './lib';
28use File::Copy;
29use File::Path;
30use File::Temp qw(tempfile);
31use File::Spec;
32use Scalar::Util qw(blessed);
33use IO::Socket;
34use Error qw(:try);
35use tahlib;
36use TahConf;
37use Tileset;
38use Server;
39use Request;
40use Upload;
41use SVG::Rasterize;
42use SVG::Rasterize::CoordinateBox;
43use English '-no_match_vars';
44use POSIX;
45
46#---------------------------------
47
48# Read the config file
49my $Config = TahConf->getConfig();
50
51# Handle the command-line
52our $Mode = shift() || '';
53my $LoopMode = (($Mode eq "loop") or ($Mode eq "upload_loop")) ? 1 : 0;
54my $RenderMode = (($Mode eq "") or ($Mode eq "xy") or ($Mode eq "loop")) ? 1 : 0;
55my $UploadMode = (($Mode eq "upload") or ($Mode eq "upload_loop")) ? 1 : 0;
56my %EnvironmentInfo;
57
58# Override *nix locales
59delete $ENV{LC_ALL};
60delete $ENV{LC_NUMERIC};
61delete $ENV{LANG};
62$ENV{LANG} = 'C';
63
64if ($RenderMode)
65{   # need to check that we can render and stuff
66    %EnvironmentInfo = $Config->CheckConfig();
67}
68else
69{   # for uploading we need only basic settings
70    %EnvironmentInfo = $Config->CheckBasicConfig();
71}
72
73# set the progress indicator variables
74our $currentSubTask;
75my $progress = 0;
76our $progressJobs = 1;
77our $progressPercent = 0;
78
79my $LastTimeVersionChecked = 0;   # version is only checked when last time was more than 10 min ago
80## NOTE: Version Check will not be run in xy mode, but on subsequent upload.
81## this will fix issues with modified, locally used clients that never ever upload
82if ($UploadMode or $LoopMode)
83{
84    if (NewClientVersion())
85    {
86        UpdateClient();
87        if ($LoopMode) 
88        {
89            reExec(-1);
90        }
91        else
92        {
93            statusMessage("tilesGen.pl has changed. Please restart new version.",1,0);
94            exit;
95        }
96    }
97}
98elsif ($RenderMode and ClientModified())
99{
100    statusMessage("! tilesGen.pl differs from svn repository. DO NOT UPLOAD to t\@h server.",1,0);
101}
102
103# Get version number from version-control system, as integer
104my $Version = '$Revision: 13755 $';
105$Version =~ s/\$Revision:\s*(\d+)\s*\$/$1/;
106printf STDERR "This is version %d (%s) of tilesgen running on %s, ID: %s\n", 
107    $Version, $Config->get("ClientVersion"), $^O, GetClientId();
108
109# autotuning complexity setting
110my $complexity = 0;
111
112# filestat: Used by reExecIfRequired.
113# This gets set to filesize/mtime/ctime of this script, and reExecIfRequired
114# checks to see if those parameters have changed since the last time it ran
115# to decide if it should reExec
116my $filestat;
117reExecIfRequired(-1); # This will not actually reExec, only set $filestat
118
119if ($LoopMode) {
120    # if this is a re-exec, we want to capture some of our status
121    # information from the command line. this feature allows setting
122    # any numeric variable by specifying "variablename=value" on the
123    # command line after the keyword "reexec". Currently unsuitable
124    # for alphanumeric variables.
125   
126    if ((shift||'') eq "reexec") {
127        my $idleSeconds; my $idleFor;
128        while(my $evalstr = shift()) {
129            die("$evalstr does not match option=value") unless $evalstr =~ /^[A-Za-z]+=\d+/;
130            eval('$'.$evalstr);
131            print STDERR "$evalstr\n" if ($Config->get("Verbose") >= 10);
132        }
133        setIdle($idleSeconds, 1);
134        setIdle($idleFor, 0);
135    }
136}
137
138# Setup SVG::Rasterize
139if( $RenderMode || $Mode eq 'startBatik' || $Mode eq 'stopBatik' ){
140    $SVG::Rasterize::object = SVG::Rasterize->new();
141    if( $Config->get("Rasterizer") ){
142        $SVG::Rasterize::object->engine( $Config->get("Rasterizer") );
143    }
144
145    print "- rasterizing using ".ref($SVG::Rasterize::object->engine)."\n";
146
147    if( $SVG::Rasterize::object->engine()->isa('SVG::Rasterize::Engine::BatikAgent') )
148    {
149        $SVG::Rasterize::object->engine()->heapsize($Config->get("BatikJVMSize"));
150        $SVG::Rasterize::object->engine()->host('localhost');
151        $SVG::Rasterize::object->engine()->port($Config->get("BatikPort"));
152    }
153
154    # Check for broken Inkscape versions
155    if( $SVG::Rasterize::object->engine()->isa('SVG::Rasterize::Engine::Inkscape') ){
156        my %brokenInkscapeVersions = (
157            "RenderStripes=0 will not work" => [0, 45, 1]
158            );
159
160        try {
161            my @version = $SVG::Rasterize::object->engine()->version();
162            die if scalar(@version) == 0;
163
164            while( my( $reason, $ver ) = each %brokenInkscapeVersions ){
165                my @brokenVersion = @{ $ver };
166
167                my $equal = 1;
168                if( $#brokenVersion == $#version ){
169                    for( my $i=0; $i < @version; $i++ ){
170                        $equal = $version[$i] eq $brokenVersion[$i];
171                        last if ! $equal;
172                    }
173                } else {
174                    $equal = 0;
175                }
176
177                if( $equal ){
178                    printf("! You have a broken version of Inkscape, %s. %s\n", join('.', @version), $reason);
179                }
180            }
181        } otherwise {
182            print "! Could not determine your Inkscape version\n";
183        };
184        print "* Take care to manually backup your inkscape user preferences\n"; 
185        print "  if you have knowingly changed them. \n";
186        print "  Some tilesets will cause inkscape to clobber that file!\n";
187#        print "  ~/.inkscape/preferences.xml\n";
188    }
189}
190
191# We need to keep parent PID so that child get the correct files after fork()
192my $parent_pid = $PID;
193my $upload_pid = -1;
194
195# keep track of the server time for current job
196my $JobTime;
197
198# If batik agent was started automatically, turn it off at exit
199our $StartedBatikAgent = 0;
200
201# Check the stylesheets for corruption and out of dateness, but only in loop mode
202# The existance check is to attempt to determine we're on a UNIX-like system
203
204## set all fault counters to 0;
205resetFault("fatal");
206resetFault("rasterizer");
207resetFault("nodata");
208resetFault("nodataROMA");
209resetFault("nodataXAPI");
210resetFault("renderer");
211resetFault("utf8");
212resetFault("upload");
213
214unlink("stopfile.txt") if $Config->get("AutoResetStopfile");
215
216# Be nice. Reduce program priority
217if( my $nice = $Config->get("Niceness") ){
218    if( $nice =~ /nice/ ){
219        $nice =~ s/nice\s*-n\s*//;
220        warn "You have Niceness set to a command, it should be only a number.\n";
221    }
222
223    if( $nice =~ /^\d+$/ ){
224        my $success=POSIX::nice($nice);
225        if( !defined($success) ){
226            printf STDERR "WARNING: Unable to apply Niceness. Will run at normal priority";
227        }
228    }
229}
230
231#---------------------------------
232## Start processing
233
234if ($Mode eq "xy")
235{
236    # ----------------------------------
237    # "xy" as first argument means you want to specify a tileset to render
238    # ----------------------------------
239
240    my $X = shift();
241    my $Y = shift();
242    my $req = new Request;
243    if (not defined $X or not defined $Y)
244    { 
245        print STDERR "Usage: $0 xy <X> <Y> [<ZOOM> [<LAYERS>]]\n";
246        print STDERR "where <X> and <Y> are the tile coordinates and \n";
247        print STDERR "<ZOOM> is an optional zoom level (defaults to 12).\n";
248        print STDERR "<LAYERS> is a comma separated list (no spaces) of the layers to render.\n";
249        print STDERR "This overrides the layers specified in the configuration.\n";
250        exit;
251    }
252    my $Zoom = shift();
253    if (not defined $Zoom)
254    {
255       $Zoom = 12;
256       $currentSubTask = "warning";
257       statusMessage(" *** No zoomlevel specified! Assuming z12 *** ",1,0);
258    }
259
260    $req->ZXY($Zoom, $X, $Y);
261
262    my $Layers = shift();
263    if (not defined $Layers) {
264        if ($Zoom >= 12) {
265            $Layers = $Config->get("Layers");
266        }
267        elsif ($Zoom >= 6) {
268            $Layers = $Config->get("LowzoomLayers");
269        }
270        else {
271            $Layers = $Config->get("WorldLayers");
272        }
273    }
274    $req->layers_str($Layers);
275
276    my $tileset = Tileset->new($req);
277    my $tilestart = time();
278    $tileset->generate();
279    autotuneComplexity($tilestart, time(), $req->complexity);
280}
281#---------------------------------
282elsif ($Mode eq "loop") 
283{
284    # ----------------------------------
285    # Continuously process requests from server
286    # ----------------------------------
287
288    # Start batik agent if it's not runnig
289    if( $SVG::Rasterize::object->engine()->isa('SVG::Rasterize::Engine::BatikAgent') ){
290        my $result = $SVG::Rasterize::object->engine()->start_agent();
291        if( $result ){
292            $StartedBatikAgent = 1;
293            statusMessage("Started Batik agent", 0, 0);
294        }
295    }
296
297    # this is the actual processing loop
298   
299    while(1) 
300    {
301        ## before we start (another) round of rendering we first check if something bad happened in the past.
302        checkFaults();
303
304        ## note: Timeouts are cumulative so if there are X timeouts from api and Y timeouts from XAPI then we wait for each timeout, one after the other
305        checkDataFaults();
306
307        # look for stopfile and exit if found
308        if (-e "stopfile.txt")
309        {
310            if ($Config->get("ForkForUpload") && $upload_pid != -1)
311            {
312                statusMessage("Waiting for previous upload process (this can take a while)",1,0);
313                waitpid($upload_pid, 0);
314            }
315            print "We suggest that you set MaxTilesetComplexity to ".int($complexity)."\n";
316            cleanUpAndDie("Stopfile found, exiting","EXIT",7); ## TODO: agree on an exit code scheme for different types of errors
317        }
318
319        # Add a basic auto-updating mechanism.
320        if (NewClientVersion()) 
321        {
322            UpdateClient();
323            reExec($upload_pid);
324        }
325
326        reExecIfRequired($upload_pid); ## check for new version of tilesGen.pl and reExec if true
327
328        ### start processing here:
329        $progressJobs++;
330        # Render stuff if we get a job from server
331        ProcessRequestsFromServer();
332        # upload results
333        uploadTilesets();
334    }
335}
336#---------------------------------
337elsif ($Mode eq "upload") 
338{   # Upload mode
339    upload();
340}
341#---------------------------------
342elsif ($Mode eq "upload_loop")
343{
344    while(1) 
345    {
346        my $startTime = time();
347        my $elapsedTime;
348
349        # before we start (another) round of uploads we first check
350        # if something bad happened in the past.
351        checkFaults();
352
353        # look for stopfile and exit if found
354        if (-e "stopfile.txt")
355        {
356            statusMessage("Stopfile found, exiting",1,0);
357            exit;
358        }
359
360        # Add a basic auto-updating mechanism.
361        if (NewClientVersion()) 
362        {
363            UpdateClient();
364            reExec(-1);
365        }
366
367        # check for new version of tilesGen.pl and reExec if true
368        reExecIfRequired(-1);
369
370        # uploading ZIP files here, returns 0 if nothing to do and -1 on error
371        my $files_uploaded = upload();
372           
373        if ($files_uploaded == 0) {
374            # no error, but no files uploaded
375            talkInSleep("waiting for new ZIP files to upload",30);
376        }
377        else {
378            $elapsedTime = time() - $startTime;
379            statusMessage(sprintf("upload finished in  %d:%02d", 
380                                  $elapsedTime/60, $elapsedTime%60),1,0);
381            $progressJobs++;
382        }
383    } #end of infinite while loop
384}
385#---------------------------------
386elsif ($Mode eq "version") 
387{
388    exit(1);
389}
390#---------------------------------
391elsif ($Mode eq "stop")
392{
393    if (open F, '>', "stopfile.txt") 
394    {
395        close F;
396        statusMessage("stop signal was sent to the currently running tilesGen.pl",1,0);
397        statusMessage("please note that it may take quite a while for it to exit",1,0);
398        exit(0);
399    }
400    else
401    {
402        statusMessage("stop signal was NOT sent to the currently running tilesGen.pl - stopfile.txt could NOT be created",1,0);
403    }
404    exit(1);
405}
406#---------------------------------
407elsif ($Mode eq "update") 
408{
409    UpdateClient();
410}
411#---------------------------------
412elsif ($Mode eq "") 
413{
414    # ----------------------------------
415    # Normal mode renders a single request from server and exits
416    # ----------------------------------
417    my ($did_something, $message) = ProcessRequestsFromServer();
418   
419    if (! $did_something)
420    {
421        statusMessage("you may safely press Ctrl-C now if you want to exit tilesGen.pl",1,0);
422        talkInSleep($message, 60);
423    }
424    statusMessage("if you want to run this program continuously, use loop mode",1,0);
425    statusMessage("please run \"tilesGen.pl upload\" now",1,0);
426}
427#---------------------------------
428elsif ($Mode eq "startBatik")
429{
430    my $result = $SVG::Rasterize::object->engine()->start_agent();
431    if( $result ){
432        $StartedBatikAgent = 1;
433        statusMessage("Started Batik agent", 0, 0);
434    } else {
435        statusMessage("Batik agent already running");
436    }
437}
438#---------------------------------
439elsif ($Mode eq "stopBatik")
440{
441    my $result = $SVG::Rasterize::object->engine()->stop_agent();
442    if( $result == 1 ){
443        statusMessage("Successfully sent stop message to Batik agent", 0, 0);
444    } elsif( $result == 0 ){
445        statusMessage("Could not contact Batik agent", 0, 0);
446    } else {
447        statusMessage($result, 0, 0);
448    }
449}
450#---------------------------------
451else {
452    # ----------------------------------
453    # "help" (or any other non understood parameter) as first argument tells how to use the program
454    # ----------------------------------
455    my $Bar = "-" x 78;
456    print "\n$Bar\nOpenStreetMap tiles\@home client\n$Bar\n";
457    print "Usage: \nNormal mode:\n  \"$0\", will download requests from server\n";
458    print "Specific area:\n  \"$0 xy <x> <y> [z [layers]]\"\n  (x and y coordinates of a\n";
459    print "zoom-12 (default) tile in the slippy-map coordinate system)\n";
460    print "See [[Slippy Map Tilenames]] on wiki.openstreetmap.org for details\n";
461    print "z is optional and can be used for low-zoom tilesets\n";
462    print "layers is a comma separated list (no spaces) of layers and overrides the config.\n";
463    print "Other modes:\n";
464    print "  $0 loop - runs continuously\n";
465    print "  $0 upload - uploads any tiles\n";
466    print "  $0 upload_loop - uploads tiles in loop mode\n";
467    print "  $0 startBatik - start batik agent\n";
468    print "  $0 stopBatik - stop batik agent\n";
469    print "  $0 version - prints out version string and exits\n";
470    print "\nGNU General Public license, version 2 or later\n$Bar\n";
471}
472
473
474#-----------------------------------------------------------------------------
475# forks to a new process when it makes sense,
476# uploads available tile data.
477# returns >=0 on success, -1 otherwise and dies if it could not fork
478#-----------------------------------------------------------------------------
479sub uploadTilesets
480{
481    my $Config = TahConf->getConfig();
482    if ($Config->get("ForkForUpload") and ($Mode eq "loop")) # makes no sense to fork upload if not looping.
483    {
484        # Upload is handled by another process, so that we can generate another tile at the same time.
485        # We still don't want to have two uploading process running at the same time, so we wait for the previous one to finish.
486        if ($upload_pid != -1)
487        {
488            statusMessage("Waiting for previous upload process to finish (this can take a while)",1,3);
489            waitpid($upload_pid, 0);
490            #FIXME: $upload_result is apparently never returned?! skip?
491            #$upload_result = $? >> 8;
492        }
493
494        $upload_pid = fork();
495        if ((not defined $upload_pid) or ($upload_pid == -1))
496        {   # exit if asked to fork but unable to
497            cleanUpAndDie("loop: could not fork, exiting","EXIT",4);
498        }
499        elsif ($upload_pid == 0)
500        {   # we are the child, so we run the upload and exit the thread
501            try {
502                upload();
503            }
504            otherwise {
505                exit 0;
506            };
507            exit 1;
508        }
509    }
510    else
511    {   ## no forking going on
512        try {
513            my $result = upload();
514
515            if ($result == -1)
516            {     # we got an error in the upload process
517                addFault("upload",1);
518            }
519            else
520            {     #reset fault counter if we uploaded successfully
521                resetFault("upload");
522            }
523        }
524        catch UploadError with {
525            my $err = shift();
526            if (!$err->value() eq "QueueFull") {
527                cleanUpAndDie("Error uploading tiles: " . $err->text(), "EXIT", 1);
528            }
529        };
530    }
531    # no error, just nothing to upload
532}
533
534#-----------------------------------------------------------------------------
535# upload() uploads all previously
536# zipped up tilesets. It returns the number of uploaded files or -1 on error
537#-----------------------------------------------------------------------------
538sub upload
539{
540    #upload all existing zip files
541    keepLog($PID,"upload","start","$progressJobs");
542
543    my $upload = new Upload;
544    my $files_uploaded = 0;
545    try {
546        $files_uploaded = $upload->uploadAllZips();
547        resetFault("upload");
548    }
549    catch UploadError with {
550        my $error = shift();
551        if ($error->value() eq "ServerError") {
552            statusMessage("Server error: " . $error->text(), 1, 0);
553            addFault("upload");
554            talkInSleep("Waiting before attempting new upload", 300) if ($LoopMode);
555        }
556        elsif ($error->value() eq "0") {
557            statusMessage("Upload error: " . $error->text(), 1, 0);
558            addFault("upload");
559            talkInSleep("Waiting before attempting new upload", 300) if ($LoopMode);
560        }
561        else {
562            $error->throw();
563        }
564    };
565
566    keepLog($PID, "upload", "stop", 0);
567    return $files_uploaded;
568}
569
570#-----------------------------------------------------------------------------
571# Ask the server what tileset needs rendering next
572#-----------------------------------------------------------------------------
573sub ProcessRequestsFromServer 
574{
575    my $Config = TahConf->getConfig();
576    if ($Config->get("LocalSlippymap"))
577    {
578        print "Config option LocalSlippymap is set. Downloading requests\n";
579        print "from the server in this mode would take them from the tiles\@home\n";
580        print "queue and never upload the results. Program aborted.\n";
581        cleanUpAndDie("ProcessRequestFromServer:LocalSlippymap set, exiting","EXIT",1);
582    }
583
584    if ($Config->get("CreateTilesetFile"))
585    {
586        print "Config option CreateTilesetFile is set. We can not upload Tileset\n";
587        print "files, yet. Downloading requests\n";
588        print "from the server in this mode would take them from the tiles\@home\n";
589        print "queue and never upload the results. Please use xy mode. Program aborted.\n";
590        cleanUpAndDie("ProcessRequestFromServer:CreateTilesetFile set, exiting","EXIT",1);
591    }
592
593    my $req;
594    my $Server = Server->new();
595    do {
596        statusMessage("Retrieving next job", 0, 3);
597        try {
598            $req = $Server->fetchRequest();
599
600            # got request, now check that it's not too complex
601            if ($Config->get('MaxTilesetComplexity')) {
602                #the setting is enabled
603                if ($req->complexity() > $Config->get('MaxTilesetComplexity')) 
604                {
605                    # too complex!
606                    statusMessage("Ignoring too complex tile (" . $req->ZXY_str() . ", "
607                    . int($req->complexity()) . " > " . int($Config->get('MaxTilesetComplexity')). ")", 1, 3);
608                    eval {
609                        $Server->putRequestBack($req, "TooComplex");
610                    }; # ignoring exceptions
611                    $req = undef;  # set to undef, need another loop
612                    talkInSleep("Waiting before new tile is requested", 15); # to avoid re-requesting the same tile
613                }
614                elsif (not $req->complexity())
615                {
616                    # unknown complexity!
617                    if ($Config->get('MaxTilesetComplexity') < $Config->get('AT_average')) 
618                    {
619                        # we have a weak client so we do not trust unknown complexity
620                        statusMessage("Ignoring unknown complexity tile (" . $req->ZXY_str() . ", "
621                           . int($req->complexity()) . " > " . int($Config->get('MaxTilesetComplexity')). ")", 1, 3);
622                        eval {
623                            $Server->putRequestBack($req, "NoComplexity");
624                        }; # ignoring exceptions
625                        $req = undef;  # set to undef, need another loop
626                        talkInSleep("Waiting before new tile is requested", 15); # to avoid re-requesting the same tile
627                    }
628                }
629            }
630            # and now check whether we found it unrenderable before
631            if (defined $req and $req->is_unrenderable()) {
632                statusMessage("Ignoring unrenderable tile (" . $req->ZXY_str . ')', 1, 3);
633                eval {
634                    $Server->putRequestBack($req, "Unrenderable");
635                }; # ignoring exceptions
636                $req = undef;   # we need to loop yet again
637                talkInSleep("Waiting before new tile is requested", 15); # to avoid re-requesting the same tile
638            }
639            # check whether there are any layers requested
640            if (defined $req and scalar($req->layers()) == 0) {
641                my $layers;
642                if ($req->Z() >= 12) {
643                    $layers = $Config->get("Layers");
644                }
645                else {
646                    $layers = $Config->get("LowzoomLayers");
647                }
648                statusMessage("Tile request with no layers, assuming default layers ($layers)", 1, 3);
649                $req->layers_str($layers);
650            }
651            # check whether we can actually render the requested zoom level for all requested layers
652            if (defined $req) {
653                my $zoom = $req->Z();
654                foreach my $layer ($req->layers()) {
655                    if (($zoom < $Config->get("${layer}_MinZoom")) or ($zoom > $Config->get("${layer}_MaxZoom"))) {
656                        statusMessage("Zoom level $zoom is out of the configured range for Layer $layer. Ignoring tile.", 1, 3);
657                        eval {
658                            $Server->putRequestBack($req, "ZoomOutOfRange ($layer z$zoom)");
659                        };
660                        $req = undef;
661                        last; # don't check any more layers
662                    }
663                }
664            }
665        }
666        catch ServerError with {
667            my $err = shift();
668            if ($err->value() eq "PermError") {
669                cleanUpAndDie($err->text(), "EXIT", 1);
670            }
671            else {
672                talkInSleep($err->text(), 60);
673            }
674        };
675    } until ($req);
676
677    # Information text to say what's happening
678    statusMessage("Got work from the server: " . $req->layers_str() . ' (' . $req->ZXY_str() . ')', 0, 6);
679
680    try {
681        my $tileset = Tileset->new($req);
682        my $tilestart = time();
683        $tileset->generate();
684        autotuneComplexity($tilestart, time(), $req->complexity);
685
686        # successfully received data, reset data faults
687        resetFault("nodata");
688        resetFault("nodataROMA");
689        resetFault("nodataXAPI");
690
691        # successfully rendered, so reset renderer faults
692        resetFault("renderer");
693        resetFault("inkscape");
694        resetFault("utf8");
695
696        # Rendered tileset, don't idle in next round
697        setIdle(0,0);
698    }
699    catch RequestError with {
700        my $err = shift();
701        cleanUpAndDie($err->text(), "EXIT", 1);
702    }
703    catch TilesetError with {
704        my $err = shift();
705        eval {
706            $Server->putRequestBack($req, $err->text()) unless $Mode eq 'xy';
707        }; # ignoring exceptions
708        if ($err->value() eq "fatal") {
709            # $err->value() is "fatal" for fatal errors
710            cleanUpAndDie($err->text(), "EXIT", 1);
711        }
712        else {
713            # $err->value() contains the error category for non-fatal errors
714            addFault($err->value(), 1);
715            statusMessage($err->text(), 1, 0);
716        }
717        talkInSleep("Waiting before new tile is requested", 15); # to avoid re-requesting the same tile
718    };
719}
720#-----------------------------------------------------------------------------
721# autotunes the complexity variable to avoid too complex tiles
722#-----------------------------------------------------------------------------
723sub autotuneComplexity #
724{
725    my $start = shift();
726    my $stop = shift();
727    my $tilecomplexity = shift();
728    my $deltaT = $stop - $start;
729
730#    my $Config = TahConf->getConfig();
731    my $timeaim = $Config->get("AT_timeaim");
732    my $minimum = $Config->get("AT_minimum");
733    my $alpha = $Config->get("AT_alpha");
734
735    if($Mode eq "xy") {
736        statusMessage ("Tile took us ".$deltaT." seconds to render",1,3);
737    } else {
738        statusMessage ("Tile of complexity ".$tilecomplexity." took us ".$deltaT." seconds to render",1,3);
739    }
740
741    if(! $complexity) { # this is the first call of this function
742        if($Config->get('MaxTilesetComplexity')) {
743            $complexity = $Config->get('MaxTilesetComplexity');
744        } elsif (($tilecomplexity > 5472) && ($deltaT > 0)) {
745            $complexity = $tilecomplexity * $timeaim / $deltaT;
746        } else {
747            $complexity = 10 * $tilecomplexity;
748        }
749    }
750
751    # empty tiles might have size 0 or 5472. if that changes, change this magic number too.
752    if (($tilecomplexity > 5472) && ($deltaT > 0)) {
753        $complexity = $alpha * ($tilecomplexity * $timeaim / $deltaT) + (1-$alpha) * $complexity;
754    }
755    $complexity = $minimum if $complexity < $minimum;
756
757    statusMessage("Suggested complexity is currently: ".int($complexity)." ",1,6);
758
759    if($Config->get('MaxTilesetComplexity')) {
760        # if MaxTilesetComplexity is not set we still do our calculations
761        # but we don't limit the client. The hint on client exit has to be enough.
762        $Config->set('MaxTilesetComplexity', $complexity);
763    }
764}
765#-----------------------------------------------------------------------------
766# Gets latest copy of client from svn repository
767# returns 1 on perceived success.
768#-----------------------------------------------------------------------------
769sub UpdateClient
770{
771    my $Config = TahConf->getConfig();
772    my $Cmd = sprintf("\"%s\" %s",
773        $Config->get("Subversion"),
774        $Config->get("SubversionUpdateCmd"));
775
776    statusMessage("Updating the Client",1,0);
777    runCommand($Cmd,$PID); # FIXME: evaluate output and handle locally changed files that need updating!
778
779    if (ClientModified())
780    {
781        return cleanUpAndDie("Auto-update failed","EXIT",1);
782    }
783    else
784    {
785        my $versionfile = "version.txt";
786        DownloadFile($Config->get("VersionCheckURL"), $versionfile ,0);
787        return 1;
788    }
789}
790
791#-----------------------------------------------------------------------------
792# Checks svn status for local code modifications
793# returns 1 if such modifications exist
794#-----------------------------------------------------------------------------
795sub ClientModified
796{
797    my $Cmd = sprintf("\"%s\" %s",
798        $Config->get("Subversion"),
799        "status -q --ignore-externals");
800
801    my $svn_status = `$Cmd`;
802
803    chomp $svn_status;
804    #$svn_status =~ s/^M.*\n?//mg;  # FFS use a future date in version.txt instead of this line.
805    if ($svn_status ne '')
806    {
807        statusMessage("svn status did not come back clean, check your installation",1,0);
808        print STDERR $svn_status;
809    }
810    return ($svn_status ne '');
811}
812
813sub NewClientVersion 
814{
815    my $Config = TahConf->getConfig();
816    return 0 if (time() - $LastTimeVersionChecked < 600);
817    my $versionfile = "version.txt";
818    my $runningVersion;
819    if (open(VERFILE, "<", $versionfile))
820    {
821        $runningVersion = <VERFILE>;
822        chomp $runningVersion;
823        close VERFILE;
824    }
825    elsif (open(VERFILE, ">", $versionfile))
826    {
827        $runningVersion = 0; 
828        print VERFILE $runningVersion;
829        close VERFILE;
830    }
831    else
832    {
833        die("can't open $versionfile");
834    }
835    # return 0;
836
837    my $curVerFile = "newversion.txt";
838    my $currentVersion;
839   
840    DownloadFile($Config->get("VersionCheckURL"), $curVerFile ,0);
841    if (open(VERFILE, "<", $curVerFile))
842    {
843        $currentVersion = <VERFILE>;
844        chomp $runningVersion;
845        close VERFILE;
846        # rename($curVerFile,$versionfile); # if enabled, this assumes the client is immediately, and successfully updated afterwards!
847    }
848    if ($currentVersion)
849    {
850        $LastTimeVersionChecked = time();
851        if ($runningVersion > $currentVersion)
852        {
853            statusMessage("\n! WARNNG: you cannot have a more current client than the server: $runningVersion > $currentVersion",1,0);
854            return 0;
855        }
856        elsif ($runningVersion == $currentVersion)
857        {
858            return 0; # no new version
859        }
860        else
861        {
862            return 1; # version on server is newer
863        }
864    }
865    else
866    {
867        statusMessage(" ! WARNING: Could not get version info from server!",1,0);
868        return 0;
869    }
870}
871
872
873#-----------------------------------------------------------------------------
874# Transform an OSM file (using osmarender) into SVG
875# returns 1 on success, 0 on failure
876#-----------------------------------------------------------------------------
877sub xml2svg 
878{
879    my $Config = TahConf->getConfig();
880    my($osmData, $bbox, $MapFeatures, $SVG, $zoom) = @_;
881    my $TSVG = "$SVG";
882    my $NoBezier = $Config->get("NoBezier") || $zoom <= 11;
883
884    if (!$NoBezier) 
885    {
886        $TSVG = "$SVG-temp.svg";
887    }
888
889    my $success = 0;
890    if ($Config->get("Osmarender") eq "XSLT")
891    {
892        my $XslFile;
893
894        $XslFile = "osmarender/xslt/osmarender.xsl";
895
896        my $Cmd = sprintf(
897          "\"%s\" tr --maxdepth %s %s -s osmfile=%s -s minlat=%s -s minlon=%s -s maxlat=%s -s maxlon=%s %s > \"%s\"",
898          $Config->get("XmlStarlet"),
899          $Config->get("XmlStarletMaxDepth"),
900          $XslFile,
901          $osmData,
902          $bbox->S, $bbox->W, $bbox->N, $bbox->E,
903          "$MapFeatures",
904          $TSVG);
905
906        statusMessage("Transforming zoom level $zoom with XSLT",0,3);
907        $success = runCommand($Cmd,$PID);
908    }
909    elsif($Config->get("Osmarender") eq "orp")
910    {
911        my $Cmd = sprintf("perl osmarender/orp/orp.pl -r %s -o %s -b %s,%s,%s,%s %s",
912          $MapFeatures,
913          $TSVG,
914          $bbox->S, $bbox->W, $bbox->N, $bbox->E,
915          $osmData);
916
917        statusMessage("Transforming zoom level $zoom with or/p",0,3);
918        $success = runCommand($Cmd,$PID);
919    }
920    else
921    {
922        die "invalid Osmarender setting in config";
923    }
924    if (!$success) {
925        statusMessage(sprintf("%s produced an error, aborting render.", $Config->get("Osmarender")),1,0);
926        return cleanUpAndDie("xml2svg failed",$Mode,3);
927    }
928
929    # look at temporary svg wether it really is a svg or just the
930    # xmlstarlet dump and exit if the latter.
931    open(SVGTEST, "<", $TSVG) || return;
932    my $TestLine = <SVGTEST>;
933    chomp $TestLine;
934    close SVGTEST;
935
936    if (grep(!/</, $TestLine))
937    {
938       statusMessage("File $TSVG doesn't look like svg, aborting render.",1,0);
939       return cleanUpAndDie("xml2svg failed",$Mode,3);
940    }
941#-----------------------------------------------------------------------------
942# Process lines to Bezier curve hinting
943#-----------------------------------------------------------------------------
944    if (!$NoBezier) 
945    {   # do bezier curve hinting
946        my $Cmd = sprintf("perl ./lines2curves.pl %s > %s",
947          $TSVG,
948          $SVG);
949        statusMessage("Beziercurvehinting zoom level $zoom",0,3);
950        runCommand($Cmd,$PID);
951#-----------------------------------------------------------------------------
952# Sanitycheck for Bezier curve hinting, no output = bezier curve hinting failed
953#-----------------------------------------------------------------------------
954        my $filesize= -s $SVG;
955        if (!$filesize) 
956        {
957            copy($TSVG,$SVG);
958            statusMessage("Error on Bezier Curve hinting, rendering without bezier curves",1,0);
959        }
960    }
961    else
962    {   # don't do bezier curve hinting
963        statusMessage("Bezier Curve hinting disabled.",0,3);
964    }
965    return 1;
966}
967
968
969#-----------------------------------------------------------------------------
970# Get the width and height (in SVG units, must be pixels) of an SVG file
971#-----------------------------------------------------------------------------
972sub getSize($)
973{
974    my $SVG = shift();
975    open(my $fpSvg,"<",$SVG);
976    while(my $Line = <$fpSvg>)
977    {
978        if($Line =~ /height=\"(.*)px\" width=\"(.*)px\"/)
979        {
980            close $fpSvg;
981            return(($1,$2,1));
982        }
983    }
984    close $fpSvg;
985    return((0,0,0));
986}
987
988#-----------------------------------------------------------------------------
989# A function to re-execute the program. 
990#
991# This function attempts to detect whether the perl script has changed
992# since it was invoked initially, and if so, just runs the new version.
993# This can be used to update the program while it is running (as it is
994# sometimes hard to hit Ctrl-C at exactly the right moment!)
995#-----------------------------------------------------------------------------
996sub reExecIfRequired
997{
998    my $child_pid = shift();## FIXME: make more general
999
1000    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,
1001        $ctime,$blksize,$blocks) = stat($0);
1002    my $de = "$size/$mtime/$ctime";
1003    if (!defined($filestat))
1004    {
1005        $filestat = $de; 
1006        return;
1007    }
1008    elsif ($filestat ne $de)
1009    {
1010        reExec($child_pid);
1011    }
1012}
1013
1014#-----------------------------------------------------------------------------
1015# A function to re-execute the program. 
1016#
1017# This function restarts the program unconditionally.
1018#-----------------------------------------------------------------------------
1019sub reExec
1020{
1021    my $child_pid = shift();## FIXME: make more general
1022    my $Config = TahConf->getConfig();
1023    # until proven to work with other systems, only attempt a re-exec
1024    # on linux.
1025    return unless ($^O eq "linux" || $^O eq "cygwin" ||  $^O eq "darwin");
1026
1027    statusMessage("tilesGen.pl has changed, re-start new version",1,0);
1028    if ($Config->get("ForkForUpload") && $child_pid != -1)  ## FIXME: make more general
1029    {
1030        statusMessage("Waiting for child process (this can take a while)",1,0);
1031        waitpid($child_pid, 0);
1032    }
1033    exec "perl", $0, $Mode, "reexec", 
1034        "progressJobs=" . $progressJobs, 
1035        "idleSeconds=" . getIdle(1), 
1036        "idleFor=" . getIdle(0) or die("could not reExec");
1037}
1038
1039#------------------------------------------------------------
1040# check for faults and die when too many have occured
1041#------------------------------------------------------------
1042sub checkFaults
1043{
1044    if (getFault("fatal") > 0) {
1045        cleanUpAndDie("Fatal error occurred during loop, exiting","EXIT",1);
1046    }
1047    elsif (getFault("rasterizer") > 5) {
1048        cleanUpAndDie("Five times rasterizer failed, exiting","EXIT",1);
1049    }
1050    elsif (getFault("renderer") > 10) {
1051        cleanUpAndDie("rendering a tileset failed 10 times in a row, exiting","EXIT",1);
1052    }
1053    elsif (getFault("upload") > 50) {
1054        cleanUpAndDie("Five times the upload failed, perhaps the server doesn't like us, exiting","EXIT",1);
1055    }
1056}
1057
1058
1059#--------------------------------------------------------------------------------------
1060# check for faults with data downloads and add delays or die when too many have occured
1061#--------------------------------------------------------------------------------------
1062sub checkDataFaults
1063{
1064    my $sleepdelay = 1;
1065    if (getFault("nodata") > 0) { # check every network condition regardless of the other network outcomes
1066        my $numfaults=getFault("nodata");
1067        if ($numfaults > 25) {
1068            cleanUpAndDie("More than 25 times no data, perhaps the server doesn't like us, exiting","EXIT",1);
1069        }
1070        else {
1071            $sleepdelay=5*(2**$numfaults); # wait 10, 20, 40, 80, ... seconds. for a total of about 6 hours
1072            $sleepdelay=600 if ($sleepdelay > 600);
1073            talkInSleep($numfaults." times no data", $sleepdelay);
1074        }
1075    }
1076    if (getFault("nodataXAPI") > 0) {
1077        my $numfaults=getFault("nodataXAPI");
1078        if ($numfaults >= 20) {
1079            cleanUpAndDie("20 times no data from XAPI, perhaps the server doesn't like us, exiting","EXIT",1); # allow XAPI more leeway
1080        }
1081        else {
1082            $sleepdelay=5*(2**$numfaults); # wait 10, 20, 49, 80 seconds
1083            $sleepdelay=600 if ($sleepdelay > 600);
1084            talkInSleep($numfaults." times no XAPI data", $sleepdelay);
1085        }
1086    }
1087}
1088
1089#sub fileUTF8ErrCheck moved to lib/tahlib.pm
Note: See TracBrowser for help on using the repository browser.