source: subversion/utils/osmfilter/osmfilter.pl @ 2016

Last change on this file since 2016 was 1842, checked in by joerg, 13 years ago

add track counter

  • Property svn:executable set to *
File size: 50.0 KB
Line 
1#!/usr/bin/perl
2# This Script converts/filters GPS-Track-Data
3# Input is one of the folowing:
4#   - Kismet-GPS File   *.gps
5#   - GpsDrive-Track    *.sav
6#   - GPX File          *.gpx
7#   - Garmin mps File   *.mps
8#   - Garmin gdb File   *.gdb
9#   - Netstumbler Files *.ns1
10#
11# Standars Filters:
12#       - are points are inside [ -90.0  , -180  , 90.0    , 180   ], # World
13#       - minimum good points          > 5 Points/Track
14#       - distance between trackpoints < 2 Km
15#       - speed between Trackpoints    < 200 Km/h
16#       - the track point is near (<20m) any OSM-segment
17#
18# Output is:
19#   - OSM File for josm *.osm ( http://openstreetmap.org/)
20#   - GPX File for josm *.gpx
21#   - 00_collection.gpx, 00_collection.osm (one File with all good tracks)
22#   - 00_filter_areas.gpx with al the area filters
23#
24# Joerg Ostertag <osm-filter.openstreetmap@ostertag.name>
25# TODO:
26#   - eliminate duplicate waypoints
27#   - area filter for waypoints
28#   - eliminate duplicate Tracks
29#   - cut out part of tracks which cover the same road
30#   - make limits (max_speed, max_line_dist, ...) configurable
31#   - add config file
32#   - write more filters:
33#      - eliminate duplicate tracksegments (driving a street up and down)
34#      - elimiate trackpoints where the GPS was standing for a longer time at one point
35#      - Filter to eliminate all waypoints
36
37BEGIN {
38    my $dir = $0;
39    $dir =~s,[^/]+/[^/]+$,,;
40    unshift(@INC,"$dir/perl");
41
42    unshift(@INC,"../perl");
43    unshift(@INC,"~/svn.openstreetmap.org/utils/perl");
44    unshift(@INC,"$ENV{HOME}/svn.openstreetmap.org/utils/perl");
45}
46
47
48use strict;
49use warnings;
50
51use Data::Dumper;
52use File::Basename;
53use File::Copy;
54use File::Path;
55use Getopt::Long;
56use HTTP::Request;
57use IO::File;
58use Pod::Usage;
59use XML::Parser;
60
61use Geo::Geometry;
62use Geo::OSM::SegmentList;
63use Geo::OSM::Tracks2OSM;
64use Geo::OSM::Write;
65use Geo::Tracks::GpsBabel;
66use Geo::Tracks::Kismet;
67use Geo::Tracks::NMEA;
68use Geo::Tracks::Tools;
69use Utils::Debug;
70use Utils::File;
71use Utils::Math;
72use Geo::GPX::File;
73
74my ($man,$help);
75#our $DEBUG =0;
76#our $VERBOSE =0;
77our $PROXY='';
78
79
80our $osm_stats         = {};
81our $osm_obj           = undef; # OSM Object currently read
82
83our $use_stdin         = 0;
84our $use_stdout        = 0;
85our $out_osm           = undef;
86our $fake_gpx_date     = undef;
87our $write_gpx_wpt     = 0;
88our $out_raw_gpx       = undef;
89our $out_upload_gpx    = undef;
90our $split_tracks      = undef;
91our @filter_area_files = ();
92our $draw_filter_areas = undef;
93our $do_filter_reduce_pt = undef;
94our $do_filter_clew   = undef;
95our $do_filter_against_osm = undef;
96our $do_filter_dup_trace_segments = undef;
97our $do_all_filters    = 0;
98our $generate_ways     = undef;
99
100our $FILTER_FILE = "$ENV{'HOME'}/.josm/filter.xml";
101
102
103##################################################################
104package OSM;
105##################################################################
106use Storable;
107use strict;
108use warnings;
109use Carp;
110
111use Geo::Geometry;
112use Geo::OSM::SegmentList;
113use Geo::Tracks::Tools;
114use Utils::Debug;
115use Utils::File;
116use Utils::Math;
117
118
119# ------------------------------------------------------------------
120# check if a nearby segment would fit
121sub is_segment_of_list_nearby($$$){
122    my $track        = shift; # Track to check
123    my $track_pos    = shift; # Track position to check
124    my $osm_segments = shift; # Segments to compare with
125    #                           List of \[$lat1,$lon1,$lat2,$lon2,$angle_north_relative]
126
127    my $max_angle = 30;
128
129    my $elem0=$track->[$track_pos-1];
130    my $elem1=$track->[$track_pos];
131    my $elem2=$track->[$track_pos+1];
132    my $skip_point=0;
133    my $min_dist = 40000;
134    my $compare_dist = $elem1->{compare_dist};
135
136    for my $segment_num ( 0 .. $#{@{$osm_segments}} ) {
137        my $segment = $osm_segments->[$segment_num];
138        # The line from or to the element has to be fairly parallel
139        next unless
140            ( abs ($elem0->{angle_n_r} - $segment->[4])  < $max_angle) ||
141            ( abs ($elem1->{angle_n_r} - $segment->[4])  < $max_angle);
142        #print STDERR "abs_angle: $angle_n_r2\n" if $DEBUG;
143
144        # Distance between line of segment($segment)  to trackpoint $elem1
145        my $dist = 1000*distance_line_point_Km($segment->[0],$segment->[1],
146                                               $segment->[2],$segment->[3],
147                                               $elem1->{lat},$elem1->{lon}
148                                               );
149        $min_dist = $dist if $dist < $min_dist;
150        next if $dist > $compare_dist; # in m
151        printf STDERR "Elem is %3.1f m away from line\n",$dist
152            if $DEBUG >5;
153        $skip_point++;
154        last;
155    }
156    # printf STDERR "Min Dist: $min_dist Meter\n";
157    return $skip_point;
158}
159
160
161# ------------------------------------------------------------------
162# check if new trackpoints are on existing osm tracks
163sub filter_against_osm($$$){
164    my $tracks       = shift; # reference to tracks list
165    my $all_osm_segments = shift;
166    my $config       = shift;
167
168    my $start_time=time();
169
170    my $filename=$tracks->{filename};
171    if ( $out_raw_gpx && $DEBUG >3 ){
172        my $new_gpx_file = "$filename-raw-pre-osm.gpx";
173        $new_gpx_file =~s/.gpx-raw-pre-osm.gpx/-raw-pre-osm.gpx/;
174        write_gpx_file($tracks,$new_gpx_file);
175    };
176
177    my $dist_osm_track = $config->{dist} || 40;
178
179    my $bounds = GPS::get_bounding_box($tracks);
180    #printf STDERR "Track Bounds: ".Dumper(\$bounds);
181    my $osm_segments = reduce_segments_list($all_osm_segments,$bounds);
182
183    enrich_tracks($tracks);
184
185    my $parsing_display_time=time();
186    my ($track_count,$point_count) =   count_data($tracks);
187    my $track_points_done=0;
188    my $track_no=0;
189    for my $track ( @{$tracks->{tracks}} ) {
190        $track_no++;
191        next if !$track;
192
193        for my $track_pos ( 0 .. $#{@{$track}} ) {
194            my $elem = $track->[$track_pos];
195            $track_points_done++;
196            if ( ! is_segment_of_list_nearby($track,$track_pos,$osm_segments)){
197                $elem->{good_point} = 1;
198                $elem->{split_track} =0;
199            } else {
200                $elem->{good_point} = 0;
201                $elem->{split_track} =1;
202            }
203            if ( ( $VERBOSE || $DEBUG ) &&
204                 ( time()-$parsing_display_time >10)
205                 )  {
206                $parsing_display_time= time();
207                print STDERR "Filter against osm track $track_no($track_count) ".mem_usage();
208                print STDERR time_estimate($start_time,$track_points_done,$point_count);
209                print STDERR "\r";
210            }
211        }
212    }
213
214    my $new_tracks = tracks_only_good_point($tracks);
215
216    print_count_data($new_tracks,"after Filtering against existing OSM Data");
217    print_time($start_time);
218    return $new_tracks;
219}
220
221##################################################################
222package GPSDrive;
223##################################################################
224use strict;
225use warnings;
226
227use Date::Parse;
228use Data::Dumper;
229
230use Geo::Geometry;
231use Utils::Debug;
232use Utils::File;
233use Utils::Math;
234
235# -----------------------------------------------------------------------------
236# Read GPSDrive Track Data
237sub read_gpsdrive_track_file($) { 
238    my $filename = shift;
239
240    my $start_time=time();
241
242    my $new_tracks={
243        filename => $filename,
244        tracks => [],
245        wpt => []
246        };
247
248    printf STDERR ("Reading $filename\n") if $VERBOSE || $DEBUG;
249    printf STDERR "$filename:   ".(-s $filename)." Bytes\n" if $DEBUG;
250
251    my $fh = data_open($filename);
252    return $new_tracks  unless $fh;
253
254    my $new_track = [];
255    while ( my $line = $fh->getline() ) {
256        chomp $line;
257        #printf STDERR "$line\n";
258        $line =~ s/^\s*//;
259        #my ($lat,$lon,$alt,$time) = ($line =~ m/\s*([\+\-\d\.]+)\s+([\+\-\d\.]+)\s+([\+\-\d\.]+)\s+(.*)/);
260        my ($lat,$lon,$alt,$time) = split(/\s+/,$line,4);
261        printf STDERR "(lat: $lat,lon: $lon, alt:$alt, time: $time)\n" if $DEBUG>1;
262        if ( ($lat>1000) || ( $lon>1000 )) { # new track
263            if ( scalar(@{$new_track}) >0 ) {
264                push(@{$new_tracks->{tracks}},$new_track);
265            }
266            $new_track = [];
267            next;
268        }
269
270        my $elem = {
271            lat => $lat, 
272            lon => $lon, 
273            alt => $alt, 
274            time => str2time($time),
275        };
276        push(@{$new_track},$elem);
277        bless($elem,"GPSDrive::gps-point");
278    }
279    push(@{$new_tracks->{tracks}},$new_track);
280    if ( $VERBOSE >1 ) {
281        printf STDERR "Read and parsed $filename";
282        print_time($start_time);
283    }
284
285    return $new_tracks;
286}
287
288
289my $GPSDRIVE_CONFIG_DIR          = "$ENV{'HOME'}/.gpsdrive";
290my $GPSDRIVE_WAYPT_FILE          = "$GPSDRIVE_CONFIG_DIR/way.txt";
291######################################################################
292# read a GpsDrive-Waypoint from the .gpsdrive/way.txt File
293my $waypoints={};
294sub get_waypoint($) {
295    my $waypoint_name = shift;
296   
297    # Look it up if it's cached?
298    if( defined ( $waypoints->{$waypoint_name} )){
299        return @{$waypoints->{$waypoint_name}};
300    }
301    # If they give just a filename, we should assume they meant the CONFIG_DIR
302    $GPSDRIVE_WAYPT_FILE = "$GPSDRIVE_CONFIG_DIR/$GPSDRIVE_WAYPT_FILE" unless ($GPSDRIVE_WAYPT_FILE =~ /\//);
303   
304    open(WAYPT,"$GPSDRIVE_WAYPT_FILE") || die "ERROR: get_waypoint Can't open: $GPSDRIVE_WAYPT_FILE: $!\n";
305    my ($name,$lat,$lon, $typ,$wlan, $action, $sqlnr, $proximity);
306    while (<WAYPT>) {
307        chomp;
308        next unless (/$waypoint_name/);
309        ($name,$lat,$lon, $typ, $wlan, $action, $sqlnr, $proximity) = split(/\s+/);
310    }
311    close(WAYPT);
312    unless (($lat) && ($lon)) {
313        printf STDERR "Unable to find waypoint '$waypoint_name' in '$GPSDRIVE_WAYPT_FILE'\n";
314        exit;
315    }
316    $waypoints->{$waypoint_name} = [$lat,$lon];
317    return ($lat,$lon,$proximity/1000);
318} #End get_waypoint
319
320##################################################################
321package GPS;
322##################################################################
323use strict;
324use warnings;
325
326use Date::Parse;
327use Data::Dumper;
328use Math::Trig;
329use Carp;
330
331use Geo::GPX::File;
332use Geo::Geometry;
333use Geo::Tracks::Tools;
334use Utils::Debug;
335use Utils::File;
336use Utils::Math;
337
338# ------------------------------------------------------------------
339# Check if the point is in the area to currently evaluate
340my $internal__filter_area_list;
341
342# -------------------------------------------------
343# Add these filters from source
344sub add_internal_filter_areas(){
345    push( @{$internal__filter_area_list},
346          (   
347               # Block a circle of <proximity> Km arround each point
348               #     { lat =>  48.175921        ,lon => 11.754312  ,proximity => .030 , block => 1 },
349               
350               # Allow Rules for square size areas
351               # Warning square areas are not tested yet
352
353               # min_lat,min_lon max_lat,max_lon,     (y,x)
354               #[ 48.0  , 11.6  , 48.4    , 12.0    ], # München
355               #[ 48.10  , 11.75  , 49.0    , 14.0    ], # Münchner-Osten-
356
357               # The rest of the World is blocked by default
358               [ -90.0  , -180  , 90.0    , 180   ], # World Allow
359               
360               ));
361}
362
363
364# -------------------------------------------------
365# Read XML Filter File from josm directory
366sub read_filter_areas_xml($){
367    my $filename = shift;
368
369    my $start_time=time();
370
371    my $new_tracks={
372        filename => $filename,
373        tracks => [],
374        wpt => []
375        };
376
377    printf STDERR "Reading Filter File $filename\n" if $VERBOSE || $DEBUG;
378    printf STDERR "$filename:   ".(-s $filename)." Bytes\n" if $DEBUG;
379
380    print STDERR "Parsing file: $filename\n" if $DEBUG;
381    my $p = XML::Parser->new( Style => 'Objects' ,
382                              );
383   
384    my $fh = data_open($filename);
385    if ( ! $fh ) {
386        warn "Could not open Filter File $filename\n";
387        return;
388    }
389    my $content = [{Kids => []}];
390    eval {
391        $content = $p->parse($fh);
392    };
393    if ( $@ ) {
394        warn "$@Error while parsing\n $filename\n";
395        printf STDERR Dumper(\$content);
396    }
397    if (not $p) {
398        print STDERR "WARNING: Could not parse filter data\n";
399        return $new_tracks;
400    }
401    if ( $DEBUG ) {
402        printf STDERR "Read and parsed $filename";
403        print_time($start_time);
404    }
405    for my $elem  ( @{$content->[0]->{Kids}} ) {
406        next unless ref($elem) eq "GPS::node";
407        for my $tag ( @{$elem->{Kids}} ) {
408            next unless defined $tag->{v};
409            $elem->{$tag->{k}}=$tag->{v};
410        };
411        delete $elem->{Kids};
412        if ( defined $elem->{'of:active'} && !  $elem->{'of:active'} ) {
413            next;
414        }
415        my $block =  $elem->{'of:block'};
416        $block = 1 unless defined $block;
417        my $lat =  $elem->{lat};
418        my $lon =  $elem->{lon};
419        my $proximity = $elem->{'of:radius'}||10000;
420        push( @{$internal__filter_area_list},
421          { lat => $lat,
422            lon => $lon,
423            proximity => $proximity,
424            block => $block, 
425        }
426              );
427    }
428}
429
430# -------------------------------------------------
431# Add all gpsdrive waypoints from this File as a filter
432# where deny specifies that its a block filter
433sub read_filter_areas($){
434    my $filename = shift;
435
436    printf STDERR "Read GpsDrive Filter File $filename\n"
437        if ( $DEBUG || $VERBOSE );
438
439    unless ( -s $filename ) {
440        printf STDERR "Filter File $filename not found\n";
441        return;
442    };
443
444    open(WAYPT,"$filename") || die "ERROR: get_waypoint Can't open: $filename: $!\n";
445    my ($name,$lat,$lon, $typ,$wlan, $action, $sqlnr, $proximity);
446    while (my $line = <WAYPT>) {
447        chomp($line);
448        ($name,$lat,$lon, $typ, $wlan, $action, $sqlnr, $proximity) = split(/\s+/,$line);
449        my $block=0;
450        next unless $name;
451        next unless $typ;
452        next unless $typ =~ m/^filter\./;
453        if ( $typ =~ m/deny/ ) {
454            $block = 1;
455        } elsif ( $typ =~ m/allow/) {
456            $block = 0;
457        } elsif ( $typ =~ m/none/) {
458            next;
459        } else {
460            warn "WARNING !!! unknown Filter type $typ for WP $name\n";
461        };
462        next unless $name;
463        push( @{$internal__filter_area_list},
464          { wp => $name, block => $block }
465              );
466    }
467    close(WAYPT);
468}
469
470# -------------------------------------------------
471# Read in all specified Filter Areas
472sub read_all_filter_areas(@){
473    my @filter_area_files=@_;
474    return unless  @filter_area_files;
475
476    if ( $filter_area_files[0] eq ''  ) {
477        shift ( @filter_area_files);
478        push ( @filter_area_files, $GPSDRIVE_WAYPT_FILE) if -s $GPSDRIVE_WAYPT_FILE;
479        push ( @filter_area_files, $FILTER_FILE) if -s $FILTER_FILE;
480        push ( @filter_area_files, 'internal');
481    };
482
483    for my $file ( @filter_area_files ) {
484        if ( $file =~ m/way[^\/]\.txt$/ ) {
485            read_filter_area();
486        } elsif ( $file =~ m/\.xml$/ ) {
487            read_filter_areas_xml($file);
488        } elsif ( $file =~ m/^internal$/ ) {
489            add_internal_filter_areas();
490        }
491        if ( $DEBUG >30 ) {
492            print "internal__filter_area_list:".Dumper(\$internal__filter_area_list);
493        }
494    }
495    print STDERR Dumper(\$internal__filter_area_list) if $DEBUG>2 || $VERBOSE>5;
496}
497
498# -------------------------------------------------
499# Check given Element against all defined area-filters
500sub check_allowed_area($$){
501    my $elem = shift;
502    my $filter_area_list = shift;
503   
504    return 1 unless @filter_area_files;
505    if ( ref($filter_area_list) ne "ARRAY" ) {
506        confess("check_allowed_area(): Filter list must be an array");
507    };
508    unless ( defined($elem->{lat}) && defined($elem->{lon}) ) {
509        confess("check_allowed_area(): Unknown Type of Element to check:".Dumper(\$elem));
510    };
511   
512    # print "check_allowed_area():".Dumper(\$elem).Dumper(\$internal__filter_area_list);
513    for my $area ( @{$filter_area_list} ) {
514        if (ref($area) eq "HASH" ) {
515            #print "check_allowed_area(HASH):".Dumper(\$elem).Dumper(\$area);
516            if ( defined ( $area->{wp} ) ) { # Get from GPSDrive ~/.gpsdrive/way.txt Waypoints
517                my $proximity;
518                ($area->{lat},$area->{lon},$proximity) = GPSDrive::get_waypoint($area->{wp});
519                $area->{proximity} ||= $proximity;
520                $area->{proximity} ||= 10;
521            }
522           
523            if ( distance_point_point_Km($area,$elem) < $area->{proximity} ) {
524                printf STDERR "check_allowed_area(".$elem->{lat}.",".$elem->{lon}.
525                    ") -> WP: $area->{wp} : block: $area->{block}\n"
526                    if $DEBUG > 30;
527                return ! $area->{block};
528            }
529        } else {
530            #print "check_allowed_area(ARRAY):".Dumper(\$elem).Dumper(\$area);
531            my ($min_lat,$min_lon, $max_lat,$max_lon ) = @{$area};
532            if ( $min_lat <= $elem->{lat} &&     $max_lat >= $elem->{lat} &&
533                 $min_lon <= $elem->{lon} &&     $max_lon >= $elem->{lon} ) {
534                if ( $DEBUG >30) {
535                    printf STDERR "Allow Square\n";
536                }
537                return 1;
538            }
539        }
540    }
541    return 1;
542}
543
544# --------------------------------------------
545# Return a tracklist whith a track for each area_filter
546sub draw_filter_areas(){
547    return [] unless $draw_filter_areas;
548    my $new_tracks={
549        filename => 'draw_filter_areas',
550        tracks => [],
551        wpt => []
552        };
553    for my $area ( @{$internal__filter_area_list} ) {
554        my $new_track = [];
555        if (ref($area) eq "HASH" ) {       
556            if ( defined ( $area->{wp} ) ) { # Get from GPSDrive way.txt Waypoints
557                my $proximity;
558                ($area->{lat},$area->{lon},$proximity) = GPSDrive::get_waypoint($area->{wp});
559
560                $area->{proximity} ||= $proximity;
561            }
562            $area->{proximity} ||= 10000;
563
564            unless ( defined $area->{lat} && defined $area->{lon}) {
565                warn "Waypoint $area->{wp} not found\n";
566            }
567           
568            my ($lat,$lon,$r) = ($area->{lat},$area->{lon},$area->{proximity}*360/40000);
569            for my $angel ( 0 .. 360 ) {
570                my $elem;
571                $elem->{lat} = $lat+sin($angel*2*pi/360)*$r;
572                $elem->{lon} = $lon+cos($angel*2*pi/360)*$r;
573                next unless ($elem->{lat} > -90) &&  ($elem->{lat} < 90);
574                next unless ($elem->{lon} > -180) &&  ($elem->{lon} < 180);
575                push(@{$new_track},$elem);
576                if ( ! ( $angel % 10 ) ) {
577                    my $dir = 1.3;
578                    $dir = 0.7 if $area->{block};
579                    my $lat1 = $lat+sin($angel*2*pi/360)*$r*$dir;
580                    my $lon1 = $lon+cos($angel*2*pi/360)*$r*$dir;
581                    next unless ($lat1 > -90) &&  ($lat1 < 90);
582                    next unless ($lon1 > -180) &&  ($lon1 < 180);
583
584                    push(@{$new_track},{lat=> $lat1,lon=>$lon1});
585                    push(@{$new_track},$elem);     
586                }
587            }
588        } else {
589            my ($min_lat,$min_lon, $max_lat,$max_lon ) = @{$area};
590            my $elem;
591            $elem->{lat} = $min_lat;
592            $elem->{lon} = $min_lon;    push(@{$new_track},$elem);
593            $elem->{lat} = $max_lat;    push(@{$new_track},$elem);
594            $elem->{lon} = $max_lon;    push(@{$new_track},$elem);
595            $elem->{lat} = $min_lat;    push(@{$new_track},$elem);
596            $elem->{lon} = $min_lon;    push(@{$new_track},$elem);
597        }
598        push(@{$new_tracks->{tracks}},$new_track);
599    }
600    return $new_tracks;
601}
602
603
604
605
606# ------------------------------------------------------------------
607# Add things like speed, time, .. to the GPS Data
608# and the split the tracks if we think it's necessary
609sub split_tracks($$){
610    my $tracks      = shift; # reference to tracks list
611    my $config      = shift;
612
613    my $start_time=time();
614
615    my $filename     = $tracks->{filename};
616
617    my $max_allowed_speed = $config->{max_speed} || 200;
618    my $max_allowed_dist  = $config->{max_dist}  || 500; # 1 Km
619    my $max_allowed_time  = $config->{max_time}  || 60;
620
621    my $track_number=0;
622    enrich_tracks($tracks);
623    for my $track ( @{$tracks->{tracks}} ) {
624        my $max_pos = $#{@{$track}};
625        for  ( my $track_pos=0; $track_pos <= $max_pos;$track_pos++ ) {
626            my $elem = $track->[$track_pos];
627            unless ( defined($elem->{lat}) && defined($elem->{lon})){
628                $elem->{good_point}=0;
629                next;
630            }
631            if ( $elem->{fix} && $elem->{fix} eq "none" ) {
632                $elem->{good_point}=0;
633                print STDERR "x ";
634                next;
635            };         
636            next if $track_pos >= $max_pos;
637
638            $elem->{time} = 0 unless defined $elem->{time};
639
640            if ( defined($elem->{time_diff}) && 
641                 ( $elem->{time_diff} > $max_allowed_time ) ) { # ---- Check for Track Split: time diff
642                $elem->{split_track} =1;
643            }
644
645            my $dist  = $elem->{dist};   # in Km
646            if ( $dist > $max_allowed_dist) {             # ---- Check for Track Split: xx Km
647                $elem->{split_track} =1;
648            }
649           
650            # --------- Speed
651            my $speed = track_point_speed($track,$track_pos);
652            if ( $speed && $speed > $max_allowed_speed) { # ---- Check for Track Split: 200 Km/h
653                $elem->{good_point} =0;
654            }
655        }
656    }
657
658    my $new_tracks = tracks_only_good_point($tracks);
659
660    print_count_data($new_tracks,"after splitting");
661    print_time($start_time);
662    return $new_tracks;
663}
664
665# ------------------------------------------------------------------
666# get bounding Box for Data
667sub get_bounding_box($){
668    my $tracks      = shift; # reference to tracks list
669
670    my $start_time=time();
671
672    my $lat_min =  90;
673    my $lat_max = -90;
674    my $lon_min =  180;
675    my $lon_max = -180;
676
677    for my $track ( @{$tracks->{tracks}} ) {
678        next if !$track;
679        for my $elem ( @{$track} ) {
680            $lat_min  = $elem->{lat}        if $lat_min > $elem->{lat};
681            $lat_max  = $elem->{lat}        if $lat_max < $elem->{lat};
682
683            $lon_min  = $elem->{lon}        if $lon_min > $elem->{lon};
684            $lon_max  = $elem->{lon}        if $lon_max < $elem->{lon};
685        }
686    }
687
688    my $used_time = time()-$start_time;
689    if ( $DEBUG>10 || ($used_time >5 )) {
690        printf STDERR "Bounds are ($lat_min,$lon_min) ($lat_max,$lon_max)";
691        print_time($start_time);
692    }
693
694    return { lat_min => $lat_min,lon_min => $lon_min,
695             lat_max => $lat_max,lon_max => $lon_max };
696}
697
698# ------------------------------------------------------------------
699# Filter tracks with points
700# check_allowed_area($elem) tells if this element is added or not
701sub filter_track_by_area($){
702    my $tracks      = shift; # reference to tracks list
703
704    return unless @filter_area_files;
705
706    my $start_time=time();
707
708    for my $track ( @{$tracks->{tracks}} ) {
709        for my $track_pos ( 0 .. $#{@{$track}} ) {
710            my $elem = $track->[$track_pos];
711            $elem->{good_point} = check_allowed_area($elem,$internal__filter_area_list);
712            $elem->{split_track} =! $elem->{good_point};
713        }
714    }
715
716    my $new_tracks = tracks_only_good_point($tracks);
717
718    print_count_data($new_tracks,"after Filtering Areas");
719    print_time($start_time); 
720    return $new_tracks;
721}
722
723
724# ------------------------------------------------------------------
725# check if a the next n points are combinable and
726# can be replaceable by one segment
727sub is_gps_combineable($$){
728    my $track    = shift;
729    my $track_pos = shift;
730
731    my $count_combine =0;
732
733    my $sum_dist=0;
734    my $sum_angle=0;
735
736    my $elem0 = $track->[$track_pos];
737    my $dist_over=0;
738    my $pos_max = $#{@{$track}};
739    for my $track_pos_test ( $track_pos .. $pos_max ) {
740        my $elem2 = $track->[$track_pos_test];
741        $sum_dist = $sum_dist + $elem2->{dist};
742        if ( $sum_dist > 300 ) { # max 300 m distanz
743            printf STDERR "Elements have $sum_dist m Distance, which would be too much\n"
744                if $DEBUG >10;
745            last;
746        }
747        if ( abs ($elem2->{angle}) > 20 ) {
748            printf STDERR "Element $track_pos_test has $elem2->{angle} ° to next elem, which would be too much\n"
749                if $DEBUG >10;
750            last;
751        }
752
753        $sum_angle +=abs($elem2->{angle});
754        if ( $sum_angle > 20 ) {
755            printf STDERR "Elements 0 .. $track_pos_test has $sum_angle ° in summ, which would be too much\n"
756                if $DEBUG >10;
757            last;
758        }
759
760        $dist_over=0;
761        for my $track_pos_test1 ( $track_pos+1 .. $track_pos_test) {
762            my $elem1 = $track->[$track_pos_test1];
763            # Distance between line of line(p0 and p2) to p1 20060810_182203.nmea
764            my $dist = distance_line_point_Km($elem0->{lat},$elem0->{lon},
765                                              $elem2->{lat},$elem2->{lon},
766                                              $elem1->{lat},$elem1->{lon}
767                                              );
768            $dist_over =  1 if $dist > 0.05; # 5 meter
769            printf STDERR "Elem  $track_pos_test1 is $dist m away from line\n"
770                if $DEBUG >10;
771            last if $dist_over;
772        }
773        last if $dist_over;
774        $count_combine++;
775    }
776
777    return 0 unless $count_combine;
778    #print "pos: $track_pos($pos_max) : combine $count_combine Dist: $sum_dist\n";
779
780    return $count_combine;
781}
782
783# ------------------------------------------------------------------
784# Filter tracks with points
785# delete points which are
786# inside a straight line of the point before and after
787sub filter_data_reduce_points($){
788    my $tracks      = shift; # reference to tracks list
789
790    return unless $do_filter_reduce_pt;
791
792    my $start_time=time();
793
794    enrich_tracks($tracks);
795    for my $track ( @{$tracks->{tracks}} ) {
796        for  ( my $track_pos=0; $track_pos <= $#{@{$track}};$track_pos++ ) {
797            my $count_combinable = is_gps_combineable($track,$track_pos);
798            set_number_bad_points($track,$track_pos,$count_combinable);
799            $track_pos+=$count_combinable;
800        }
801    }
802   
803    my $new_tracks = tracks_only_good_point($tracks);
804
805    print_count_data($new_tracks,"after Data Reduce Points ");
806    print_time($start_time);
807    return $new_tracks;
808}
809
810
811# ------------------------------------------------------------------
812# remove Waypoints which are duplicate
813sub filter_duplicate_wpt($){
814    my $tracks      = shift; # reference to tracks list
815
816    my @new_wpt;
817    my %wpt_by_name;
818    sub compare_wpt($$){
819        my $wpt1 = shift;
820        my $wpt2 = shift;
821        # later we do a full compare
822        for my $type ( qw ( name lat lon ele )) {
823            return 0
824                if $wpt1->{$type} && $wpt2->{$type} && $wpt1->{$type} ne $wpt2->{$type} ;
825        }
826        return 1;
827    }
828
829    for my $elem ( @{$tracks->{wpt}} ) {
830        my $name = $elem->{name};
831        #printf STDERR Dumper(\$elem);
832        if ( defined($name) && $name ) {
833            if ( defined $wpt_by_name{$name}) {
834                my $found =0;
835                for my $wpt1( @{$wpt_by_name{$name}} ) {
836                    if ( compare_wpt( $elem,$wpt1 ) ) {
837                        $found=1;
838                        last;
839                    }
840                }
841                if ( $found ) {
842                    printf STDERR "wpt($name) is duplicate ignoring\n"
843                        if $VERBOSE >5 || $DEBUG;
844                    next;
845                }
846                push(@{$wpt_by_name{$name}}, $elem);
847            } else {
848                $wpt_by_name{$name} = [ $elem ];
849            }
850            push(@new_wpt,$elem);
851        } else {
852            warn "unnamed Waypoint".Dumper(\$elem)."\n" if $VERBOSE || $DEBUG;
853        }
854    }
855
856    $tracks->{wpt}=\@new_wpt;
857}
858
859
860
861# Adds a segment to the segment list by providing
862# a track and the position where to get the segment from
863sub add_trackpoint2segmentlist($$$){
864    my $segment_list = shift;
865    my $track = shift;
866    my $track_pos = shift;
867
868    my $elem0 = $track->[$track_pos];
869    my $elem1 = $track->[$track_pos+1];
870
871    my @segment;
872    ($segment[0],$segment[1],$segment[2],$segment[3]) =
873        ($elem0->{lat},$elem0->{lon},$elem1->{lat},$elem1->{lon});
874    $segment[4] = angle_north_relative(
875                                   { lat => $segment[0] , lon => $segment[1] },
876                                   { lat => $segment[2] , lon => $segment[3] });
877    push (@{$segment_list},\@segment);
878}
879
880# ------------------------------------------------------------------
881# check if new trackpoints are ner already existing ones
882sub filter_dup_trace_segments($$){
883    my $tracks       = shift; # reference to tracks list
884    my $config       = shift;
885
886    my $dist_old2track = $config->{dist} || 20;
887    my $start_time=time();
888
889    my $bounds = GPS::get_bounding_box($tracks);
890
891    my $count_points = 0;
892    my $new_points = 0;
893    my $last_compare_dist=0;
894
895    enrich_tracks($tracks);
896    my $parsing_display_time=time();
897    my ($track_count,$point_count) =   count_data($tracks);
898    my $track_points_done=0;
899
900    my $segment_list=[];
901    for my $track_no ( 0 .. $#{@{$tracks->{tracks}}} ) {
902        my $track = $tracks->{tracks}->[$track_no];
903        next if !$track;
904        my $sliding_track_pos=0;
905        my $pos_max= $#{@{$track}};
906        for my $track_pos ( 1 .. $pos_max ) {
907            my $elem = $track->[$track_pos];
908            $elem->{good_point} =
909                ! OSM::is_segment_of_list_nearby($track,$track_pos,$segment_list);
910            $elem->{split_track} =! $elem->{good_point};
911
912            my $track_angle=0;
913            while ( ( abs($track_angle = track_part_angle($track,$sliding_track_pos,$track_pos-1)) > 140 ) 
914                    && ($sliding_track_pos<$track_pos-5)
915                    && track_part_distance($track,$sliding_track_pos,$track_pos-1)>100) {
916                add_trackpoint2segmentlist($segment_list,$track,$sliding_track_pos);
917                $sliding_track_pos ++;
918                print STDERR "Track: $track_no ".
919                    "sliding_track_pos: $sliding_track_pos/".($sliding_track_pos-$track_pos).
920                    " ($track_pos,$pos_max)     track_angle: $track_angle lets me assume a turn\n" 
921                    if $DEBUG > 1;
922            }
923            if ( ( $track_pos - $sliding_track_pos ) > 140 ) {
924                #add_trackpoint2segmentlist($segment_list,$track,$sliding_track_pos);
925                #$sliding_track_pos ++;
926
927                # TODO: Immer nur dann kopieren, wenn der
928                # winkel sich um mehr als 160 Grad geändert hat
929            }
930#           print "Track: $track_no sliding_track_pos: $sliding_track_pos ($pos_max)    track_angle: $track_angle\n";
931            #print Dumper(\@sliding_track_list);
932            $track_points_done++;
933            if ( ( $VERBOSE || $DEBUG ) &&
934                 ( time()-$parsing_display_time >.9)
935                 )  {
936                $parsing_display_time= time();
937                print STDERR "Filter dup Trackseg ".mem_usage();
938                print STDERR time_estimate($start_time,$track_points_done,$point_count);
939                print STDERR "\r";
940            }
941        }
942        while ( $sliding_track_pos < $pos_max ) {
943            add_trackpoint2segmentlist($segment_list,$track,$sliding_track_pos);
944            $sliding_track_pos ++;
945        }
946
947    }
948
949    my $new_tracks = tracks_only_good_point($tracks);
950
951    print_count_data($new_tracks,"after Filtering my own Tracks.");
952    print_time($start_time);
953   
954    return $new_tracks;
955}
956
957
958
959# ------------------------------------------------------------------
960# check if distance between two  trackpoints is 0 Km
961sub filter_null_dist($){
962    my $tracks       = shift; # reference to tracks list
963
964    my $start_time=time();
965
966    enrich_tracks($tracks);
967    for my $track ( @{$tracks->{tracks}} ) {
968        next if !$track;
969       
970        for  ( my $track_pos=1; $track_pos <= $#{@{$track}};$track_pos++ ) {
971            my $elem0  =$track->[$track_pos-1];
972            my $elem1  =$track->[$track_pos];
973            next if $elem0->{lat} !=  $elem1->{lat};
974            next if $elem0->{lon} !=  $elem1->{lon};
975            $elem1->{good_point} =0;
976        }
977
978    }
979
980    my $new_tracks = tracks_only_good_point($tracks);
981    print_count_data($new_tracks,"after Filtering null distance");
982    print_time($start_time);
983    return $new_tracks;
984}
985
986
987
988
989# ------------------------------------------------------------------
990# scan over the elements and return the number of elements which
991# are inside a boundingbox with a specified length
992sub find_max_points_in_bbox_for_track($$$){
993    my $track     = shift; # reference to tracks list
994    my $track_pos = shift; # First Trackpoint to look at
995    my $max_dist  = shift; # Height/width in km for bounding box
996
997    my $lat_min =  90;
998    my $lat_max = -90;
999    my $lon_min =  180;
1000    my $lon_max = -180;
1001
1002    my $count=0;
1003
1004    my $last_track_point = $#{@{$track}};
1005    for my $track_pos_test ( $track_pos .. $#{@{$track}} ) {
1006        my $elem  =$track->[$track_pos_test];
1007        $lat_min  = $elem->{lat}            if $lat_min > $elem->{lat};
1008        $lat_max  = $elem->{lat}            if $lat_max < $elem->{lat};
1009       
1010        $lon_min  = $elem->{lon}            if $lon_min > $elem->{lon};
1011        $lon_max  = $elem->{lon}            if $lon_max < $elem->{lon};
1012       
1013        my $dist = distance_point_point_Km(
1014                                                 { lat => $lat_min,lon => $lon_min},
1015                                                 { lat => $lat_max,lon => $lon_max} );
1016        last if $dist>$max_dist;
1017        $count++;
1018    }
1019    #print STDERR "Bbox($count)\n" if $DEBUG>1;
1020    return $count;
1021}
1022
1023# ------------------------------------------------------------------
1024# check if a the next n points are a GPS inacuracy clew
1025sub is_gps_clew($$){
1026    my $track        = shift;
1027    my $track_pos    = shift;
1028
1029    my $max_angle = 90;
1030    my $skip_point=0;
1031
1032    my $count_clew=0;
1033    # We work with a bounding circle too
1034    # we just have to return a number of segments which belong to the clew
1035    # This reduces the cost for this action dramatically
1036    my $bbox=find_max_points_in_bbox_for_track($track,$track_pos,0.050);
1037
1038    my $pos_max = $#{@{$track}};
1039    for my $track_pos_test ( $track_pos .. min($track_pos+$bbox,$pos_max) ) {
1040        #print "track_pos_test($track_pos): $track_pos_test\n";
1041        my $elem0 = $track->[$track_pos_test];
1042        my $elem1 = $track->[$track_pos_test+1];
1043        last if $elem0->{dist} > $elem0->{compare_dist};
1044        last if abs ($elem0->{angle}) < 20;
1045        last if abs ($elem0->{angle})+abs($elem1->{angle})<70;
1046        last if abs ($elem0->{angle})<70 && abs($elem1->{angle})<70;
1047        $count_clew++;
1048    }
1049
1050    return 0 if $count_clew < 5;
1051    #print "pos: $track_pos($pos_max) : clew $count_clew bbox: $bbox\n";
1052    return $count_clew;
1053}
1054
1055# ------------------------------------------------------------------
1056# check if new trackpoints are on existing duplicate to other gpx tracks
1057sub filter_gps_clew($$){
1058    my $tracks       = shift; # reference to tracks list
1059    my $config       = shift;
1060
1061    my $start_time=time();
1062
1063    if ( $out_raw_gpx && $DEBUG >3 ){
1064        my $filename=$tracks->{filename};
1065        my $new_gpx_file = "$filename-raw-pre-clew.gpx";
1066        $new_gpx_file =~s/.gpx-raw-pre-clew.gpx/-raw-pre-clew.gpx/;
1067        write_gpx_file($tracks,$new_gpx_file);
1068    };
1069
1070    my $dist_osm_track = $config->{dist} || 40;
1071
1072    enrich_tracks($tracks);
1073    my $parsing_display_time=time();
1074    my ($track_count,$point_count) =   count_data($tracks);
1075    my $track_points_done=0;
1076    for my $track ( @{$tracks->{tracks}} ) {
1077        next if !$track;
1078       
1079        for  ( my $track_pos=0; $track_pos <= $#{@{$track}};$track_pos++ ) {
1080            my $count_clew = is_gps_clew($track,$track_pos);
1081            set_number_bad_points($track,$track_pos,$count_clew);
1082            $track_points_done++;
1083            if ( ( $VERBOSE || $DEBUG ) &&
1084                 ( time()-$parsing_display_time >.9)
1085                 )  {
1086                $parsing_display_time= time();
1087                print STDERR "Filter clew ".mem_usage();
1088                print STDERR time_estimate($start_time,$track_points_done,$point_count);
1089                print STDERR "\r";
1090            }
1091        }
1092        for  ( my $track_pos=0; $track_pos <= $#{@{$track}};$track_pos++ ) {
1093            my $elem = $track->[$track_pos];
1094            $elem->{split_track} =! $elem->{good_point};
1095        }
1096    }
1097
1098    #printf STDERR "Good Points: %d\n",count_good_point($tracks);
1099
1100    my $new_tracks = tracks_only_good_point($tracks);
1101
1102    print_count_data($new_tracks,"after Filtering Clews");
1103    print_time($start_time);
1104    return $new_tracks;
1105}
1106
1107
1108
1109# ------------------------------------------------------------------
1110
1111
1112
1113###########################################
1114
1115
1116
1117########################################################################################
1118########################################################################################
1119########################################################################################
1120#
1121#                     Main
1122#
1123########################################################################################
1124########################################################################################
1125########################################################################################
1126package main;
1127
1128use strict;
1129use warnings;
1130
1131# *****************************************************************************
1132sub convert_Data(){
1133    my $all_tracks={
1134        filename => '',
1135        tracks => [],
1136        wpt => []
1137        };
1138    my $all_raw_tracks={
1139        filename => '',
1140        tracks => [],
1141        wpt => []
1142        };
1143    my $single_file = ( @ARGV ==1 );
1144    my $start_time  = time();
1145
1146    if ( $use_stdin ) {
1147        push(@ARGV,"-");
1148    }
1149    if ( @ARGV < 1){
1150        printf STDERR "Need Filename(s) to convert\n";
1151        printf STDERR "use: osmfilter.pl -h for more help\n";
1152        exit 1;
1153    }
1154
1155    GPS::read_all_filter_areas(@filter_area_files);
1156
1157   
1158    my $osm_segments;
1159    if ( $do_filter_against_osm ) {
1160        $osm_segments = load_segment_list($do_filter_against_osm);
1161    }
1162   
1163
1164   
1165    my $count_files_converted=0;
1166    while ( my $filename = shift @ARGV ) {
1167        my $new_tracks;
1168        if ( $filename =~ m/\*/ ){
1169            printf STDERR "$filename: Skipping for read. Filename has wildcard.\n\n";
1170            next;
1171        };
1172        if ( ( $filename =~ m/-raw(|-osm|-pre-osm|-pre-clew)\.gpx$/ ) ||
1173             ( $filename =~ m/-converted\.gpx(\.gz|\.bz2)?$/ ) ||
1174             ( $filename =~ m/-combination.*\.gpx(\.gz|\.bz2)?$/ ) ||
1175             ( $filename =~ m/00_combination\.gpx(\.gz|\.bz2)?$/ ) ||
1176             ( $filename =~ m/00_filter_areas\.gpx(\.gz|\.bz2)?$/ ) 
1177             ){
1178            printf STDERR "$filename: Skipping for read. These are my own files.\n\n";
1179            next;
1180        }
1181
1182        if ( ! -s $filename ) {
1183            printf STDERR "$filename: Skipping for read. Cannot be found.\n\n";
1184            next;
1185           
1186        }
1187
1188        my ( $extention ) = ( $filename =~ m/\.([^\.]+)(\.gz|\.bz2)?$/ );
1189        printf STDERR "$filename has extention '$extention'\n" if $DEBUG>1;
1190        if ( $filename eq '-' ) {
1191            $new_tracks = read_gpx_file($filename);
1192        } elsif ( $filename =~ m/^gpsbabel:(\S+):(\S+)$/ ) {
1193            my ($type,$name) = ($1,$2);
1194            $new_tracks = read_track_GpsBabel($name,$type);   
1195        } elsif ( $extention eq "gps" ) {
1196            $new_tracks = read_kismet_file($filename);
1197        } elsif ( $extention eq "gpx" ) {
1198            $new_tracks = read_gpx_file($filename);
1199        } elsif ( $extention eq "mps" ) {
1200            $new_tracks = read_track_GpsBabel($filename,"mapsource");
1201        } elsif ( $extention eq "gdb" ) {
1202            $new_tracks = read_track_GpsBabel($filename,"gdb");
1203        } elsif ( $extention eq "ns1" ) {
1204            $new_tracks = read_track_GpsBabel($filename,"netstumbler");
1205        } elsif ( $extention eq "nmea" ) {
1206            $new_tracks = read_track_NMEA($filename);
1207        } elsif ( $extention eq "TXT" ) { # This is the NAVI-GPS extention
1208            $new_tracks = read_track_NMEA($filename);
1209        } elsif ( $extention eq "sav" ) {
1210            $new_tracks = GPSDrive::read_gpsdrive_track_file($filename);
1211        } else {
1212            warn "$filename: !!! Skipping because of unknown Filetype ($extention) for reading\n";
1213            next;
1214        }
1215        my ($track_read_count,$point_read_count) =   count_data($new_tracks);
1216        if ( $VERBOSE || $DEBUG) {
1217            my $comment = "read";
1218            printf STDERR "%-35s: %5d Points in %d Tracks $comment",$filename,$point_read_count,$track_read_count;
1219            print  STDERR "\n";
1220        }
1221
1222        if ( $out_raw_gpx ){
1223            my $new_gpx_file = "$filename-raw.gpx";
1224            $new_gpx_file =~s/.gpx-raw.gpx/-raw.gpx/;
1225            write_gpx_file($new_tracks,$new_gpx_file);
1226
1227            add_tracks($all_raw_tracks,$new_tracks);
1228        };
1229               
1230        $new_tracks = GPS::filter_null_dist( $new_tracks );
1231
1232        if ( @filter_area_files ) {
1233            $new_tracks = GPS::filter_track_by_area($new_tracks);
1234        }
1235
1236        if ( $split_tracks ) {
1237            $new_tracks = GPS::split_tracks($new_tracks,
1238                                        { max_speed => 200 });
1239        }
1240
1241        if ( $do_filter_clew ) {
1242            $new_tracks = GPS::filter_gps_clew( $new_tracks,
1243                                              { dist => 30 });
1244        };
1245
1246        if ( $do_filter_dup_trace_segments ) {
1247            $new_tracks = GPS::filter_dup_trace_segments( $new_tracks,{});
1248        };
1249
1250        if ( $do_filter_against_osm ) {
1251            $new_tracks = OSM::filter_against_osm( $new_tracks,$osm_segments,
1252                                              { dist => 30 });
1253        };
1254
1255        if ( $split_tracks ) {
1256            $new_tracks = GPS::split_tracks($new_tracks,
1257                                        { max_speed => 200 });
1258        }
1259
1260        if ( $do_filter_reduce_pt ) {
1261            $new_tracks = GPS::filter_data_reduce_points($new_tracks);
1262        }
1263
1264        my ($track_count,$point_count);
1265        ($track_count,$point_count) =   count_data($new_tracks);
1266        $count_files_converted ++ if $point_count && $track_count;
1267
1268        if ( $track_count > 0 ) {
1269            my $new_gpx_file = "$filename-converted.gpx";
1270            $new_gpx_file =~s/.gpx-converted.gpx/-converted.gpx/;
1271            write_gpx_file($new_tracks,$new_gpx_file)
1272                if $single_file;
1273           
1274            if ( $out_osm ){
1275                my $new_osm_file = "$filename.osm";
1276                my $osm = tracks2osm($new_tracks);
1277                write_osm_file($new_osm_file,$osm)
1278                }
1279        } else {
1280            printf STDERR "%-35s:No resulting Tracks so nothing is written\n",$filename;
1281        }
1282
1283        add_tracks($all_tracks,$new_tracks);
1284        if ( $point_count && $track_count ) {
1285            printf STDERR "%-35s:       ",$filename;
1286            printf STDERR "%5d(%5d) Points in %3d(%3d) Tracks added\n",
1287            $point_count,$point_read_count,$track_count,$track_read_count;
1288
1289        }
1290        if ( $VERBOSE || $DEBUG ) {
1291            printf STDERR "\n";
1292        }
1293    }
1294
1295    GPS::filter_duplicate_wpt($all_tracks);
1296   
1297
1298    if ( $count_files_converted ) {
1299            print_count_data($all_tracks,"after complete processing");
1300            print_time($start_time);
1301           
1302            if (  $out_osm ) {
1303                my $osm = tracks2osm($all_tracks);
1304                write_osm_file("00_combination.osm",$osm);
1305            }
1306           
1307            if ( $use_stdout ) {
1308                write_gpx_file($all_tracks,'-');
1309            } else {
1310                write_gpx_file($all_tracks,"00_combination.gpx");
1311                if ( $out_raw_gpx ){
1312                    write_gpx_file($all_raw_tracks,"00_combination-raw.gpx");
1313                }
1314                if ( $out_upload_gpx ){
1315                    my $mem1= $out_upload_gpx;
1316                    my $mem2 = $fake_gpx_date;
1317                    $out_upload_gpx=0;
1318                    $fake_gpx_date=1;
1319                    write_gpx_file($all_raw_tracks,"00_combination-upload.gpx");
1320                    $out_upload_gpx=$mem1;
1321                    $fake_gpx_date=$mem2;
1322                }
1323            };
1324           
1325        } else {
1326            if ( $VERBOSE || $DEBUG ) {
1327                print STDERR "No files Converted; so nothing written\n";
1328            }
1329        }
1330   
1331    if ( $draw_filter_areas ) {
1332        my $filter_areas = GPS::draw_filter_areas();
1333        write_gpx_file($filter_areas,"00_filter_areas.gpx");
1334    }
1335
1336    if ( $VERBOSE) {
1337        printf STDERR "Converting $count_files_converted Files";
1338        print_time($start_time);
1339    }
1340}
1341
1342# ------------------------------------------------------------------
1343
1344# Set defaults and get options from command line
1345Getopt::Long::Configure('no_ignore_case');
1346GetOptions ( 
1347             'debug:+'              => \$DEBUG,     
1348             'd:+'                  => \$DEBUG,     
1349             'verbose:+'            => \$VERBOSE,
1350             'v:+'                  => \$VERBOSE,
1351             'MAN'                  => \$man, 
1352             'man'                  => \$man, 
1353             'h|help|x'             => \$help, 
1354
1355             'stdin'                => \$use_stdin,
1356             'stdout'               => \$use_stdout,
1357             'proxy=s'              => \$PROXY,
1358
1359             'out-osm!'             => \$out_osm,
1360             'out-raw-gpx!'         => \$out_raw_gpx,
1361             'out-upload-gpx!'      => \$out_upload_gpx,
1362             'split-tracks!'        => \$split_tracks,
1363             'filter-against-osm:s' => \$do_filter_against_osm,
1364             'osm:s'                => \$do_filter_against_osm,
1365             'filter-dup-seg!'      => \$do_filter_dup_trace_segments,
1366             'filter-clew!'         => \$do_filter_clew,
1367             'filter-reduce!'       => \$do_filter_reduce_pt,
1368             'filter-area:s@'       => \@filter_area_files,
1369             'generate_ways!'       => \$generate_ways,
1370             'filter-all'           => \$do_all_filters,
1371             'fake-gpx-date!'       => \$fake_gpx_date,
1372             'write-gpx-wpt!'       => \$write_gpx_wpt,
1373             'draw_filter_areas!'   => \$draw_filter_areas,
1374             )
1375    or pod2usage(1);
1376
1377if ( $do_all_filters ) {
1378    $out_osm               = 1 unless defined $out_osm;
1379    $out_raw_gpx           = 1 unless defined $out_raw_gpx;
1380    $out_upload_gpx        = 1 unless defined $out_upload_gpx;
1381    $split_tracks          = 1 unless defined $split_tracks;
1382    $do_filter_reduce_pt   = 1 unless defined $do_filter_reduce_pt;
1383    $do_filter_clew        = 1 unless defined $do_filter_clew;
1384    $do_filter_against_osm = 1 unless defined $do_filter_against_osm;
1385    $do_filter_dup_trace_segments =1 unless defined $do_filter_dup_trace_segments;
1386    @filter_area_files || push(@filter_area_files,"");
1387}
1388
1389$fake_gpx_date ||=0;
1390
1391pod2usage(1) if $help;
1392pod2usage(-verbose=>2) if $man;
1393
1394convert_Data();
1395
1396##################################################################
1397# Usage/manual
1398
1399__END__
1400
1401=head1 NAME
1402
1403B<osmfilter.pl> Version 0.05
1404
1405=head1 DESCRIPTION
1406
1407B<osm-filter.pl> is a program to convert and filter Track Files
1408to a *.gpx and *.osm File. This File then can then be loaded into josm,
1409corrected and then uploaded to OSM.
1410
1411This Programm is completely experimental, but some Data
1412can already be retrieved with it.
1413Since the script is still in Alpha Stage, please make backups
1414of your source gpx/kismet,... Files.
1415
1416So: Have Fun, improve it and send me lots of new fixes/patches/features
1417    and new filters :-))
1418
1419
1420This description is still preleminary, since the script is
1421still in the stage of development, but for me it was already
1422very usefull. Any hints/suggestions/patches are welcome.
1423Most of the threasholds are currently hardcoded in the source.
1424But since it's a perl script it shouldn't be too much effort
1425to change them. If you think we need some of these in a config-file
1426i can try to add a config file to the script too.
1427
1428The Idea behind the osm-filter is:
1429 - reading some different input formats.
1430       Input is currently one of the folowing:
1431         - Kismet-GPS File   *.gps
1432         - GpsDrive-Track    *.sav
1433         - GPX File          *.gpx
1434         - Garmin mps File   *.mps
1435         - Garmin gdb File   *.gdb
1436 - Then the Data is optionally filtered by area filters.
1437   For this you can define squares- and circle- shaped areas
1438   where the data is accepted or dropped.
1439   For now the areas can only be defined in the Source,
1440   but this will change in the future.
1441   This can be for example be used to:
1442     - eliminate your home position where your gps always
1443       walk arround in circles.
1444     - eliminate areas already mapped completely
1445     - limit your editing/uploading to a special area
1446   The areas are currently defined like
1447     { lat =>  48.1111  ,lon => 11.7111    ,proximity => .10  , block => 1 },
1448     { wp => "MyHome"   ,proximity => 6, block => 1 },
1449   Where MyHome is a GPSDrive waypoint taken from ~/.gpsdrive/way.txt
1450    proximity is defaulted by the proximity in the way.txt File (column 8)
1451    block is defaulted with 0
1452    If you want to see the filter areas in the resulting gpx file you can
1453    use the option    --draw_filter_areas. This will draw in the check areas
1454    as seperate tracks.
1455 - Then osm-filter then enriches the Data for internal use by adding:
1456        - speed of each segment (if necessary)
1457        - distance to last point
1458        - angle to last segment (Which would represent steering wheel angle)
1459 - Then osm-filter is splitting the tracks if necessary.
1460   This is needed if you have for example gathered Tracks
1461   with a Garmin handheld. There the Tracks get combined
1462   even if you had no reception or switched of the unit inbetween.
1463   The decission to split the tracks is done by checking if:
1464      - time between points is too high ( > 60 seconds for now )
1465      - Speed is too high ( > 200 Km/h for now )
1466      - Distance between 2 point is too high ( >2 Km for now)
1467   Then each Track with less than 3 points is discarded.
1468 - if you use the option
1469       --generate_ways
1470   osm-filter tries to determin continuous ways by looking
1471   at the angle to the last segment, the speed and distance.
1472
1473 - This is now done for all input Files. So you can also use
1474   multiple input files as source and combine them to ine large
1475   output File.
1476
1477 - After this all now existing data iw written to a gpx file.
1478 - If you add the option  --out-osm. osm-filter tries
1479   to generate an *.osm file out of this Data.
1480
1481=head1 SYNOPSIS
1482
1483B<Common usages:>
1484
1485osm-filter.pl [--man] [-d] [-v] [-h][--out-osm] [--limit-area] <File1.gps> [<File2.sav>,<File2.ns1>,...]
1486
1487!!!Please be carefull this is still a beta Version.
1488   Make Backups of your valuable source Files
1489   and never upload the created data without first checking it!!!
1490
1491=head1 OPTIONS
1492
1493=over 2
1494
1495=item B<--man> Complete documentation
1496
1497This shows the Complete documentation
1498
1499=item B<--out-raw-gpx>
1500
1501Write raw converted output to filename-raw.gpx
1502
1503=item B<--out-upload-gpx>
1504
1505Write raw converted output to filename-upload.gpx.
1506Without timestamps and without Waypoints.
1507
1508=item B<--out-osm>
1509
1510*.osm files will only be generated if this option is set.
1511
1512There is still a Bug/"Design Flaw" so all single .osm Files might
1513always be a collection of all previous read Files.
1514
1515There will also be written a file named
1516 ./00_combination.osm
1517
1518
1519=item B<--split-tracks>
1520
1521Split tracks it they have gaps of more than
1522 - 1 Minute
1523 - 1 Km
1524 - 200 Km/h
1525
1526=item B<--filter-against-osm> |  B<--osm>
1527
1528This loads the osm.csv and checks if the
1529track-points already exist as an osm-segment.
1530It checks if any of the points are near (<20m) any 
1531of the osm-segments.
1532And the OSM Segment and the track segments have an
1533angle of less than 30 Degrees.
1534If so they will be dropped.
1535
1536The file osm.csv is the segments file used by osm-pdf-atlas.
1537It's containing all osm-segments from planet.osm in a rather simple way.
1538The probably easiest way to create it is to go to the
1539directory
1540   svn.openstreetmap.org/utils/osm-pdf-atlas
1541and once call "create.sh".
1542
1543If you provide a filename this file is read instead of the osm.csv file.
1544The Filename provided can be a csv-file or a standard osm-file.
1545
1546you can use --osm=0 in combination with --filter-all to not let osm-filter
1547check against osm Data.
1548
1549=item B<--filter-dup-seg>
1550
1551This Filter checks for points near an already existing Trackline.
1552If so it is removed
1553
1554Currently it's only a stub, so anyone can sit down and programm the compare routine.
1555
1556=item B<--filter-reduce>
1557
1558The ammount of Datapoints is reduced.
1559This is done by looking at three trackpoints in a row. For now I calculate the
1560distance between the line of point 1 and 3 to the point in the
1561middle. If this distance is small enough (currently 4 meter) the
1562middle point is dropped, because it doesn't really improve the track.
1563
1564
1565=item B<--limit-area>
1566
1567Use the area filters
1568
1569By default the Files ~/.gpsdrive/way.txt is read
1570and all waypoints starting with filter.
1571are added as filter areas.
1572    filter.deny will be filtered out
1573    filter.allow will be left in the resulting files
1574    filter.none nothing will be done here
1575after this by default the file  ~/josm/filter.xml is read.
1576An example would look lie:
1577<?xml version="1.0"?>
1578<osm>
1579  <node id="-10001"  lat="48.411430"  lon="9.492400" >     
1580    <tag k="name" v="de_Dottingen"/>
1581    <tag k="of:radius" v="2000" />
1582    <tag k="of:block" v="1" />
1583    <tag k="of:active" v="1" />
1584  </node>
1585  <node id="-10002"  lat="48.138004"  lon="11.557109" >
1586    <tag k="name" v="de_Muenchen"/>
1587    <tag k="of:radius" v="5000" />
1588    <tag k="of:block" v="1" />
1589    <tag k="of:active" v="0" />
1590  </node>
1591</osm>
1592
1593If you want to define squares you have to define them for now
1594in the Source at the definition of
1595  $internal__filter_area_list =
1596AND: they are not tested :-(
1597
1598The default area-filter rule is allow the rest.
1599
1600=item B<--filter-clew>
1601
1602Filter out these little nasty gps accuracies if you are standing still
1603
1604=item B<--filter-all>
1605
1606Switch on all of the above filters
1607
1608=item B<--draw_filter_areas>
1609
1610Draw the filter_areas into the file 00_filter_areas.gpx file
1611by creating a track with the border of each filter_area
1612
1613=item B<--generate_ways>
1614
1615Try to generate ways inside the OSM structure.
1616Still only testing
1617
1618
1619=item B<--fake-gpx-date>
1620
1621This eliminates the date for while writing gpx data.
1622
1623=item B<--write-gpx-wpt>
1624
1625Only if this option is set, Waypoints are written to any of the gpx Files.
1626Default is on.
1627
1628=item <File1.gps> [<File2.gps>,...]
1629
1630 The Files to read and proccess
1631
1632 Input is one of the folowing:
1633   - Kismet-GPS File   *.gps
1634   - GpsDrive-Track    *.sav
1635   - GPX File          *.gpx
1636   - Garmin mps File   *.mps
1637   - Garmin gdb File   *.gdb
1638   - Netstumbler Files *.ns1
1639   - NMEA              *.nmea
1640   - via gpsbabel gpsbabel:<type>:*
1641
1642For each File read a File *-converted.gpx will be written
1643All input filenames ending with -converted.gpx will be skiped.
1644
1645To read all Files in a specified directory at once do the following:
1646
1647 find <kismet_dir>/log -name "*.gps" | xargs ./osm-filter.pl
1648
1649If you define multiple Files a summary File will automagically be written:
1650 ./00_combination.gpx
1651
1652=item B<--stdin>
1653
1654use stdin to read GPX track file
1655
1656=item B<--stdout>
1657
1658use stdout to write filtered  tracks as GPX
1659
1660=back
1661
1662=head1 COPYRIGHT
1663
1664Copyright 2006, Jörg Ostertag
1665
1666This program is free software; you can redistribute it and/or
1667modify it under the terms of the GNU General Public License
1668as published by the Free Software Foundation; either version 2
1669of the License, or (at your option) any later version.
1670
1671This program is distributed in the hope that it will be useful,
1672but WITHOUT ANY WARRANTY; without even the implied warranty of
1673MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
1674GNU General Public License for more details.
1675
1676You should have received a copy of the GNU General Public License
1677along with this program; if not, write to the Free Software
1678Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
1679
1680=head1 AUTHOR
1681
1682Jörg Ostertag (osmfilter-for-openstreetmap@ostertag.name)
1683
1684=head1 SEE ALSO
1685
1686http://www.openstreetmap.org/
1687
1688=cut
Note: See TracBrowser for help on using the repository browser.