source: subversion/applications/utils/import/linz2osm/mp2osm_linz_jr.py @ 16228

Revision 16228, 17.8 KB checked in by joerichards, 5 years ago (diff)

updated after initial release

  • Property svn:executable set to *
Line 
1#!/usr/bin/python
2# mp2osm_linz_jr.py
3#
4# derived from 'mp2osm_ukraine.py'
5# modified by simon@mungewell.org
6# modified by Karl Newman (User:SiliconFiend) to preserve routing topology and parse RouteParam
7# modified by Joe Richards (joe@penski.net) to include SUFI IDs and support most LINZ/NZOGPS feature types
8# license: GPL V2 or later
9# if you make modifications, please email your version to the previous authors (as per the GPL)
10
11import xml.etree.ElementTree as ET
12
13attribution = 'LINZ & NZ Open GIS'
14file_mp = open('short.mp')
15
16# flags and global variable
17poi = False
18polyline = False
19polygon = False
20roadid = ''
21rNodeToOsmId = {} # map routing node ids to OSM node ids
22
23# debug/stats counters
24poi_counter = 0
25polyline_counter = 0
26polygon_counter = 0
27
28osm = ET.Element('osm', version='0.6', generator='mp2osm_linz_jr' )
29osm.text = '\n  '
30osm.tail = '\n'
31source = ET.Element('tag', k='source',v=attribution)
32source.tail = '\n    '
33nodeid = -1
34# Define the mapping from Garmin type codes to OSM tags
35# Note that single items in parentheses need a trailing comma
36poitagmap = {
37
38     #JR: catCities
39     (0x0100,0x100): {'place': 'city','linz:comment': 'size >10M'},        # added by JR.  np
40     (0x0200,0x200): {'place': 'city', 'linz:comment': 'size 5-10M'},      # added by JR.  np
41     (0x0300,0x300): {'place': 'city', 'linz:comment': 'size 2-5M'},       # added by JR.  np
42     (0x0400,0x400): {'place': 'city', 'linz:comment': 'size 1-2M'},       # added by JR - pres
43     (0x0500,0x500): {'place': 'city', 'linz:comment': 'size 0.5-1M'},     # added by JR - pres
44     (0x0600,0x600): {'place': 'city', 'linz:comment': 'size 200-500K'},   # added by JR - pres
45     (0x0700,0x700): {'place': 'city', 'linz:comment': 'size 100-200K'},   # added by JR - pres
46     (0x0800,0x800): {'place': 'city', 'linz:comment': 'size 50-100K'},    # added by JR - pres
47     (0x0900,0x900): {'place': 'city', 'linz:comment': 'size 20-50K'},     # added by JR - pres
48     (0x0a00,0xa00): {'place': 'town', 'linz:comment': 'size 10-20K'},     # added by JR - pres
49     (0x0b00,0xb00): {'place': 'town', 'linz:comment': 'size 5-10K'},       # added by JR - pres
50     (0x0c00,0xc00): {'place': 'town', 'linz:comment': 'size 2-5K'},       # added by JR - pres
51     (0x0d00,0xd00): {'place': 'town', 'linz:comment': 'size 1-2K'},       # added by JR - pres
52     (0x0e00,0xe00): {'place': 'village', 'linz:comment': 'size 500-1K'},  # added by JR - pres
53     (0x0f00,0xf00): {'place': 'village', 'linz:comment': 'size 200-500'}, # added by JR - pres
54     (0x1000,): {'place': 'hamlet', 'linz:comment': 'size 100-200'},  # added by JR - pres
55     (0x1100,): {'place': 'hamlet', 'linz:comment': 'size 100-200'},  # added by JR - pres
56
57     (0x1200,): {'leisure': 'marina' },                               # added by JR - Raglan Wharf.  Spreadsheet suggests hamlet (0-100 size) but not in our dataset
58
59     (0x1612, 0x1c00, 0x6400): {'highway': 'gate'},
60     (0x160f,): {'man_made': 'beacon', 'mark_type': 'white'},         # added by JR - pres
61
62     (0x1c01,): {'man_made': 'ship_wreck'},                           # added by JR - pres
63     (0x1c09,): {'linz:comment': 'FIXMEFIXME  rock, awash, reef'},                   # added by JR - pres - TODO ammend
64
65     (0x1e00,): {'place': 'region'},                                  # added by JR - pres, e.g. Southland   
66     (0x2b00,): {'tourism': 'hotel'},
67     (0x2b01,): {'tourism': 'motel'},
68     (0x2b03,): {'tourism': 'caravan_site'},
69     (0x2c06,): {'leisure': 'park'}, # added by JR
70     (0x2c08,): {'leisure': 'park'}, # added by JR
71     (0x2d05,): {'leisure': 'golf_course'}, # added by JR
72     (0x2d06,): {'sport': 'skiing'}, # added by JR
73     (0x2f04,): {'aeroway': 'aerodrome'}, # added by JR
74     (0x2e02,): {'shop': 'supermarket'},
75     (0x2f08,): {'amenity': 'bus_station'},
76     (0x2f09,): {'waterway': 'boatyard'}, # added by JR
77     (0x3002,): {'amenity': 'hospital'}, # added by JR
78     (0x4400,): {'amenity': 'fuel'},
79     (0x4700,): {'leisure': 'slipway'},
80     (0x4800,): {'tourism': 'campsite'},
81     (0x4900,): {'leisure': 'park'},
82     (0x4a00,): {'tourism': 'picnic_site'},
83     (0x4c00,): {'tourism': 'information'},
84     (0x4d00,): {'amenity': 'parking'},
85     (0x4e00,): {'amenity': 'toilets'},
86     (0x5100,): {'amenity': 'telephone'},
87     (0x5200,): {'tourism': 'viewpoint'},
88     (0x5400,): {'sport': 'swimming'},
89     (0x5500,): {'waterway': 'dam'}, # added by JR
90     (0x5904,): {'aeroway': 'helipad'},
91     (0x5905,): {'aeroway': 'aerodrome'},
92     (0x5904,): {'aeroway': 'helipad'},
93     (0x5a00,): {'distance_marker': 'yes'}, # Not approved
94     (0x6401,): {'bridge': 'yes'}, # Apply to points?
95     (0x6401,): {'building': 'yes'},
96     (0x6402,): {'amenity': 'shelter', 'building': 'yes', 'tourism': 'alpine_hut'}, # added by JR -trekking huts, bivouacs, shelters, cabins
97     (0x6403,): {'amenity': 'grave_yard'}, #added by JR
98     (0x6505,): {'amenity': 'public_building', 'building': 'yes'},  # added by JR, only ex is Pumphouse
99     (0x6406,): {'mountain_pass': 'yes', 'place': 'locality'}, # by JR: was highway=crossing, now mountain_pass=yes (for pass,col,saddle) TODO check
100     (0x640c,): {'man_made': 'mineshaft'},
101     (0x640a,): {'place': 'locality'}, # by JR - seems to apply to a lot of diff features, and is Garmin default for a POI
102     (0x640d,): {'man_made': 'pumping_rig', 'type': 'oil'},
103     (0x6411,): {'man_made': 'tower'},
104     (0x6412,): {'highway': 'track', 'place': 'locality'}, # by JR - start of trek/walk- used to be 'trailhead' which is not even a proposed value
105     (0x6413,): {'natural': 'cave_entrance'}, # by JR - used to be tunnel=yes but appears to caves - TODO check
106     (0x6500, 0x650d): {'natural': 'water'},
107     (0x6501,): {'natural': 'glacier'}, # added by JR - TODO tourism=attraction too?
108     (0x6508,): {'waterway': 'waterfall'},
109     (0x650c,): {'place': 'islet'},  # added by JR - v2
110     (0x6511,): {'natural': 'spring'}, # added by JR - natural spring source
111     (0x6513,): {'natural': 'wetland'}, # added by JR
112     (0x6600,): {'place': 'locality', 'natural': 'hill', 'linz:comment': 'FIXME - hill?'}, # added by JR.. TODO check these, most seem to be in northland
113     (0x6603,): {'landuse':'basin'}, # added by JR - TODO check since this is natural basin poi
114     (0x6605,): {'natural': 'bench'},
115     (0x6607,): {'natural': 'cliff'}, # added by JR - cliffs, bluffs etc
116     (0x660c,): {'place': 'island'}, # added by JR - seems to be small islands as well as large
117     (0x660a,): {'natural': 'wood'}, # added by JR - include bush and wild overgrowth
118     (0x6611,): {'natural': 'mountain_range'}, # added by JR - TODO does not exist as tag
119     (0x6613,): {'natural': 'ridge'}, # added by JR - TODO does not exist as tag
120
121     (0x660d,): {'place': 'locality', 'natural': 'water'}, # added by JR - poi which are lake names
122     (0x6616,): {'natural': 'peak'},
123     (0x6617,): {'place': 'locality', 'natural': 'gully', 'linz:comment': 'FIXME: gorge/gully'}, # added by JR - gully/gorge does not exist, but giving it a placename might work
124    }
125polylinetagmap = {
126     # TODO FIXME all these highway types, and links need to be checked
127     (0x1,): {'highway': 'motorway'},   # added by JR
128     (0x2,): {'highway': 'trunk'},
129     (0x3,): {'highway': 'primary'},
130     (0x4,): {'highway': 'secondary'},
131     (0x5,): {'highway': 'tertiary'},
132     (0x6,): {'highway': 'residential'},
133     (0x7,): {'highway': 'service'}, # added by JR - TODO confirm in other places, example is Auckland airport's Cyril Kay Rd.. cf 0x7 as polygon which is diff.
134     (0x8,): {'highway': 'primary_link'}, # added by JR - TODO FIXME what is a turning point (U-Turn etc) on a big road? this is actually on a tertiary road when i looked
135     (0x9,): {'highway': 'secondary_link'}, # added by JR - TODO FIXME in the LINZ dataset is this actually used to connect secondary roads, or other types?
136     (0xa,): {'highway': 'track', 'surface': 'unpaved'},
137     (0xb,): {'highway': 'trunk_link'}, # added by JR
138        (0xc,): {'junction': 'roundabout'}, # added by JR
139        (0xd,): {'highway': 'cycleway'}, # added by JR
140     (0x14,): {'railway': 'rail'}, # added by JR
141     (0x16,): {'highway': 'footway'}, # added by JR
142     (0x18,): {'waterway': 'stream'},
143     (0x1b,): {'route': 'ferry'}, # added by JR
144
145     # boundaries http://wiki.openstreetmap.org/wiki/Key:admin_level#10_admin_level_values_for_specific_countries
146     (0x1c,): {'boundary': 'administrative', 'admin_level': '4'}, # added by JR, seems to be e.g. Otago/Canterbury, Auckland/Waikato TODO check this
147     (0x1d,): {'boundary': 'administrative', 'admin_level': '6'}, # added by JR, seems to be e.g. Timaru district council, TODO check this
148
149     (0x1e,): {'landuse': 'forest', 'leisure': 'nature_reserve'}, # added by JR
150     (0x1f,): {'waterway': 'canal'},                              # added by JR, was 'river', but the only ones present are in Chch canals
151     (0x29,): {'power': 'line'}
152    }
153polygontagmap = {
154     (0x2,): {'landuse': 'residential'},     # added by JR - pres
155     (0x20,): {'tourism': 'zoo', 'tourism': 'attraction'}, # added by JR - only Hamilton Zoo
156     (0x5,): {'amenity': 'parking', 'area': 'yes'},
157     (0x7,): {'landuse': 'aeroway'}, # added by JR - to describe outlines of runways and airport areas, cf. polyline
158     (0x8,): {'landuse': 'retail', 'building': 'shops'}, # added by JR, cf. 0x8 polyline which is a ramp
159     (0xd,): {'landuse': 'reservation', 'area': 'yes'}, # reservation is not even a proposed value TODO JR: this looks weird to me
160     (0x13,): {'building': 'yes', 'area': 'yes'}, # added by JR - buildings, including hospitals or other random ones, esp present in Waikato
161        (0x14,0x15): {'natural': 'wood', 'area': 'yes'}, # added by JR - v2
162     (0x17,): {'leisure': 'park', 'area': 'yes'}, # added by JR - city park
163     (0x18,): {'leisure': 'golf_course', 'area': 'yes'},
164     (0x19,): {'leisure': 'sports_centre', 'area': 'yes', 'linz:todo': 'check type'}, # added by JR, TODO (check, sports_centre implies a building, but is at least rendered green). examples found in LINZ include sports parks, race courses, stadiums, cricket 'domains'..
165     (0x1a,): {'landuse': 'cemetery', 'area': 'yes'}, # added by JR, TODO is ok, or should it be amenity=graveyard
166     (0x28,): {'comment': 'JR:ocean'}, # added by JR, TODO work out what this is about, should we using coastlines
167     (0x3c, 0x3e, 0x40, 0x41): {'natural': 'water', 'area': 'yes'}, # 0x3e added by JR
168     (0x47, 0x48, 0x49): {'waterway': 'riverbank', 'area': 'yes'}, # 0x47 added by JR
169     (0x4c,): {'waterway': 'intermittent', 'area': 'yes'},
170     (0x50,): {'natural': 'wood', 'area': 'yes'}, # added by JR, seems to be for nature reserves too
171     (0x51,): {'natural': 'wetland', 'area': 'yes'} # added by JR, was marsh
172    }
173
174sufi=None
175found=False
176for line in file_mp:
177    print line
178    # Marker for start of sections
179    if line.startswith(';sufi'):
180     sufi = line.split('=')[1].strip();
181     
182    if line.startswith(('[POI]','[RGN10]','[RGN20]')):
183        node = ET.Element('node', visible='true', id=str(nodeid))
184        nodeid -= 1
185        node.append(source)
186        poi = True
187        elementtagmap = poitagmap
188        poi_counter += 1
189        if sufi is not None:
190                sufitag = ET.Element('tag', k='linz:sufi',v=sufi)
191                sufitag.tail = '\n' 
192                node.append(sufitag)
193                sufi=None
194
195    if line.startswith(('[POLYLINE]','[RGN40]')):
196        node = ET.Element('way', visible='true', id=str(nodeid))
197        nodeid -= 1
198        node.append(source)
199        polyline = True
200        elementtagmap = polylinetagmap
201        rnodes = {} # Track routing nodes for current polyline
202        polyline_counter += 1
203        if sufi is not None:
204                sufitag = ET.Element('tag', k='linz:sufi',v=sufi)
205                sufitag.tail = '\n' 
206                node.append(sufitag)
207                sufi=None
208
209
210
211    if line.startswith(('[POLYGON]','[RGN80]')):
212        node = ET.Element('way', visible='true', id=str(nodeid))
213        nodeid -= 1
214        node.append(source)
215        polygon = True
216        elementtagmap = polygontagmap
217        polygon_counter += 1
218        if sufi is not None:
219                sufitag = ET.Element('tag', k='linz:sufi',v=sufi)
220                sufitag.tail = '\n' 
221                node.append(sufitag)
222                sufi=None
223
224
225
226    # parsing data
227    if poi or polyline or polygon:
228       
229        if line.startswith('Label'):
230            label = line.split('=')[1].strip()
231            # Now strip out control codes such as ~[0x2f]
232            codestart = label.find('~[')
233            if codestart != -1:
234                codeend = label.find(']',codestart)
235                if codeend != -1:
236                    label = label[0:codestart] + ' ' + label[codeend+1:]
237            tag = ET.Element('tag', k='name',v=label.strip().title()) # convert to title case
238            tag.tail = '\n    '
239            node.append(tag)
240        if line.startswith('Type'):
241            typecode = line.split('=')[1].strip()
242            tag = ET.Element('tag', k='linz:garmin_type',v=typecode)
243            tag.tail = '\n    '
244            node.append(tag)
245            typecode = int(typecode, 16)
246            for codes, taglist in elementtagmap.iteritems():
247                if typecode in codes:
248                    found=True
249                    for key, value in taglist.iteritems():
250                        tag = ET.Element('tag', k=key, v=value)
251                        tag.tail = '\n    '
252                        node.append(tag)
253            if found==False :
254                print 'ptang missing 0x%x poi:%s line:%s poly:%s sufi:%s' % (typecode, poi, polyline, polygon, sufi)
255                found=False;
256        if line.startswith('RoadID'):
257            roadid = line.split('=')[1].strip()
258            tag = ET.Element('tag', k='linz:RoadID',v=roadid)
259            tag.tail = '\n    '
260            node.append(tag)
261        if line.startswith('RouteParam'):
262            rparams = line.split('=')[1].split(',')
263            # speedval has speeds in km/h corresponding to RouteParam speed value index
264            speedval = [8, 20, 40, 56, 72, 93, 108, 128]
265            speed = ET.Element('tag', k='maxspeed', v=str(speedval[int(rparams[0])]))
266            speed.tail = '\n    '
267            node.append(speed)
268            rclass = ET.Element('tag', k='linz:garmin_road_class', v=str(rparams[1]))
269            rclass.tail = '\n    '
270            node.append(rclass)
271            for att, attval in zip(('oneway', 'toll'), rparams[2:3]):
272                if int(attval):
273                    attrib = ET.Element('tag', k=att, v='true')
274                    attrib.tail = '\n    '
275                    node.append(attrib)
276            # Note: taxi is not an approved access key
277            vehicles = ['emergency', 'goods', 'motorcar', 'psv', 'taxi', 'foot', 'bicycle', 'hgv']
278            for veh, res in zip(vehicles, rparams[4:]):
279                vehtag = ET.Element('tag', k=veh, v=('yes', 'no')[int(res)])
280                vehtag.tail = '\n    '
281                node.append(vehtag)
282
283        # Get nodes from all zoom levels (ie. Data0, Data1, etc)
284        # TODO: Only grab the lowest-numbered data line (highest-resolution) and ignore the rest
285        if line.startswith('Data'):
286            if poi:
287                coords = line.split('=')[1].strip()
288                coords = coords.split(',')
289                node.set('lat',str(float(coords[0][1:])))
290                node.set('lon',str(float(coords[1][:-1])))
291            if polyline or polygon:
292                # Just grab the line and parse it later when the [END] element is encountered
293                coords = line.split('=')[1].strip() + ','
294                # TODO: parse out "holes" in a polygon by reading multiple Data0 lines and
295                # constructing a multipolygon relation
296        if line.startswith('Nod'):
297            if polyline:
298                # Store the point index and routing node id for later use
299                nod = line.split('=')[1].strip().split(',', 2)
300                rnodes[nod[0]] = nod[1]
301        if line.startswith('[END]'):
302            if polyline or polygon:
303                # Have to write out nodes as they are parsed
304                nodidx = 0
305                nodIds = []
306                reused = False
307                while coords != '':
308                    coords = coords.split(',', 2)
309                    if str(nodidx) in rnodes:
310                        if rnodes[str(nodidx)] in rNodeToOsmId:
311                            curId = rNodeToOsmId[str(rnodes[str(nodidx)])]
312                            reused = True
313                        else:
314                            curId = nodeid
315                            nodeid -= 1
316                            rNodeToOsmId[str(rnodes[str(nodidx)])] = curId
317                    else:
318                        curId = nodeid
319                        nodeid -= 1
320                    nodIds.append(curId)
321                    # Don't write another node element if we reused an existing one
322                    if not reused:
323                        nodes = ET.Element('node', visible='true', id=str(curId), lat=str(float(coords[0][1:])), lon=str(float(coords[1][:-1])))
324                        nodes.text = '\n    '
325                        nodes.tail = '\n  '
326                        osm.append(nodes)
327                    coords = coords[2]
328                    reused = False
329                    nodidx += 1
330                nodidx = 0
331                for ndid in nodIds:
332                    nd = ET.Element('nd', ref=str(ndid))
333                    nd.tail = '\n    '
334                    node.append(nd)
335            if polygon:
336                nd = ET.Element('nd', ref=str(nodIds[0]))
337                nd.tail = '\n    '
338                node.append(nd)
339
340            poi = False
341            polyline = False
342            polygon = False
343            roadid = ''
344            rnodes = {} # Clear out routing nodes to prepare for next entity
345
346            node.text = '\n    '
347            node.tail = '\n  '
348           
349            osm.append(node)
350
351# writing to file
352f = open('out.osm', 'w')
353f.write(ET.tostring(osm))
354
355# dump some stats
356print '======'
357print 'Totals'
358print '======'
359print 'POI', poi_counter
360print 'POLYLINE', polyline_counter
361print 'POLYGON', polygon_counter
362print 'Last nodeid', nodeid
363
Note: See TracBrowser for help on using the repository browser.