source: subversion/applications/utils/export/osm2pgsql/output-gazetteer.c @ 15118

Last change on this file since 15118 was 14843, checked in by tomhughes, 11 years ago

Initial work on generating a gazetteer database.

File size: 13.2 KB
Line 
1#include <stdio.h>
2#include <stdlib.h>
3#include <string.h>
4
5#include <libpq-fe.h>
6
7#include "osmtypes.h"
8#include "middle.h"
9#include "output.h"
10#include "output-gazetteer.h"
11#include "pgsql.h"
12#include "reprojection.h"
13#include "build_geometry.h"
14
15#define BUFFER_SIZE 4096
16
17#define SRID (project_getprojinfo()->srs)
18
19#define CREATE_PLACE_TABLE                      \
20   "CREATE TABLE place ("                       \
21   "  osm_type CHAR(1) NOT NULL,"               \
22   "  osm_id BIGINT NOT NULL,"                  \
23   "  class TEXT NOT NULL,"                     \
24   "  type TEXT NOT NULL,"                      \
25   "  name TEXT NOT NULL,"                      \
26   "  rank INTEGER NOT NULL"                    \
27   ")"
28
29#define CREATE_PLACE_ID_INDEX \
30   "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id)"
31
32#define CREATE_PLACE_TYPE_INDEX \
33   "CREATE INDEX place_type_idx ON place USING BTREE (class, type)"
34
35#define CREATE_PLACE_NAME_INDEX \
36   "CREATE INDEX place_name_idx ON place USING GIN (TO_TSVECTOR('simple', name))"
37
38#define CREATE_PLACE_GEOMETRY_INDEX \
39   "CREATE INDEX place_geometry_idx ON place USING GIST (geometry)"
40
41#define TAGINFO_NODE 0x1u
42#define TAGINFO_WAY  0x2u
43#define TAGINFO_AREA 0x4u
44
45static const struct taginfo {
46   const char   *name;
47   const char   *value;
48   unsigned int flags;
49} taginfo[] = {
50   { "amenity",  NULL,      TAGINFO_NODE|TAGINFO_WAY|TAGINFO_AREA },
51   { "highway",  NULL,      TAGINFO_WAY                           },
52   { "historic", NULL,      TAGINFO_NODE|TAGINFO_WAY|TAGINFO_AREA },
53   { "place",    NULL,      TAGINFO_NODE                          },
54   { "railway",  "station", TAGINFO_NODE                          },
55   { "railway",  "halt",    TAGINFO_NODE                          },
56   { "tourism",  NULL,      TAGINFO_NODE|TAGINFO_WAY|TAGINFO_AREA },
57   { "waterway", NULL,      TAGINFO_WAY                           },
58   { NULL,       NULL,      0                                     }
59};
60
61static const struct output_options *Options = NULL;
62static PGconn *Connection = NULL;
63static int CopyActive = 0;
64static char Buffer[BUFFER_SIZE];
65static unsigned int BufferLen = 0;
66
67static void require_slim_mode(void)
68{
69   if (!Options->slim)
70   {
71      fprintf(stderr, "Cannot apply diffs unless in slim mode\n");
72      exit_nicely();
73   }
74
75   return;
76}
77
78static void copy_data(const char *sql)
79{
80   unsigned int sqlLen = strlen(sql);
81
82   /* Make sure we have an active copy */
83   if (!CopyActive)
84   {
85      pgsql_exec(Connection, PGRES_COPY_IN, "COPY place FROM STDIN");
86      CopyActive = 1;
87   }
88
89   /* If the combination of old and new data is too big, flush old data */
90   if (BufferLen + sqlLen > BUFFER_SIZE - 10)
91   {
92      pgsql_CopyData("place", Connection, Buffer);
93      BufferLen = 0;
94   }
95
96   /*
97    * If new data by itself is too big, output it immediately,
98    * otherwise just add it to the buffer.
99    */
100   if (sqlLen > BUFFER_SIZE - 10)
101   {
102      pgsql_CopyData("Place", Connection, sql);
103      sqlLen = 0;
104   }
105   else if (sqlLen > 0)
106   {
107      strcpy(Buffer + BufferLen, sql);
108      BufferLen += sqlLen;
109      sqlLen = 0;
110   }
111
112   /* If we have completed a line, output it */
113   if (BufferLen > 0 && Buffer[BufferLen-1] == '\n')
114   {
115      pgsql_CopyData("place", Connection, Buffer);
116      BufferLen = 0;
117   }
118
119   return;
120}
121
122static void stop_copy(void)
123{
124   PGresult *res;
125
126   /* Do we have a copy active? */
127   if (!CopyActive) return;
128
129   /* Terminate the copy */
130   if (PQputCopyEnd(Connection, NULL) != 1)
131   {
132      fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
133      exit_nicely();
134   }
135
136   /* Check the result */
137   res = PQgetResult(Connection);
138   if (PQresultStatus(res) != PGRES_COMMAND_OK)
139   {
140      fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
141      PQclear(res);
142      exit_nicely();
143   }
144
145   /* Discard the result */
146   PQclear(res);
147
148   /* We no longer have an active copy */
149   CopyActive = 0;
150
151   return;
152}
153
154static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *names, struct keyval *places)
155{
156   int area = 0;
157   struct keyval *item;
158
159   /* Initialise the result lists */
160   initList(names);
161   initList(places);
162
163   /* Loop over the tags */
164   while ((item = popItem(tags)) != NULL)
165   {
166      /* If this is a name tag, add it to the name list */
167      if (strcmp(item->key, "ref") == 0 ||
168          strcmp(item->key, "iata") == 0 ||
169          strcmp(item->key, "icao") == 0 ||
170          strcmp(item->key, "name") == 0 ||
171          strcmp(item->key, "old_name") == 0 ||
172          strcmp(item->key, "loc_name") == 0 ||
173          strcmp(item->key, "alt_name") == 0 ||
174          strncmp(item->key, "name:", 5) == 0)
175      {
176         pushItem(names, item);
177      }
178      else
179      {
180         const struct taginfo *t;
181
182         /* If this is a tag we want then add it to the place list */
183         for (t = taginfo; t->name != NULL; t++)
184         {
185            if ((t->flags & flags) != 0)
186            {
187               if (strcmp(t->name, item->key) == 0 &&
188                   (t->value == NULL || strcmp(t->value, item->value) == 0))
189               {
190                  if ((t->flags & TAGINFO_AREA) != 0) area = 1;
191
192                  pushItem(places, item);
193
194                  break;
195               }
196            }
197         }
198
199         /* Free the tag if we didn't want it */
200         if (t->name == NULL) freeItem(item);
201      }
202   }
203
204   return area;
205}
206
207static void add_place(char osm_type, int osm_id, const char *class, const char *type, const char *name, int rank, const char *wkt)
208{
209   char sql[2048];
210
211   /* Output a copy line for this place */
212   sprintf(sql, "%c\t%d\t", osm_type, osm_id);
213   copy_data(sql);
214   escape(sql, sizeof(sql), class);
215   copy_data(sql);
216   copy_data("\t");
217   escape(sql, sizeof(sql), type);
218   copy_data(sql);
219   copy_data("\t");
220   escape(sql, sizeof(sql), name);
221   copy_data(sql);
222   sprintf(sql, "\t%d\tSRID=%d;", rank, SRID);
223   copy_data(sql);
224   copy_data(wkt);
225   copy_data("\n");
226
227   return;
228}
229
230static void delete_place(char osm_type, int osm_id)
231{
232   /* Stop any active copy */
233   stop_copy();
234
235   /* Delete all places for this object */
236   pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %d", osm_type, osm_id);
237
238   return;
239}
240
241static int gazetteer_out_start(const struct output_options *options)
242{
243   /* Save option handle */
244   Options = options;
245
246   /* Connection to the database */
247   Connection = PQconnectdb(options->conninfo);
248
249   /* Check to see that the backend connection was successfully made */
250   if (PQstatus(Connection) != CONNECTION_OK)
251   {
252      fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(Connection));
253      exit_nicely();
254   }
255
256   /* Start a transaction */
257   pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN");
258
259   /* (Re)create the table unless we are appending */
260   if (!options->append)
261   {
262      /* Drop any existing table */
263      pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place");
264
265      /* Create the new table */
266      pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE);
267      pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX);
268      pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TYPE_INDEX);
269      pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_NAME_INDEX);
270      pgsql_exec(Connection, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('place', 'geometry', %d, 'GEOMETRY', 2)", SRID);
271      pgsql_exec(Connection, PGRES_COMMAND_OK, "ALTER TABLE place ALTER COLUMN geometry SET NOT NULL");
272      pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_GEOMETRY_INDEX);
273   }
274
275   /* Setup middle layer */
276   options->mid->start(options);
277
278   return 0;
279}
280
281static void gazetteer_out_stop(void)
282{
283   /* Process any remaining ways and relations */
284//   Options->mid->iterate_ways( gazetteer_out_way );
285//   Options->mid->iterate_relations( gazetteer_process_relation );
286
287   /* No longer need to access middle layer */
288   Options->mid->stop();
289
290   /* Stop any active copy */
291   stop_copy();
292
293   /* Commit transaction */
294   pgsql_exec(Connection, PGRES_COMMAND_OK, "COMMIT");
295
296   /* Analyse the table */
297   pgsql_exec(Connection, PGRES_COMMAND_OK, "ANALYZE place");
298
299   return;
300}
301
302static void gazetteer_out_cleanup(void)
303{
304   return;
305}
306
307static int gazetteer_add_node(int id, double lat, double lon, struct keyval *tags)
308{
309   struct keyval names;
310   struct keyval places;
311
312   /* Split the tags */
313   split_tags(tags, TAGINFO_NODE, &names, &places);
314
315   /* Feed this node to the middle layer */
316   Options->mid->nodes_set(id, lat, lon, tags);
317
318   /* Are we interested in this item? */
319   if (listHasData(&names) && listHasData(&places))
320   {
321      struct keyval *name;
322      struct keyval *place;
323
324      for (name = firstItem(&names); name; name = nextItem(&names, name))
325      {
326         for (place = firstItem(&places); place; place = nextItem(&places, place))
327         {
328            int  rank = 0;
329            char wkt[128];
330
331            if (strcasecmp(place->key, "place") == 0)
332            {
333               if (strcasecmp(place->value, "city") == 0) rank = 5;
334               else if (strcasecmp(place->value, "town") == 0) rank = 4;
335               else if (strcasecmp(place->value, "village") == 0) rank = 3;
336               else if (strcasecmp(place->value, "hamlet") == 0) rank = 2;
337               else if (strcasecmp(place->value, "suburb") == 0) rank = 1;
338            }
339
340            sprintf(wkt, "POINT(%.15g %.15g)", lon, lat);
341
342            add_place('N', id, place->key, place->value, name->value, rank, wkt);
343         }
344      }
345   }
346
347   /* Free tag lists */
348   resetList(&names);
349   resetList(&places);
350
351   return 0;
352}
353
354static int gazetteer_add_way(int id, int *ndv, int ndc, struct keyval *tags)
355{
356   struct keyval names;
357   struct keyval places;
358   int area;
359
360   /* Split the tags */
361   area = split_tags(tags, TAGINFO_WAY, &names, &places);
362
363   /* Are we interested in this item? */
364   if (listHasData(&names) && listHasData(&places))
365   {
366      struct osmNode *nodev;
367      int nodec;
368      char *wkt;
369      double dummy;
370
371      /* Feed this way to the middle layer */
372      Options->mid->ways_set(id, ndv, ndc, tags, 0);
373
374      /* Fetch the node details */
375      nodev = malloc(ndc * sizeof(struct osmNode));
376      nodec = Options->mid->nodes_get_list(nodev, ndv, ndc);
377
378      /* Get the geometry of the object */
379      if ((wkt = get_wkt_simple(nodev, nodec, area, &dummy, &dummy, &dummy)) != NULL && strlen(wkt) > 0)
380      {
381         struct keyval *name;
382         struct keyval *place;
383
384         for (name = firstItem(&names); name; name = nextItem(&names, name))
385         {
386            for (place = firstItem(&places); place; place = nextItem(&places, place))
387            {
388               add_place('W', id, place->key, place->value, name->value, 0, wkt);
389            }
390         }
391      }
392
393      /* Free the geometry */
394      free(wkt);
395
396      /* Free the nodes */
397      free(nodev);
398   }
399
400   /* Free tag lists */
401   resetList(&names);
402   resetList(&places);
403
404   return 0;
405}
406
407static int gazetteer_add_relation(int id, struct member *membv, int membc, struct keyval *tags)
408{
409   return 0;
410}
411
412static int gazetteer_delete_node(int id)
413{
414   /* Make sure we are in slim mode */
415   require_slim_mode();
416
417   /* Delete all references to this node */
418   delete_place('N', id);
419
420   /* Feed this delete to the middle layer */
421   Options->mid->nodes_delete(id);
422
423   return 0;
424}
425
426static int gazetteer_delete_way(int id)
427{
428   /* Make sure we are in slim mode */
429   require_slim_mode();
430
431   /* Delete all references to this way */
432   delete_place('W', id);
433
434   /* Feed this delete to the middle layer */
435   Options->mid->ways_delete(id);
436
437   return 0;
438}
439
440static int gazetteer_delete_relation(int id)
441{
442   /* Make sure we are in slim mode */
443   require_slim_mode();
444
445   /* Feed this delete to the middle layer */
446   Options->mid->relations_delete(id);
447
448   return 0;
449}
450
451static int gazetteer_modify_node(int id, double lat, double lon, struct keyval *tags)
452{
453   /* Make sure we are in slim mode */
454   require_slim_mode();
455
456   /* Delete all references to this node... */
457   gazetteer_delete_node(id);
458
459   /* ...then add it back again */
460   gazetteer_add_node(id, lat, lon, tags);
461
462   /* Feed this change to the middle layer */
463   Options->mid->node_changed(id);
464
465   return 0;
466}
467
468static int gazetteer_modify_way(int id, int *ndv, int ndc, struct keyval *tags)
469{
470   /* Make sure we are in slim mode */
471   require_slim_mode();
472
473   /* Delete all references to this way... */
474   gazetteer_delete_way(id);
475
476   /* ...then add it back again */
477   gazetteer_add_way(id, ndv, ndc, tags);
478
479   /* Feed this change to the middle layer */
480   Options->mid->way_changed(id);
481
482   return 0;
483}
484
485static int gazetteer_modify_relation(int id, struct member *membv, int membc, struct keyval *tags)
486{
487   /* Make sure we are in slim mode */
488   require_slim_mode();
489
490   /* Feed this change to the middle layer */
491   Options->mid->relation_changed(id);
492
493   return 0;
494}
495
496struct output_t out_gazetteer = {
497   .start = gazetteer_out_start,
498   .stop = gazetteer_out_stop,
499   .cleanup = gazetteer_out_cleanup,
500
501   .node_add = gazetteer_add_node,
502   .way_add = gazetteer_add_way,
503   .relation_add = gazetteer_add_relation,
504
505   .node_modify = gazetteer_modify_node,
506   .way_modify = gazetteer_modify_way,
507   .relation_modify = gazetteer_modify_relation,
508
509   .node_delete = gazetteer_delete_node,
510   .way_delete = gazetteer_delete_way,
511   .relation_delete = gazetteer_delete_relation
512};
Note: See TracBrowser for help on using the repository browser.