source: subversion/applications/utils/export/osm2pgsql/mapnik-osm-updater.sh @ 9252

Last change on this file since 9252 was 8186, checked in by guenther, 12 years ago
  • adding poi key
  • Property svn:executable set to *
File size: 20.5 KB
Line 
1#!/bin/bash
2
3export osm_username="osm"
4export database_name="gis"
5export planet_dir="/home/$osm_username/osm/planet"
6export planet_file="$planet_dir/planet.osm.bz2"
7export sql_dump="$planet_dir/planet.osm.sql.bz2"
8export osm2pgsql_cmd=`which osm2pgsql`
9export gpsdrive_poitypes_cmd=`which gpsdrive-update-mapnik-poitypes.pl`
10export log_dir=/var/log
11test -x "$osm2pgsql_cmd" || osm2pgsql_cmd="$HOME/svn.openstreetmap.org/applications/utils/export/osm2pgsql/osm2pgsql"
12
13osm_planet_mirror_cmd='../../planet-mirror/planet-mirror.pl'
14test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="$HOME/svn.openstreetmap.org/applications/utils/planet-mirror/planet-mirror.pl"
15test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd=`which osm-planet-mirror`
16
17test -n "$1" || help=1
18quiet=" -q "
19verbose=1
20
21for arg in "$@" ; do
22    case $arg in
23        --all-planet) # Do all the creation steps listed below from planet file
24            create_osm_user=1
25            mirror=1
26            check_newer_planet=
27            drop=1
28            create_db=1
29            create_db_user=1
30            grant_all_rights_to_user_osm=1
31            planet_fill=1
32            db_add_gpsdrive_poitypes=1
33            create_db_users=${create_db_users:-*}
34            grant_db_users=${grant_db_users:-*}
35            ;;
36
37        --all-planet-geofabrik=*) #     Use Planet Extract from Frederics GeoFabrik.de Page as planet File and import
38                #               Use ? for a list of possible files
39                #               Example: europe/germany/baden-wuerttemberg
40            dir_country=${arg#*=}
41            country=`basename $dir_country`
42            planet_file="$planet_dir/${country}.osm.bz2"
43            mirror_geofabrik=${dir_country}
44            create_osm_user=1
45            mirror=
46            check_newer_planet=
47            drop=1
48            create_db=1
49            create_db_user=1
50            grant_all_rights_to_user_osm=1
51            planet_fill=1
52            db_add_gpsdrive_poitypes=1
53            create_db_users=${create_db_users:-*}
54            grant_db_users=${grant_db_users:-*}
55            ;;
56
57        --all-planet-update) #  Do all the creation steps listed below from planet file with up to date cheching
58            create_osm_user=1
59            mirror=1
60            check_newer_planet=1
61            drop=1
62            create_db=1
63            create_db_user=1
64            grant_all_rights_to_user_osm=1
65            planet_fill=1
66            db_add_gpsdrive_poitypes=1
67            create_db_users=${create_db_users:-*}
68            grant_db_users=${grant_db_users:-*}
69            ;;
70
71        --all-from-dump) #      Do all the creation steps listed below
72                #       from planet-dump file
73                #       !!! all-from-dump is not completely tested yet
74            create_osm_user=1
75            mirror_dump=1
76            drop=1
77            create_db=1
78            create_db_user=1
79            grant_all_rights_to_user_osm=1
80            create_db_users=${create_db_users:-*}
81            fill_from_dump="$sql_dump"
82            grant_db_users=${grant_db_users:-*}
83            db_add_gpsdrive_poitypes=1
84            ;;
85
86        --all-create) #         Do all the creation steps listed below only no data
87                      # import and no planet mirroring
88            create_osm_user=1
89            drop=1
90            create_db=1
91            create_db_user=1
92            grant_all_rights_to_user_osm=1
93            create_db_users=${create_db_users:-*}
94            grant_db_users=${grant_db_users:-*}
95            ;;
96
97        --create-osm-user) #    create the osm-user needed
98                #       This means creating a user 'osm' and his home directory
99                #       with useradd, mkdir, chmod and chown
100            create_osm_user=1
101            ;;
102       
103        --mirror) #             mirror planet File (http://planet.openstreetmap.org/)
104            mirror=1
105            ;;
106
107        --no-mirror) #          do not mirror planet File
108            mirror=
109            ;;
110
111        --check-newer-planet) # Check if Planet File is newer then stampfile.
112            #           If yes: Continue
113            check_newer_planet=1
114            ;;
115
116        --drop) #               drop the old Database (gis) and Database-user (osm)
117            drop=1
118            ;;
119
120        --create-db) #          create the database (gis)
121            #           with this command only the database is created,
122            #           but no tables inside it
123            create_db=1
124            ;;
125       
126        --create-db-user) #     create the database-user (osm)
127            create_db_user=1
128            ;;
129       
130        --grant-all2osm-user) # grant all rights for the database to the DB-User osm
131            grant_all_rights_to_user_osm=1
132            ;;
133
134        --create-db-users=*) #Create a Database user for all users specified.
135            #           To create a db-user for all available system-user
136            #           specify *. (Except root))
137            create_db_users=${arg#*=}
138            ;;
139       
140        --grant-db-users=*) #   Grant database-users all rights (including write, ...)
141            #           to the gis Database !!! This has to be changed in the
142            #           future, normally only the osm user needs update rights
143            grant_db_users=${arg#*=}
144            ;;
145
146        --add-gpsdrive-types) # add GpsDrive POI-Types to points table
147            db_add_gpsdrive_poitypes=1
148            ;;
149
150        --planet-fill) #        fill database from planet File
151            planet_fill=1
152            ;;
153
154        --mirror-dump) #        mirror the planet.sql dump File
155            mirror_dump=1
156            ;;
157
158        --no-mirror-dump) #     Do not mirror the planet.sql dump File
159            mirror_dump=
160            ;;
161
162        --fill-from-dump=*) #   fill database from Dump File
163            fill_from_dump=${arg#*=}
164            ;;
165
166        --mapnik-dump=*) #      Dump Content of Mapnik Database to a File (.sql|.sql.bz))
167            postgis_mapnik_dump=${arg#*=}
168            ;;
169       
170        --db-table-create) #    Create tables in Database with osm2pgsql
171            db_table_create=1
172            ;;
173
174        --count-db) #           Count entries in Database. This is to check
175            #           if the database really contains entries
176            #           if you set an  empty user with the option osm_username=''
177            #           the current user is used
178            count_db=1
179            ;;
180
181        -h)
182            help=1
183            ;;
184
185        --help)
186            help=1
187            ;;
188
189        -help)
190            help=1
191            ;;
192
193        --debug) #              switch on debugging
194            debug=1
195            verbose=1
196            quiet=""
197            ;;
198
199        -debug)
200            debug=1
201            verbose=1
202            quiet=""
203            ;;
204       
205
206        --nv) #                 be a little bit less verbose
207            verbose=''
208            ;;
209
210        --planet-dir=*) #       define Directory for Planet-File
211            planet_dir=${arg#*=}
212            planet_file="$planet_dir/planet.osm.bz2"
213            ;;
214
215        --planet-file=*) #      define Planet-File including Directory
216            planet_file=${arg#*=}
217            ;;
218       
219        --osm-username=*) #     Define username to use for DB creation and planet
220            #           download
221            #           !! You shouldn't use your username or root as the
222            #           !! download and install user.
223            #           This username is the download and install user.
224            #           The osm-user normally only should have the planet files
225            #           in hishome directory and nothing else. By default
226            #           the osm-username is 'osm'
227            osm_username=${arg#*=}
228
229            if [ "$osm_username" = "$USER" ] ; then
230                echo 
231                echo "!!! Don't use your own login account as the osm_username!!" 1>&2
232                echo 
233                exit 1
234            fi
235
236            if [ "$osm_username" = "root" ] ; then
237                echo 
238                echo "!!! Don't use the root account as the osm_username!!" 1>&2
239                echo 
240                exit 1
241            fi
242
243            planet_dir="/home/$osm_username/osm/planet"
244            planet_file="$planet_dir/planet.osm.bz2"
245            ;;
246       
247        --osm2pgsql-cmd=*) #    The path to the osm2pgsql command
248            #           It can be found at
249            #           svn.openstreetmap.org/applications/utils/export/osm2pgsql/
250            #           and has to be compiled. Alternatively you can install
251            #           the Debian Package openstreetmap-utils
252            osm2pgsql_cmd=${arg#*=}
253                if ! [ -x "$osm2pgsql_cmd" ]; then
254                    echo "Cannot execute '$osm2pgsql_cmd'" 1>&2
255                    exit -1
256                fi
257                ;;
258
259        --database-name=*) #    use this name for the database default is 'gis'
260            database_name=${arg#*=}
261            ;;
262
263        *)
264            echo ""
265            echo "!!!!!!!!! Unknown option $arg"
266            echo ""
267            help=1
268            ;;
269    esac
270done
271
272if [ -n "$help" ] ; then
273    # extract options from case commands above
274    options=`grep -E -e esac -e '\s*--.*\).*#' $0 | sed '/esac/,$d;s/.*--/ [--/; s/=\*)/=val]/; s/)[\s ]/]/; s/#.*\s*//; s/[\n/]//g;'`
275    options=`for a in $options; do echo -n " $a" ; done`
276    echo "$0 $options"
277    echo "
278!!! Warning: This Script is for now a quick hack to make setting up
279!!! Warning: My databases easier. Please check if it really works for you!!
280!!! Warning: Especially when using different Database names or username, ...
281!!! Warning: not every combination of values except the default is tested.
282
283    This script tries to install the mapnik database.
284    For this it first creates a new user osm on the system
285    and mirrors the current planet to his home directory.
286    Then this planet is imported into the postgis Database from a
287    newly created user named osm
288
289    This script uses sudo. So you either have to have sudo right or you'll
290    have to start the script as root. The users needed will be postgres and osm
291    "
292    # extract options + description from case commands above
293    grep -E  -e esac -e '--.*\).*#' -e '^[\t\s  ]+#' $0 | \
294        grep -v /bin/bash | sed '/esac/,$d;s/.*--/  --/;s/=\*)/=val/;s/)//;s/#//;' 
295    exit;
296fi
297
298
299if [ -n "$osm_username" ] ; then
300    sudo_cmd="sudo -u $osm_username"
301else
302    sudo_cmd=''
303fi
304
305export import_stamp_file=${log_dir}/osm2pgsql_postgis-$database_name.stamp
306export import_log=${log_dir}/osm2pgsql_postgis-$database_name.log
307
308
309if [ -n "$debug" ] ; then
310        echo "Planet File: `ls -l $planet_file`"
311        echo "Import Stamp : `ls -l $import_stamp_file`"
312fi
313
314
315############################################
316# Create a user on the system
317############################################
318if [ -n "$create_osm_user" ] ; then
319    test -n "$verbose" && echo "----- Check if we already have an user '$osm_username'"
320   
321    if ! id "$osm_username" >/dev/null; then
322        echo "create '$osm_username' User"
323        useradd "$osm_username"
324    fi
325   
326    mkdir -p "/home/$osm_username/osm/planet"
327    # The user itself should be allowed to read/write all his own files
328    # in the ~/osm/ Directory
329    chown "$osm_username" "/home/$osm_username"
330    chown -R "$osm_username" "/home/$osm_username/osm"
331    chmod +rwX "/home/$osm_username"
332    chmod -R +rwX "/home/$osm_username/osm"
333
334    # Everyone on the system is allowed to read the planet.osm Files
335    chmod -R a+rX "/home/$osm_username/osm"
336fi
337
338
339############################################
340# Mirror the planet-dump File for Europe
341############################################
342if [ -n "$mirror_geofabrik" ] ; then
343    geofabrik_basedir="http://download.geofabrik.de/osm"
344    if [ "$mirror_geofabrik" = "?" ]; then
345        echo "Possible Values are:"
346        for sub_dir in "" "europe/" "europe/germany/"; do
347            wget -q  --level=0 -O - "$geofabrik_basedir/$sub_dir" | grep 'OpenStreetMap data' | \
348                perl -ne 'm/.*href="([^"]+)\.osm.bz2"/;print "  '$sub_dir'$1\n"'
349        done
350        exit 1 
351    fi
352    planet_source_file="${geofabrik_basedir}/${mirror_geofabrik}.osm.bz2"
353    test -n "$verbose" && echo "----- Mirroring planet File $planet_source_file"
354    wget -v --mirror "$planet_source_file" \
355        --no-directories --directory-prefix=$planet_dir/
356fi
357
358
359############################################
360# Mirror the newest planet File from planet.openstreetmap.org
361############################################
362if [ -n "$mirror" ] ; then
363    test -n "$verbose" && echo "----- Mirroring planet File"
364    if ! [ -x "$osm_planet_mirror_cmd" ]; then
365        echo "Cannot execute '$osm_planet_mirror_cmd'" 1>&2
366        exit -1
367    fi
368    if ! $sudo_cmd $osm_planet_mirror_cmd -v -v --planet-dir=$planet_dir ; then 
369        echo "Cannot Mirror Planet File" 1>&2
370        exit 1
371    fi
372    if ! [ -s $planet_file ] ; then
373        echo "File $planet_file is missing"
374        exit -1
375    fi
376
377
378fi
379
380############################################
381# Check if Planet File is newer than import Stamp
382############################################
383if [ -n "$check_newer_planet" ] ; then
384    if [ "$planet_file" -nt "$import_stamp_file" ] ; then
385        if [ -n "$verbose" ] ; then
386            echo "----- New File needs updating"
387            echo "Planet File: `ls -l $planet_file`"
388            echo "Import Stamp : `ls -l $import_stamp_file`"
389        fi
390    else
391        echo "Postgis Database already Up To Date"
392        echo "`ls -l $import_stamp_file`"
393        exit 0
394    fi
395fi
396
397############################################
398# Drop the old Database and Database-user
399############################################
400if [ -n "$drop" ] ; then
401    test -n "$verbose" && echo "----- Drop complete Database '$database_name' and user '$osm_username'"
402    echo "CHECKPOINT" | sudo -u postgres psql $quiet
403    sudo -u postgres dropdb $quiet -Upostgres   "$database_name"
404    sudo -u postgres dropuser $quiet -Upostgres "$osm_username"
405fi
406
407############################################
408# Create db
409############################################
410if [ -n "$create_db" ] ; then
411    test -n "$verbose" && echo
412    test -n "$verbose" && echo "----- Create Database '$database_name'"
413    if ! sudo -u postgres createdb -Upostgres  $quiet  -EUTF8 "$database_name"; then
414        echo "Creation Failed"
415        exit -1
416    fi
417    if ! sudo -u postgres createlang plpgsql "$database_name"; then
418        echo "Creation Failed"
419        exit -1
420    fi
421
422    if ! sudo -u postgres psql $quiet -Upostgres "$database_name" </usr/share/postgresql-8.2-postgis/lwpostgis.sql ; then
423        echo "Creation Failed"
424        exit -1
425    fi
426fi
427
428############################################
429# Create db-user
430############################################
431if [ -n "$create_db_user" ] ; then
432    test -n "$verbose" && echo "----- Create Database-user '$osm_username'"
433    sudo -u postgres createuser -Upostgres  $quiet -S -D -R "$osm_username"  || exit -1
434fi
435
436if [ -n "$grant_all_rights_to_user_osm" ] ; then
437    test -n "$verbose" && echo 
438    test -n "$verbose" && echo "----- Grant rights on Database '$database_name' for '$osm_username'"
439    (
440        echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";" 
441        echo "GRANT ALL on geometry_columns TO \"$osm_username\";"
442        echo "GRANT ALL on spatial_ref_sys TO \"$osm_username\";" 
443        echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";" 
444    ) | sudo -u postgres psql $quiet -Upostgres "$database_name"
445fi
446
447############################################
448# Create a Database user for all users specified (*) or available on the system. Except root
449############################################
450if [ -n "$create_db_users" ] ; then
451
452    if [ "$create_db_users" = "*" ] ; then
453        echo "Create DB User for every USER"
454        create_db_users=''
455        # try to see if all users above uid=1000 are interesting
456        all_users=`cat /etc/passwd | sed 's/:/ /g' | while read user pwd uid rest ; do test "$uid" -ge "1000" || continue; echo $user; done`
457        echo "all_users: $all_users"
458        for user in $all_users ; do 
459            echo $user | grep -q -e root && continue
460            echo $user | grep -q -e "$osm_username" && continue
461            echo $user | grep -q -e "nobody" && continue
462            echo "$create_db_users" | grep -q  " $user " && continue
463            create_db_users=" $create_db_users $user "
464        done
465    fi
466
467# This is not good; this probably broke my postgres installation
468# dpkg  --purge postgresql-8.2
469# Stopping PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more
470# apt-get -f install postgresql-8.2
471# Starting PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more
472#if false ; then
473    for user in $create_db_users; do
474            echo "      Create DB User for $user"
475        sudo -u postgres createuser $quiet -Upostgres --no-superuser --no-createdb --no-createrole "$user"
476    done
477#fi
478
479fi
480
481############################################
482# Grant all rights on the gis Database to all system users or selected users in the system
483############################################
484if [ -n "$grant_db_users" ] ; then
485
486    if [ "$grant_db_users" = "*" ] ; then
487        echo "-------- GRANT Rights to every USER"
488        grant_db_users=''
489        for user in `users` ; do 
490            echo "$user" | grep -q "root" && continue
491            echo " $grant_db_users " | grep -q " $user " && continue
492            grant_db_users="$grant_db_users $user"
493        done
494    fi
495
496    test -n "$verbose" && echo "Granting rights to users: '$grant_db_users'"
497
498    for user in $grant_db_users; do
499        echo "Granting all rights to user '$user' for Database '$database_name'"
500        (
501            echo "GRANT ALL on geometry_columns TO \"$user\";"
502            echo "GRANT ALL ON SCHEMA PUBLIC TO \"$user\";"
503            echo "GRANT ALL on spatial_ref_sys TO \"$user\";"
504            echo "GRANT ALL on TABLE planet_osm_line TO \"$user\";"
505            echo "GRANT ALL on TABLE planet_osm_point TO \"$user\";"
506            )| sudo -u postgres psql $quiet -Upostgres "$database_name" || true
507    done
508fi
509
510
511############################################
512# Create Database tables with osm2pgsql
513############################################
514if [ -n "$db_table_create" ] ; then
515    if ! [ -x "$osm2pgsql_cmd" ]; then
516        echo "Cannot execute '$osm2pgsql_cmd'" 1>&2
517        exit -1
518    fi
519    echo ""
520    echo "--------- Unpack and import $planet_file"
521    cd /usr/share/openstreetmap/
522    $sudo_cmd $osm2pgsql_cmd --create "$database_name"
523fi
524
525
526############################################
527# Fill Database from planet File
528############################################
529if [ -n "$planet_fill" ] ; then
530    if ! [ -x "$osm2pgsql_cmd" ]; then
531        echo "Cannot execute '$osm2pgsql_cmd'" 1>&2
532        exit -1
533    fi
534    echo ""
535    echo "--------- Unpack and import $planet_file"
536    echo "Import started: `date`" >>"$import_log"
537    cd /usr/share/openstreetmap/
538    $sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file
539    rc=$?
540    if [ "$rc" -gt "0" ]; then
541        echo "`date`: Import With Error $rc:" >> "$import_log"
542        echo "`ls -l $planet_file` import --> rc($rc)" >> "$import_log"
543        echo "!!!!!!!! ERROR while running '$sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file'"
544        echo "Creation with for Database "$database_name" from planet-file '$planet_file' with '$osm2pgsql_cmd' Failed"
545        echo "see Logfile for more Information:"
546        echo "less $import_log"
547        exit -1
548    fi
549    echo "`date`: Import Done: `ls -l $planet_file` import --> $rc" >> "$import_log"
550    echo "`date`: `ls -l $planet_file` import --> $rc" >>$import_stamp_file
551    touch --reference=$planet_file $import_stamp_file
552fi
553
554
555############################################
556# Add GpsDrive POI-Types to points Table
557############################################
558if [ -n "$db_add_gpsdrive_poitypes" ] ; then
559    if ! [ -x "$gpsdrive_poitypes_cmd" ]; then
560        echo "Cannot execute '$gpsdrive_poitypes_cmd'" 1>&2
561        exit -1
562    fi
563    echo ""
564    echo "--------- Adding GpsDrive POI-Types to Database"
565    sudo -u postgres $gpsdrive_poitypes_cmd -n
566fi
567
568
569############################################
570# Dump the complete Database
571############################################
572if [ -n "$postgis_mapnik_dump" ] ; then
573        # get Database Content with Dump
574    postgis_mapnik_dump_dir=`dirname $postgis_mapnik_dump`
575        mkdir -p "$postgis_mapnik_dump_dir"
576        case "$postgis_mapnik_dump" in
577            *.bz2)
578                $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | bzip2 >"$postgis_mapnik_dump"
579                ;;
580            *.gz)
581                $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | gzip >"$postgis_mapnik_dump"
582                ;;
583            *)
584                $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" >"$postgis_mapnik_dump"
585                ;;
586        esac
587    if [ "$?" -gt "0" ]; then
588        echo "Error While dumping Database"
589    fi
590fi
591
592############################################
593# Mirror the planet-dump File from planet.openstreetmap.de
594############################################
595if [ -n "$mirror_dump" ] ; then
596    test -n "$verbose" && echo "----- Mirroring planet-dump File"
597    wget -v --mirror http://planet.openstreetmap.de/planet.osm.sql.bz2 \
598        --no-directories --directory-prefix=$planet_dir/
599fi
600
601
602############################################
603# Fill Database from Dump File
604############################################
605if [ -n "$fill_from_dump" ] ; then
606    echo ""
607    echo "--------- Import from Dump '$fill_from_dump'"
608    sudo -u postgres createdb -T template0 $database_name
609    case "$fill_from_dump" in
610        *.bz2)
611            test -n "$verbose" && echo "Uncompress File ..."
612            bzip2 -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name"
613            ;;
614        *.gz)
615            test -n "$verbose" && echo "Uncompress File ..."
616            gzip -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name"
617            ;;
618        *)
619            test -n "$verbose" && echo "Import uncompressed File ..."
620            $sudo_cmd psql $quiet "$database_name" <"$fill_from_dump"
621            ;;
622    esac
623    if [ "$?" -gt "0" ]; then
624        echo "Error While reding Dump into Database"
625    fi
626fi
627
628
629############################################
630# Check number of entries in Database
631############################################
632if [ -n "$count_db" ] ; then
633    echo ""
634    echo "--------- Check Number of lines in Database '$database_name'"
635
636    # Get the Table names
637    if [ -n "$osm_username" ]; then
638        table_owner=" AND tableowner ='$osm_username' ";
639    fi
640    table_names=`echo "SELECT tablename from pg_catalog.pg_tables where schemaname = 'public' $tableowner;" | \
641        $sudo_cmd psql   "$database_name" -h /var/run/postgresql | grep -E -e '^ planet'`
642
643    echo "Counting entries in all Tables (" $table_names ")"
644    for table in $table_names; do
645        echo -n "Table $table   = "
646        echo "SELECT COUNT(*) from $table;" | \
647            $sudo_cmd psql  gis -h /var/run/postgresql | grep -v -e count -e '------' -e '1 row' | head -1
648    done
649fi
650
Note: See TracBrowser for help on using the repository browser.