From a5d9e32d9b756fa164a55ef00b0dcce82b108062 Mon Sep 17 00:00:00 2001 From: dracic Date: Mon, 14 Jul 2025 17:37:39 +0200 Subject: [PATCH] OSM vector tile server setup with postgis and tegola - Set up docker-compose with PostgreSQL/PostGIS and Tegola services - Added initialization scripts for database setup and OSM data import - Configured Tegola with vector tile layers for roads and buildings - Included README with setup instructions and troubleshooting guide --- README.md | 140 +++++++++++++++++++++++++ docker-compose.yml | 56 ++++++++++ init-scripts/01-init-database.sql | 23 ++++ init-scripts/02-install-osm2pgsql.sh | 28 +++++ init-scripts/03-download-import-osm.sh | 83 +++++++++++++++ osm-data/.gitkeep | 0 tegola_config.toml | 40 +++++++ 7 files changed, 370 insertions(+) create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 init-scripts/01-init-database.sql create mode 100644 init-scripts/02-install-osm2pgsql.sh create mode 100644 init-scripts/03-download-import-osm.sh create mode 100644 osm-data/.gitkeep create mode 100644 tegola_config.toml diff --git a/README.md b/README.md new file mode 100644 index 0000000..6941950 --- /dev/null +++ b/README.md @@ -0,0 +1,140 @@ +# Hands-on Coding Workshop - Serving and Displaying Vector Tiles with PostgreSQL, Tegola, and OpenLayers + +This project sets up a complete OpenStreetMap (OSM) vector tile server using Tegola and PostgreSQL with PostGIS. + +## Services + +### PostgreSQL with PostGIS +- **Image**: `kartoza/postgis:15-3.3` +- **Database**: `osm` +- **User/Password**: `postgres/postgres` +- **Port**: `5432` +- **Extensions**: PostGIS, PostGIS Topology, hstore + +### Tegola Vector Tile Server +- **Image**: `gospatial/tegola:latest` +- **Port**: `8080` (mapped to container port 8080) +- **Configuration**: `tegola_config.toml` + +## Setup and Usage + +### 1. Start the Services +```bash +docker-compose up -d +``` + +### 2. Monitor the Setup Process +The PostgreSQL container will automatically: +1. Initialize the `osm` database +2. Install PostGIS and hstore extensions +3. Install osm2pgsql package +4. Download OSM data for Bosnia and Herzegovina +5. Import the data using osm2pgsql + +Monitor the progress: +```bash +docker-compose logs -f postgres +``` + +### 3. Verify the Setup +Check if all services are healthy: +```bash +docker-compose ps +``` +Wait until all services are in the `healthy` state. + +Connect to PostgreSQL to verify data: +```bash +docker exec -it workshop-postgres psql -U postgres -h 127.0.0.1 -d osm +``` + +Check table counts: +```sql +SELECT 'planet_osm_point' as table_name, count(*) as row_count FROM planet_osm_point +UNION ALL +SELECT 'planet_osm_line' as table_name, count(*) as row_count FROM planet_osm_line +UNION ALL +SELECT 'planet_osm_polygon' as table_name, count(*) as row_count FROM planet_osm_polygon; +``` + +### 4. Access Vector Tiles +Once setup is complete, access vector tiles at: +- Tegola capabilities: `http://localhost:8080/capabilities` +- Map tiles: `http://localhost:8080/maps/osm/{z}/{x}/{y}.mvt` + +## Data + +### OSM Data Source +- **Region**: Bosnia and Herzegovina +- **Source**: Geofabrik (https://download.geofabrik.de/europe/bosnia-herzegovina-latest.osm.pbf) +- **Update**: Manual (re-run import script to update) + +### Available Layers +- **Roads**: `osm.roads` (zoom levels 9-16) +- **Buildings**: `osm.buildings` (zoom levels 12-18) + +## File Structure +``` +. +├── docker-compose.yml # Docker services configuration +├── tegola_config.toml # Tegola server configuration +├── init-scripts/ # PostgreSQL initialization scripts +│ ├── 01-init-database.sql # Database and extensions setup +│ ├── 02-install-osm2pgsql.sh # Install osm2pgsql package +│ └── 03-download-import-osm.sh # Download and import OSM data +├── osm-data/ # OSM data storage directory +└── README.md # This file +``` + +## Troubleshooting + +### Container Issues +```bash +# Check container status +docker-compose ps + +# View logs +docker-compose logs postgres +docker-compose logs tegola + +# Restart services +docker-compose restart +``` + +### Database Issues +```bash +# Connect to database +docker exec -it workshop-postgres psql -U postgres -h 127.0.0.1 -d osm + +# Check extensions +\dx + +# Check tables +\dt +``` + +### Re-import Data +To re-import OSM data with fresh data: +```bash +# Stop services +docker-compose down + +# Remove old data +docker volume rm list-workshop_postgres_data +rm -rf osm-data/*.pbf + +# Start services (will trigger fresh import) +docker-compose up -d +``` + +## Performance Notes + +- Initial setup may take 5-10 minutes depending on internet speed and system performance +- OSM data file size: ~50-100MB for Bosnia and Herzegovina +- Database size after import: ~500MB-1GB +- Memory usage: ~512MB cache for osm2pgsql import + +## Map Center +The default map center is set to Mostar (17.8078, 43.3430) in `tegola_config.toml`. You may want to update this to a location within Bosnia and Herzegovina: +- Sarajevo: `[18.4131, 43.8563, 12.0]` +- Banja Luka: `[17.1910, 44.7666, 12.0]` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ea33a00 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +services: + # PostgreSQL with PostGIS Service + postgres: + image: kartoza/postgis:15-3.3 + container_name: workshop-postgres + restart: unless-stopped + environment: + POSTGRES_DB: osm + POSTGRES_USER: postgres + POSTGRES_PASS: postgres + PGDATA: /var/lib/postgresql/data/pgdata + DEFAULT_ENCODING: "UTF8" + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init-scripts:/docker-entrypoint-initdb.d + - ./osm-data:/osm-data + networks: + - workshop_net + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d osm"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 40s + + # Tegola Vector Tile Server Service + tegola: + image: gospatial/tegola:latest + container_name: workshop-tegola + restart: unless-stopped + command: ["serve", "--config", "/etc/tegola/config.toml"] + volumes: + - ./tegola_config.toml:/etc/tegola/config.toml + networks: + - workshop_net + ports: + - "8080:8080" + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/capabilities"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s + +volumes: + postgres_data: + driver: local + +networks: + workshop_net: + driver: bridge \ No newline at end of file diff --git a/init-scripts/01-init-database.sql b/init-scripts/01-init-database.sql new file mode 100644 index 0000000..f250151 --- /dev/null +++ b/init-scripts/01-init-database.sql @@ -0,0 +1,23 @@ +-- Initialize OSM database with PostGIS and hstore extensions +-- This script runs automatically when PostgreSQL container starts for the first time + +-- Connect to the osm database +\c osm; + +-- Enable PostGIS extension +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS postgis_topology; + +-- Enable hstore extension for key-value storage +CREATE EXTENSION IF NOT EXISTS hstore; + +-- Grant necessary permissions +GRANT ALL PRIVILEGES ON DATABASE osm TO postgres; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres; + +-- Set up proper ownership +ALTER DATABASE osm OWNER TO postgres; + +-- Display confirmation +SELECT 'OSM database initialized successfully with PostGIS and hstore extensions' AS status; \ No newline at end of file diff --git a/init-scripts/02-install-osm2pgsql.sh b/init-scripts/02-install-osm2pgsql.sh new file mode 100644 index 0000000..0d1c3dc --- /dev/null +++ b/init-scripts/02-install-osm2pgsql.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Install osm2pgsql and required tools +# This script runs during PostgreSQL container initialization + +set -e + +echo "Installing osm2pgsql and dependencies..." + +# Update package list +apt-get update + +# Install required packages +apt-get install -y \ + osm2pgsql \ + wget \ + curl \ + unzip \ + ca-certificates \ + --no-install-recommends + +# Clean up to reduce image size +apt-get clean +rm -rf /var/lib/apt/lists/* + +echo "osm2pgsql installation completed successfully" + +# Verify installation +osm2pgsql --version \ No newline at end of file diff --git a/init-scripts/03-download-import-osm.sh b/init-scripts/03-download-import-osm.sh new file mode 100644 index 0000000..d3d7a4b --- /dev/null +++ b/init-scripts/03-download-import-osm.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Download and import OSM data for Bosnia and Herzegovina +# This script runs during PostgreSQL container initialization + +set -e + +echo "Starting OSM data download and import process..." + +# Configuration +OSM_URL="https://download.geofabrik.de/europe/bosnia-herzegovina-latest.osm.pbf" +OSM_FILE="/osm-data/bosnia-herzegovina-latest.osm.pbf" +DB_NAME="osm" +DB_USER="postgres" +DB_HOST="127.0.0.1" +DB_PORT="5432" + +# Wait for PostgreSQL to be ready +echo "Waiting for PostgreSQL to be ready..." +until pg_isready -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME; do + echo "PostgreSQL is not ready yet, waiting..." + sleep 5 +done + +echo "PostgreSQL is ready, proceeding with OSM data processing..." + +# Download OSM data if not already present +if [ ! -f "$OSM_FILE" ]; then + echo "Downloading OSM data from $OSM_URL..." + wget -O "$OSM_FILE" "$OSM_URL" + echo "Download completed successfully" +else + echo "OSM file already exists, skipping download" +fi + +# Verify file was downloaded successfully +if [ ! -f "$OSM_FILE" ]; then + echo "Error: OSM file not found after download attempt" + exit 1 +fi + +echo "OSM file size: $(du -h $OSM_FILE | cut -f1)" + +# Import data using osm2pgsql +echo "Starting OSM data import with osm2pgsql..." + +# osm2pgsql command with optimized settings for vector tiles +osm2pgsql \ + --create \ + --database $DB_NAME \ + --username $DB_USER \ + --host $DB_HOST \ + --port $DB_PORT \ + --hstore \ + --style /usr/share/osm2pgsql/default.style \ + --multi-geometry \ + --number-processes 2 \ + --cache 512 \ + --flat-nodes /osm-data/nodes.cache \ + "$OSM_FILE" + +echo "OSM data import completed successfully" + +# Verify import by checking table counts +echo "Verifying import - checking table row counts:" +psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c " + SELECT + 'planet_osm_point' as table_name, count(*) as row_count + FROM planet_osm_point + UNION ALL + SELECT + 'planet_osm_line' as table_name, count(*) as row_count + FROM planet_osm_line + UNION ALL + SELECT + 'planet_osm_polygon' as table_name, count(*) as row_count + FROM planet_osm_polygon + UNION ALL + SELECT + 'planet_osm_roads' as table_name, count(*) as row_count + FROM planet_osm_roads; +" + +echo "OSM data download and import process completed successfully!" \ No newline at end of file diff --git a/osm-data/.gitkeep b/osm-data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tegola_config.toml b/tegola_config.toml new file mode 100644 index 0000000..d7a78af --- /dev/null +++ b/tegola_config.toml @@ -0,0 +1,40 @@ +[webserver] +port = ":8080" + +[webserver.headers] +Access-Control-Allow-Origin = "*" +Cache-Control = "s-maxage=3600" + +[[providers]] +name = "osm" +type = "mvt_postgis" +uri = "postgres://postgres:postgres@postgres:5432/osm?sslmode=disable" +srid = 3857 + + [[providers.layers]] + name = "roads" + geometry_fieldname = "way" + geometry_type = "linestring" + id_fieldname = "osm_id" + sql = "SELECT osm_id, highway, name, ST_AsMVTGeom(way, !BBOX!) AS way FROM planet_osm_line WHERE highway IS NOT NULL AND ST_IsValid(way) AND way && !BBOX!" + + [[providers.layers]] + name = "buildings" + geometry_fieldname = "way" + geometry_type = "polygon" + id_fieldname = "osm_id" + sql = "SELECT osm_id, building, name, ST_AsMVTGeom(way, !BBOX!) AS way FROM planet_osm_polygon WHERE building IS NOT NULL AND ST_IsValid(way) AND way && !BBOX!" + +[[maps]] +name = "osm" +center = [17.8078, 43.3430, 12.0] # Mostar, adjust as needed + + [[maps.layers]] + provider_layer = "osm.roads" + min_zoom = 9 + max_zoom = 16 + + [[maps.layers]] + provider_layer = "osm.buildings" + min_zoom = 12 + max_zoom = 18 \ No newline at end of file