Merge pull request #499 from hotosm/feature/splitdb

Multi-database support

.github/workflows/tests.yml

@@ -23,6 +23,6 @@ jobs:
       compose_service: underpass
       compose_command: '"make check -j $(nproc)"'
       tag_override: ci
-      # TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }}
+      # TODO update postgis image to use github repo var ${{ vars.UNDERPASSDB_TAG }}
       cache_extra_imgs: |
         "docker.io/postgis/postgis:15-3.3-alpine"

.gitignore

@@ -43,3 +43,4 @@ unconfig.h.in
 **/__pycache__/
 .vscode
 data
+data_osm

config/default.yaml

@@ -2,6 +2,8 @@
 config:
     - underpass_db_url:
         - underpass:underpass@localhost:5432/underpass
+    - underpass_osm_db_url:
+        - underpass:underpass@localhost:5432/underpass
     - planet_servers:
         - planet.maps.mail.ru
     - destdir_base:

docker-compose.yml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
 #
 # This file is part of Underpass.
 #
@@ -20,10 +20,10 @@
 version: "3"
 
 services:
-  # Database
-  postgis:
-    image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine}
-    container_name: "underpass_postgis"
+  # Database for Underpass
+  underpass_db:
+    image: postgis/postgis:${UNDERPASS_DB_TAG:-15-3.3-alpine}
+    container_name: "underpass_db"
     ports:
       - "${DB_PORT:-5439}:5432"
     environment:
@@ -41,6 +41,28 @@ services:
     networks:
       internal:
 
+# Un-comment for starting a second database
+# Database for OSM Raw Data
+  # osm_db:
+  #   image: postgis/postgis:${OSM_DB_TAG:-15-3.3-alpine}
+  #   container_name: "osm_db"
+  #   ports:
+  #     - "${DB_PORT:-5440}:5432"
+  #   environment:
+  #     - POSTGRES_DB=osm
+  #     - POSTGRES_USER=underpass
+  #     - POSTGRES_PASSWORD=underpass
+  #   volumes:
+  #     - ./data_osm:/var/lib/postgresql/data
+  #   restart: on-failure
+  #   logging:
+  #     driver: "json-file"
+  #     options:
+  #       max-size: "200k"
+  #       max-file: "10"
+  #   networks:
+  #     internal:
+
   # Underpass
   underpass:
     image: "ghcr.io/hotosm/underpass:${TAG_OVERRIDE:-debug}"
@@ -51,13 +73,16 @@ services:
      target: ${TAG_OVERRIDE:-debug}
      args:
         APP_VERSION: ${APP_VERSION:-debug}
-    depends_on: [postgis]
+    depends_on: [underpass_db]
     environment:
-      - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass
+      - REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@underpass_db/underpass
+      - REPLICATOR_OSM_DB_URL=underpass:underpass@underpass_db/underpass
+
     command: tail -f /dev/null
+    # Un-comment for debugging
     # volumes:
-      # - ${PWD}:/code
-      # - ./replication:/code/build/replication
+    #   - ${PWD}:/code
+    #   - ./replication:/usr/local/lib/underpass/data/replication
     networks:
       internal:
 
@@ -79,26 +104,8 @@ services:
     networks:
       internal:
     environment:
-      - UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass
-
-  # Underpass UI
-  ui:
-    image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}"
-    container_name: "underpass_ui"
-    build:
-     context: .
-     dockerfile: docker/underpass-ui.dockerfile
-     target: debug
-     args:
-        APP_VERSION: ${APP_VERSION:-debug}
-    # # Mount underpass-ui repo
-    # volumes:
-    #   - ../underpass-ui/src:/code/src
-    #   - ../underpass-ui/playground:/code/playground
-    ports:
-      - "${UI_PORT:-8080}:5000"
-    networks:
-      internal:
+      - UNDERPASS_API_DB=postgresql://underpass:underpass@underpass/underpass
+      - UNDERPASS_API_OSM_DB=postgresql://underpass:underpass@underpass/underpass
 
 networks:
   internal:

docker/underpass-config.yaml

@@ -1,7 +1,9 @@
 # Underpass config file
 config:
+    - underpass_osm_db_url:
+        - underpass:underpass@underpass_db:5432/underpass
     - underpass_db_url:
-        - underpass@postgis/underpass
+        - underpass:underpass@underpass_db:5432/underpass
     - planet_servers:
         - planet.maps.mail.ru
     - destdir_base:

docker/underpass.dockerfile

@@ -48,6 +48,7 @@ RUN set -ex \
         "librange-v3-dev" \
         "libtool" \
         "osm2pgsql" \
+        "rsync" \
     && rm -rf /var/lib/apt/lists/*
 
 

python/dbapi/api/config.py

@@ -0,0 +1,6 @@
+# Results per page on raw featues queries
+RESULTS_PER_PAGE = 500
+# Results per page on list featues queries
+RESULTS_PER_PAGE_LIST = 10
+# Print debug messages
+DEBUG=False

python/dbapi/api/db.py

@@ -19,21 +19,23 @@
 
 import asyncpg
 import json
+from .config import DEBUG
 
-class UnderpassDB():
-    # Default Underpass local DB configuration
-    # This might be replaced by an .ini config file
+class DB():
+    # Default DB configuration
 
     def __init__(self, connectionString = None):
-        self.connectionString = connectionString or "postgresql://underpass:underpass@postgis/underpass"
+        self.connectionString = connectionString or "postgresql://underpass:underpass@localhost:5432/underpass"
         self.pool = None
+        # Extract the name of the database
+        self.name = self.connectionString[self.connectionString.rfind('/') + 1:]
 
     async def __enter__(self):
         await self.connect()
 
     async def connect(self):
         """ Connect to the database """
-        print("Connecting to DB ...")
+        print("Connecting to DB ... " + self.connectionString if DEBUG else "")
         if not self.pool:
             try:
                 self.pool = await asyncpg.create_pool(
@@ -50,16 +52,23 @@ def close(self):
         if self.pool is not None:
             self.pool.close()
 
-    async def run(self, query, singleObject = False):
+    async def run(self, query, singleObject = False, asJson=False):
+        if DEBUG:
+            print("Running query ...")
         if not self.pool:
             await self.connect()
         if self.pool:
             try:
                 conn = await self.pool.acquire()
                 result = await conn.fetch(query)
-                if singleObject:
-                    return result[0]
-                return json.loads((result[0]['result']))
+                if asJson:
+                    if singleObject:
+                        return result[0]['result']
+                    return result[0]['result']
+                else:
+                    if singleObject:
+                        return result[0]
+                    return result
             except Exception as e: 
                 print("\n******* \n" + query + "\n******* \n")
                 print(e)

python/dbapi/api/filters.py

@@ -1,21 +1,55 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of Underpass.
+#
+#     Underpass is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     Underpass is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with Underpass.  If not, see <https://www.gnu.org/licenses/>.
 
 def tagsQueryFilter(tagsQuery, table):
     query = ""
     tags = tagsQuery.split(",")
     keyValue = tags[0].split("=")
 
     if len(keyValue) == 2:
-        query += "{0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
+        query += "{table}.tags->>'{key}' ~* '^{value}'".format(
+            table=table,
+            key=keyValue[0],
+            value=keyValue[1]
+        )
     else:
-        query += "{0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])
+        query += "{table}.tags->>'{key}' IS NOT NULL".format(
+            table=table, 
+            key=keyValue[0]
+        )
 
     for tag in tags[1:]:
         keyValue = tag.split("=")
         if len(keyValue) == 2:
-            query += "OR {0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
+            query += "OR {table}.tags->>'{key}' ~* '^{value}'".format(
+                table=table,
+                key=keyValue[0],
+                value=keyValue[1]
+            )
         else:
-            query += "OR {0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])
+            query += "OR {table}.tags->>'{key}' IS NOT NULL".format(
+                table=table,
+                key=keyValue[0]
+            )
     return query
 
 def hashtagQueryFilter(hashtag, table):
-    return "'{0}' = ANY (hashtags)".format(hashtag)
+    return "'{hashtag}' = ANY (hashtags)".format(
+        hashtag=hashtag
+    )

python/dbapi/api/queryHelper.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 #
-# Copyright (c) 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
 #
 # This file is part of Underpass.
 #
@@ -17,14 +17,14 @@
 #     You should have received a copy of the GNU General Public License
 #     along with Underpass.  If not, see <https://www.gnu.org/licenses/>.
 
-RESULTS_PER_PAGE = 25
-
 def hashtags(hashtagsList):
-    return "EXISTS ( SELECT * from unnest(hashtags) as h where {0} )".format(
-        ' OR '.join(
-            map(lambda x: "h ~* '^{0}'".format(x), hashtagsList)
+    return "EXISTS ( SELECT * from unnest(hashtags) as h where {condition} )".format(
+        condition=' OR '.join(
+            map(lambda x: "h ~* '^{hashtag}'".format(hashtag=x), hashtagsList)
         )
     )
 
 def bbox(wktMultipolygon):
-    return "ST_Intersects(bbox, ST_GeomFromText('{0}', 4326))".format(wktMultipolygon)
+    return "ST_Intersects(bbox, ST_GeomFromText('{area}', 4326))".format(
+        area=wktMultipolygon
+    )