Create a version of TM with sandboxed OSM for training data generation

Dockerfile

@@ -81,9 +81,11 @@ FROM runtime as debug
 RUN pip install --user --no-warn-script-location \
     --no-cache-dir debugpy==1.6.7
 EXPOSE 5678/tcp
-CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678", \
+# Required for debugpy and gevent to work nicely together
+ENV GEVENT_SUPPORT=True
+CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", \
     "-m", "gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
-    "--reload", "--log-level", "error"]
+    "--workers", "1", "--reload", "--log-level", "error"]
 
 
 

backend/api/system/authentication.py

@@ -79,7 +79,7 @@ def get(self):
             description: A problem occurred negotiating with the OSM API
         """
 
-        token_url = f"{EnvironmentConfig.OSM_SERVER_URL}/oauth2/token"
+        token_url = f"{EnvironmentConfig.OSM_SERVER_URL_BACKEND}/oauth2/token"
         authorization_code = request.args.get("code", None)
         if authorization_code is None:
             return {"SubCode": "InvalidData", "Error": "Missing code parameter"}, 400

backend/config.py

@@ -42,6 +42,7 @@ class EnvironmentConfig:
 
     # OSM API, Nomimatim URLs
     OSM_SERVER_URL = os.getenv("OSM_SERVER_URL", "https://www.openstreetmap.org")
+    OSM_SERVER_URL_BACKEND = os.getenv("OSM_SERVER_URL_BACKEND", "https://www.openstreetmap.org")
     OSM_NOMINATIM_SERVER_URL = os.getenv(
         "OSM_NOMINATIM_SERVER_URL", "https://nominatim.openstreetmap.org"
     )
@@ -192,7 +193,7 @@ class EnvironmentConfig:
     }
 
     # Connection to OSM authentification system
-    OAUTH_API_URL = "{}/api/0.6/".format(OSM_SERVER_URL)
+    OAUTH_API_URL = "{}/api/0.6/".format(OSM_SERVER_URL_BACKEND)
     OAUTH_CLIENT_ID = os.getenv("TM_CLIENT_ID", None)
     OAUTH_CLIENT_SECRET = os.getenv("TM_CLIENT_SECRET", None)
     OAUTH_SCOPE = os.getenv("TM_SCOPE", "read_prefs write_api")

backend/services/users/osm_service.py

@@ -21,7 +21,7 @@ def get_osm_details_for_user(user_id: int) -> UserOSMDTO:
         :raises OSMServiceError
         """
         osm_user_details_url = (
-            f"{current_app.config['OSM_SERVER_URL']}/api/0.6/user/{user_id}.json"
+            f"{current_app.config['OSM_SERVER_URL_BACKEND']}/api/0.6/user/{user_id}.json"
         )
         response = requests.get(osm_user_details_url)
 

contrib/tm-training/docker-compose.yml

@@ -0,0 +1,115 @@
+version: "3"
+
+networks:
+  tm-net: 
+    name: tm-net
+
+volumes:
+  osm-tmp:
+  osm-storage:
+  osm-db-data:
+  certs:
+  certbot_data:
+
+services:
+services:
+  proxy:
+    image: "ghcr.io/hotosm/osm-sandbox/proxy:2024-04-29"
+    profiles: [""]
+    depends_on:
+      osm:
+        condition: service_started
+    ports:
+      - ${OSM_DEV_PORT:-4433}:80
+    networks:
+      - tm-net
+    restart: "unless-stopped"
+
+  proxy-ssl:
+    image: "ghcr.io/hotosm/osm-sandbox/proxy:latest"
+    profiles: [public]
+    depends_on:
+      osm:
+        condition: service_started
+      certbot:
+        condition: service_completed_successfully
+    volumes:
+      - certs:/etc/letsencrypt
+      - certbot_data:/var/www/certbot
+    environment:
+      DOMAIN: ${DOMAIN}
+    ports:
+      - 80:80
+      - 443:443
+    networks:
+      - tm-net
+    restart: "unless-stopped"
+
+  osm:
+    image: ghcr.io/hotosm/osm-sandbox:2024-04-29
+    environment:
+      PROTOCOL: http${DOMAIN:+s}
+      DOMAIN: ${DOMAIN:-127.0.0.1:4433}
+      ADMIN_EMAIL: ${ADMIN_EMAIL:-admin@hotosm.org}
+      ADMIN_PASS: ${ADMIN_PASS:-Password1234}
+      ID_EDITOR_REDIRECT_URI: http${DOMAIN:+s}://${DOMAIN:-127.0.0.1:4433}
+    volumes:
+      # Mount a tmp directory that will persist between runs
+      - osm-tmp:/app/tmp
+      # Mount a storage directory that will persist between runs
+      - osm-storage:/app/storage
+      # Mount local setting overrides
+      # - ./settings.local.yml:/app/config/settings.local.yml:ro
+    tmpfs:
+      /tmp/pids/
+    networks:
+      - tm-net
+    depends_on:
+      osm-db:
+        condition: service_healthy
+    restart: unless-stopped
+
+  osm-db:
+    image: docker.io/postgres:14
+    environment:
+      POSTGRES_DB: openstreetmap
+      POSTGRES_PASSWORD: openstreetmap
+      POSTGRES_USER: openstreetmap
+    volumes:
+      - osm-db-data:/var/lib/postgresql/data
+    networks:
+      - tm-net
+    restart: unless-stopped
+    healthcheck:
+      test: pg_isready -U openstreetmap -d openstreetmap
+      start_period: 5s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  mail:
+    image: "ixdotai/smtp:v0.5.2"
+    volumes:
+      - ./rsa.private:/etc/exim4/dkim.key.temp:ro
+    environment:
+      - MAILNAME=${DOMAIN:-hotosm.org}
+      - DKIM_KEY_PATH=/etc/exim4/dkim.key.temp
+    networks:
+      - tm-net
+    restart: unless-stopped
+
+  certbot:
+    image: "ghcr.io/hotosm/osm-sandbox/proxy:certs-init"
+    profiles: [public]
+    volumes:
+      - certs:/etc/letsencrypt
+      - certbot_data:/var/www/certbot
+    environment:
+      DOMAIN: ${DOMAIN}
+      CERT_EMAIL: ${CERT_EMAIL}
+    ports:
+      - 80:80
+      - 443:443
+    networks:
+      - osm-net
+    restart: "on-failure:2"

docker-compose.yml

@@ -31,11 +31,14 @@ services:
     image: ghcr.io/hotosm/tasking-manager/backend:main
     build:
       context: .
+      # target: debug
     depends_on:
       tm-db:
         condition: service_healthy
       tm-migration:
         condition: service_completed_successfully
+    volumes:
+      - ./backend:/usr/src/app/backend:ro
     env_file:
       - tasking-manager.env
     restart: unless-stopped
@@ -70,15 +73,6 @@ services:
         condition: service_healthy
     env_file:
       - tasking-manager.env
-    deploy:
-      replicas: ${API_REPLICAS:-1}
-      resources:
-        limits:
-          cpus: "1"
-          memory: 1500M
-        reservations:
-          cpus: "1"
-          memory: 100M
     networks:
       - tm-net
 
@@ -112,6 +106,8 @@ services:
     build:
       context: .
       dockerfile: "./scripts/docker/Dockerfile.frontend_development"
+    volumes:
+      - ./frontend/src:/usr/src/app/src:ro
     env_file:
       - tasking-manager.env
     labels:

example.env

@@ -36,6 +36,7 @@ TM_ORG_GITHUB=https://github.com/hotosm
 # Information about the OSM server - Customize your server here
 # By default, it's the public OpenStreetMap.org server
 OSM_SERVER_URL=https://www.openstreetmap.org
+# OSM_SERVER_URL_BACKEND=http://osm:3000
 OSM_NOMINATIM_SERVER_URL=https://nominatim.openstreetmap.org
 OSM_REGISTER_URL=https://www.openstreetmap.org/user/new
 

scripts/docker/Dockerfile.frontend_development

@@ -1,5 +1,6 @@
 FROM node:16
 
+ENV CHOKIDAR_USEPOLLING=true
 WORKDIR /usr/src/app
 
 COPY ./frontend .