Pre-generated mbtile files for each task area in a project

src/backend/app/projects/project_crud.py

@@ -66,13 +66,17 @@
 from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson, timestamp
 from app.organization import organization_crud
 from app.projects import project_schemas
-from app.s3 import add_obj_to_bucket, get_obj_from_bucket
+from app.s3 import (
+    add_file_to_bucket,
+    add_obj_to_bucket,
+    get_bucket_path,
+    get_obj_from_bucket,
+)
 from app.tasks import tasks_crud
 from app.users import user_crud
 
 QR_CODES_DIR = "QR_codes/"
 TASK_GEOJSON_DIR = "geojson/"
-TILESDIR = "/opt/tiles"
 
 
 async def get_projects(
@@ -647,11 +651,14 @@ async def get_data_extract_url(
     try:
         result = requests.post(query_url, data=json.dumps(query), headers=headers)
         result.raise_for_status()
-    except requests.exceptions.HTTPError:
+    except requests.exceptions.HTTPError as e:
         error_dict = result.json()
         error_dict["status_code"] = result.status_code
         log.error(f"Failed to get extract from raw data api: {error_dict}")
-        return error_dict
+        raise HTTPException(
+            status_code=error_dict.get("status_code"),
+            detail=error_dict.get("detail"),
+        ) from e
 
     task_id = result.json()["task_id"]
 
@@ -2094,29 +2101,141 @@ async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str)
 
 
 # NOTE defined as non-async to run in separate thread
-def get_project_tiles(
+def init_project_basemaps(
     db: Session,
     project_id: int,
     background_task_id: uuid.UUID,
     source: str,
+) -> None:
+    """Init basemaps for the project and task areas.
+
+    A PMTiles archive is created for the entire project, plus
+    individual mbtile archives for each task area.
+
+    Args:
+        db (Session): SQLAlchemy db session.
+        project_id (int): Associated project ID.
+        background_task_id (uuid.UUID): Pre-generated background task ID.
+        source (str): Source to use for basemap tiles.
+
+    Returns:
+        None
+    """
+    # Generate PMTiles for project area
+    generate_project_or_task_basemap(
+        db,
+        project_id,
+        background_task_id,
+        source,
+        output_format="pmtiles",
+    )
+
+    # Generate MBTiles for each task
+    get_tasks_async = async_to_sync(tasks_crud.get_task_id_list)
+    task_list = get_tasks_async(db, project_id)
+
+    # TODO optimise to run with threadpool
+    for task_id in task_list:
+        generate_project_or_task_basemap(
+            db,
+            project_id,
+            background_task_id,
+            source,
+            output_format="mbtiles",
+            task_id=task_id,
+        )
+
+
+# NOTE defined as non-async to run in separate thread
+def generate_project_or_task_basemap(
+    db: Session,
+    project_id: int,
+    background_task_id: uuid.UUID,
+    source: str = "esri",
+    output_format: str = "pmtiles",
+    tms: str = None,
+    task_id: int = None,
+) -> None:
+    """For a given project or task area, generate a basemap.
+
+    Wrapper that extracts the project or task bbox prior to calling
+    generate_basemap_for_bbox function.
+
+    Args:
+        db (Session): SQLAlchemy db session.
+        project_id (int): ID of project to create tiles for.
+        background_task_id (uuid.UUID): UUID of background task to track.
+        source (str): Tile source ("esri", "bing", "topo", "google", "oam").
+        output_format (str, optional): Default "mbtiles".
+            Other options: "pmtiles", "sqlite3".
+        tms (str, optional): Default None. Custom TMS provider URL.
+        task_id (bool): If set, create for a task boundary only.
+
+    Returns:
+        None
+    """
+    if not task_id:
+        # Project Outline
+        log.debug(f"Getting bbox for project: {project_id}")
+    else:
+        # Task Outline
+        log.debug(f"Getting bbox for task: {task_id}")
+
+    query = text(
+        f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
+                    ST_YMin(ST_Envelope(outline)) AS min_lat,
+                    ST_XMax(ST_Envelope(outline)) AS max_lon,
+                    ST_YMax(ST_Envelope(outline)) AS max_lat
+            FROM {'tasks' if task_id else 'projects'}
+            WHERE id = {task_id if task_id else project_id};"""
+    )
+
+    result = db.execute(query)
+    db_bbox = result.fetchone()
+    if db_bbox:
+        log.debug(f"Extracted bbox: {db_bbox}")
+    else:
+        log.error(f"Failed to get bbox from project: {project_id}")
+
+    generate_basemap_for_bbox(
+        db,
+        project_id,
+        db_bbox,
+        background_task_id,
+        source,
+        output_format,
+        tms,
+        task_id,
+    )
+
+
+# NOTE defined as non-async to run in separate thread
+def generate_basemap_for_bbox(
+    db: Session,
+    project_id: int,
+    bbox: tuple,
+    background_task_id: uuid.UUID,
+    source: str,
     output_format: str = "mbtiles",
     tms: str = None,
+    task_id: int = None,
 ):
-    """Get the tiles for a project.
+    """Get basemap tiles for a given bounding box.
 
     Args:
         db (Session): SQLAlchemy db session.
         project_id (int): ID of project to create tiles for.
+        bbox (tuple): the bounding box for generate for.
         background_task_id (uuid.UUID): UUID of background task to track.
         source (str): Tile source ("esri", "bing", "topo", "google", "oam").
         output_format (str, optional): Default "mbtiles".
             Other options: "pmtiles", "sqlite3".
         tms (str, optional): Default None. Custom TMS provider URL.
+        task_id (bool): If set, create for a task boundary only.
     """
     zooms = "12-19"
-    tiles_path_id = uuid.uuid4()
-    tiles_dir = f"{TILESDIR}/{tiles_path_id}"
-    outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
+    tiles_dir = "opt/tiles"
+    outfile = f"/tmp/{project_id}_{uuid.uuid4()}.{output_format}"
 
     tile_path_instance = db_models.DbTilesPath(
         project_id=project_id,
@@ -2130,36 +2249,9 @@ def get_project_tiles(
         db.add(tile_path_instance)
         db.commit()
 
-        # Project Outline
-        log.debug(f"Getting bbox for project: {project_id}")
-        query = text(
-            f"""SELECT ST_XMin(ST_Envelope(outline)) AS min_lon,
-                        ST_YMin(ST_Envelope(outline)) AS min_lat,
-                        ST_XMax(ST_Envelope(outline)) AS max_lon,
-                        ST_YMax(ST_Envelope(outline)) AS max_lat
-                FROM projects
-                WHERE id = {project_id};"""
-        )
-
-        result = db.execute(query)
-        project_bbox = result.fetchone()
-        log.debug(f"Extracted project bbox: {project_bbox}")
+        # Get coords from bbox
+        min_lon, min_lat, max_lon, max_lat = bbox
 
-        if project_bbox:
-            min_lon, min_lat, max_lon, max_lat = project_bbox
-        else:
-            log.error(f"Failed to get bbox from project: {project_id}")
-
-        log.debug(
-            "Creating basemap with params: "
-            f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
-            f"outfile={outfile} | "
-            f"zooms={zooms} | "
-            f"outdir={tiles_dir} | "
-            f"source={source} | "
-            f"xy={False} | "
-            f"tms={tms}"
-        )
         create_basemap_file(
             boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
             outfile=outfile,
@@ -2171,7 +2263,24 @@ def get_project_tiles(
         )
         log.info(f"Basemap created for project ID {project_id}: {outfile}")
 
+        get_bucket_path_sync = async_to_sync(get_bucket_path)
+        project_s3_path = get_bucket_path_sync(db, project_id)
+
+        if task_id:
+            s3_tile_path = f"{project_s3_path}/basemaps/{task_id}.{output_format}"
+        else:
+            s3_tile_path = f"{project_s3_path}/basemap.{output_format}"
+
+        add_file_to_bucket(
+            settings.S3_BUCKET_NAME,
+            s3_tile_path,
+            outfile,
+        )
+
         tile_path_instance.status = 4
+        tile_path_instance.path = (
+            f"{settings.S3_DOWNLOAD_ROOT}/" f"{settings.S3_BUCKET_NAME}{s3_tile_path}"
+        )
         db.commit()
 
         # Update background task status to COMPLETED
@@ -2185,6 +2294,7 @@ def get_project_tiles(
         log.error(f"Tiles generation process failed for project id {project_id}")
 
         tile_path_instance.status = 2
+        tile_path_instance.path = ""
         db.commit()
 
         # Update background task status to FAILED

src/backend/app/projects/project_routes.py

@@ -20,7 +20,6 @@
 import json
 import os
 import uuid
-from pathlib import Path
 from typing import Optional
 
 import geojson
@@ -35,7 +34,7 @@
     Response,
     UploadFile,
 )
-from fastapi.responses import FileResponse, JSONResponse
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
 from loguru import logger as log
 from osm_fieldwork.make_data_extract import getChoices
 from osm_fieldwork.xlsforms import xlsforms_path
@@ -992,6 +991,10 @@ async def download_features(project_id: int, db: Session = Depends(database.get_
 async def generate_project_tiles(
     background_tasks: BackgroundTasks,
     project_id: int,
+    task_id: str = Query(
+        None,
+        description="Optional task id to generate for",
+    ),
     source: str = Query(
         ..., description="Select a source for tiles", enum=TILES_SOURCE
     ),
@@ -1009,34 +1012,75 @@ async def generate_project_tiles(
     Args:
         background_tasks (BackgroundTasks): FastAPI bg tasks, provided automatically.
         project_id (int): ID of project to create tiles for.
+        task_id (int): Optional task ID (task area) to generate for.
         source (str): Tile source ("esri", "bing", "topo", "google", "oam").
         format (str, optional): Default "mbtiles". Other options: "pmtiles", "sqlite3".
         tms (str, optional): Default None. Custom TMS provider URL.
         db (Session): The database session, provided automatically.
 
     Returns:
-        str: Success message that tile generation started.
+        dict: Success message that tile generation started.
     """
     # Create task in db and return uuid
     log.debug(
         "Creating generate_project_tiles background task "
         f"for project ID: {project_id}"
     )
     background_task_id = await project_crud.insert_background_task_into_database(
-        db, project_id=project_id
+        db, name="generate tiles", project_id=project_id
     )
 
     background_tasks.add_task(
-        project_crud.get_project_tiles,
+        project_crud.generate_project_or_task_basemap,
         db,
         project_id,
         background_task_id,
         source,
         format,
         tms,
+        task_id,
     )
 
-    return {"Message": "Tile generation started"}
+    return JSONResponse(status_code=200, content={"success": True})
+
+
+@router.get("/tiles/{project_id}/init")
+async def init_project_tiles(
+    background_tasks: BackgroundTasks,
+    project_id: int,
+    source: str = Query(
+        ..., description="Select a source for tiles", enum=TILES_SOURCE
+    ),
+    db: Session = Depends(database.get_db),
+):
+    """Init all basemaps for a project, optional on project creation.
+
+    Args:
+        project_id (int): ID of project to create tiles for.
+        source (str): Tile source ("esri", "bing", "topo", "google", "oam").
+        db (Session): Autogenerated FastAPI database session.
+        background_tasks (BackgroundTasks): Autogenerated FastAPI dependency.
+
+    Returns:
+        dict: Success message that tile generation started.
+    """
+    # Create task in db and return uuid
+    log.debug(
+        f"Creating init_project_tiles background task for project ID: {project_id}"
+    )
+    background_task_id = await project_crud.insert_background_task_into_database(
+        db, name="init tiles", project_id=project_id
+    )
+
+    background_tasks.add_task(
+        project_crud.init_project_basemaps,
+        db,
+        project_id,
+        background_task_id,
+        source,
+    )
+
+    return JSONResponse(status_code=200, content={"success": True})
 
 
 @router.get("/tiles_list/{project_id}/")
@@ -1063,18 +1107,7 @@ async def download_tiles(tile_id: int, db: Session = Depends(database.get_db)):
         .first()
     )
     log.info(f"User requested download for tiles: {tiles_path.path}")
-
-    project_id = tiles_path.project_id
-    project_name = await project_crud.get_project(db, project_id).project_name_prefix
-    filename = Path(tiles_path.path).name.replace(
-        f"{project_id}_", f"{project_name.replace(' ', '_')}_"
-    )
-    log.debug(f"Sending tile archive to user: {filename}")
-
-    return FileResponse(
-        tiles_path.path,
-        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
-    )
+    return RedirectResponse(tiles_path.path)
 
 
 @router.get("/boundary_in_osm/{project_id}/")