Dev/mp api (#3)

* update app api with respect to marketplace capabilities * add functionality to disable direct upload to cache * update docker compose files * update setup.cfg * update setup.cfg, update main.py, update docker compose * update README
simphony · Sep 28, 2023 · 0deefa6 · 0deefa6
1 parent 2ca2670
commit 0deefa6
Show file tree

Hide file tree

Showing 12 changed files with 462 additions and 153 deletions.
diff --git a/.env.example b/.env.example
@@ -1,8 +1,6 @@
-AUTHENTICAION_DEPENDENCIES=False
 BUILD_VERSION=v2.0.0
 WRAPPER_TAG=v1.0.0
 BASE_IMAGE=registry.gitlab.cc-asp.fraunhofer.de/reaxpro/simphony-catalytic
-DOCKER_BUILD_TARGET=develop
 INSTALL_SLURM=no
 SLURM_VERSION=23.02.4
 
@@ -11,7 +9,7 @@ WRAPPER_DEPS_EXTRA=git+https://github.com/SCM-NV/PLAMS@7661960a9db53249a0b77935d
 
 PYTHON_BIN=python3
 REAXPRO_FASTAPI_HOST=0.0.0.0
-REAXPRO_FASTAPI_PORT=8081
+REAXPRO_FASTAPI_PORT=8080
 REAXPRO_WORKER_CONCURRENCY=5
 
 REAXPRO_SCHEMAS="osp.models.multiscale.co_pt111_full:COPt111FullscaleModel| osp.models.multiscale.co_pt111_full:COPt111FromMesoScaleModel | osp.models.multiscale.co_pt111_meso:PESExploration | osp.models.multiscale.co_pt111_meso:COPt111MesoscaleModel  | osp.models.catalytic.co_catalyticfoam:COCatalyticFOAMModel | osp.models.ams.energy_landscape_refinement:EnergyLandscapeRefinement |  osp.models.zacros.co_pyzacros:COpyZacrosModel"

diff --git a/README.md b/README.md
@@ -5,6 +5,7 @@ A simple [FastAPI-framework](https://fastapi.tiangolo.com/) for running asynchro
 ## Authors
 
 [Matthias Büschelberger](mailto:matthias.bueschelberger@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM)
+[Kiran Kumaraswamy](mailto:kiran.kumaraswamy@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM)
 
 ## Documentation
 

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -2,13 +2,13 @@ version: "3"
 
 services:
   fastapi:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-${DOCKER_BUILD_TARGET:-develop}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-develop"
     build:
       context: "."
       dockerfile: Dockerfile.api
-      target: "${DOCKER_BUILD_TARGET:-develop}"
+      target: develop
       args:
-        - INSTALL_SLURM=$INSTALL_SLURM
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
         - SLURM_VERSION=$SLURM_VERSION
         - WRAPPER_DEPS_INSTALL=$WRAPPER_DEPS_INSTALL
         - WRAPPER_DEPS_EXTRA=$WRAPPER_DEPS_EXTRA
@@ -41,15 +41,15 @@ services:
       - reaxpro
 
   headworker:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-${DOCKER_BUILD_TARGET:-develop}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-develop"
     build:
       context: "."
       dockerfile: Dockerfile.api
-      target: "${DOCKER_BUILD_TARGET:-develop}"
+      target: develop
       args:
         - WRAPPER_DEPS_INSTALL=$WRAPPER_DEPS_INSTALL
         - WRAPPER_DEPS_EXTRA=$WRAPPER_DEPS_EXTRA
-        - INSTALL_SLURM=$INSTALL_SLURM
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
         - SLURM_VERSION=$SLURM_VERSION
     environment:
       REAXPRO_WORKER_NAME: simphony-workflows
@@ -77,14 +77,15 @@ services:
       - reaxpro
 
   catalyticfoam:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}-${DOCKER_BUILD_TARGET:-develop}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}-develop"
     build:
       context: "."
-      target: "${DOCKER_BUILD_TARGET:-develop}"
+      target: develop
       dockerfile: Dockerfile.catalyticfoam
       args:
         - BASE_IMAGE=$BASE_IMAGE
         - WRAPPER_TAG=$WRAPPER_TAG
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
         - SLURM_VERSION=$SLURM_VERSION
     environment:
       REAXPRO_WORKER_NAME: simphony-catalyticfoam

diff --git a/docker-compose.marketplace.yml b/docker-compose.marketplace.yml
@@ -0,0 +1,99 @@
+version: "3"
+
+services:
+  fastapi:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
+    ports:
+      - "${REAXPRO_FASTAPI_PORT:-8080}:8080"
+    environment:
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_SCHEMAS: ${REAXPRO_SCHEMAS}
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_FASTAPI_HOST: ${REAXPRO_FASTAPI_HOST:-'0.0.0.0'}
+      REAXPRO_FASTAPI_PORT: ${REAXPRO_FASTAPI_PORT:-8080}
+      REAXPRO_AUTHENTICATION_DEPENDENCIES: ${AUTHENTICATION_DEPENDENCIES:-True}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      KEYCLOAK_HOST: ${KEYCLOAK_HOST}
+      KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID}
+      KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET}
+      KEYCLOAK_REALM_NAME: ${KEYCLOAK_REALM_NAME}
+      APP_MODE: server
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  headworker:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
+    environment:
+      REAXPRO_WORKER_NAME: simphony-workflows
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_WORKER_MAPPING: ${REAXPRO_WORKER_MAPPING}
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_WORKER_CONCURRENCY: ${REAXPRO_WORKER_CONCURRENCY:-5}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      APP_MODE: worker
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  catalyticfoam:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}-production"
+    environment:
+      REAXPRO_WORKER_NAME: simphony-catalyticfoam
+      REAXPRO_WRAPPER_NAME: osp.wrappers.simcatalyticfoam.simcatalyticfoam:SimCatalyticFoamSession
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_WORKER_CONCURRENCY: ${REAXPRO_WORKER_CONCURRENCY:-5}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      APP_MODE: worker
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  redis:
+    image: redis:latest
+    volumes:
+      - redis-data:/data
+    networks:
+      - reaxpro
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000
+    environment:
+      MINIO_ROOT_USER: ${REAXPRO_MINIO_USER}
+      MINIO_ROOT_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+    volumes:
+      - minio-data:/data
+    command: server /data
+    networks:
+      - reaxpro
+
+volumes:
+  redis-data:
+  minio-data:
+
+networks:
+  reaxpro:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -0,0 +1,125 @@
+version: "3"
+
+services:
+  fastapi:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-prod"
+    build:
+      context: "."
+      dockerfile: Dockerfile.api
+      target: production
+      args:
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
+        - SLURM_VERSION=$SLURM_VERSION
+        - WRAPPER_DEPS_INSTALL=$WRAPPER_DEPS_INSTALL
+        - WRAPPER_DEPS_EXTRA=$WRAPPER_DEPS_EXTRA
+    ports:
+      - "${REAXPRO_FASTAPI_PORT:-8080}:${REAXPRO_FASTAPI_PORT:-8080}"
+    environment:
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_REDIS_TEST_DB: 9
+      REAXPRO_AUTHENTICATION_DEPENDENCIES: ${AUTHENTICATION_DEPENDENCIES}
+      REAXPRO_SCHEMAS: ${REAXPRO_SCHEMAS}
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_FASTAPI_HOST: ${REAXPRO_FASTAPI_HOST:-'0.0.0.0'}
+      REAXPRO_FASTAPI_PORT: ${REAXPRO_FASTAPI_PORT:-8080}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      APP_MODE: server
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  headworker:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
+    build:
+      context: "."
+      dockerfile: Dockerfile.api
+      target: production
+      args:
+        - WRAPPER_DEPS_INSTALL=$WRAPPER_DEPS_INSTALL
+        - WRAPPER_DEPS_EXTRA=$WRAPPER_DEPS_EXTRA
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
+        - SLURM_VERSION=$SLURM_VERSION
+    environment:
+      REAXPRO_WORKER_NAME: simphony-workflows
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_REDIS_TEST_DB: 9
+      REAXPRO_WORKER_MAPPING: ${REAXPRO_WORKER_MAPPING}
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_WORKER_CONCURRENCY: ${REAXPRO_WORKER_CONCURRENCY:-5}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      APP_MODE: worker
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  catalyticfoam:
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}-production"
+    build:
+      context: "."
+      target: production
+      dockerfile: Dockerfile.catalyticfoam
+      args:
+        - BASE_IMAGE=$BASE_IMAGE
+        - WRAPPER_TAG=$WRAPPER_TAG
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
+        - SLURM_VERSION=$SLURM_VERSION
+    environment:
+      REAXPRO_WORKER_NAME: simphony-catalyticfoam
+      REAXPRO_WRAPPER_NAME: osp.wrappers.simcatalyticfoam.simcatalyticfoam:SimCatalyticFoamSession
+      REAXPRO_REDIS_TYPE: redis
+      REAXPRO_REDIS_HOST: redis
+      REAXPRO_REDIS_PORT: 6379
+      REAXPRO_REDIS_DB: 3
+      REAXPRO_REDIS_TEST_DB: 9
+      REAXPRO_MINIO_ENDPOINT: minio:9000
+      REAXPRO_MINIO_USER: ${REAXPRO_MINIO_USER}
+      REAXPRO_MINIO_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+      REAXPRO_WORKER_CONCURRENCY: ${REAXPRO_WORKER_CONCURRENCY:-5}
+      PYTHON_BIN: ${PYTHON_BIN:-python3}
+      APP_MODE: worker
+    depends_on:
+      - redis
+      - minio
+    networks:
+      - reaxpro
+
+  redis:
+    image: redis:latest
+    volumes:
+      - redis-data:/data
+    networks:
+      - reaxpro
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000
+    environment:
+      MINIO_ROOT_USER: ${REAXPRO_MINIO_USER}
+      MINIO_ROOT_PASSWORD: ${REAXPRO_MINIO_PASSWORD}
+    volumes:
+      - minio-data:/data
+    command: server /data
+    networks:
+      - reaxpro
+
+volumes:
+  redis-data:
+  minio-data:
+
+networks:
+  reaxpro:
diff --git a/docker-compose.singularity.yml b/docker-compose.singularity.yml
@@ -2,15 +2,15 @@ version: "3"
 
 services:
   reaxpro:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-singularity-${DOCKER_BUILD_TARGET:-develop}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-singularity-production"
     build:
       context: "."
       dockerfile: Dockerfile.singularity
-      target: "${DOCKER_BUILD_TARGET:-develop}"
+      target: "production"
       args:
         - BASE_IMAGE=$BASE_IMAGE
         - WRAPPER_TAG=$WRAPPER_TAG
-        - INSTALL_SLURM=$INSTALL_SLURM
+        - INSTALL_SLURM=${INSTALL_SLURM:-no}
         - SLURM_VERSION=$SLURM_VERSION
     ports:
       - "${REAXPRO_FASTAPI_PORT:-8081}:${REAXPRO_FASTAPI_PORT:-8081}"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3"
 
 services:
   fastapi:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
     ports:
       - "${REAXPRO_FASTAPI_PORT:-8080}:8080"
     environment:
@@ -26,7 +26,7 @@ services:
       - reaxpro
 
   headworker:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
     environment:
       REAXPRO_WORKER_NAME: simphony-workflows
       REAXPRO_REDIS_TYPE: redis
@@ -47,7 +47,7 @@ services:
       - reaxpro
 
   catalyticfoam:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:catalyticfoam-${BUILD_VERSION:-latest}-production"
     environment:
       REAXPRO_WORKER_NAME: simphony-catalyticfoam
       REAXPRO_WRAPPER_NAME: osp.wrappers.simcatalyticfoam.simcatalyticfoam:SimCatalyticFoamSession
@@ -68,7 +68,7 @@ services:
       - reaxpro
 
   ams:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
     environment:
       REAXPRO_WORKER_NAME: simphony-ams
       REAXPRO_WRAPPER_NAME: osp.wrappers.simams.simams_session:SimamsSession
@@ -100,7 +100,7 @@ services:
       - reaxpro
 
   zacros:
-    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}"
+    image: "registry.gitlab.cc-asp.fraunhofer.de/reaxpro/fastapi-celery:${BUILD_VERSION:-latest}-production"
     environment:
       REAXPRO_WORKER_NAME: simphony-zacros
       REAXPRO_WRAPPER_NAME: osp.wrappers.simzacros.simzacros_session:SimzacrosSession