Merge pull request #454 from sunbeam-labs/453-slurm-executor-still-no…

…t-getting-job-name

Use hack to create slurm_extra resource within each rule

.github/workflows/integration-tests.yml

@@ -44,6 +44,12 @@ jobs:
                 sunbeam init --data_fp tests/data/reads/ --profile slurm projects/test/
 
                 sunbeam run --default-resources slurm_account=runner --profile projects/test/ test
+
+            - name: Check output
+              shell: bash
+              run: |
+                cd $GITHUB_WORKSPACE
+                ls -l . | grep "slurm_test_"
     
     test-apptainer:
         name: Apptainer Test

environment.yml

@@ -5,3 +5,5 @@ dependencies:
   - snakemake =8
   - git # Ensure sunbeam extend works even with tar installation of main pipeline
   - python =3.12.0
+  - pip:
+    - docker

install.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 __conda_url=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-__version_tag=$(if git describe --tags >/dev/null 2>&1 ; then git describe --tags; else echo v4.3.7; fi) # <--- Update this on each version release
+__version_tag=$(if git describe --tags >/dev/null 2>&1 ; then git describe --tags; else echo v4.4.0; fi) # <--- Update this on each version release
 __version_tag="${__version_tag:1}" # Remove the 'v' prefix
 
 read -r -d '' __usage <<-'EOF'
@@ -167,7 +167,7 @@ function install_environment () {
 function install_env_vars () {
     activate_sunbeam
     mkdir -p ${CONDA_PREFIX}/etc/conda/activate.d
-    echo -ne "#/bin/sh\nexport SUNBEAM_DIR=${__sunbeam_dir}\nexport SUNBEAM_VER=${__version_tag}\nexport SUNBEAM_MIN_MEM_MB=1000\nexport SUNBEAM_MIN_RUNTIME=60" > \
+    echo -ne "#/bin/sh\nexport SUNBEAM_DIR=${__sunbeam_dir}\nexport SUNBEAM_VER=${__version_tag}\nexport SUNBEAM_MIN_MEM_MB=8000\nexport SUNBEAM_MIN_RUNTIME=60" > \
 	 ${CONDA_PREFIX}/etc/conda/activate.d/env_vars.sh
     mkdir -p ${CONDA_PREFIX}/etc/conda/deactivate.d
     echo -ne "#/bin/sh\nunset SUNBEAM_DIR\nunset SUNBEAM_VER\nunset SUNBEAM_MIN_MEM_MB\nunset SUNBEAM_MIN_RUNTIME" > \

src/sunbeamlib/slurm_profile.yaml

@@ -23,7 +23,6 @@ default-resources:
   - mem_mb=8000
   - runtime=15
   - disk_mb=1000
-  - slurm_extra="--job-name={rule} --output=slurm_{rule}_%j.out"
 # set-threads: map rule names to threads
 set-threads:
   - diamond_reads=8

workflow/Snakefile

@@ -5,6 +5,7 @@
 # Created: 2016-04-28
 #
 import configparser
+import docker
 import os
 import re
 import shutil
@@ -23,8 +24,18 @@ from sunbeamlib.parse import read_seq_ids
 from sunbeamlib.post import *
 from sunbeamlib.reports import *
 
+client = docker.from_env()
+image_name = f"ctbushman/sunbeam:v{__version__}"
+try:
+    client.images.get_registry_data(image_name)
+except docker.errors.NotFound:
+    sys.stderr.write(
+        f"WARNING: {image_name} not found on DockerHub, using latest tag instead.\n"
+    )
+    image_name = "ctbushman/sunbeam:latest"
+
 
-containerized: f"docker://ctbushman/sunbeam:v{__version__}"
+containerized: f"docker://{image_name}"
 
 
 # Disallow slashes in our sample names during Snakemake's wildcard evaluation.
@@ -182,6 +193,17 @@ localrules:
     samples,
 
 
+for rule_name in rules._rules:
+    rule_obj = getattr(rules, rule_name).rule
+    wcs = rule_obj._wildcard_names
+    if wcs:
+        rule_obj.resources["slurm_extra"] = (
+            lambda wc, rule_name=rule_name: f"--output=slurm_{rule_name}_{'_'.join(wc)}_%j"
+        )
+    else:
+        rule_obj.resources["slurm_extra"] = f"--output=slurm_{rule_name}_%j"
+
+
 onstart:
     try:
         shutil.rmtree(BENCHMARK_FP)

workflow/rules/qc.smk

@@ -219,6 +219,9 @@ rule remove_low_complexity:
         LOG_FP / "remove_low_complexity_{sample}_{rp}.log",
     benchmark:
         BENCHMARK_FP / "remove_low_complexity_{sample}_{rp}.tsv"
+    resources:
+        mem_mb=lambda wc, input: max(MIN_MEM_MB, 2 * input.size_mb),
+        runtime=lambda wc: max(MIN_RUNTIME, 120),
     conda:
         "../envs/reports.yml"
     script: