Merge pull request #458 from sunbeam-labs/457-remove-slurm_executor-hack

Remove hack

.github/workflows/integration-tests.yml

@@ -49,7 +49,7 @@ jobs:
               shell: bash
               run: |
                 cd $GITHUB_WORKSPACE
-                ls -l . | grep "slurm_test_"
+                ls -l .snakemake/slurm_logs/ | grep "rule_test"
     
     test-apptainer:
         name: Apptainer Test
@@ -77,4 +77,4 @@ jobs:
     
                 sunbeam init --data_fp tests/data/reads/ --profile apptainer projects/test/
 
-                sunbeam run --profile projects/test/
+                sunbeam run --profile projects/test/ --docker_tag latest

docs/commands.rst

@@ -46,7 +46,7 @@ Sunbeam Commands
     Executes the Sunbeam pipeline by calling Snakemake.
 
     .. code-block:: shell
-        sunbeam run [-h] [-m] [-s PATH] [--target_list [TARGETS, ...]] [--include [INCLUDES, ...]] [--exclude [EXCLUDE, ...]] -- <snakemake options>
+        sunbeam run [-h] [-m] [-s PATH] [--target_list [TARGETS, ...]] [--include [INCLUDES, ...]] [--exclude [EXCLUDE, ...]] [--docker_tag TAG] <snakemake options>
 
     .. tip::
         The ``--target_list`` option is deprecated. Pass the targets directly to ``sunbeam run`` instead.
@@ -66,7 +66,8 @@ Sunbeam Commands
         --target_list: A list of targets to run successively. (DEPRECATED)
         --include: List of extensions to include in run.
         --exclude: List of extensions to exclude from run, use 'all' to exclude all extensions.
-        <snakemake options>: You can pass further arguments to Snakemake after ``--``, e.g: ``$ sunbeam run -- --cores 12``. See http://snakemake.readthedocs.io for more information.
+        --docker_tag: Tag to use for internal environment docker images. Try 'latest' if the default tag doesn't work.
+        <snakemake options>: You can pass further arguments to Snakemake, e.g: ``$ sunbeam run --cores 12``. See http://snakemake.readthedocs.io for more information.
 
     .. tip::
         The ``--profile`` option is a snakemake option but should be used whenever using ``sunbeam run``. The main sunbeam snakefile requires a config object to be defined and the profile created by ``sunbeam init`` will always specify a config file to get that from.

docs/examples.rst

@@ -54,7 +54,7 @@ Then you submit the job:
 
     sbatch run_sunbeam.sh
 
-Once this run completes, you will have a directory called ``/projects/my_project/sunbeam_output/`` that contains all of the output from the run and ``slurm_*`` files wherever you ran the main script from that contains logs for each job. Look in ``/projects/my_project/sunbeam_output/assembly/contigs/`` for the assembled contigs.
+Once this run completes, you will have a directory called ``/projects/my_project/sunbeam_output/`` that contains all of the output from the run and ``.snakemake/slurm_logs/rule_*/`` directories wherever you ran the main script from that contain logs for each job. Look in ``/projects/my_project/sunbeam_output/assembly/contigs/`` for the assembled contigs.
 
 Using Containerized Environments
 --------------------------------

pyproject.toml

@@ -104,7 +104,6 @@ classifiers = [  # Optional
 # For an analysis of this field vs pip's requirements files see:
 # https://packaging.python.org/discussions/install-requires-vs-requirements/
 dependencies = [ # Optional
-  "docker",
   "more-itertools",
   "pyyaml",
 ]

src/sunbeamlib/__init__.py

@@ -37,20 +37,9 @@ def __str__(self) -> str:
 
 
 def get_docker_str(repo: str, user: str = "sunbeamlabs") -> str:
-    # Docker import needs to live here to avoid circular imports
-    # pyproject.toml needs __version__ to be defined before installing dependencies
-    import docker
-
-    client = docker.from_env()
-    image_name = f"{user}/{repo}:{__version__}"
-    try:
-        client.images.get_registry_data(image_name)
-        return f"docker://{image_name}"
-    except docker.errors.NotFound:
-        sys.stderr.write(
-            f"WARNING: {image_name} not found on DockerHub, using latest tag instead.\n"
-        )
-        return f"docker://{user}/{repo}:latest"
+    docker_tag = os.environ.get("SUNBEAM_DOCKER_TAG", __version__)
+
+    return f"docker://{user}/{repo}:{docker_tag}"
 
 
 def load_sample_list(

src/sunbeamlib/script_run.py

@@ -4,6 +4,8 @@
 import subprocess
 from pathlib import Path
 
+from sunbeamlib import __version__
+
 
 def main(argv=sys.argv):
     epilog_str = (
@@ -51,6 +53,11 @@ def main(argv=sys.argv):
         default=[],
         help="List of extensions to exclude from run, use 'all' to exclude all extensions",
     )
+    parser.add_argument(
+        "--docker_tag",
+        default=__version__,
+        help="The tag to use when pulling docker images for the core pipeline environments, defaults to sunbeam's current version ($SUNBEAM_VER), a good alternative is 'latest' for the latest stable release",
+    )
 
     # The remaining args (after --) are passed to Snakemake
     args, remaining = parser.parse_known_args(argv)
@@ -83,6 +90,8 @@ def main(argv=sys.argv):
     if args.exclude:
         os.environ["SUNBEAM_EXTS_EXCLUDE"] = ", ".join(args.exclude)
 
+    os.environ["SUNBEAM_DOCKER_TAG"] = args.docker_tag
+
     snakemake_args = (
         [
             "snakemake",

workflow/Snakefile

@@ -179,17 +179,6 @@ localrules:
     samples,
 
 
-for rule_name in rules._rules:
-    rule_obj = getattr(rules, rule_name).rule
-    wcs = rule_obj._wildcard_names
-    if wcs:
-        rule_obj.resources["slurm_extra"] = (
-            lambda wc, rule_name=rule_name: f"--output=slurm_{rule_name}_{'_'.join(wc)}_%j"
-        )
-    else:
-        rule_obj.resources["slurm_extra"] = f"--output=slurm_{rule_name}_%j"
-
-
 onstart:
     try:
         shutil.rmtree(BENCHMARK_FP)

workflow/rules/qc.smk

@@ -112,7 +112,7 @@ rule trimmomatic_unpaired:
         TRAILING:{Cfg[qc][trailing]} \
         SLIDINGWINDOW:{params.sw_start}:{params.sw_end} \
         MINLEN:{Cfg[qc][minlen]} \
-        > >(tee {log}) 2> >(tee {log} >&2)
+        2>&1 | tee {log}
         """
 
 
@@ -156,7 +156,7 @@ rule trimmomatic_paired:
         TRAILING:{Cfg[qc][trailing]} \
         SLIDINGWINDOW:{params.sw_start}:{params.sw_end} \
         MINLEN:{Cfg[qc][minlen]} \
-        > >(tee {log}) 2> >(tee {log} >&2)
+        2>&1 | tee {log}
         """