snakemake-executor-plugin-aws-basic-batch 0.1.0__tar.gz → 0.2.0__tar.gz

snakemake_executor_plugin_aws_basic_batch-0.2.0/.github/workflows/docker.yml

@@ -0,0 +1,48 @@
+name: Publish to GHCR
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v5
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

{snakemake_executor_plugin_aws_basic_batch-0.1.0 → snakemake_executor_plugin_aws_basic_batch-0.2.0}/.gitignore

@@ -5,6 +5,7 @@
 examples/simple-workflow/.snakemake/
 examples/terraform/**/.terraform/
 examples/terraform/*/terraform.tfvars
+examples/terraform/*/terraform.tfstate
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

{snakemake_executor_plugin_aws_basic_batch-0.1.0 → snakemake_executor_plugin_aws_basic_batch-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snakemake-executor-plugin-aws-basic-batch
-Version: 0.1.0
+Version: 0.2.0
 Project-URL: Repository, https://github.com/radusuciu/snakemake-executor-plugin-aws-basic-batch
 Author-email: Radu Suciu <radusuciu@gmail.com>
 License-File: LICENSE

{snakemake_executor_plugin_aws_basic_batch-0.1.0 → snakemake_executor_plugin_aws_basic_batch-0.2.0}/README.md

@@ -31,6 +31,30 @@ Optional coordinator-specific settings:
 - `--aws-basic-batch-coordinator-queue` - Job queue for the coordinator (defaults to main queue)
 - `--aws-basic-batch-coordinator-job-definition` - Job definition for the coordinator (defaults to main job definition)
 
+## Per-Job Resource Customization
+
+Override CPU, memory, queue, or job definition on a per-rule basis using Snakemake's resource system:
+
+```python
+rule compute_heavy:
+    output: "result.txt"
+    resources:
+        aws_batch_vcpu=4,
+        aws_batch_mem_mb=8192,
+        aws_batch_job_queue="high-memory-queue"
+    shell: "python compute.py > {output}"
+```
+
+| Resource | Description | Default |
+|----------|-------------|---------|
+| `aws_batch_vcpu` | Number of vCPUs | 1 |
+| `aws_batch_mem_mb` | Memory in MiB | 1024 |
+| `aws_batch_job_queue` | Job queue ARN/name | `--aws-basic-batch-job-queue` |
+| `aws_batch_job_definition` | Job definition ARN/name | `--aws-basic-batch-job-definition` |
+| `aws_batch_task_timeout` | Job timeout in seconds (min: 60) | `--aws-basic-batch-task-timeout` |
+
+These values override the base job definition's resource configuration at submission time via AWS Batch's `containerOverrides.resourceRequirements`.
+
 ## Requirements
 
 - Workflow and dependencies must be included in the container image

snakemake_executor_plugin_aws_basic_batch-0.2.0/examples/simple-workflow/.dockerignore

@@ -0,0 +1,5 @@
+.snakemake/
+README.md
+justfile
+.last-job-id
+terraform/

snakemake_executor_plugin_aws_basic_batch-0.2.0/examples/simple-workflow/Dockerfile

@@ -0,0 +1,56 @@
+# syntax=docker/dockerfile:1
+
+# Example workflow image using multi-stage build with uv
+# This serves as a template for building your own workflow images
+
+ARG PYTHON_VERSION=3.13
+ARG COORDINATOR_IMAGE=ghcr.io/radusuciu/snakemake-executor-plugin-aws-basic-batch:latest
+
+# =============================================================================
+# Builder stage: install dependencies with uv
+# =============================================================================
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
+
+WORKDIR /app
+
+# Copy dependency files first (better layer caching)
+COPY pyproject.toml uv.lock ./
+
+# Sync dependencies
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --no-dev
+
+# =============================================================================
+# Runtime stage: minimal image with workflow
+# =============================================================================
+FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime
+
+# Create non-root user
+RUN groupadd --gid 1000 snakemake && \
+    useradd --uid 1000 --gid 1000 --create-home snakemake
+
+# Copy virtual environment from builder
+COPY --from=builder --chown=snakemake:snakemake /app/.venv /app/.venv
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+USER snakemake
+WORKDIR /workflow
+
+# Copy workflow files
+COPY --chown=snakemake:snakemake Snakefile ./
+
+CMD ["snakemake", "--help"]
+
+# =============================================================================
+# Coordinator stage: base plugin image with workflow files
+# =============================================================================
+FROM ${COORDINATOR_IMAGE} AS coordinator
+
+# Copy workflow files
+COPY --chown=snakemake:snakemake Snakefile ./

{snakemake_executor_plugin_aws_basic_batch-0.1.0 → snakemake_executor_plugin_aws_basic_batch-0.2.0}/examples/simple-workflow/README.md

@@ -24,37 +24,26 @@ terraform apply
 
 ### 2. Deploy Workflow Infrastructure
 
-The workflow module creates workflow-specific resources (job definition, ECR repository). It requires outputs from the coordinator:
+The workflow module creates workflow-specific resources (compute environment, job queue, job definition, ECR repositories for both workflow and coordinator images). It requires outputs from the coordinator:
 
 ```bash
-# From the simple-workflow example directory:
-just tf-init
-just tf-apply-new
-
-# Or manually:
-cd examples/terraform/simple-workflow
-terraform init
-terraform apply \
-  -var="job_queue_arn=$(terraform -chdir=../coordinator output -raw job_queue_arn)" \
-  -var="job_role_arn=$(terraform -chdir=../coordinator output -raw job_role_arn)" \
-  -var="execution_role_arn=$(terraform -chdir=../coordinator output -raw execution_role_arn)" \
-  -var="log_group_name=$(terraform -chdir=../coordinator output -raw log_group_name)" \
-  -var="log_group_arn=$(terraform -chdir=../coordinator output -raw log_group_arn)" \
-  -var="bucket_arn=$(terraform -chdir=../coordinator output -raw bucket_arn)"
+just example::tf-init
+just example::tf-apply-new
 ```
 
-### 3. Note the Outputs
+### 3. Build and Push Images
 
-After deployment, get the values needed for snakemake:
+Build and push both the coordinator and workflow images to ECR:
 
 ```bash
-# From examples/terraform/coordinator:
-terraform output  # region, job_queue_name, coordinator_job_definition_name, bucket_name
-
-# From examples/terraform/simple-workflow:
-terraform output  # job_definition_name
+just example::ecr-login
+just example::build-push
 ```
 
+This builds two images from the same Dockerfile:
+- **Coordinator image**: Base plugin image with workflow files (Snakefile)
+- **Workflow image**: Minimal image with workflow dependencies
+
 ## Running the Workflow
 
 ### Using Justfile (Recommended)
@@ -62,25 +51,23 @@ terraform output  # job_definition_name
 The justfile provides shortcuts for common operations:
 
 ```bash
-cd examples/simple-workflow
-
 # Submit coordinator job
-just run
+just example::run
 
 # Monitor job status
-just watch
+just example::watch
 
 # View logs
-just logs
+just example::logs
 
 # Check status
-just status
+just example::status
 ```
 
 Generate a snakemake command with your deployed infrastructure values:
 
 ```bash
-just tf-snakemake-cmd
+just example::tf-snakemake-cmd
 ```
 
 ### Option 1: Standard Mode (local orchestration)
@@ -105,8 +92,10 @@ snakemake --executor aws-basic-batch \
   --aws-basic-batch-region <region> \
   --aws-basic-batch-job-queue <queue-name> \
   --aws-basic-batch-job-definition <job-def-name> \
-  --aws-basic-batch-coordinator \
+  --aws-basic-batch-coordinator true \
   --aws-basic-batch-coordinator-job-definition <coordinator-job-def-name> \
+  --aws-basic-batch-coordinator-queue <coordinator-queue-name> \
+  --aws-basic-batch-coordinator-image <coordinator-image-uri> \
   --default-storage-provider s3 \
   --default-storage-prefix s3://<bucket-name>
 ```

{snakemake_executor_plugin_aws_basic_batch-0.1.0 → snakemake_executor_plugin_aws_basic_batch-0.2.0}/examples/simple-workflow/justfile

@@ -1,72 +1,95 @@
 # Simple Workflow Example
-# Requires the base image (radusuciu/snakemake-executor-plugin-aws-basic-batch) to be available
-# either from Docker Hub or built locally via `just build-base` from the project root.
+# Both coordinator and workflow images are built from this Dockerfile and pushed to ECR.
 
-coordinator_tf_dir := "../terraform/coordinator"
-workflow_tf_dir := "../terraform/simple-workflow"
+tf_dir := "../terraform"
 
 # =============================================================================
 # Build and Push
 # =============================================================================
 
-# Build the example workflow image
-build:
+# Build the workflow image
+build-workflow:
     #!/usr/bin/env bash
     set -euo pipefail
-    ecr_repo=$(terraform -chdir={{workflow_tf_dir}} output -raw ecr_repository_url)
-    docker build -t "$ecr_repo:latest" .
+    workflow_ecr=$(terraform -chdir={{tf_dir}} output -raw workflow_ecr_repository_url)
+    docker build --target runtime -t "$workflow_ecr:latest" .
+
+# Build the coordinator image (includes workflow files)
+build-coordinator:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    coordinator_ecr=$(terraform -chdir={{tf_dir}} output -raw coordinator_ecr_repository_url)
+    docker build --target coordinator -t "$coordinator_ecr:latest" .
+
+# Build both images
+build: build-workflow build-coordinator
 
 # Login to ECR
 ecr-login:
     #!/usr/bin/env bash
     set -euo pipefail
-    region=$(terraform -chdir={{coordinator_tf_dir}} output -raw region)
-    ecr_repo=$(terraform -chdir={{workflow_tf_dir}} output -raw ecr_repository_url)
-    aws ecr get-login-password --region "$region" | docker login --username AWS --password-stdin "$ecr_repo"
+    region=$(terraform -chdir={{tf_dir}} output -raw region)
+    workflow_ecr=$(terraform -chdir={{tf_dir}} output -raw workflow_ecr_repository_url)
+    ecr_host="${workflow_ecr%%/*}"
+    aws ecr get-login-password --region "$region" | docker login --username AWS --password-stdin "$ecr_host"
+
+# Push the workflow image to ECR
+push-workflow: ecr-login
+    #!/usr/bin/env bash
+    set -euo pipefail
+    workflow_ecr=$(terraform -chdir={{tf_dir}} output -raw workflow_ecr_repository_url)
+    docker push "$workflow_ecr:latest"
 
-# Push the example workflow image to ECR
-push: ecr-login
+# Push the coordinator image to ECR
+push-coordinator: ecr-login
     #!/usr/bin/env bash
     set -euo pipefail
-    ecr_repo=$(terraform -chdir={{workflow_tf_dir}} output -raw ecr_repository_url)
-    docker push "$ecr_repo:latest"
+    coordinator_ecr=$(terraform -chdir={{tf_dir}} output -raw coordinator_ecr_repository_url)
+    docker push "$coordinator_ecr:latest"
 
-# Build and push example image
+# Push both images to ECR
+push: push-workflow push-coordinator
+
+# Build and push both images
 build-push: build push
 
 # =============================================================================
 # Run and Monitor
 # =============================================================================
 
+# Directory containing this justfile
+workflow_dir := justfile_directory()
+
 # Submit coordinator job via snakemake plugin
 run *args:
     #!/usr/bin/env bash
     set -euo pipefail
-    cd {{coordinator_tf_dir}}
-    region=$(terraform output -raw region)
-    job_queue=$(terraform output -raw job_queue_name)
-    coordinator_job_def=$(terraform output -raw coordinator_job_definition_name)
-    bucket=$(terraform output -raw bucket_name)
-    workflow_job_def=$(terraform -chdir=../simple-workflow output -raw job_definition_name)
-    cd - > /dev/null
+    region=$(terraform -chdir={{tf_dir}} output -raw region)
+    coordinator_queue=$(terraform -chdir={{tf_dir}} output -raw coordinator_job_queue_name)
+    workflow_queue=$(terraform -chdir={{tf_dir}} output -raw workflow_job_queue_name)
+    bucket=$(terraform -chdir={{tf_dir}} output -raw bucket_name)
+    workflow_job_def=$(terraform -chdir={{tf_dir}} output -raw workflow_job_definition_name)
+    coordinator_job_def=$(terraform -chdir={{tf_dir}} output -raw coordinator_job_definition_name)
     snakemake \
+        --snakefile "{{workflow_dir}}/Snakefile" \
         --executor aws-basic-batch \
         --aws-basic-batch-coordinator true \
         --aws-basic-batch-region "$region" \
-        --aws-basic-batch-job-queue "$job_queue" \
+        --aws-basic-batch-job-queue "$workflow_queue" \
         --aws-basic-batch-job-definition "$workflow_job_def" \
         --aws-basic-batch-coordinator-job-definition "$coordinator_job_def" \
+        --aws-basic-batch-coordinator-queue "$coordinator_queue" \
         --default-storage-provider s3 \
         --default-storage-prefix "s3://$bucket" \
         --jobs 10 \
         --forceall \
-        {{args}} 2>&1 | tee /dev/stderr | grep -oP 'Coordinator job submitted: \K[a-f0-9-]+' > .last-job-id || true
+        {{args}} 2>&1 | tee /dev/stderr | grep -oP 'Coordinator job submitted: \K[a-f0-9-]+' > "{{workflow_dir}}/.last-job-id" || true
 
 # Check status of last submitted job
 status:
     #!/usr/bin/env bash
     set -euo pipefail
-    job_id=$(cat .last-job-id 2>/dev/null || echo "")
+    job_id=$(cat "{{workflow_dir}}/.last-job-id" 2>/dev/null || echo "")
     if [ -z "$job_id" ]; then
         echo "No job ID found. Run 'just run' first."
         exit 1
@@ -79,7 +102,7 @@ status:
 logs:
     #!/usr/bin/env bash
     set -euo pipefail
-    job_id=$(cat .last-job-id 2>/dev/null || echo "")
+    job_id=$(cat "{{workflow_dir}}/.last-job-id" 2>/dev/null || echo "")
     if [ -z "$job_id" ]; then
         echo "No job ID found. Run 'just run' first."
         exit 1
@@ -90,7 +113,7 @@ logs:
         echo "No log stream available yet. Job may still be starting."
         exit 1
     fi
-    log_group=$(terraform -chdir={{coordinator_tf_dir}} output -raw log_group_name)
+    log_group=$(terraform -chdir={{tf_dir}} output -raw log_group_name)
     aws logs get-log-events \
         --log-group-name "$log_group" \
         --log-stream-name "$log_stream" \
@@ -100,7 +123,7 @@ logs:
 watch:
     #!/usr/bin/env bash
     set -euo pipefail
-    job_id=$(cat .last-job-id 2>/dev/null || echo "")
+    job_id=$(cat "{{workflow_dir}}/.last-job-id" 2>/dev/null || echo "")
     if [ -z "$job_id" ]; then
         echo "No job ID found. Run 'just run' first."
         exit 1
@@ -118,57 +141,45 @@ watch:
 
 # =============================================================================
 # Infrastructure
-# Requires coordinator to be deployed first via: terraform -chdir=../terraform/coordinator apply
 # =============================================================================
 
-# Initialize workflow terraform
+# Initialize terraform
 tf-init:
-    terraform -chdir={{workflow_tf_dir}} init
+    terraform -chdir={{tf_dir}} init
 
-# Run terraform command with coordinator outputs as variables
-[private]
-tf-with-coordinator-vars cmd *args:
-    #!/usr/bin/env bash
-    set -euo pipefail
-    cd {{coordinator_tf_dir}}
-    exec terraform -chdir=../simple-workflow {{cmd}} \
-        -var="job_queue_arn=$(terraform output -raw job_queue_arn)" \
-        -var="job_role_arn=$(terraform output -raw job_role_arn)" \
-        -var="execution_role_arn=$(terraform output -raw execution_role_arn)" \
-        -var="log_group_name=$(terraform output -raw log_group_name)" \
-        -var="log_group_arn=$(terraform output -raw log_group_arn)" \
-        -var="bucket_arn=$(terraform output -raw bucket_arn)" \
-        {{args}}
-
-# Plan workflow infrastructure
-tf-plan *args: (tf-with-coordinator-vars "plan" args)
-
-# Apply workflow infrastructure
-tf-apply-new *args: (tf-with-coordinator-vars "apply" args)
-
-# Destroy workflow infrastructure
-tf-destroy *args: (tf-with-coordinator-vars "destroy" args)
-
-# Show workflow terraform outputs
+# Plan infrastructure
+tf-plan *args:
+    terraform -chdir={{tf_dir}} plan {{args}}
+
+# Apply infrastructure
+tf-apply *args:
+    terraform -chdir={{tf_dir}} apply {{args}}
+
+# Destroy infrastructure
+tf-destroy *args:
+    terraform -chdir={{tf_dir}} destroy {{args}}
+
+# Show terraform outputs
 tf-output *args:
-    terraform -chdir={{workflow_tf_dir}} output {{args}}
+    terraform -chdir={{tf_dir}} output {{args}}
 
 # Generate snakemake command using deployed infrastructure
 tf-snakemake-cmd:
     #!/usr/bin/env bash
     set -euo pipefail
-    cd {{coordinator_tf_dir}}
-    region=$(terraform output -raw region)
-    job_queue=$(terraform output -raw job_queue_name)
-    coordinator_job_def=$(terraform output -raw coordinator_job_definition_name)
-    bucket=$(terraform output -raw bucket_name)
-    workflow_job_def=$(terraform -chdir=../simple-workflow output -raw job_definition_name)
+    region=$(terraform -chdir={{tf_dir}} output -raw region)
+    coordinator_queue=$(terraform -chdir={{tf_dir}} output -raw coordinator_job_queue_name)
+    workflow_queue=$(terraform -chdir={{tf_dir}} output -raw workflow_job_queue_name)
+    bucket=$(terraform -chdir={{tf_dir}} output -raw bucket_name)
+    workflow_job_def=$(terraform -chdir={{tf_dir}} output -raw workflow_job_definition_name)
+    coordinator_job_def=$(terraform -chdir={{tf_dir}} output -raw coordinator_job_definition_name)
 
     echo "snakemake --executor aws-basic-batch \\"
     echo "    --aws-basic-batch-region $region \\"
-    echo "    --aws-basic-batch-job-queue $job_queue \\"
+    echo "    --aws-basic-batch-job-queue $workflow_queue \\"
     echo "    --aws-basic-batch-job-definition $workflow_job_def \\"
     echo "    --aws-basic-batch-coordinator true \\"
     echo "    --aws-basic-batch-coordinator-job-definition $coordinator_job_def \\"
+    echo "    --aws-basic-batch-coordinator-queue $coordinator_queue \\"
     echo "    --default-storage-provider s3 \\"
     echo "    --default-storage-prefix s3://$bucket"