snakemake-executor-plugin-aws-basic-batch 0.1.0__tar.gz

snakemake_executor_plugin_aws_basic_batch-0.1.0/.dockerignore

@@ -0,0 +1,14 @@
+.venv/
+.git/
+.github/
+.claude/
+*.pyc
+__pycache__/
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.coverage
+examples/
+tests/
+docs/

snakemake_executor_plugin_aws_basic_batch-0.1.0/.gitattributes

@@ -0,0 +1 @@
+uv.lock merge=binary linguist-generated=true

snakemake_executor_plugin_aws_basic_batch-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Check formatting
+        run: uv run ruff format --check .
+
+      - name: Lint
+        run: uv run ruff check .
+
+  testing:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Run pytest
+        run: uv run coverage run -m pytest
+
+      - name: Run Coverage
+        run: uv run coverage report -m

snakemake_executor_plugin_aws_basic_batch-0.1.0/.github/workflows/conventional-prs.yml

@@ -0,0 +1,16 @@
+name: PR
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - edited
+      - synchronize
+
+jobs:
+  title-format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v6
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

snakemake_executor_plugin_aws_basic_batch-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,23 @@
+name: Publish to PyPI
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

snakemake_executor_plugin_aws_basic_batch-0.1.0/.gitignore

@@ -0,0 +1,215 @@
+# Just workflow tracking
+.last-job-id
+
+# workflow example files
+examples/simple-workflow/.snakemake/
+examples/terraform/**/.terraform/
+examples/terraform/*/terraform.tfvars
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

snakemake_executor_plugin_aws_basic_batch-0.1.0/Dockerfile

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+
+# Minimal Snakemake executor image for AWS Batch
+# Uses uv for fast, reproducible dependency installation
+
+ARG PYTHON_VERSION=3.13
+
+# =============================================================================
+# Builder stage: install dependencies with uv
+# =============================================================================
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
+
+# Copy uv from the official image
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# Set uv environment variables for reproducible builds
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy
+
+WORKDIR /app
+
+# Install dependencies first (better layer caching)
+# Only copy files needed for dependency resolution
+COPY pyproject.toml uv.lock* ./
+
+# Sync dependencies without installing the project itself
+# This layer is cached unless pyproject.toml or uv.lock changes
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --no-dev
+
+# Copy source code and install the project in non-editable mode
+COPY src/ ./src/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-editable --no-dev
+
+# =============================================================================
+# Runtime stage: minimal image with just the virtual environment
+# =============================================================================
+FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime
+
+LABEL org.opencontainers.image.title="snakemake-executor-plugin-aws-basic-batch" \
+      org.opencontainers.image.description="Minimal Snakemake image with AWS Batch executor plugin" \
+      org.opencontainers.image.source="https://github.com/radusuciu/snakemake-executor-plugin-aws-basic-batch" \
+      org.opencontainers.image.authors="Radu Suciu <radusuciu@gmail.com>"
+
+# Create non-root user for security
+RUN groupadd --gid 1000 snakemake && \
+    useradd --uid 1000 --gid 1000 --create-home snakemake
+
+# Copy the prepared virtual environment from builder
+COPY --from=builder --chown=snakemake:snakemake /app/.venv /app/.venv
+
+# Add virtual environment to PATH
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Switch to non-root user
+USER snakemake
+
+WORKDIR /workflow
+
+# Default command shows help
+CMD ["snakemake", "--help"]

snakemake_executor_plugin_aws_basic_batch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Radu Suciu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

snakemake_executor_plugin_aws_basic_batch-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: snakemake-executor-plugin-aws-basic-batch
+Version: 0.1.0
+Project-URL: Repository, https://github.com/radusuciu/snakemake-executor-plugin-aws-basic-batch
+Author-email: Radu Suciu <radusuciu@gmail.com>
+License-File: LICENSE
+Requires-Python: <4.0,>=3.11
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: snakemake-interface-common<2,>=1.22.0
+Requires-Dist: snakemake-interface-executor-plugins<10,>=9.3.9
+Requires-Dist: snakemake-storage-plugin-s3>=0.3.6
+Requires-Dist: snakemake>=9.14.0

snakemake_executor_plugin_aws_basic_batch-0.1.0/README.md

@@ -0,0 +1,37 @@
+# snakemake-executor-plugin-aws-basic-batch
+
+A Snakemake executor plugin for AWS Batch that uses pre-configured job definitions.
+
+Unlike the [standard AWS Batch plugin](https://github.com/snakemake/snakemake-executor-plugin-aws-batch) which dynamically creates job definitions, this "basic" plugin relies on existing job definitions. This allows all resource configuration to be managed externally (e.g., via Terraform/CloudFormation). Additionally, workflow files and dependencies must bundled in the container image.
+
+## Usage
+
+```bash
+snakemake --executor aws-basic-batch \
+  --aws-basic-batch-region us-east-1 \
+  --aws-basic-batch-job-queue my-queue \
+  --aws-basic-batch-job-definition my-job-def \
+  --default-storage-provider s3 \
+  --default-storage-prefix s3://my-bucket/workdir
+```
+
+## Coordinator Mode
+
+Run the entire workflow as a fire-and-forget AWS Batch job:
+
+```bash
+snakemake --executor aws-basic-batch \
+  --aws-basic-batch-coordinator true \
+  ...
+```
+
+The coordinator job runs Snakemake itself on AWS Batch, submitting and monitoring rule jobs. Your terminal can disconnect after submission.
+
+Optional coordinator-specific settings:
+- `--aws-basic-batch-coordinator-queue` - Job queue for the coordinator (defaults to main queue)
+- `--aws-basic-batch-coordinator-job-definition` - Job definition for the coordinator (defaults to main job definition)
+
+## Requirements
+
+- Workflow and dependencies must be included in the container image
+- Job definitions should have appropriate IAM roles for S3 access and Batch job submission (for coordinator mode)

snakemake_executor_plugin_aws_basic_batch-0.1.0/examples/simple-workflow/.dockerignore

@@ -0,0 +1,2 @@
+.snakemake/
+README.md

snakemake_executor_plugin_aws_basic_batch-0.1.0/examples/simple-workflow/Dockerfile

@@ -0,0 +1,7 @@
+# Example workflow image based on the snakemake-executor-plugin-aws-basic-batch base
+FROM radusuciu/snakemake-executor-plugin-aws-basic-batch:latest
+
+# Copy the example workflow
+COPY --chown=snakemake:snakemake Snakefile /workflow/Snakefile
+
+WORKDIR /workflow

snakemake_executor_plugin_aws_basic_batch-0.1.0/examples/simple-workflow/README.md

@@ -0,0 +1,125 @@
+# Simple Workflow Example
+
+A minimal Snakemake workflow to test the AWS Batch executor plugin.
+
+## What It Does
+
+1. **create_input**: Generates sample text files
+2. **process**: Processes each sample in parallel (one Batch job per sample)
+3. **aggregate**: Combines all processed results into a summary
+
+## Prerequisites
+
+Deploy the terraform infrastructure in two steps:
+
+### 1. Deploy Coordinator Infrastructure
+
+The coordinator module creates shared resources (VPC, job queue, IAM roles, S3 bucket, coordinator job definition):
+
+```bash
+cd examples/terraform/coordinator
+terraform init
+terraform apply
+```
+
+### 2. Deploy Workflow Infrastructure
+
+The workflow module creates workflow-specific resources (job definition, ECR repository). It requires outputs from the coordinator:
+
+```bash
+# From the simple-workflow example directory:
+just tf-init
+just tf-apply-new
+
+# Or manually:
+cd examples/terraform/simple-workflow
+terraform init
+terraform apply \
+  -var="job_queue_arn=$(terraform -chdir=../coordinator output -raw job_queue_arn)" \
+  -var="job_role_arn=$(terraform -chdir=../coordinator output -raw job_role_arn)" \
+  -var="execution_role_arn=$(terraform -chdir=../coordinator output -raw execution_role_arn)" \
+  -var="log_group_name=$(terraform -chdir=../coordinator output -raw log_group_name)" \
+  -var="log_group_arn=$(terraform -chdir=../coordinator output -raw log_group_arn)" \
+  -var="bucket_arn=$(terraform -chdir=../coordinator output -raw bucket_arn)"
+```
+
+### 3. Note the Outputs
+
+After deployment, get the values needed for snakemake:
+
+```bash
+# From examples/terraform/coordinator:
+terraform output  # region, job_queue_name, coordinator_job_definition_name, bucket_name
+
+# From examples/terraform/simple-workflow:
+terraform output  # job_definition_name
+```
+
+## Running the Workflow
+
+### Using Justfile (Recommended)
+
+The justfile provides shortcuts for common operations:
+
+```bash
+cd examples/simple-workflow
+
+# Submit coordinator job
+just run
+
+# Monitor job status
+just watch
+
+# View logs
+just logs
+
+# Check status
+just status
+```
+
+Generate a snakemake command with your deployed infrastructure values:
+
+```bash
+just tf-snakemake-cmd
+```
+
+### Option 1: Standard Mode (local orchestration)
+
+Your machine runs Snakemake and submits jobs to AWS Batch:
+
+```bash
+snakemake --executor aws-basic-batch \
+  --aws-basic-batch-region <region> \
+  --aws-basic-batch-job-queue <queue-name> \
+  --aws-basic-batch-job-definition <job-def-name> \
+  --default-storage-provider s3 \
+  --default-storage-prefix s3://<bucket-name>
+```
+
+### Option 2: Coordinator Mode (fire-and-forget)
+
+Submit a coordinator job that runs the entire workflow on Batch:
+
+```bash
+snakemake --executor aws-basic-batch \
+  --aws-basic-batch-region <region> \
+  --aws-basic-batch-job-queue <queue-name> \
+  --aws-basic-batch-job-definition <job-def-name> \
+  --aws-basic-batch-coordinator \
+  --aws-basic-batch-coordinator-job-definition <coordinator-job-def-name> \
+  --default-storage-provider s3 \
+  --default-storage-prefix s3://<bucket-name>
+```
+
+### Local Test (no AWS)
+
+```bash
+snakemake --cores 1
+```
+
+## Expected Output
+
+After completion, you'll have:
+- `data/sample_A.txt`, `data/sample_B.txt`, `data/sample_C.txt`
+- `results/sample_A.processed.txt`, etc.
+- `results/summary.txt` - aggregated output showing all processed samples

snakemake_executor_plugin_aws_basic_batch-0.1.0/examples/simple-workflow/Snakefile

@@ -0,0 +1,75 @@
+# Simple example workflow for testing snakemake-executor-plugin-aws-basic-batch
+#
+# This workflow:
+# 1. Creates sample input files
+# 2. Processes them in parallel on AWS Batch
+# 3. Aggregates results
+#
+# Run locally (for testing):
+#   snakemake --cores 1
+#
+# Run on AWS Batch:
+#   snakemake --executor aws-basic-batch \
+#     --aws-basic-batch-region <region> \
+#     --aws-basic-batch-job-queue <queue-name> \
+#     --aws-basic-batch-job-definition <job-def-name> \
+#     --default-storage-provider s3 \
+#     --default-storage-prefix s3://<your-bucket>/workflow
+
+SAMPLES = ["sample_A", "sample_B", "sample_C"]
+
+
+rule all:
+    input:
+        "results/summary.txt",
+
+
+rule create_input:
+    """Generate sample input data."""
+    output:
+        "data/{sample}.txt",
+    shell:
+        """
+        echo "Input data for {wildcards.sample}" > {output}
+        echo "Generated at: $(date)" >> {output}
+        echo "Hostname: $(hostname)" >> {output}
+        """
+
+
+rule process:
+    """Process each sample (runs on AWS Batch in parallel)."""
+    input:
+        "data/{sample}.txt",
+    output:
+        "results/{sample}.processed.txt",
+    shell:
+        """
+        echo "=== Processing {wildcards.sample} ===" > {output}
+        echo "Started at: $(date)" >> {output}
+        echo "Running on: $(hostname)" >> {output}
+        echo "" >> {output}
+        echo "Input contents:" >> {output}
+        cat {input} >> {output}
+        echo "" >> {output}
+        echo "Processing complete at: $(date)" >> {output}
+        """
+
+
+rule aggregate:
+    """Combine all processed results."""
+    input:
+        expand("results/{sample}.processed.txt", sample=SAMPLES),
+    output:
+        "results/summary.txt",
+    shell:
+        """
+        echo "=== Workflow Summary ===" > {output}
+        echo "Generated at: $(date)" >> {output}
+        echo "Total samples processed: {params}" >> {output}
+        echo "" >> {output}
+        for f in {input}; do
+            echo "--- $f ---" >> {output}
+            cat "$f" >> {output}
+            echo "" >> {output}
+        done
+        """