snakemake-executor-plugin-aws-batch-flex 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. snakemake_executor_plugin_aws_batch_flex/__init__.py +8 -0
  2. snakemake_executor_plugin_aws_batch_flex/bootstrap.py +376 -0
  3. snakemake_executor_plugin_aws_batch_flex/bundle.py +142 -0
  4. snakemake_executor_plugin_aws_batch_flex/check.py +129 -0
  5. snakemake_executor_plugin_aws_batch_flex/condacache.py +184 -0
  6. snakemake_executor_plugin_aws_batch_flex/executor.py +725 -0
  7. snakemake_executor_plugin_aws_batch_flex/headjob.py +244 -0
  8. snakemake_executor_plugin_aws_batch_flex/hints.py +75 -0
  9. snakemake_executor_plugin_aws_batch_flex/hostenv.py +221 -0
  10. snakemake_executor_plugin_aws_batch_flex/imageinfo.py +207 -0
  11. snakemake_executor_plugin_aws_batch_flex/jobdefs.py +221 -0
  12. snakemake_executor_plugin_aws_batch_flex/jobfixes.py +65 -0
  13. snakemake_executor_plugin_aws_batch_flex/jobmonitor.py +146 -0
  14. snakemake_executor_plugin_aws_batch_flex/naming.py +16 -0
  15. snakemake_executor_plugin_aws_batch_flex/orphans.py +51 -0
  16. snakemake_executor_plugin_aws_batch_flex/patches.py +196 -0
  17. snakemake_executor_plugin_aws_batch_flex/preflight.py +165 -0
  18. snakemake_executor_plugin_aws_batch_flex/profiles.py +98 -0
  19. snakemake_executor_plugin_aws_batch_flex/remote.py +409 -0
  20. snakemake_executor_plugin_aws_batch_flex/settings.py +482 -0
  21. snakemake_executor_plugin_aws_batch_flex/sizing.py +75 -0
  22. snakemake_executor_plugin_aws_batch_flex/sources.py +83 -0
  23. snakemake_executor_plugin_aws_batch_flex/tags.py +76 -0
  24. snakemake_executor_plugin_aws_batch_flex/transfer.py +193 -0
  25. snakemake_executor_plugin_aws_batch_flex/usage.py +208 -0
  26. snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/METADATA +343 -0
  27. snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/RECORD +30 -0
  28. snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/WHEEL +4 -0
  29. snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/entry_points.txt +3 -0
  30. snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,343 @@
1
+ Metadata-Version: 2.5
2
+ Name: snakemake-executor-plugin-aws-batch-flex
3
+ Version: 0.1.0
4
+ Summary: Snakemake executor plugin for AWS Batch that runs every rule in its own container image
5
+ Project-URL: Homepage, https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex
6
+ Project-URL: Documentation, https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex#getting-started
7
+ Project-URL: Issues, https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/issues
8
+ Project-URL: Changelog, https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/CHANGELOG.md
9
+ Author: Stephen Smith
10
+ License: MIT
11
+ License-File: LICENSE
12
+ Keywords: aws,aws-batch,bioinformatics,executor,plugin,snakemake
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: POSIX :: Linux
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: boto3>=1.34
21
+ Requires-Dist: pyyaml
22
+ Requires-Dist: snakemake-interface-common<2,>=1.17.4
23
+ Requires-Dist: snakemake-interface-executor-plugins<10,>=9.3.2
24
+ Requires-Dist: snakemake-interface-storage-plugins<5,>=4.0.0
25
+ Provides-Extra: test
26
+ Requires-Dist: pytest; extra == 'test'
27
+ Requires-Dist: pytest-xdist; extra == 'test'
28
+ Requires-Dist: ruff; extra == 'test'
29
+ Requires-Dist: snakemake; extra == 'test'
30
+ Requires-Dist: snakemake-storage-plugin-fs; extra == 'test'
31
+ Requires-Dist: snakemake-storage-plugin-s3; extra == 'test'
32
+ Description-Content-Type: text/markdown
33
+
34
+ # snakemake-executor-plugin-aws-batch-flex
35
+
36
+ Run [Snakemake](https://snakemake.readthedocs.io) workflows on [AWS Batch](https://aws.amazon.com/batch/),
37
+ with each step running in whatever container image you choose: a stock image from Docker Hub,
38
+ quay.io/biocontainers, your own ECR repository, and so on. The images don't need Snakemake or
39
+ Python inside them.
40
+
41
+ In plain terms: you write a workflow on your own computer, and this plugin sends each step to
42
+ AWS, where machines are started for you, run the step in its container, save the results to S3
43
+ (Amazon's file storage) and shut down again when there's nothing left to do. You pay only while
44
+ machines are running.
45
+
46
+ How it works, briefly: a copy of Snakemake is built into the machine image (an "AMI") that the
47
+ AWS machines start from, and every job gets read access to it. That's why your containers don't
48
+ need it. On Fargate (AWS's serverless option, no machines to manage) Snakemake is installed into
49
+ the container when the job starts instead.
50
+
51
+ **How it differs from the aws-batch plugin.** The other AWS Batch executor in the Snakemake
52
+ plugin catalog,
53
+ [snakemake-executor-plugin-aws-batch](https://github.com/snakemake/snakemake-executor-plugin-aws-batch), runs
54
+ `snakemake` inside each job's container, so every image a rule uses needs Snakemake and the S3
55
+ storage plugin installed; the usual biocontainers and stock images can't be used as they are.
56
+ This plugin brings Snakemake to the job instead, so each rule can use any image, and conda envs
57
+ work in images without conda. It also sets up the AWS side in one command, mixes EC2, Spot and
58
+ Fargate queues in one run, can run Snakemake itself on AWS (`smkflex run`), retries jobs that
59
+ ran out of memory with more, and reports what each rule used, with the memory and threads to give it.
60
+
61
+ This page takes you from nothing to a finished test run ([Getting started](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex#getting-started)),
62
+ then shows how to run your own workflow and what you can set on each rule. The rest is in three
63
+ more pages:
64
+
65
+ - [Guide](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md): published pipelines, moving from a cluster, long runs with `smkflex`,
66
+ conda, the usage table, Spot and Fargate, private images, and troubleshooting;
67
+ - [Reference](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/reference.md): every profile setting and `smkflex` command, IAM, what has been
68
+ tested, and the limits;
69
+ - [Setup and upkeep](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/setup.md): university and company accounts, updating the machine image,
70
+ setting up by hand, and removing everything.
71
+
72
+ ## Getting started
73
+
74
+ These steps take you from nothing to a finished test run, in under an hour. They assume you're
75
+ comfortable typing commands into a terminal, but not that you know AWS or Snakemake.
76
+
77
+ It has been tested on Linux. On Windows, use [WSL](https://learn.microsoft.com/windows/wsl/install)
78
+ and run everything inside it. macOS should work but hasn't been tried.
79
+
80
+ ### 1. What you need
81
+
82
+ - An AWS account. Setting this up costs very little: building the machine image runs one small
83
+ machine for about 10 minutes, and keeping the image costs well under $1 a month. The test run
84
+ in step 5 costs a few cents. If your account comes from your university or company, read
85
+ [University and company accounts](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/setup.md#university-and-company-accounts) first.
86
+ - Somewhere to run commands: your laptop is fine. When you run `snakemake` it has to stay on
87
+ (and online) until the workflow finishes, because it's what tells AWS what to do next;
88
+ `smkflex run` hands that to AWS instead (see [Long runs](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#long-runs-smkflex)).
89
+
90
+ ### 2. Install the tools
91
+
92
+ Install [Miniforge](https://github.com/conda-forge/miniforge#install) if you don't already have
93
+ `conda`. Then get this repository and create an environment with everything you need (Python,
94
+ Snakemake, Terraform, the AWS command-line tool and `snakedeploy`):
95
+
96
+ ```bash
97
+ git clone https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex.git
98
+ cd snakemake-executor-plugin-aws-batch-flex
99
+ conda env create -f environment.yml
100
+ conda activate smk-aws
101
+ pip install -e .
102
+ ```
103
+
104
+ Run `conda activate smk-aws` again in every new terminal before the steps below.
105
+
106
+ Check it worked: `snakemake --version` should print a version number (9.x), and
107
+ `snakemake --help | grep aws-batch-flex` should list this plugin's settings.
108
+
109
+ ### 3. Connect to your AWS account
110
+
111
+ The command-line tools need an access key. In the [AWS console](https://console.aws.amazon.com/):
112
+
113
+ 1. Go to **IAM** > **Users** > **Create user**. Give it a name (e.g. `snakemake-admin`).
114
+ 2. On the permissions page choose **Attach policies directly** and tick **AdministratorAccess**.
115
+ (Setup creates machines, roles, a queue and a bucket, which needs broad permissions. Don't
116
+ use your account's root login for this.)
117
+ 3. Open the new user > **Security credentials** > **Create access key** > **Command Line
118
+ Interface (CLI)**. Copy the access key ID and secret access key.
119
+
120
+ Then in your terminal:
121
+
122
+ ```bash
123
+ aws configure
124
+ ```
125
+
126
+ Paste the two keys when asked. For the region, pick the one closest to you, e.g. `eu-west-1`
127
+ (Ireland) or `us-east-1` (N. Virginia), and use the same one everywhere below. Leave the output
128
+ format empty.
129
+
130
+ Check it worked: `aws sts get-caller-identity` should print your account number. (If you log in
131
+ through a university or company page instead, follow
132
+ [University and company accounts](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/setup.md#university-and-company-accounts) for this step.)
133
+
134
+ ### 4. Set up AWS
135
+
136
+ One script does the rest of the setup. It finds your account's network, builds a machine image
137
+ with Snakemake on it (plus conda and pandas, which most published pipelines need), creates the
138
+ queue, roles and bucket with Terraform, and installs a Snakemake profile called `smkflex` in
139
+ `~/.config/snakemake/smkflex/`. A profile is a settings file: it tells Snakemake to use this
140
+ plugin, your queue and your bucket, and because it's installed by name, `snakemake --profile
141
+ smkflex` works in any folder. Use the region you picked in step 3:
142
+
143
+ ```bash
144
+ bash scripts/setup.sh --region eu-west-1
145
+ ```
146
+
147
+ It asks twice before creating anything. The first time is before building the machine image,
148
+ which takes 10 to 20 minutes. The second is after showing what Terraform will create (about 20
149
+ new resources, nothing changed or removed). At the end it checks the whole setup and says
150
+ `All good.`. If it stops halfway, fix what it says and run the same command again: it carries on
151
+ where it stopped.
152
+
153
+ Options: `--fargate` also creates a Fargate queue and `--spot` a cheaper Spot queue (see
154
+ [Spot and Fargate queues](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#spot-and-fargate-queues)), `--extra-package NAME` adds a Python
155
+ package to the image, and `--no-conda` leaves conda out. `bash scripts/setup.sh --help` lists the
156
+ rest. To do each step by hand instead, see [Setting up by hand](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/setup.md#setting-up-by-hand).
157
+
158
+ Nothing is running (or costing money) yet: machines start when jobs arrive and stop when the
159
+ queue is empty.
160
+
161
+ ### 5. Run the test workflow
162
+
163
+ ```bash
164
+ cd examples/smoke-test
165
+ snakemake --profile smkflex
166
+ ```
167
+
168
+ What happens: Snakemake uploads the workflow to S3 and submits two jobs. The first time, AWS has
169
+ to start a machine, so the jobs show as waiting for a few minutes. Then each job runs in its own
170
+ container (`debian:12`, which has no Python) and writes its result to S3. When it's finished
171
+ you'll see `3 of 3 steps (100%) done`, followed by a table of what the run used.
172
+
173
+ Download the results from the bucket and look at them:
174
+
175
+ ```bash
176
+ smkflex pull results --profile smkflex
177
+ cat results/hello.txt results/sum.txt
178
+ ```
179
+
180
+ The first says hello from Debian and `image has NO python3`; the second is `45`. A few minutes
181
+ later the machine shuts itself down.
182
+
183
+ ### 6. Set a budget alert
184
+
185
+ Before running real pipelines, set a budget alert, so a mistake (a huge job, or a forgotten
186
+ setting) costs an email rather than a surprise bill. In the AWS console, **Billing and Cost
187
+ Management** > **Budgets** > **Create budget** > **Use a template** > **Monthly cost budget**:
188
+ enter an amount (e.g. $50) and your email address. AWS then mails you at 85% and 100% of it, and
189
+ when the month's forecast goes over. It takes two minutes and costs nothing.
190
+
191
+ As a rough guide: compute costs about $0.05 per vCPU-hour on normal instances (a third of that
192
+ on Spot), so a job using 4 vCPUs for an hour costs about $0.20. The published RNA-seq pipeline
193
+ in [Running a published pipeline](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#running-a-published-pipeline) ran its 53 test jobs for about
194
+ $0.10; a real run with 20 samples of 30 million reads should be a few dollars (an estimate, not
195
+ measured). S3 storage is about $0.02 per GB per month. The table at the end of every run shows
196
+ what that run used (see [What a run used](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#what-a-run-used)).
197
+
198
+ That's the setup done. Next, depending on what you're running:
199
+
200
+ - a workflow of your own: [Your own workflow](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex#your-own-workflow);
201
+ - a pipeline from the Snakemake workflow catalog: [Running a published pipeline](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#running-a-published-pipeline);
202
+ - a pipeline you've been running on a cluster: read [Your own workflow](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex#your-own-workflow),
203
+ then [Coming from a cluster](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#coming-from-a-cluster).
204
+
205
+ ## Running your workflows
206
+
207
+ ### Your own workflow
208
+
209
+ A Snakemake workflow is a file called `Snakefile` made of rules. Each rule says what files it
210
+ makes (`output`), what it needs (`input`), which container to use, and the command to run:
211
+
212
+ ```python
213
+ rule count_lines:
214
+ input: "data/reads.txt"
215
+ output: "results/count.txt"
216
+ container: "docker://debian:12"
217
+ resources: mem_mb=1000
218
+ shell: "wc -l < {input} > {output}"
219
+ ```
220
+
221
+ Snakemake works out the order from the file names, and with this plugin each rule runs as a
222
+ Batch job. The [Snakemake tutorial](https://snakemake.readthedocs.io/en/stable/tutorial/tutorial.html)
223
+ explains rules, wildcards and the rest; everything in it works here, with the differences listed
224
+ under [Limits](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/reference.md#limits).
225
+
226
+ Each workflow lives in its own project folder, and its files live in your S3 bucket: jobs read
227
+ their inputs from there, never from your computer, so upload them first, and write their outputs
228
+ there. Put the `Snakefile` in a new folder and,
229
+ in that folder:
230
+
231
+ ```bash
232
+ smkflex init # gives this project its own folder in the bucket
233
+ smkflex push data # uploads data/reads.txt to it
234
+ snakemake --profile smkflex # runs the workflow
235
+ smkflex pull results # downloads results/ from it
236
+ ```
237
+
238
+ `smkflex init` writes the project's own settings file, `profiles/default/config.yaml`, with the
239
+ project's folder in the bucket (`s3://YOUR-BUCKET/snakemake/<folder name>`, so each project's
240
+ `results/` stay apart), and adds `data/`, `results/` and the like to the folder's `.gitignore`.
241
+ Snakemake loads that file by itself, on top of the `smkflex` profile, so it's also the place for
242
+ anything this workflow needs (memory, conda, retries, see [Tuning jobs](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#tuning-jobs)); the
243
+ `smkflex` profile keeps what's the same for every workflow (your queue, bucket and roles).
244
+
245
+ `data` is only this example's folder: push whatever files and folders your rules and sample
246
+ sheets name (`smkflex push reads samples.tsv`), and pull whichever outputs you want
247
+ (`smkflex pull results/qc`). `push` and `pull` skip files that are already there with the same
248
+ size (`--dry-run` only lists them) and also take absolute paths (see
249
+ [Coming from a cluster](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/guide.md#coming-from-a-cluster)); paths that go up out of the project folder
250
+ (`../raw`) can't be mapped to storage, so use an absolute path for those. If something you push
251
+ isn't in `.gitignore`, `push` says so, since `smkflex run` would otherwise send it along with
252
+ the workflow.
253
+ `export SNAKEMAKE_PROFILE=smkflex` saves typing `--profile` every time. `examples/conformance/`
254
+ has one small workflow per feature (scripts, conda, checkpoints, Fargate, ...) if you want to
255
+ see more.
256
+
257
+ ### Resources per rule
258
+
259
+ Everything about a job is set on its rule. An example with every option in the table below (a real
260
+ rule only needs the few it uses):
261
+
262
+ ```python
263
+ rule align:
264
+ input: "data/{sample}.fq"
265
+ output: "results/{sample}.sam"
266
+ container: "docker://quay.io/biocontainers/bwa:0.7.17--h7132678_9"
267
+ threads: 8
268
+ retries: 4
269
+ resources:
270
+ mem_mb=16000,
271
+ runtime=120, # minutes
272
+ batch_shm_mb=4096, # EC2 only
273
+ disk_mb=100000, # Fargate only
274
+ gpu=1, # needs a queue with GPU machines
275
+ batch_queue="smkflex-big-queue",
276
+ batch_platform="ec2", # only if the queue can't be looked up
277
+ batch_tags="stage=align,project=rnaseq",
278
+ batch_spot_retries=3, # Spot queues
279
+ batch_priority=500, # fair-share queues
280
+ batch_script_python="host", # script: rules only
281
+ # batch_entrypoint='["python3"]', # only if the image's registry can't be read
282
+ # batch_job_definition="my-jobdef", # instead of the plugin's own; it then sets the image
283
+ shell: "bwa mem -t {threads} ref.fa {input} > {output}"
284
+ ```
285
+
286
+ | On the rule | What it does on AWS |
287
+ |---|---|
288
+ | `container: "docker://..."` | The image the job runs in. Any registry works (Docker Hub, quay.io, ghcr.io, ECR...); `docker://` is optional. Rules without one use `aws-batch-flex-default-image`. Images with an `ENTRYPOINT` work too: Batch can't change it, so the plugin reads it from the registry and gives the jobs a command it runs. |
289
+ | `threads: 8` | Number of vCPUs requested. On EC2, a job bigger than the largest machine the queue may start waits forever (the plugin warns after `runnable-warn-minutes`). On Fargate, vCPUs and memory are rounded up to the nearest size Fargate offers (at most 16 vCPUs and 120 GiB). |
290
+ | `mem_mb=16000` (or `mem_mib`) | Memory requested, a hard limit. Snakemake itself runs inside the job too, so 256 MiB is added (`mem-overhead-mib`). Without it (many published pipelines don't set it), `default-resources` applies, or else 2048 MiB. A job that goes over it is stopped; with `--retries 2` (or `retries: 2` in the profile) it's retried with twice the memory, then four times (`oom-memory-factor`). The end-of-run table shows what each rule really used. |
291
+ | `retries: 2` | Not a resource but a rule setting (or `retries: 2` in the profile for every rule): how many times Snakemake runs a failed job again. A job that ran out of memory gets more on each retry (`oom-memory-factor`), and `attempt` lets a rule raise anything itself, e.g. `mem_mb=lambda wildcards, attempt: 4000 * attempt` (then the rule's own figure is used, not doubled on top). Jobs that lost their Spot machine are restarted by Batch instead (`batch_spot_retries`) and don't use these up. |
292
+ | `runtime=120` | Minutes before Batch stops the job (timeout); `"2h"` also works. To give every rule a timeout, put `runtime` in `default-resources`. |
293
+ | `batch_shm_mb=4096` | Size of `/dev/shm` (Docker's default is 64 MB), for tools that use shared memory, e.g. bwa-mem2 or PyTorch data loaders. What's stored there counts towards `mem_mb`. EC2 only. `shared_memory_size_mb` also works. |
294
+ | `disk_mb=100000` (or `disk_mib`) | Fargate only: disk space for the job, up to 200 GiB, rounded up to 25 GiB steps. Without it, Snakemake asks for twice the size of the job's input files, and jobs get at least 20 GiB (or `ephemeral-storage-gib`). On EC2 it's ignored: jobs share their machine's disk (see [Limits](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/reference.md#limits)). |
295
+ | `gpu=1` | Number of GPUs (needs a queue with GPU instances; untested). |
296
+ | `batch_queue="other-queue"` | Send this rule to a different queue than the profile's. EC2, Spot and Fargate queues can be mixed in one workflow; the plugin works out which is which. |
297
+ | `batch_platform="fargate"` | Force the platform instead of detecting it from the queue (only needed if the plugin isn't allowed to look the queue up). |
298
+ | `batch_tags="k=v,k2=v2"` | Extra tags on this rule's jobs, e.g. for cost reports. Overrides profile tags with the same key. |
299
+ | `batch_spot_retries=0` | How often Batch retries this rule's job if its Spot instance is taken away (0-9; default from the profile, 2). `aws_batch_spot_attempts` (retries + 1) also works. |
300
+ | `batch_priority=500` | Scheduling priority, 0-9999, higher first. Only on queues with a fair-share policy. `aws_batch_scheduling_priority` also works. |
301
+ | `batch_job_definition="name"` | Use an existing job definition for this rule (see the `job-definition` setting). |
302
+ | `batch_script_python="host"` | For Python `script:` rules: use Snakemake's own Python instead of the image's. |
303
+ | `batch_entrypoint='["python3"]'` | The image's `ENTRYPOINT`, for when the plugin can't read it from the registry (it then warns at the start): a private registry without `registry-credentials`, or no network to it. `'[]'` for none. |
304
+
305
+ To change resources without editing the Snakefile (for a published pipeline, say), use
306
+ Snakemake's `set-resources`, `set-threads` and `default-resources` in the project's
307
+ `profiles/default/config.yaml`:
308
+
309
+ ```yaml
310
+ default-resources: [mem_mb=4000] # rules that set no memory
311
+ max-threads: 8 # caps rules written for big cluster nodes
312
+ set-resources:
313
+ align:
314
+ batch_queue: smkflex-big-queue
315
+ mem_mb: 32000
316
+ ```
317
+
318
+ Snakemake features that work as usual (each has a small example in `examples/conformance/`):
319
+ `shell:` commands; `script:` files in Python, R or bash; `run:` blocks; `conda:` environments and
320
+ wrappers (`wrapper:`); `retries: 2`, and resources that grow with each attempt, e.g.
321
+ `mem_mb=lambda wildcards, attempt: 4000 * attempt`; `group:` (grouped rules run as one Batch job,
322
+ so they must share an image), `checkpoint`, `temp()`, `directory()`, `shadow:`, `log:` and
323
+ `benchmark:`.
324
+
325
+ ### Most-used settings
326
+
327
+ Plugin settings go in a profile as `aws-batch-flex-NAME: value` (or on the command line as
328
+ `--aws-batch-flex-NAME value`); `setup.sh` writes the usual ones. The ones people change most:
329
+
330
+ | Setting (`aws-batch-flex-...`) | What it's for |
331
+ |---|---|
332
+ | `queue` | The Batch queue jobs go to. |
333
+ | `default-image` | The image for rules that have no `container:`. |
334
+ | `conda-cache` | Reuse built conda envs: per machine (default), or kept in S3 for later runs. |
335
+ | `oom-memory-factor` | How much more memory a job gets when it's retried after running out (default 2×). |
336
+ | `spot-retries` | How many times Batch restarts a job whose Spot machine was taken back. |
337
+ | `tags` | Extra tags on every job, e.g. to see costs by project. |
338
+
339
+ All of them, with their defaults, are in the [Reference](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/docs/reference.md#profile-settings).
340
+
341
+ ## License
342
+
343
+ MIT, see [LICENSE](https://github.com/SSmith-NUIG/snakemake-executor-plugin-aws-batch-flex/blob/main/LICENSE).
@@ -0,0 +1,30 @@
1
+ snakemake_executor_plugin_aws_batch_flex/__init__.py,sha256=IaA5sFRnG1S-L7AfxLRHYH5iLa2YOuYaf1eGfj700is,393
2
+ snakemake_executor_plugin_aws_batch_flex/bootstrap.py,sha256=TPxveniyGoeQLLiNzGenFvzMxxvy_67mfRVwbRe0-Fs,16930
3
+ snakemake_executor_plugin_aws_batch_flex/bundle.py,sha256=0iYdR6QLBx-_DawAv0RI4UJokIci43DTlYQvAI5fUks,6572
4
+ snakemake_executor_plugin_aws_batch_flex/check.py,sha256=waQLzSH2-nhMtlWBHUcVdqm6WyAD5-j9DggTVG7emhM,6379
5
+ snakemake_executor_plugin_aws_batch_flex/condacache.py,sha256=Gv83M200P-DvrkgbEOGHp_slv1ldax24HLZZOJCuN7Q,7347
6
+ snakemake_executor_plugin_aws_batch_flex/executor.py,sha256=0yI0akVDd7jo0TiZDOherLDpcQnN1eZfQg7DtoMz7fw,34807
7
+ snakemake_executor_plugin_aws_batch_flex/headjob.py,sha256=AK9LpqVTayg2F4BiNaQiL3cuWQcmIl27lVxvdHErhrc,8535
8
+ snakemake_executor_plugin_aws_batch_flex/hints.py,sha256=ZWabgkHF7PZKd1HU7ThCvLzozH4Yj4Wu6zUtsO9Ld5k,4794
9
+ snakemake_executor_plugin_aws_batch_flex/hostenv.py,sha256=Flov0jppi3ov4-ZG-qe3wMym_KH9EQeKZfyYx2NBUb4,9801
10
+ snakemake_executor_plugin_aws_batch_flex/imageinfo.py,sha256=-7Xr1gz2XfrgngNtoIpB2NY3kE-U6bMmmTqlF1FIgFk,10472
11
+ snakemake_executor_plugin_aws_batch_flex/jobdefs.py,sha256=UyZAQk_dXijHfPGSDnFcz4WY4Xnlhte4Rs95QGnzYIE,9244
12
+ snakemake_executor_plugin_aws_batch_flex/jobfixes.py,sha256=Hh-lxScLYwit7l-_zvf_JMPNr_EQGduSTcLu04D1qMU,2754
13
+ snakemake_executor_plugin_aws_batch_flex/jobmonitor.py,sha256=MI-iwqXefYGgQpNvhu61L_Rc35nN25_D3MEij3Hx424,5179
14
+ snakemake_executor_plugin_aws_batch_flex/naming.py,sha256=bUOBos191s6RkcqvE3G63Gjo1RWmwQvCLczTv0CRK_Q,612
15
+ snakemake_executor_plugin_aws_batch_flex/orphans.py,sha256=AKKtyxmIvZn96TUd1R2bY1Ui1U1qzsiowS6d4rq7z9c,2155
16
+ snakemake_executor_plugin_aws_batch_flex/patches.py,sha256=Yc3oubCC1KQKkMsGxJ1b3kMJCvWceAnmSijTt2rf2iM,9498
17
+ snakemake_executor_plugin_aws_batch_flex/preflight.py,sha256=tijLDpkOL1dvl7Dure2BKro9btekjzDBgxXpVilsIWM,7934
18
+ snakemake_executor_plugin_aws_batch_flex/profiles.py,sha256=0AZ4tC6Y7Qufwv8iq1TxZ9f65TncBYMnQL0K1fSfS8g,4013
19
+ snakemake_executor_plugin_aws_batch_flex/remote.py,sha256=v0ZolXefM7ntqLCulKRVnCYIA_BHNLfeYN7-bU0G1h8,20231
20
+ snakemake_executor_plugin_aws_batch_flex/settings.py,sha256=AHa4pnfIEmOEEbABzngP2-dCb8e1X5vwlA9kjN2aAoY,19507
21
+ snakemake_executor_plugin_aws_batch_flex/sizing.py,sha256=y5FydaAfboMrwtkaguW7Vj5SsBWXdRdJ5b4ZuVmn6a8,2494
22
+ snakemake_executor_plugin_aws_batch_flex/sources.py,sha256=storqscP60B0Ws_JF8QpcdtLVqPax_xklHMqro5LgNc,3850
23
+ snakemake_executor_plugin_aws_batch_flex/tags.py,sha256=kEM53RdqrhQPHrpoaNEric9JOVdyXmR0W0afNWvoULQ,2500
24
+ snakemake_executor_plugin_aws_batch_flex/transfer.py,sha256=K8z2hhMESuTm82JAi9D8jlXHTChLzUe1qA-Y9CreqiQ,8666
25
+ snakemake_executor_plugin_aws_batch_flex/usage.py,sha256=NXvuxV3BZJvoKtwdKhYIA2C1sAAk_XuqtyEEq48cKaE,11183
26
+ snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/METADATA,sha256=a7UTJpDusgJo-Ek0EiqjIUmNE4OVQWr6iT9A-IS8Qw8,21919
27
+ snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
28
+ snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/entry_points.txt,sha256=mJmQ6XSjNHpTd77Sq1E2NpJ4BveLpqtjdBanp8_Ht3w,167
29
+ snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/licenses/LICENSE,sha256=Hibr4Ms3FAwzr2RXkjai7WjC4nfp5ZNeu1hJxRaEi4g,1070
30
+ snakemake_executor_plugin_aws_batch_flex-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ smkflex = snakemake_executor_plugin_aws_batch_flex.remote:main
3
+ snakemake-batch-flex-host-env = snakemake_executor_plugin_aws_batch_flex.hostenv:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Stephen Smith
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.