absconda 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. absconda-0.1.0/LICENSE +21 -0
  2. absconda-0.1.0/PKG-INFO +296 -0
  3. absconda-0.1.0/README.md +275 -0
  4. absconda-0.1.0/pyproject.toml +54 -0
  5. absconda-0.1.0/setup.cfg +4 -0
  6. absconda-0.1.0/src/absconda/__init__.py +11 -0
  7. absconda-0.1.0/src/absconda/__main__.py +4 -0
  8. absconda-0.1.0/src/absconda/_templates/default/Dockerfile.j2 +2 -0
  9. absconda-0.1.0/src/absconda/_templates/default/main.j2 +21 -0
  10. absconda-0.1.0/src/absconda/_templates/fragments/builder_stage.j2 +45 -0
  11. absconda-0.1.0/src/absconda/_templates/fragments/export_block.j2 +3 -0
  12. absconda-0.1.0/src/absconda/_templates/fragments/requirements_runtime.j2 +12 -0
  13. absconda-0.1.0/src/absconda/_templates/fragments/runtime_stage.j2 +17 -0
  14. absconda-0.1.0/src/absconda/_templates/fragments/single_stage.j2 +67 -0
  15. absconda-0.1.0/src/absconda/_templates/fragments/tarball_runtime.j2 +19 -0
  16. absconda-0.1.0/src/absconda/cli.py +1281 -0
  17. absconda-0.1.0/src/absconda/config.py +227 -0
  18. absconda-0.1.0/src/absconda/environment.py +339 -0
  19. absconda-0.1.0/src/absconda/modules.py +143 -0
  20. absconda-0.1.0/src/absconda/policy.py +259 -0
  21. absconda-0.1.0/src/absconda/remote.py +696 -0
  22. absconda-0.1.0/src/absconda/templates.py +179 -0
  23. absconda-0.1.0/src/absconda/wrappers.py +203 -0
  24. absconda-0.1.0/src/absconda.egg-info/PKG-INFO +296 -0
  25. absconda-0.1.0/src/absconda.egg-info/SOURCES.txt +36 -0
  26. absconda-0.1.0/src/absconda.egg-info/dependency_links.txt +1 -0
  27. absconda-0.1.0/src/absconda.egg-info/entry_points.txt +2 -0
  28. absconda-0.1.0/src/absconda.egg-info/requires.txt +11 -0
  29. absconda-0.1.0/src/absconda.egg-info/top_level.txt +1 -0
  30. absconda-0.1.0/tests/test_cli.py +391 -0
  31. absconda-0.1.0/tests/test_environment.py +51 -0
  32. absconda-0.1.0/tests/test_examples.py +41 -0
  33. absconda-0.1.0/tests/test_integration_smoke.py +214 -0
  34. absconda-0.1.0/tests/test_modules.py +273 -0
  35. absconda-0.1.0/tests/test_policy.py +113 -0
  36. absconda-0.1.0/tests/test_remote.py +188 -0
  37. absconda-0.1.0/tests/test_templates.py +98 -0
  38. absconda-0.1.0/tests/test_wrappers.py +236 -0
absconda-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Garvan Institute of Medical Research
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,296 @@
1
+ Metadata-Version: 2.4
2
+ Name: absconda
3
+ Version: 0.1.0
4
+ Summary: Generate Dockerfiles and container build workflows from Conda environments
5
+ Author: John Reeves
6
+ License: MIT
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Requires-Dist: click<8.2,>=8.1
11
+ Requires-Dist: typer[all]<0.13,>=0.12
12
+ Requires-Dist: rich<14,>=13.7
13
+ Requires-Dist: pydantic<3,>=2.7
14
+ Requires-Dist: pyyaml<7,>=6.0.1
15
+ Requires-Dist: jinja2<4,>=3.1
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest<9,>=8.3; extra == "dev"
18
+ Requires-Dist: pytest-cov<6,>=5; extra == "dev"
19
+ Requires-Dist: ruff>=0.6; extra == "dev"
20
+ Dynamic: license-file
21
+
22
+ # Absconda
23
+
24
+ [![PyPI version](https://badge.fury.io/py/absconda.svg)](https://badge.fury.io/py/absconda)
25
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
26
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
27
+
28
+ **Turn conda environments into optimized container images for development and HPC deployment.**
29
+
30
+ Absconda bridges the gap between conda's reproducible environments and container-based workflows. Define your scientific computing environment once with conda, then deploy it anywhere—Docker, Singularity/Apptainer, HPC clusters—with production-ready optimizations.
31
+
32
+ ## Key Features
33
+
34
+ 🚀 **Multi-Stage Builds** - Automatic optimization that reduces image sizes by 40-60%
35
+ 🔐 **Policy Validation** - Enforce security and compliance rules organization-wide
36
+ 🏗️ **Remote Builders** - Offload builds to cloud instances with automatic provisioning
37
+ 🧪 **HPC Integration** - First-class Singularity support with module files and wrappers
38
+ 📦 **R + renv Support** - Combine conda environments with R package management
39
+ 🎯 **Flexible Deployment** - Multiple modes: full-env, tarball, requirements, export-explicit
40
+ 🔧 **Custom Templates** - Jinja2-based system for advanced customization
41
+
42
+ ## Quick Start
43
+
44
+ ### Installation
45
+
46
+ ```bash
47
+ # Install from PyPI
48
+ pip install absconda
49
+
50
+ # Or with pipx (recommended)
51
+ pipx install absconda
52
+
53
+ # Verify installation
54
+ absconda --version
55
+ ```
56
+
57
+ ### Basic Usage
58
+
59
+ **1. Create a conda environment file:**
60
+
61
+ ```yaml
62
+ # environment.yaml
63
+ name: my-analysis
64
+ channels:
65
+ - conda-forge
66
+ - bioconda
67
+ dependencies:
68
+ - python=3.11
69
+ - numpy=1.26
70
+ - pandas=2.1
71
+ - scikit-learn=1.3
72
+ ```
73
+
74
+ **2. Build a Docker image:**
75
+
76
+ ```bash
77
+ absconda build \
78
+ --file environment.yaml \
79
+ --repository ghcr.io/myorg/my-analysis \
80
+ --tag latest \
81
+ --push
82
+ ```
83
+
84
+ **3. Use the image:**
85
+
86
+ ```bash
87
+ docker run --rm ghcr.io/myorg/my-analysis:latest python -c "import numpy; print(numpy.__version__)"
88
+ ```
89
+
90
+ **For HPC with Singularity:**
91
+
92
+ ```bash
93
+ # Build Docker image and convert to Singularity
94
+ absconda build \
95
+ --file environment.yaml \
96
+ --repository ghcr.io/myorg/my-analysis \
97
+ --tag latest \
98
+ --push
99
+
100
+ # Convert to Singularity
101
+ absconda singularity \
102
+ --image ghcr.io/myorg/my-analysis:latest \
103
+ --output my-analysis.sif
104
+
105
+ # Generate HPC module
106
+ absconda module \
107
+ --image my-analysis.sif \
108
+ --module-path /apps/modules/my-analysis/1.0
109
+ ```
110
+
111
+ See the [Quick Start Guide](docs/getting-started/quickstart.md) for a complete walkthrough.
112
+
113
+ ## Documentation
114
+
115
+ ### For New Users
116
+
117
+ - **[Installation](docs/getting-started/installation.md)** - Get up and running
118
+ - **[Quick Start](docs/getting-started/quickstart.md)** - 5-minute tutorial
119
+ - **[Core Concepts](docs/getting-started/concepts.md)** - Understanding absconda
120
+
121
+ ### Guides
122
+
123
+ - **[Basic Usage](docs/guides/basic-usage.md)** - Essential workflows
124
+ - **[Building Images](docs/guides/building-images.md)** - Docker build process
125
+ - **[HPC Deployment](docs/guides/hpc-deployment.md)** - Singularity on HPC clusters
126
+ - **[Remote Builders](docs/guides/remote-builders.md)** - Cloud-based builds
127
+ - **[R + renv Integration](docs/guides/renv-integration.md)** - Combining conda and renv
128
+ - **[Requirements Mode](docs/guides/requirements-mode.md)** - Deployment mode comparison
129
+ - **[Advanced Templating](docs/guides/advanced-templating.md)** - Custom Dockerfiles
130
+
131
+ ### How-To Guides
132
+
133
+ - **[Multi-Stage Builds](docs/how-to/multi-stage-builds.md)** - Optimize image size
134
+ - **[Custom Base Images](docs/how-to/custom-base-images.md)** - GPU and specialized bases
135
+ - **[Secrets Management](docs/how-to/secrets-and-auth.md)** - Handle credentials safely
136
+ - **[CI/CD Integration](docs/how-to/ci-cd-integration.md)** - Automate with GitHub Actions
137
+
138
+ ### Examples
139
+
140
+ Complete working examples with explanations:
141
+
142
+ - **[Minimal Python](docs/examples/minimal-python.md)** - Simple Python environment
143
+ - **[Data Science Stack](docs/examples/data-science.md)** - NumPy/pandas/scikit-learn
144
+ - **[R + Bioconductor](docs/examples/r-bioconductor.md)** - RNA-seq analysis
145
+ - **[GPU + PyTorch](docs/examples/gpu-pytorch.md)** - Deep learning with CUDA
146
+ - **[HPC Workflow](docs/examples/hpc-singularity.md)** - Complete HPC deployment
147
+
148
+ ### Reference
149
+
150
+ - **[CLI Reference](docs/reference/cli.md)** - Complete command documentation
151
+ - **[Environment Files](docs/reference/environment-files.md)** - YAML specification
152
+ - **[Configuration](docs/reference/configuration.md)** - System configuration
153
+ - **[Policies](docs/reference/policies.md)** - Policy validation system
154
+
155
+ ### For Contributors
156
+
157
+ - **[Contributing](docs/development/contributing.md)** - How to contribute
158
+ - **[Testing](docs/development/testing.md)** - Running and writing tests
159
+ - **[Architecture](docs/architecture/)** - Technical design documentation
160
+
161
+ ## Why Absconda?
162
+
163
+ ### The Problem
164
+
165
+ Scientific computing has conflicting requirements:
166
+ - **Reproducibility**: Need exact package versions
167
+ - **Portability**: Must run on laptops, HPC clusters, cloud
168
+ - **Performance**: Large conda environments create huge containers
169
+ - **HPC Reality**: Singularity/Apptainer, not Docker; module systems, not Docker Compose
170
+
171
+ ### The Solution
172
+
173
+ Absconda solves this by:
174
+
175
+ 1. **Starting with conda** - Use the ecosystem you already know
176
+ 2. **Optimizing automatically** - Multi-stage builds reduce image sizes by 40-60%
177
+ 3. **Targeting HPC** - Native Singularity support with modules and wrappers
178
+ 4. **Enforcing policies** - Organization-wide security and compliance
179
+ 5. **Enabling remote builds** - Build on powerful cloud instances, not your laptop
180
+
181
+ ### Compared to Alternatives
182
+
183
+ | Feature | Absconda | repo2docker | docker-conda | Manual Dockerfile |
184
+ |---------|----------|-------------|--------------|-------------------|
185
+ | Multi-stage optimization | ✅ Automatic | ❌ No | ❌ No | ⚠️ Manual |
186
+ | Singularity integration | ✅ First-class | ❌ No | ❌ No | ⚠️ Manual |
187
+ | HPC modules | ✅ Built-in | ❌ No | ❌ No | ⚠️ Manual |
188
+ | Policy enforcement | ✅ Yes | ❌ No | ❌ No | ❌ No |
189
+ | Remote builders | ✅ Yes | ❌ No | ❌ No | ❌ No |
190
+ | R + renv support | ✅ Yes | ⚠️ Limited | ❌ No | ⚠️ Manual |
191
+ | Custom templates | ✅ Jinja2 | ❌ No | ❌ No | ✅ Full control |
192
+
193
+ ## Real-World Example
194
+
195
+ Complete workflow for deploying to NCI Gadi HPC:
196
+
197
+ ```bash
198
+ # 1. Build optimized Docker image (locally or on GCP)
199
+ absconda build \
200
+ --file rnaseq-env.yaml \
201
+ --repository ghcr.io/lab/rnaseq \
202
+ --tag v1.0 \
203
+ --remote-builder gcp-builder \
204
+ --push
205
+
206
+ # 2. Convert to Singularity on HPC
207
+ ssh gadi.nci.org.au
208
+ singularity pull rnaseq.sif docker://ghcr.io/lab/rnaseq:v1.0
209
+
210
+ # 3. Generate module file
211
+ absconda module \
212
+ --image /apps/rnaseq/v1.0/rnaseq.sif \
213
+ --module-path /apps/Modules/modulefiles/rnaseq/1.0 \
214
+ --wrapper-dir /apps/rnaseq/v1.0/wrappers
215
+
216
+ # 4. Use in PBS job
217
+ module load rnaseq/1.0
218
+ python analysis.py # Uses containerized environment transparently
219
+ ```
220
+
221
+ The result: reproducible, optimized, compliant environments deployed consistently from development through production.
222
+
223
+ ## Use Cases
224
+
225
+ ### 🔬 Research Computing
226
+
227
+ - Reproducible analysis pipelines
228
+ - Sharing environments with collaborators
229
+ - Publishing with computational papers
230
+ - Archive environments for long-term reproducibility
231
+
232
+ ### 🏢 Multi-User HPC
233
+
234
+ - Centralized environment management
235
+ - Policy enforcement across teams
236
+ - Module system integration
237
+ - Singularity deployment at scale
238
+
239
+ ### ☁️ Cloud + HPC Hybrid
240
+
241
+ - Build in cloud (fast, powerful VMs)
242
+ - Deploy to HPC (Singularity)
243
+ - Consistent environments across platforms
244
+ - Cost-optimized with remote builders
245
+
246
+ ### 🧬 Bioinformatics
247
+
248
+ - Bioconda + R/Bioconductor workflows
249
+ - GPU-accelerated analysis
250
+ - Large-scale genomics pipelines
251
+ - Compliance with data policies
252
+
253
+ ## Project Status
254
+
255
+ Absconda is production-ready and actively maintained. It powers scientific computing workflows for research teams at the Garvan Institute and beyond.
256
+
257
+ **Current version**: 0.1.0
258
+ **Python support**: 3.10, 3.11, 3.12
259
+ **License**: MIT
260
+
261
+ ## Getting Help
262
+
263
+ - **Documentation**: [docs/](docs/)
264
+ - **Issues**: [GitHub Issues](https://github.com/swarbricklab/absconda/issues)
265
+ - **Discussions**: [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
266
+
267
+ ## Contributing
268
+
269
+ Contributions welcome! See [Contributing Guide](docs/development/contributing.md).
270
+
271
+ **Areas needing help**:
272
+ - Documentation improvements
273
+ - Example workflows
274
+ - Testing on different platforms
275
+ - Feature requests and feedback
276
+
277
+ ## Acknowledgments
278
+
279
+ Developed at the [Garvan Institute of Medical Research](https://www.garvan.org.au/) by the Swarbrick Lab.
280
+
281
+ Built with:
282
+ - [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) - Fast conda package manager
283
+ - [Jinja2](https://jinja.palletsprojects.com/) - Template engine
284
+ - [Singularity/Apptainer](https://apptainer.org/) - HPC containers
285
+ - [Terraform](https://www.terraform.io/) - Infrastructure as code
286
+
287
+ ## License
288
+
289
+ MIT License - see [LICENSE](LICENSE) for details.
290
+
291
+ ---
292
+
293
+ **Ready to get started?** → [Installation Guide](docs/getting-started/installation.md)
294
+
295
+ **Have questions?** → [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
296
+
@@ -0,0 +1,275 @@
1
+ # Absconda
2
+
3
+ [![PyPI version](https://badge.fury.io/py/absconda.svg)](https://badge.fury.io/py/absconda)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
6
+
7
+ **Turn conda environments into optimized container images for development and HPC deployment.**
8
+
9
+ Absconda bridges the gap between conda's reproducible environments and container-based workflows. Define your scientific computing environment once with conda, then deploy it anywhere—Docker, Singularity/Apptainer, HPC clusters—with production-ready optimizations.
10
+
11
+ ## Key Features
12
+
13
+ 🚀 **Multi-Stage Builds** - Automatic optimization that reduces image sizes by 40-60%
14
+ 🔐 **Policy Validation** - Enforce security and compliance rules organization-wide
15
+ 🏗️ **Remote Builders** - Offload builds to cloud instances with automatic provisioning
16
+ 🧪 **HPC Integration** - First-class Singularity support with module files and wrappers
17
+ 📦 **R + renv Support** - Combine conda environments with R package management
18
+ 🎯 **Flexible Deployment** - Multiple modes: full-env, tarball, requirements, export-explicit
19
+ 🔧 **Custom Templates** - Jinja2-based system for advanced customization
20
+
21
+ ## Quick Start
22
+
23
+ ### Installation
24
+
25
+ ```bash
26
+ # Install from PyPI
27
+ pip install absconda
28
+
29
+ # Or with pipx (recommended)
30
+ pipx install absconda
31
+
32
+ # Verify installation
33
+ absconda --version
34
+ ```
35
+
36
+ ### Basic Usage
37
+
38
+ **1. Create a conda environment file:**
39
+
40
+ ```yaml
41
+ # environment.yaml
42
+ name: my-analysis
43
+ channels:
44
+ - conda-forge
45
+ - bioconda
46
+ dependencies:
47
+ - python=3.11
48
+ - numpy=1.26
49
+ - pandas=2.1
50
+ - scikit-learn=1.3
51
+ ```
52
+
53
+ **2. Build a Docker image:**
54
+
55
+ ```bash
56
+ absconda build \
57
+ --file environment.yaml \
58
+ --repository ghcr.io/myorg/my-analysis \
59
+ --tag latest \
60
+ --push
61
+ ```
62
+
63
+ **3. Use the image:**
64
+
65
+ ```bash
66
+ docker run --rm ghcr.io/myorg/my-analysis:latest python -c "import numpy; print(numpy.__version__)"
67
+ ```
68
+
69
+ **For HPC with Singularity:**
70
+
71
+ ```bash
72
+ # Build Docker image and convert to Singularity
73
+ absconda build \
74
+ --file environment.yaml \
75
+ --repository ghcr.io/myorg/my-analysis \
76
+ --tag latest \
77
+ --push
78
+
79
+ # Convert to Singularity
80
+ absconda singularity \
81
+ --image ghcr.io/myorg/my-analysis:latest \
82
+ --output my-analysis.sif
83
+
84
+ # Generate HPC module
85
+ absconda module \
86
+ --image my-analysis.sif \
87
+ --module-path /apps/modules/my-analysis/1.0
88
+ ```
89
+
90
+ See the [Quick Start Guide](docs/getting-started/quickstart.md) for a complete walkthrough.
91
+
92
+ ## Documentation
93
+
94
+ ### For New Users
95
+
96
+ - **[Installation](docs/getting-started/installation.md)** - Get up and running
97
+ - **[Quick Start](docs/getting-started/quickstart.md)** - 5-minute tutorial
98
+ - **[Core Concepts](docs/getting-started/concepts.md)** - Understanding absconda
99
+
100
+ ### Guides
101
+
102
+ - **[Basic Usage](docs/guides/basic-usage.md)** - Essential workflows
103
+ - **[Building Images](docs/guides/building-images.md)** - Docker build process
104
+ - **[HPC Deployment](docs/guides/hpc-deployment.md)** - Singularity on HPC clusters
105
+ - **[Remote Builders](docs/guides/remote-builders.md)** - Cloud-based builds
106
+ - **[R + renv Integration](docs/guides/renv-integration.md)** - Combining conda and renv
107
+ - **[Requirements Mode](docs/guides/requirements-mode.md)** - Deployment mode comparison
108
+ - **[Advanced Templating](docs/guides/advanced-templating.md)** - Custom Dockerfiles
109
+
110
+ ### How-To Guides
111
+
112
+ - **[Multi-Stage Builds](docs/how-to/multi-stage-builds.md)** - Optimize image size
113
+ - **[Custom Base Images](docs/how-to/custom-base-images.md)** - GPU and specialized bases
114
+ - **[Secrets Management](docs/how-to/secrets-and-auth.md)** - Handle credentials safely
115
+ - **[CI/CD Integration](docs/how-to/ci-cd-integration.md)** - Automate with GitHub Actions
116
+
117
+ ### Examples
118
+
119
+ Complete working examples with explanations:
120
+
121
+ - **[Minimal Python](docs/examples/minimal-python.md)** - Simple Python environment
122
+ - **[Data Science Stack](docs/examples/data-science.md)** - NumPy/pandas/scikit-learn
123
+ - **[R + Bioconductor](docs/examples/r-bioconductor.md)** - RNA-seq analysis
124
+ - **[GPU + PyTorch](docs/examples/gpu-pytorch.md)** - Deep learning with CUDA
125
+ - **[HPC Workflow](docs/examples/hpc-singularity.md)** - Complete HPC deployment
126
+
127
+ ### Reference
128
+
129
+ - **[CLI Reference](docs/reference/cli.md)** - Complete command documentation
130
+ - **[Environment Files](docs/reference/environment-files.md)** - YAML specification
131
+ - **[Configuration](docs/reference/configuration.md)** - System configuration
132
+ - **[Policies](docs/reference/policies.md)** - Policy validation system
133
+
134
+ ### For Contributors
135
+
136
+ - **[Contributing](docs/development/contributing.md)** - How to contribute
137
+ - **[Testing](docs/development/testing.md)** - Running and writing tests
138
+ - **[Architecture](docs/architecture/)** - Technical design documentation
139
+
140
+ ## Why Absconda?
141
+
142
+ ### The Problem
143
+
144
+ Scientific computing has conflicting requirements:
145
+ - **Reproducibility**: Need exact package versions
146
+ - **Portability**: Must run on laptops, HPC clusters, cloud
147
+ - **Performance**: Large conda environments create huge containers
148
+ - **HPC Reality**: Singularity/Apptainer, not Docker; module systems, not Docker Compose
149
+
150
+ ### The Solution
151
+
152
+ Absconda solves this by:
153
+
154
+ 1. **Starting with conda** - Use the ecosystem you already know
155
+ 2. **Optimizing automatically** - Multi-stage builds reduce image sizes by 40-60%
156
+ 3. **Targeting HPC** - Native Singularity support with modules and wrappers
157
+ 4. **Enforcing policies** - Organization-wide security and compliance
158
+ 5. **Enabling remote builds** - Build on powerful cloud instances, not your laptop
159
+
160
+ ### Compared to Alternatives
161
+
162
+ | Feature | Absconda | repo2docker | docker-conda | Manual Dockerfile |
163
+ |---------|----------|-------------|--------------|-------------------|
164
+ | Multi-stage optimization | ✅ Automatic | ❌ No | ❌ No | ⚠️ Manual |
165
+ | Singularity integration | ✅ First-class | ❌ No | ❌ No | ⚠️ Manual |
166
+ | HPC modules | ✅ Built-in | ❌ No | ❌ No | ⚠️ Manual |
167
+ | Policy enforcement | ✅ Yes | ❌ No | ❌ No | ❌ No |
168
+ | Remote builders | ✅ Yes | ❌ No | ❌ No | ❌ No |
169
+ | R + renv support | ✅ Yes | ⚠️ Limited | ❌ No | ⚠️ Manual |
170
+ | Custom templates | ✅ Jinja2 | ❌ No | ❌ No | ✅ Full control |
171
+
172
+ ## Real-World Example
173
+
174
+ Complete workflow for deploying to NCI Gadi HPC:
175
+
176
+ ```bash
177
+ # 1. Build optimized Docker image (locally or on GCP)
178
+ absconda build \
179
+ --file rnaseq-env.yaml \
180
+ --repository ghcr.io/lab/rnaseq \
181
+ --tag v1.0 \
182
+ --remote-builder gcp-builder \
183
+ --push
184
+
185
+ # 2. Convert to Singularity on HPC
186
+ ssh gadi.nci.org.au
187
+ singularity pull rnaseq.sif docker://ghcr.io/lab/rnaseq:v1.0
188
+
189
+ # 3. Generate module file
190
+ absconda module \
191
+ --image /apps/rnaseq/v1.0/rnaseq.sif \
192
+ --module-path /apps/Modules/modulefiles/rnaseq/1.0 \
193
+ --wrapper-dir /apps/rnaseq/v1.0/wrappers
194
+
195
+ # 4. Use in PBS job
196
+ module load rnaseq/1.0
197
+ python analysis.py # Uses containerized environment transparently
198
+ ```
199
+
200
+ The result: reproducible, optimized, compliant environments deployed consistently from development through production.
201
+
202
+ ## Use Cases
203
+
204
+ ### 🔬 Research Computing
205
+
206
+ - Reproducible analysis pipelines
207
+ - Sharing environments with collaborators
208
+ - Publishing with computational papers
209
+ - Archive environments for long-term reproducibility
210
+
211
+ ### 🏢 Multi-User HPC
212
+
213
+ - Centralized environment management
214
+ - Policy enforcement across teams
215
+ - Module system integration
216
+ - Singularity deployment at scale
217
+
218
+ ### ☁️ Cloud + HPC Hybrid
219
+
220
+ - Build in cloud (fast, powerful VMs)
221
+ - Deploy to HPC (Singularity)
222
+ - Consistent environments across platforms
223
+ - Cost-optimized with remote builders
224
+
225
+ ### 🧬 Bioinformatics
226
+
227
+ - Bioconda + R/Bioconductor workflows
228
+ - GPU-accelerated analysis
229
+ - Large-scale genomics pipelines
230
+ - Compliance with data policies
231
+
232
+ ## Project Status
233
+
234
+ Absconda is production-ready and actively maintained. It powers scientific computing workflows for research teams at the Garvan Institute and beyond.
235
+
236
+ **Current version**: 0.1.0
237
+ **Python support**: 3.10, 3.11, 3.12
238
+ **License**: MIT
239
+
240
+ ## Getting Help
241
+
242
+ - **Documentation**: [docs/](docs/)
243
+ - **Issues**: [GitHub Issues](https://github.com/swarbricklab/absconda/issues)
244
+ - **Discussions**: [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
245
+
246
+ ## Contributing
247
+
248
+ Contributions welcome! See [Contributing Guide](docs/development/contributing.md).
249
+
250
+ **Areas needing help**:
251
+ - Documentation improvements
252
+ - Example workflows
253
+ - Testing on different platforms
254
+ - Feature requests and feedback
255
+
256
+ ## Acknowledgments
257
+
258
+ Developed at the [Garvan Institute of Medical Research](https://www.garvan.org.au/) by the Swarbrick Lab.
259
+
260
+ Built with:
261
+ - [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) - Fast conda package manager
262
+ - [Jinja2](https://jinja.palletsprojects.com/) - Template engine
263
+ - [Singularity/Apptainer](https://apptainer.org/) - HPC containers
264
+ - [Terraform](https://www.terraform.io/) - Infrastructure as code
265
+
266
+ ## License
267
+
268
+ MIT License - see [LICENSE](LICENSE) for details.
269
+
270
+ ---
271
+
272
+ **Ready to get started?** → [Installation Guide](docs/getting-started/installation.md)
273
+
274
+ **Have questions?** → [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
275
+
@@ -0,0 +1,54 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "absconda"
7
+ version = "0.1.0"
8
+ description = "Generate Dockerfiles and container build workflows from Conda environments"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ authors = [{ name = "John Reeves" }]
12
+ license = { text = "MIT" }
13
+ dependencies = [
14
+ "click>=8.1,<8.2",
15
+ "typer[all]>=0.12,<0.13",
16
+ "rich>=13.7,<14",
17
+ "pydantic>=2.7,<3",
18
+ "pyyaml>=6.0.1,<7",
19
+ "jinja2>=3.1,<4",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ dev = [
24
+ "pytest>=8.3,<9",
25
+ "pytest-cov>=5,<6",
26
+ "ruff>=0.6",
27
+ ]
28
+
29
+ [project.scripts]
30
+ absconda = "absconda.cli:app"
31
+
32
+ [tool.setuptools.packages.find]
33
+ where = ["src"]
34
+
35
+ [tool.setuptools.package-data]
36
+ "absconda" = ["_templates/**/*.j2"]
37
+
38
+ [tool.pytest.ini_options]
39
+ addopts = "-ra"
40
+ pythonpath = ["src"]
41
+ markers = [
42
+ "integration: marks slower smoke tests that exercise external CLIs",
43
+ ]
44
+
45
+ [tool.ruff]
46
+ line-length = 100
47
+ src = ["src", "tests"]
48
+
49
+ [tool.ruff.lint]
50
+ select = ["E", "F", "I", "B"]
51
+
52
+ [tool.ruff.format]
53
+ quote-style = "double"
54
+ indent-style = "space"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,11 @@
1
+ """Absconda package metadata."""
2
+
3
+ from importlib.metadata import PackageNotFoundError
4
+ from importlib.metadata import version as _pkg_version
5
+
6
+ try:
7
+ __version__ = _pkg_version("absconda")
8
+ except PackageNotFoundError: # pragma: no cover - during local development
9
+ __version__ = "0.0.0"
10
+
11
+ __all__ = ["__version__"]
@@ -0,0 +1,4 @@
1
+ from .cli import app
2
+
3
+ if __name__ == "__main__": # pragma: no cover
4
+ app()
@@ -0,0 +1,2 @@
1
+ FROM scratch
2
+ # Placeholder to keep Dockerfile linters happy. Real template lives in main.j2.
@@ -0,0 +1,21 @@
1
+ # Generated by Absconda
2
+
3
+ {% if tarball_mode %}
4
+ {# Tarball mode: simpler runtime-only template #}
5
+ {% include "fragments/tarball_runtime.j2" %}
6
+ {% elif requirements_mode %}
7
+ {# Requirements mode: pip-based Python environment #}
8
+ {% include "fragments/requirements_runtime.j2" %}
9
+ {% elif multi_stage %}
10
+ {% include "fragments/builder_stage.j2" %}
11
+
12
+ {% include "fragments/runtime_stage.j2" %}
13
+ {% else %}
14
+ {% include "fragments/single_stage.j2" %}
15
+ {% endif %}
16
+
17
+ {% if not requirements_mode %}
18
+ {% include "fragments/export_block.j2" %}
19
+ {% endif %}
20
+
21
+ CMD {{ runtime_command }}