absconda 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- absconda-0.1.0/LICENSE +21 -0
- absconda-0.1.0/PKG-INFO +296 -0
- absconda-0.1.0/README.md +275 -0
- absconda-0.1.0/pyproject.toml +54 -0
- absconda-0.1.0/setup.cfg +4 -0
- absconda-0.1.0/src/absconda/__init__.py +11 -0
- absconda-0.1.0/src/absconda/__main__.py +4 -0
- absconda-0.1.0/src/absconda/_templates/default/Dockerfile.j2 +2 -0
- absconda-0.1.0/src/absconda/_templates/default/main.j2 +21 -0
- absconda-0.1.0/src/absconda/_templates/fragments/builder_stage.j2 +45 -0
- absconda-0.1.0/src/absconda/_templates/fragments/export_block.j2 +3 -0
- absconda-0.1.0/src/absconda/_templates/fragments/requirements_runtime.j2 +12 -0
- absconda-0.1.0/src/absconda/_templates/fragments/runtime_stage.j2 +17 -0
- absconda-0.1.0/src/absconda/_templates/fragments/single_stage.j2 +67 -0
- absconda-0.1.0/src/absconda/_templates/fragments/tarball_runtime.j2 +19 -0
- absconda-0.1.0/src/absconda/cli.py +1281 -0
- absconda-0.1.0/src/absconda/config.py +227 -0
- absconda-0.1.0/src/absconda/environment.py +339 -0
- absconda-0.1.0/src/absconda/modules.py +143 -0
- absconda-0.1.0/src/absconda/policy.py +259 -0
- absconda-0.1.0/src/absconda/remote.py +696 -0
- absconda-0.1.0/src/absconda/templates.py +179 -0
- absconda-0.1.0/src/absconda/wrappers.py +203 -0
- absconda-0.1.0/src/absconda.egg-info/PKG-INFO +296 -0
- absconda-0.1.0/src/absconda.egg-info/SOURCES.txt +36 -0
- absconda-0.1.0/src/absconda.egg-info/dependency_links.txt +1 -0
- absconda-0.1.0/src/absconda.egg-info/entry_points.txt +2 -0
- absconda-0.1.0/src/absconda.egg-info/requires.txt +11 -0
- absconda-0.1.0/src/absconda.egg-info/top_level.txt +1 -0
- absconda-0.1.0/tests/test_cli.py +391 -0
- absconda-0.1.0/tests/test_environment.py +51 -0
- absconda-0.1.0/tests/test_examples.py +41 -0
- absconda-0.1.0/tests/test_integration_smoke.py +214 -0
- absconda-0.1.0/tests/test_modules.py +273 -0
- absconda-0.1.0/tests/test_policy.py +113 -0
- absconda-0.1.0/tests/test_remote.py +188 -0
- absconda-0.1.0/tests/test_templates.py +98 -0
- absconda-0.1.0/tests/test_wrappers.py +236 -0
absconda-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Garvan Institute of Medical Research
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
absconda-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: absconda
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Generate Dockerfiles and container build workflows from Conda environments
|
|
5
|
+
Author: John Reeves
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: click<8.2,>=8.1
|
|
11
|
+
Requires-Dist: typer[all]<0.13,>=0.12
|
|
12
|
+
Requires-Dist: rich<14,>=13.7
|
|
13
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
14
|
+
Requires-Dist: pyyaml<7,>=6.0.1
|
|
15
|
+
Requires-Dist: jinja2<4,>=3.1
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest<9,>=8.3; extra == "dev"
|
|
18
|
+
Requires-Dist: pytest-cov<6,>=5; extra == "dev"
|
|
19
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# Absconda
|
|
23
|
+
|
|
24
|
+
[](https://badge.fury.io/py/absconda)
|
|
25
|
+
[](https://opensource.org/licenses/MIT)
|
|
26
|
+
[](https://www.python.org/downloads/)
|
|
27
|
+
|
|
28
|
+
**Turn conda environments into optimized container images for development and HPC deployment.**
|
|
29
|
+
|
|
30
|
+
Absconda bridges the gap between conda's reproducible environments and container-based workflows. Define your scientific computing environment once with conda, then deploy it anywhere—Docker, Singularity/Apptainer, HPC clusters—with production-ready optimizations.
|
|
31
|
+
|
|
32
|
+
## Key Features
|
|
33
|
+
|
|
34
|
+
🚀 **Multi-Stage Builds** - Automatic optimization that reduces image sizes by 40-60%
|
|
35
|
+
🔐 **Policy Validation** - Enforce security and compliance rules organization-wide
|
|
36
|
+
🏗️ **Remote Builders** - Offload builds to cloud instances with automatic provisioning
|
|
37
|
+
🧪 **HPC Integration** - First-class Singularity support with module files and wrappers
|
|
38
|
+
📦 **R + renv Support** - Combine conda environments with R package management
|
|
39
|
+
🎯 **Flexible Deployment** - Multiple modes: full-env, tarball, requirements, export-explicit
|
|
40
|
+
🔧 **Custom Templates** - Jinja2-based system for advanced customization
|
|
41
|
+
|
|
42
|
+
## Quick Start
|
|
43
|
+
|
|
44
|
+
### Installation
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# Install from PyPI
|
|
48
|
+
pip install absconda
|
|
49
|
+
|
|
50
|
+
# Or with pipx (recommended)
|
|
51
|
+
pipx install absconda
|
|
52
|
+
|
|
53
|
+
# Verify installation
|
|
54
|
+
absconda --version
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Basic Usage
|
|
58
|
+
|
|
59
|
+
**1. Create a conda environment file:**
|
|
60
|
+
|
|
61
|
+
```yaml
|
|
62
|
+
# environment.yaml
|
|
63
|
+
name: my-analysis
|
|
64
|
+
channels:
|
|
65
|
+
- conda-forge
|
|
66
|
+
- bioconda
|
|
67
|
+
dependencies:
|
|
68
|
+
- python=3.11
|
|
69
|
+
- numpy=1.26
|
|
70
|
+
- pandas=2.1
|
|
71
|
+
- scikit-learn=1.3
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**2. Build a Docker image:**
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
absconda build \
|
|
78
|
+
--file environment.yaml \
|
|
79
|
+
--repository ghcr.io/myorg/my-analysis \
|
|
80
|
+
--tag latest \
|
|
81
|
+
--push
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**3. Use the image:**
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
docker run --rm ghcr.io/myorg/my-analysis:latest python -c "import numpy; print(numpy.__version__)"
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
**For HPC with Singularity:**
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# Build Docker image and convert to Singularity
|
|
94
|
+
absconda build \
|
|
95
|
+
--file environment.yaml \
|
|
96
|
+
--repository ghcr.io/myorg/my-analysis \
|
|
97
|
+
--tag latest \
|
|
98
|
+
--push
|
|
99
|
+
|
|
100
|
+
# Convert to Singularity
|
|
101
|
+
absconda singularity \
|
|
102
|
+
--image ghcr.io/myorg/my-analysis:latest \
|
|
103
|
+
--output my-analysis.sif
|
|
104
|
+
|
|
105
|
+
# Generate HPC module
|
|
106
|
+
absconda module \
|
|
107
|
+
--image my-analysis.sif \
|
|
108
|
+
--module-path /apps/modules/my-analysis/1.0
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
See the [Quick Start Guide](docs/getting-started/quickstart.md) for a complete walkthrough.
|
|
112
|
+
|
|
113
|
+
## Documentation
|
|
114
|
+
|
|
115
|
+
### For New Users
|
|
116
|
+
|
|
117
|
+
- **[Installation](docs/getting-started/installation.md)** - Get up and running
|
|
118
|
+
- **[Quick Start](docs/getting-started/quickstart.md)** - 5-minute tutorial
|
|
119
|
+
- **[Core Concepts](docs/getting-started/concepts.md)** - Understanding absconda
|
|
120
|
+
|
|
121
|
+
### Guides
|
|
122
|
+
|
|
123
|
+
- **[Basic Usage](docs/guides/basic-usage.md)** - Essential workflows
|
|
124
|
+
- **[Building Images](docs/guides/building-images.md)** - Docker build process
|
|
125
|
+
- **[HPC Deployment](docs/guides/hpc-deployment.md)** - Singularity on HPC clusters
|
|
126
|
+
- **[Remote Builders](docs/guides/remote-builders.md)** - Cloud-based builds
|
|
127
|
+
- **[R + renv Integration](docs/guides/renv-integration.md)** - Combining conda and renv
|
|
128
|
+
- **[Requirements Mode](docs/guides/requirements-mode.md)** - Deployment mode comparison
|
|
129
|
+
- **[Advanced Templating](docs/guides/advanced-templating.md)** - Custom Dockerfiles
|
|
130
|
+
|
|
131
|
+
### How-To Guides
|
|
132
|
+
|
|
133
|
+
- **[Multi-Stage Builds](docs/how-to/multi-stage-builds.md)** - Optimize image size
|
|
134
|
+
- **[Custom Base Images](docs/how-to/custom-base-images.md)** - GPU and specialized bases
|
|
135
|
+
- **[Secrets Management](docs/how-to/secrets-and-auth.md)** - Handle credentials safely
|
|
136
|
+
- **[CI/CD Integration](docs/how-to/ci-cd-integration.md)** - Automate with GitHub Actions
|
|
137
|
+
|
|
138
|
+
### Examples
|
|
139
|
+
|
|
140
|
+
Complete working examples with explanations:
|
|
141
|
+
|
|
142
|
+
- **[Minimal Python](docs/examples/minimal-python.md)** - Simple Python environment
|
|
143
|
+
- **[Data Science Stack](docs/examples/data-science.md)** - NumPy/pandas/scikit-learn
|
|
144
|
+
- **[R + Bioconductor](docs/examples/r-bioconductor.md)** - RNA-seq analysis
|
|
145
|
+
- **[GPU + PyTorch](docs/examples/gpu-pytorch.md)** - Deep learning with CUDA
|
|
146
|
+
- **[HPC Workflow](docs/examples/hpc-singularity.md)** - Complete HPC deployment
|
|
147
|
+
|
|
148
|
+
### Reference
|
|
149
|
+
|
|
150
|
+
- **[CLI Reference](docs/reference/cli.md)** - Complete command documentation
|
|
151
|
+
- **[Environment Files](docs/reference/environment-files.md)** - YAML specification
|
|
152
|
+
- **[Configuration](docs/reference/configuration.md)** - System configuration
|
|
153
|
+
- **[Policies](docs/reference/policies.md)** - Policy validation system
|
|
154
|
+
|
|
155
|
+
### For Contributors
|
|
156
|
+
|
|
157
|
+
- **[Contributing](docs/development/contributing.md)** - How to contribute
|
|
158
|
+
- **[Testing](docs/development/testing.md)** - Running and writing tests
|
|
159
|
+
- **[Architecture](docs/architecture/)** - Technical design documentation
|
|
160
|
+
|
|
161
|
+
## Why Absconda?
|
|
162
|
+
|
|
163
|
+
### The Problem
|
|
164
|
+
|
|
165
|
+
Scientific computing has conflicting requirements:
|
|
166
|
+
- **Reproducibility**: Need exact package versions
|
|
167
|
+
- **Portability**: Must run on laptops, HPC clusters, cloud
|
|
168
|
+
- **Performance**: Large conda environments create huge containers
|
|
169
|
+
- **HPC Reality**: Singularity/Apptainer, not Docker; module systems, not Docker Compose
|
|
170
|
+
|
|
171
|
+
### The Solution
|
|
172
|
+
|
|
173
|
+
Absconda solves this by:
|
|
174
|
+
|
|
175
|
+
1. **Starting with conda** - Use the ecosystem you already know
|
|
176
|
+
2. **Optimizing automatically** - Multi-stage builds reduce image sizes by 40-60%
|
|
177
|
+
3. **Targeting HPC** - Native Singularity support with modules and wrappers
|
|
178
|
+
4. **Enforcing policies** - Organization-wide security and compliance
|
|
179
|
+
5. **Enabling remote builds** - Build on powerful cloud instances, not your laptop
|
|
180
|
+
|
|
181
|
+
### Compared to Alternatives
|
|
182
|
+
|
|
183
|
+
| Feature | Absconda | repo2docker | docker-conda | Manual Dockerfile |
|
|
184
|
+
|---------|----------|-------------|--------------|-------------------|
|
|
185
|
+
| Multi-stage optimization | ✅ Automatic | ❌ No | ❌ No | ⚠️ Manual |
|
|
186
|
+
| Singularity integration | ✅ First-class | ❌ No | ❌ No | ⚠️ Manual |
|
|
187
|
+
| HPC modules | ✅ Built-in | ❌ No | ❌ No | ⚠️ Manual |
|
|
188
|
+
| Policy enforcement | ✅ Yes | ❌ No | ❌ No | ❌ No |
|
|
189
|
+
| Remote builders | ✅ Yes | ❌ No | ❌ No | ❌ No |
|
|
190
|
+
| R + renv support | ✅ Yes | ⚠️ Limited | ❌ No | ⚠️ Manual |
|
|
191
|
+
| Custom templates | ✅ Jinja2 | ❌ No | ❌ No | ✅ Full control |
|
|
192
|
+
|
|
193
|
+
## Real-World Example
|
|
194
|
+
|
|
195
|
+
Complete workflow for deploying to NCI Gadi HPC:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# 1. Build optimized Docker image (locally or on GCP)
|
|
199
|
+
absconda build \
|
|
200
|
+
--file rnaseq-env.yaml \
|
|
201
|
+
--repository ghcr.io/lab/rnaseq \
|
|
202
|
+
--tag v1.0 \
|
|
203
|
+
--remote-builder gcp-builder \
|
|
204
|
+
--push
|
|
205
|
+
|
|
206
|
+
# 2. Convert to Singularity on HPC
|
|
207
|
+
ssh gadi.nci.org.au
|
|
208
|
+
singularity pull rnaseq.sif docker://ghcr.io/lab/rnaseq:v1.0
|
|
209
|
+
|
|
210
|
+
# 3. Generate module file
|
|
211
|
+
absconda module \
|
|
212
|
+
--image /apps/rnaseq/v1.0/rnaseq.sif \
|
|
213
|
+
--module-path /apps/Modules/modulefiles/rnaseq/1.0 \
|
|
214
|
+
--wrapper-dir /apps/rnaseq/v1.0/wrappers
|
|
215
|
+
|
|
216
|
+
# 4. Use in PBS job
|
|
217
|
+
module load rnaseq/1.0
|
|
218
|
+
python analysis.py # Uses containerized environment transparently
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
The result: reproducible, optimized, compliant environments deployed consistently from development through production.
|
|
222
|
+
|
|
223
|
+
## Use Cases
|
|
224
|
+
|
|
225
|
+
### 🔬 Research Computing
|
|
226
|
+
|
|
227
|
+
- Reproducible analysis pipelines
|
|
228
|
+
- Sharing environments with collaborators
|
|
229
|
+
- Publishing with computational papers
|
|
230
|
+
- Archive environments for long-term reproducibility
|
|
231
|
+
|
|
232
|
+
### 🏢 Multi-User HPC
|
|
233
|
+
|
|
234
|
+
- Centralized environment management
|
|
235
|
+
- Policy enforcement across teams
|
|
236
|
+
- Module system integration
|
|
237
|
+
- Singularity deployment at scale
|
|
238
|
+
|
|
239
|
+
### ☁️ Cloud + HPC Hybrid
|
|
240
|
+
|
|
241
|
+
- Build in cloud (fast, powerful VMs)
|
|
242
|
+
- Deploy to HPC (Singularity)
|
|
243
|
+
- Consistent environments across platforms
|
|
244
|
+
- Cost-optimized with remote builders
|
|
245
|
+
|
|
246
|
+
### 🧬 Bioinformatics
|
|
247
|
+
|
|
248
|
+
- Bioconda + R/Bioconductor workflows
|
|
249
|
+
- GPU-accelerated analysis
|
|
250
|
+
- Large-scale genomics pipelines
|
|
251
|
+
- Compliance with data policies
|
|
252
|
+
|
|
253
|
+
## Project Status
|
|
254
|
+
|
|
255
|
+
Absconda is production-ready and actively maintained. It powers scientific computing workflows for research teams at the Garvan Institute and beyond.
|
|
256
|
+
|
|
257
|
+
**Current version**: 0.1.0
|
|
258
|
+
**Python support**: 3.10, 3.11, 3.12
|
|
259
|
+
**License**: MIT
|
|
260
|
+
|
|
261
|
+
## Getting Help
|
|
262
|
+
|
|
263
|
+
- **Documentation**: [docs/](docs/)
|
|
264
|
+
- **Issues**: [GitHub Issues](https://github.com/swarbricklab/absconda/issues)
|
|
265
|
+
- **Discussions**: [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
|
|
266
|
+
|
|
267
|
+
## Contributing
|
|
268
|
+
|
|
269
|
+
Contributions welcome! See [Contributing Guide](docs/development/contributing.md).
|
|
270
|
+
|
|
271
|
+
**Areas needing help**:
|
|
272
|
+
- Documentation improvements
|
|
273
|
+
- Example workflows
|
|
274
|
+
- Testing on different platforms
|
|
275
|
+
- Feature requests and feedback
|
|
276
|
+
|
|
277
|
+
## Acknowledgments
|
|
278
|
+
|
|
279
|
+
Developed at the [Garvan Institute of Medical Research](https://www.garvan.org.au/) by the Swarbrick Lab.
|
|
280
|
+
|
|
281
|
+
Built with:
|
|
282
|
+
- [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) - Fast conda package manager
|
|
283
|
+
- [Jinja2](https://jinja.palletsprojects.com/) - Template engine
|
|
284
|
+
- [Singularity/Apptainer](https://apptainer.org/) - HPC containers
|
|
285
|
+
- [Terraform](https://www.terraform.io/) - Infrastructure as code
|
|
286
|
+
|
|
287
|
+
## License
|
|
288
|
+
|
|
289
|
+
MIT License - see [LICENSE](LICENSE) for details.
|
|
290
|
+
|
|
291
|
+
---
|
|
292
|
+
|
|
293
|
+
**Ready to get started?** → [Installation Guide](docs/getting-started/installation.md)
|
|
294
|
+
|
|
295
|
+
**Have questions?** → [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
|
|
296
|
+
|
absconda-0.1.0/README.md
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
# Absconda
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/py/absconda)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
|
|
7
|
+
**Turn conda environments into optimized container images for development and HPC deployment.**
|
|
8
|
+
|
|
9
|
+
Absconda bridges the gap between conda's reproducible environments and container-based workflows. Define your scientific computing environment once with conda, then deploy it anywhere—Docker, Singularity/Apptainer, HPC clusters—with production-ready optimizations.
|
|
10
|
+
|
|
11
|
+
## Key Features
|
|
12
|
+
|
|
13
|
+
🚀 **Multi-Stage Builds** - Automatic optimization that reduces image sizes by 40-60%
|
|
14
|
+
🔐 **Policy Validation** - Enforce security and compliance rules organization-wide
|
|
15
|
+
🏗️ **Remote Builders** - Offload builds to cloud instances with automatic provisioning
|
|
16
|
+
🧪 **HPC Integration** - First-class Singularity support with module files and wrappers
|
|
17
|
+
📦 **R + renv Support** - Combine conda environments with R package management
|
|
18
|
+
🎯 **Flexible Deployment** - Multiple modes: full-env, tarball, requirements, export-explicit
|
|
19
|
+
🔧 **Custom Templates** - Jinja2-based system for advanced customization
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
### Installation
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# Install from PyPI
|
|
27
|
+
pip install absconda
|
|
28
|
+
|
|
29
|
+
# Or with pipx (recommended)
|
|
30
|
+
pipx install absconda
|
|
31
|
+
|
|
32
|
+
# Verify installation
|
|
33
|
+
absconda --version
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### Basic Usage
|
|
37
|
+
|
|
38
|
+
**1. Create a conda environment file:**
|
|
39
|
+
|
|
40
|
+
```yaml
|
|
41
|
+
# environment.yaml
|
|
42
|
+
name: my-analysis
|
|
43
|
+
channels:
|
|
44
|
+
- conda-forge
|
|
45
|
+
- bioconda
|
|
46
|
+
dependencies:
|
|
47
|
+
- python=3.11
|
|
48
|
+
- numpy=1.26
|
|
49
|
+
- pandas=2.1
|
|
50
|
+
- scikit-learn=1.3
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**2. Build a Docker image:**
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
absconda build \
|
|
57
|
+
--file environment.yaml \
|
|
58
|
+
--repository ghcr.io/myorg/my-analysis \
|
|
59
|
+
--tag latest \
|
|
60
|
+
--push
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
**3. Use the image:**
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
docker run --rm ghcr.io/myorg/my-analysis:latest python -c "import numpy; print(numpy.__version__)"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
**For HPC with Singularity:**
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# Build Docker image and convert to Singularity
|
|
73
|
+
absconda build \
|
|
74
|
+
--file environment.yaml \
|
|
75
|
+
--repository ghcr.io/myorg/my-analysis \
|
|
76
|
+
--tag latest \
|
|
77
|
+
--push
|
|
78
|
+
|
|
79
|
+
# Convert to Singularity
|
|
80
|
+
absconda singularity \
|
|
81
|
+
--image ghcr.io/myorg/my-analysis:latest \
|
|
82
|
+
--output my-analysis.sif
|
|
83
|
+
|
|
84
|
+
# Generate HPC module
|
|
85
|
+
absconda module \
|
|
86
|
+
--image my-analysis.sif \
|
|
87
|
+
--module-path /apps/modules/my-analysis/1.0
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
See the [Quick Start Guide](docs/getting-started/quickstart.md) for a complete walkthrough.
|
|
91
|
+
|
|
92
|
+
## Documentation
|
|
93
|
+
|
|
94
|
+
### For New Users
|
|
95
|
+
|
|
96
|
+
- **[Installation](docs/getting-started/installation.md)** - Get up and running
|
|
97
|
+
- **[Quick Start](docs/getting-started/quickstart.md)** - 5-minute tutorial
|
|
98
|
+
- **[Core Concepts](docs/getting-started/concepts.md)** - Understanding absconda
|
|
99
|
+
|
|
100
|
+
### Guides
|
|
101
|
+
|
|
102
|
+
- **[Basic Usage](docs/guides/basic-usage.md)** - Essential workflows
|
|
103
|
+
- **[Building Images](docs/guides/building-images.md)** - Docker build process
|
|
104
|
+
- **[HPC Deployment](docs/guides/hpc-deployment.md)** - Singularity on HPC clusters
|
|
105
|
+
- **[Remote Builders](docs/guides/remote-builders.md)** - Cloud-based builds
|
|
106
|
+
- **[R + renv Integration](docs/guides/renv-integration.md)** - Combining conda and renv
|
|
107
|
+
- **[Requirements Mode](docs/guides/requirements-mode.md)** - Deployment mode comparison
|
|
108
|
+
- **[Advanced Templating](docs/guides/advanced-templating.md)** - Custom Dockerfiles
|
|
109
|
+
|
|
110
|
+
### How-To Guides
|
|
111
|
+
|
|
112
|
+
- **[Multi-Stage Builds](docs/how-to/multi-stage-builds.md)** - Optimize image size
|
|
113
|
+
- **[Custom Base Images](docs/how-to/custom-base-images.md)** - GPU and specialized bases
|
|
114
|
+
- **[Secrets Management](docs/how-to/secrets-and-auth.md)** - Handle credentials safely
|
|
115
|
+
- **[CI/CD Integration](docs/how-to/ci-cd-integration.md)** - Automate with GitHub Actions
|
|
116
|
+
|
|
117
|
+
### Examples
|
|
118
|
+
|
|
119
|
+
Complete working examples with explanations:
|
|
120
|
+
|
|
121
|
+
- **[Minimal Python](docs/examples/minimal-python.md)** - Simple Python environment
|
|
122
|
+
- **[Data Science Stack](docs/examples/data-science.md)** - NumPy/pandas/scikit-learn
|
|
123
|
+
- **[R + Bioconductor](docs/examples/r-bioconductor.md)** - RNA-seq analysis
|
|
124
|
+
- **[GPU + PyTorch](docs/examples/gpu-pytorch.md)** - Deep learning with CUDA
|
|
125
|
+
- **[HPC Workflow](docs/examples/hpc-singularity.md)** - Complete HPC deployment
|
|
126
|
+
|
|
127
|
+
### Reference
|
|
128
|
+
|
|
129
|
+
- **[CLI Reference](docs/reference/cli.md)** - Complete command documentation
|
|
130
|
+
- **[Environment Files](docs/reference/environment-files.md)** - YAML specification
|
|
131
|
+
- **[Configuration](docs/reference/configuration.md)** - System configuration
|
|
132
|
+
- **[Policies](docs/reference/policies.md)** - Policy validation system
|
|
133
|
+
|
|
134
|
+
### For Contributors
|
|
135
|
+
|
|
136
|
+
- **[Contributing](docs/development/contributing.md)** - How to contribute
|
|
137
|
+
- **[Testing](docs/development/testing.md)** - Running and writing tests
|
|
138
|
+
- **[Architecture](docs/architecture/)** - Technical design documentation
|
|
139
|
+
|
|
140
|
+
## Why Absconda?
|
|
141
|
+
|
|
142
|
+
### The Problem
|
|
143
|
+
|
|
144
|
+
Scientific computing has conflicting requirements:
|
|
145
|
+
- **Reproducibility**: Need exact package versions
|
|
146
|
+
- **Portability**: Must run on laptops, HPC clusters, cloud
|
|
147
|
+
- **Performance**: Large conda environments create huge containers
|
|
148
|
+
- **HPC Reality**: Singularity/Apptainer, not Docker; module systems, not Docker Compose
|
|
149
|
+
|
|
150
|
+
### The Solution
|
|
151
|
+
|
|
152
|
+
Absconda solves this by:
|
|
153
|
+
|
|
154
|
+
1. **Starting with conda** - Use the ecosystem you already know
|
|
155
|
+
2. **Optimizing automatically** - Multi-stage builds reduce image sizes by 40-60%
|
|
156
|
+
3. **Targeting HPC** - Native Singularity support with modules and wrappers
|
|
157
|
+
4. **Enforcing policies** - Organization-wide security and compliance
|
|
158
|
+
5. **Enabling remote builds** - Build on powerful cloud instances, not your laptop
|
|
159
|
+
|
|
160
|
+
### Compared to Alternatives
|
|
161
|
+
|
|
162
|
+
| Feature | Absconda | repo2docker | docker-conda | Manual Dockerfile |
|
|
163
|
+
|---------|----------|-------------|--------------|-------------------|
|
|
164
|
+
| Multi-stage optimization | ✅ Automatic | ❌ No | ❌ No | ⚠️ Manual |
|
|
165
|
+
| Singularity integration | ✅ First-class | ❌ No | ❌ No | ⚠️ Manual |
|
|
166
|
+
| HPC modules | ✅ Built-in | ❌ No | ❌ No | ⚠️ Manual |
|
|
167
|
+
| Policy enforcement | ✅ Yes | ❌ No | ❌ No | ❌ No |
|
|
168
|
+
| Remote builders | ✅ Yes | ❌ No | ❌ No | ❌ No |
|
|
169
|
+
| R + renv support | ✅ Yes | ⚠️ Limited | ❌ No | ⚠️ Manual |
|
|
170
|
+
| Custom templates | ✅ Jinja2 | ❌ No | ❌ No | ✅ Full control |
|
|
171
|
+
|
|
172
|
+
## Real-World Example
|
|
173
|
+
|
|
174
|
+
Complete workflow for deploying to NCI Gadi HPC:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
# 1. Build optimized Docker image (locally or on GCP)
|
|
178
|
+
absconda build \
|
|
179
|
+
--file rnaseq-env.yaml \
|
|
180
|
+
--repository ghcr.io/lab/rnaseq \
|
|
181
|
+
--tag v1.0 \
|
|
182
|
+
--remote-builder gcp-builder \
|
|
183
|
+
--push
|
|
184
|
+
|
|
185
|
+
# 2. Convert to Singularity on HPC
|
|
186
|
+
ssh gadi.nci.org.au
|
|
187
|
+
singularity pull rnaseq.sif docker://ghcr.io/lab/rnaseq:v1.0
|
|
188
|
+
|
|
189
|
+
# 3. Generate module file
|
|
190
|
+
absconda module \
|
|
191
|
+
--image /apps/rnaseq/v1.0/rnaseq.sif \
|
|
192
|
+
--module-path /apps/Modules/modulefiles/rnaseq/1.0 \
|
|
193
|
+
--wrapper-dir /apps/rnaseq/v1.0/wrappers
|
|
194
|
+
|
|
195
|
+
# 4. Use in PBS job
|
|
196
|
+
module load rnaseq/1.0
|
|
197
|
+
python analysis.py # Uses containerized environment transparently
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
The result: reproducible, optimized, compliant environments deployed consistently from development through production.
|
|
201
|
+
|
|
202
|
+
## Use Cases
|
|
203
|
+
|
|
204
|
+
### 🔬 Research Computing
|
|
205
|
+
|
|
206
|
+
- Reproducible analysis pipelines
|
|
207
|
+
- Sharing environments with collaborators
|
|
208
|
+
- Publishing with computational papers
|
|
209
|
+
- Archive environments for long-term reproducibility
|
|
210
|
+
|
|
211
|
+
### 🏢 Multi-User HPC
|
|
212
|
+
|
|
213
|
+
- Centralized environment management
|
|
214
|
+
- Policy enforcement across teams
|
|
215
|
+
- Module system integration
|
|
216
|
+
- Singularity deployment at scale
|
|
217
|
+
|
|
218
|
+
### ☁️ Cloud + HPC Hybrid
|
|
219
|
+
|
|
220
|
+
- Build in cloud (fast, powerful VMs)
|
|
221
|
+
- Deploy to HPC (Singularity)
|
|
222
|
+
- Consistent environments across platforms
|
|
223
|
+
- Cost-optimized with remote builders
|
|
224
|
+
|
|
225
|
+
### 🧬 Bioinformatics
|
|
226
|
+
|
|
227
|
+
- Bioconda + R/Bioconductor workflows
|
|
228
|
+
- GPU-accelerated analysis
|
|
229
|
+
- Large-scale genomics pipelines
|
|
230
|
+
- Compliance with data policies
|
|
231
|
+
|
|
232
|
+
## Project Status
|
|
233
|
+
|
|
234
|
+
Absconda is production-ready and actively maintained. It powers scientific computing workflows for research teams at the Garvan Institute and beyond.
|
|
235
|
+
|
|
236
|
+
**Current version**: 0.1.0
|
|
237
|
+
**Python support**: 3.10, 3.11, 3.12
|
|
238
|
+
**License**: MIT
|
|
239
|
+
|
|
240
|
+
## Getting Help
|
|
241
|
+
|
|
242
|
+
- **Documentation**: [docs/](docs/)
|
|
243
|
+
- **Issues**: [GitHub Issues](https://github.com/swarbricklab/absconda/issues)
|
|
244
|
+
- **Discussions**: [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
|
|
245
|
+
|
|
246
|
+
## Contributing
|
|
247
|
+
|
|
248
|
+
Contributions welcome! See [Contributing Guide](docs/development/contributing.md).
|
|
249
|
+
|
|
250
|
+
**Areas needing help**:
|
|
251
|
+
- Documentation improvements
|
|
252
|
+
- Example workflows
|
|
253
|
+
- Testing on different platforms
|
|
254
|
+
- Feature requests and feedback
|
|
255
|
+
|
|
256
|
+
## Acknowledgments
|
|
257
|
+
|
|
258
|
+
Developed at the [Garvan Institute of Medical Research](https://www.garvan.org.au/) by the Swarbrick Lab.
|
|
259
|
+
|
|
260
|
+
Built with:
|
|
261
|
+
- [micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) - Fast conda package manager
|
|
262
|
+
- [Jinja2](https://jinja.palletsprojects.com/) - Template engine
|
|
263
|
+
- [Singularity/Apptainer](https://apptainer.org/) - HPC containers
|
|
264
|
+
- [Terraform](https://www.terraform.io/) - Infrastructure as code
|
|
265
|
+
|
|
266
|
+
## License
|
|
267
|
+
|
|
268
|
+
MIT License - see [LICENSE](LICENSE) for details.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
**Ready to get started?** → [Installation Guide](docs/getting-started/installation.md)
|
|
273
|
+
|
|
274
|
+
**Have questions?** → [GitHub Discussions](https://github.com/swarbricklab/absconda/discussions)
|
|
275
|
+
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "absconda"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Generate Dockerfiles and container build workflows from Conda environments"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [{ name = "John Reeves" }]
|
|
12
|
+
license = { text = "MIT" }
|
|
13
|
+
dependencies = [
|
|
14
|
+
"click>=8.1,<8.2",
|
|
15
|
+
"typer[all]>=0.12,<0.13",
|
|
16
|
+
"rich>=13.7,<14",
|
|
17
|
+
"pydantic>=2.7,<3",
|
|
18
|
+
"pyyaml>=6.0.1,<7",
|
|
19
|
+
"jinja2>=3.1,<4",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
dev = [
|
|
24
|
+
"pytest>=8.3,<9",
|
|
25
|
+
"pytest-cov>=5,<6",
|
|
26
|
+
"ruff>=0.6",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
absconda = "absconda.cli:app"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.package-data]
|
|
36
|
+
"absconda" = ["_templates/**/*.j2"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
addopts = "-ra"
|
|
40
|
+
pythonpath = ["src"]
|
|
41
|
+
markers = [
|
|
42
|
+
"integration: marks slower smoke tests that exercise external CLIs",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.ruff]
|
|
46
|
+
line-length = 100
|
|
47
|
+
src = ["src", "tests"]
|
|
48
|
+
|
|
49
|
+
[tool.ruff.lint]
|
|
50
|
+
select = ["E", "F", "I", "B"]
|
|
51
|
+
|
|
52
|
+
[tool.ruff.format]
|
|
53
|
+
quote-style = "double"
|
|
54
|
+
indent-style = "space"
|
absconda-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Absconda package metadata."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError
|
|
4
|
+
from importlib.metadata import version as _pkg_version
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
__version__ = _pkg_version("absconda")
|
|
8
|
+
except PackageNotFoundError: # pragma: no cover - during local development
|
|
9
|
+
__version__ = "0.0.0"
|
|
10
|
+
|
|
11
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Generated by Absconda
|
|
2
|
+
|
|
3
|
+
{% if tarball_mode %}
|
|
4
|
+
{# Tarball mode: simpler runtime-only template #}
|
|
5
|
+
{% include "fragments/tarball_runtime.j2" %}
|
|
6
|
+
{% elif requirements_mode %}
|
|
7
|
+
{# Requirements mode: pip-based Python environment #}
|
|
8
|
+
{% include "fragments/requirements_runtime.j2" %}
|
|
9
|
+
{% elif multi_stage %}
|
|
10
|
+
{% include "fragments/builder_stage.j2" %}
|
|
11
|
+
|
|
12
|
+
{% include "fragments/runtime_stage.j2" %}
|
|
13
|
+
{% else %}
|
|
14
|
+
{% include "fragments/single_stage.j2" %}
|
|
15
|
+
{% endif %}
|
|
16
|
+
|
|
17
|
+
{% if not requirements_mode %}
|
|
18
|
+
{% include "fragments/export_block.j2" %}
|
|
19
|
+
{% endif %}
|
|
20
|
+
|
|
21
|
+
CMD {{ runtime_command }}
|