dosho 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dosho-0.1.0a1/.gitignore +9 -0
- dosho-0.1.0a1/AGENTS.md +124 -0
- dosho-0.1.0a1/CONTRIBUTING.md +76 -0
- dosho-0.1.0a1/LICENSE +21 -0
- dosho-0.1.0a1/PKG-INFO +66 -0
- dosho-0.1.0a1/README.md +51 -0
- dosho-0.1.0a1/pyproject.toml +61 -0
- dosho-0.1.0a1/src/dosho/__init__.py +11 -0
- dosho-0.1.0a1/src/dosho/_builder.py +84 -0
- dosho-0.1.0a1/src/dosho/cabs/__init__.py +84 -0
- dosho-0.1.0a1/src/dosho/cabs/aoflagger.py +69 -0
- dosho-0.1.0a1/src/dosho/cabs/casaplotms.py +62 -0
- dosho-0.1.0a1/src/dosho/cabs/casatasks.py +475 -0
- dosho-0.1.0a1/src/dosho/cabs/crystalball.py +56 -0
- dosho-0.1.0a1/src/dosho/cabs/cubical.py +252 -0
- dosho-0.1.0a1/src/dosho/cabs/mosaic_queen.py +97 -0
- dosho-0.1.0a1/src/dosho/cabs/owlcat_plotelev.py +41 -0
- dosho-0.1.0a1/src/dosho/cabs/quartical.py +133 -0
- dosho-0.1.0a1/src/dosho/cabs/ragavi.py +113 -0
- dosho-0.1.0a1/src/dosho/cabs/shadems.py +179 -0
- dosho-0.1.0a1/src/dosho/cabs/simms.py +303 -0
- dosho-0.1.0a1/src/dosho/cabs/sofia2.py +525 -0
- dosho-0.1.0a1/src/dosho/cabs/tricolour.py +55 -0
- dosho-0.1.0a1/src/dosho/cabs/wsclean.py +296 -0
- dosho-0.1.0a1/src/dosho/images.py +28 -0
- dosho-0.1.0a1/src/dosho/registry.py +71 -0
dosho-0.1.0a1/.gitignore
ADDED
dosho-0.1.0a1/AGENTS.md
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# dosho -- design conventions
|
|
2
|
+
|
|
3
|
+
`dosho` ("a shinobi's tool bag") is the native cab repository for shinobi
|
|
4
|
+
(stimela-ninja, Stimela 3.0). cult-cargo and scabha did the real work of
|
|
5
|
+
cataloguing this ecosystem's tools first, and dosho's cab set leans on
|
|
6
|
+
that prior art throughout -- it's a new repository, not a rejection of
|
|
7
|
+
the old one. It exists because cult-cargo's YAML cab format -- built for
|
|
8
|
+
Stimela 2.0/scabha -- carries assumptions shinobi deliberately doesn't
|
|
9
|
+
carry forward: `dynamic_schema` (a Python function imported and
|
|
10
|
+
*executed* at cab-load time to compute a real tool's schema),
|
|
11
|
+
package-scoped `_include` composition, and dtype coverage gaps that
|
|
12
|
+
silently degrade to `str`. A compatibility loader could paper over those,
|
|
13
|
+
but maintaining one indefinitely was costing more than it was saving. See
|
|
14
|
+
stimela-ninja's own `AGENTS.md` for the full design philosophy this repo
|
|
15
|
+
inherits; this file only states what's specific to authoring cabs here.
|
|
16
|
+
|
|
17
|
+
## Core rule
|
|
18
|
+
|
|
19
|
+
**Cabs are authored directly in Python, not parsed from a YAML dialect.**
|
|
20
|
+
Every tool is a `shinobi.Cab` object (or several) built in a plain Python
|
|
21
|
+
module under `src/dosho/cabs/`, using shinobi's existing
|
|
22
|
+
`Cab`/`Policies`/`ParamMeta`/`ParamPattern` machinery -- see
|
|
23
|
+
`dosho._builder.define_cab` for the boilerplate-reduction helper. There is
|
|
24
|
+
no YAML authoring path and none should be added: a YAML dialect is exactly
|
|
25
|
+
the kind of "grows its own semantics" surface stimela-ninja's `AGENTS.md`
|
|
26
|
+
warns against.
|
|
27
|
+
|
|
28
|
+
Not every tool can be a `Cab`, though: shinobi only ever executes
|
|
29
|
+
`flavour="binary"` cabs (real standalone executables, argv-built and
|
|
30
|
+
shelled out to). A tool that's actually a Python-package function call --
|
|
31
|
+
no standalone binary at all (CASA tasks are the running example:
|
|
32
|
+
`casatasks.listobs`, `casaplotms.plotms`) -- is instead a
|
|
33
|
+
`@shinobi.pystep`-decorated function (a `StepRef`, not a `Cab`), calling
|
|
34
|
+
`ctx.import_func("<task>", "<package>")` *inside the running container at
|
|
35
|
+
step-execution time*. That's architecturally distinct from -- and doesn't
|
|
36
|
+
violate -- "never import a cab package" below: the import happens at
|
|
37
|
+
execution time inside the container, on ordinary trusted Python the
|
|
38
|
+
pystep author wrote directly, not shinobi interpreting untrusted cab data
|
|
39
|
+
on the host at load time. Both shapes live side by side under
|
|
40
|
+
`src/dosho/cabs/`, and both are first-class for `Recipe.add_step` -- a
|
|
41
|
+
pipeline author (or `dosho.get(name)`) doesn't need to know or care which
|
|
42
|
+
one a given tool is. See `dosho/cabs/casatasks.py` for the pattern.
|
|
43
|
+
|
|
44
|
+
## Never import/execute an external Cab's own schema-generation code
|
|
45
|
+
|
|
46
|
+
External cabs that generate parameters dynamically via embedded functions
|
|
47
|
+
are strictly forbidden, as are cabs whose plain-text source shinobi would
|
|
48
|
+
have to execute to make sense of -- see stimela-ninja's "Never
|
|
49
|
+
eval()/exec() a cab's `command`". Instead:
|
|
50
|
+
|
|
51
|
+
- **Per-instance dynamic parameter families** (CubiCal's `g1-solvable`,
|
|
52
|
+
QuartiCal's `K.time_interval`, one family per caller-chosen term name)
|
|
53
|
+
are expressed as a hand-authored `ParamPattern` -- transcribed once from
|
|
54
|
+
the tool's own template/docs, not generated. This is static data, not
|
|
55
|
+
code.
|
|
56
|
+
- **Dynamic output paths** (wsclean's `{prefix}-MFS-image.fits`-shaped
|
|
57
|
+
outputs) are expressed as a `ParamMeta.implicit` string template,
|
|
58
|
+
resolved by shinobi's `_fill_outputs` via plain `str.format` against a
|
|
59
|
+
step's own validated inputs -- never eval, never an expression language.
|
|
60
|
+
Only the handful of outputs a real pipeline actually wires as a
|
|
61
|
+
dependency need a resolved `implicit` template; anything more exotic
|
|
62
|
+
stays validation-only via `output_patterns`.
|
|
63
|
+
|
|
64
|
+
If a tool's dynamic behavior can't be expressed this way, don't invent a
|
|
65
|
+
new mechanism speculatively -- leave the field/output out and come back
|
|
66
|
+
when a real pipeline needs it (same "as small and boring as possible" gate
|
|
67
|
+
as stimela-ninja itself).
|
|
68
|
+
|
|
69
|
+
## Container images
|
|
70
|
+
|
|
71
|
+
No image-building infrastructure here (no cult-cargo-style
|
|
72
|
+
bundle-manifest.md + Dockerfile tree) -- cult-cargo already solves that
|
|
73
|
+
problem well; it just isn't this repo's job, since dosho is a cab-schema
|
|
74
|
+
repo, not an image-building one. `src/dosho/images.py` is the single place
|
|
75
|
+
image references are pinned (reusing existing `quay.io/stimela2/*` images
|
|
76
|
+
where they already exist); bumping a tool's version is editing one
|
|
77
|
+
constant there. The repo's own git tag versions the cab set as a whole.
|
|
78
|
+
|
|
79
|
+
## Repo layout
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
src/dosho/
|
|
83
|
+
__init__.py # re-exports get(), list_cabs()
|
|
84
|
+
registry.py # registered (real, possibly hyphenated) name ->
|
|
85
|
+
# "dosho.cabs:<attribute>"; get()/list_cabs(),
|
|
86
|
+
# registered under the "shinobi.cabs" entry-point group
|
|
87
|
+
# -- for a caller that only knows the tool's name at
|
|
88
|
+
# *runtime* (the CLI, shinobi.cabs discovery)
|
|
89
|
+
images.py # pinned {tool: "quay.io/stimela2/<tool>:<tag>"} constants
|
|
90
|
+
_builder.py # define_cab() helper over Cab(...) + shinobi.loaders.build_model
|
|
91
|
+
cabs/
|
|
92
|
+
__init__.py # re-exports every tool-level object by name, so
|
|
93
|
+
# `from dosho.cabs import wsclean` works directly --
|
|
94
|
+
# the write-time-known counterpart to registry.py
|
|
95
|
+
<tool>.py # single-command tool: exports one object named after
|
|
96
|
+
# the tool itself (e.g. wsclean.py's `wsclean = ...`)
|
|
97
|
+
<family>.py # multi-command tool: one module-level object per
|
|
98
|
+
# sub-command (e.g. casatasks.py's `listobs`,
|
|
99
|
+
# `mstransform`, ...; simms.py's `skysim`/
|
|
100
|
+
# `telsim`/`simms_classic`)
|
|
101
|
+
tests/
|
|
102
|
+
test_registry.py
|
|
103
|
+
test_<tool>.py # one per ported cab: round-trips a representative param
|
|
104
|
+
# set through build_argv and checks real CLI token shape
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Registering a new tool touches two places: the module itself under
|
|
108
|
+
`cabs/`, and both `cabs/__init__.py` (direct-import re-export) and
|
|
109
|
+
`registry.py` (`_ENTRIES`, the runtime-lookup name -- which may differ
|
|
110
|
+
from the Python attribute name if the tool's real name isn't a valid
|
|
111
|
+
identifier, e.g. `"simms-skysim"` -> attribute `skysim`).
|
|
112
|
+
|
|
113
|
+
## Before adding a cab
|
|
114
|
+
|
|
115
|
+
Port from the real tool's own `--help`/docs (or, for a `@shinobi.pystep`
|
|
116
|
+
wrapper, the real Python package's own function signature), cross-checked
|
|
117
|
+
against the matching cult-cargo YAML (if one exists) as a useful second
|
|
118
|
+
source -- but not copied from it blindly, since cult-cargo's own schema
|
|
119
|
+
for the hard cases (wsclean, cubical, quartical) has the known gaps
|
|
120
|
+
dosho exists to close. Every
|
|
121
|
+
ported tool gets a test: for a `Cab`, round-trip a representative param
|
|
122
|
+
set through `build_argv` and check the real CLI token shape; for a
|
|
123
|
+
pystep, check its `inputs_model` schema shape and that it wires into a
|
|
124
|
+
`Recipe` -- not just that the object constructs without error.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Contributing to dosho
|
|
2
|
+
|
|
3
|
+
Thanks for your interest in contributing! **dosho** ("a shinobi's tool
|
|
4
|
+
bag") is the native cab repository for
|
|
5
|
+
[shinobi](https://github.com/SpheMakh/stimela-ninja) (stimela-ninja,
|
|
6
|
+
Stimela 3.0). It's early software, so the most valuable contributions
|
|
7
|
+
right now are new tool ports, bug reports, focused fixes, tests,
|
|
8
|
+
documentation, and feedback on the design.
|
|
9
|
+
|
|
10
|
+
## Scope and philosophy
|
|
11
|
+
|
|
12
|
+
Every tool must contain a single executable, a Cab's `command`
|
|
13
|
+
or a `@shinobi.pystep` function. No other tool attribute/section
|
|
14
|
+
will be executed, i.e, no `exec()/eval()` executions or
|
|
15
|
+
dynamic parameter creation at loadtime. Tools must be fully defined
|
|
16
|
+
when they are authored. The `ParamPattern` construct should be more
|
|
17
|
+
than enough for most cases, and anything beyond that should be
|
|
18
|
+
configured in the Python recipe's logic. See
|
|
19
|
+
**[`AGENTS.md`](https://github.com/SpheMakh/dosho/blob/main/AGENTS.md)**
|
|
20
|
+
for the full design rationale and tool-authoring conventions; read it
|
|
21
|
+
before porting a new tool or touching `dosho/_builder.py`/
|
|
22
|
+
`dosho/registry.py`.
|
|
23
|
+
|
|
24
|
+
Our core philosophy: **avoid unnecessary complexity like the plague**.
|
|
25
|
+
|
|
26
|
+
## Ways to contribute
|
|
27
|
+
|
|
28
|
+
- **Port a new tool** -- see `AGENTS.md`'s "Before adding a cab" section.
|
|
29
|
+
Every ported tool needs a test that checks real CLI/schema shape, not
|
|
30
|
+
just that the object constructs without error.
|
|
31
|
+
- **Report bugs** and request features via
|
|
32
|
+
[issues](https://github.com/SpheMakh/dosho/issues).
|
|
33
|
+
- **Improve documentation** under `docs/` or the docstrings that feed the
|
|
34
|
+
API reference.
|
|
35
|
+
- **Sizable change/addition** -- if you're considering a larger change,
|
|
36
|
+
opening an issue to discuss it first is a great way to align before
|
|
37
|
+
writing code.
|
|
38
|
+
|
|
39
|
+
## Development setup
|
|
40
|
+
|
|
41
|
+
The project uses [uv](https://docs.astral.sh/uv/). `stimela-ninja` is
|
|
42
|
+
published on PyPI, so a plain `uv sync` resolves it like any other
|
|
43
|
+
dependency:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
git clone https://github.com/SpheMakh/dosho.git
|
|
47
|
+
cd dosho
|
|
48
|
+
uv sync --group dev
|
|
49
|
+
uv run pytest
|
|
50
|
+
uv run ruff check .
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
If you're working against an unreleased `stimela-ninja` change, clone
|
|
54
|
+
it next to this repo and layer it in for the duration of a command with
|
|
55
|
+
`uv run --with-editable ../stimela-ninja -- <command>` instead.
|
|
56
|
+
|
|
57
|
+
## Testing
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
uv run pytest -q
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Every ported tool's test round-trips a representative param set through
|
|
64
|
+
`shinobi.policies.build_argv` (for a `Cab`) or checks its
|
|
65
|
+
`inputs_model`/`Recipe` wiring (for a pystep) against the real tool's
|
|
66
|
+
known CLI/signature shape -- most tools here (`casatasks`, `casaplotms`,
|
|
67
|
+
CASA in general) aren't installed in the test environment, so tests never
|
|
68
|
+
actually dispatch/execute them.
|
|
69
|
+
|
|
70
|
+
## Documentation
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
uv sync --group docs
|
|
74
|
+
uv run sphinx-build -b html docs docs/_build/html
|
|
75
|
+
open docs/_build/html/index.html
|
|
76
|
+
```
|
dosho-0.1.0a1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sphesihle Makhathini
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dosho-0.1.0a1/PKG-INFO
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dosho
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: A shinobi's tool bag -- the native shinobi (stimela-ninja) cab repository
|
|
5
|
+
Project-URL: Homepage, https://github.com/SpheMakh/dosho
|
|
6
|
+
Project-URL: Repository, https://github.com/SpheMakh/dosho
|
|
7
|
+
Project-URL: Issues, https://github.com/SpheMakh/dosho/issues
|
|
8
|
+
Project-URL: Documentation, https://dosho.readthedocs.io
|
|
9
|
+
Author-email: Sphesihle Makhathini <sphemakh@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: stimela-ninja>=0.1.0b2
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# dosho
|
|
17
|
+
|
|
18
|
+
*A shinobi's tool bag.*
|
|
19
|
+
|
|
20
|
+
`dosho` is the native cab repository for
|
|
21
|
+
[shinobi](https://github.com/SpheMakh/stimela-ninja) (stimela-ninja,
|
|
22
|
+
Stimela 3.0). Every tool is authored directly in Python -- a
|
|
23
|
+
`shinobi.Cab` object for a real binary, or a `@shinobi.pystep`-produced
|
|
24
|
+
`StepRef` for a Python-package tool with no standalone binary (CASA
|
|
25
|
+
tasks) -- instead of a YAML dialect, so there's no `dynamic_schema`-style
|
|
26
|
+
Python-execution step at cab-load time and no dtype coverage gaps.
|
|
27
|
+
|
|
28
|
+
See [`AGENTS.md`](./AGENTS.md) for the design rationale and
|
|
29
|
+
tool-authoring conventions.
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
Know the tool at write-time? Import it directly:
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from dosho.cabs import wsclean
|
|
37
|
+
from dosho.cabs.casatasks import listobs
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Only know the name at runtime? Use the string-keyed registry:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import dosho
|
|
44
|
+
|
|
45
|
+
wsclean = dosho.get("wsclean")
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Or, from shinobi itself (any installed `shinobi.cabs`-entry-point
|
|
49
|
+
provider, `dosho` included):
|
|
50
|
+
|
|
51
|
+
```console
|
|
52
|
+
$ ninja cabs list
|
|
53
|
+
$ ninja cabs show wsclean
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Status
|
|
57
|
+
|
|
58
|
+
`dosho` currently hosts 14 real-binary `Cab`s (`wsclean`, `cubical`,
|
|
59
|
+
`quartical`, `aoflagger`, `tricolour`, `crystalball`, `owlcat_plotelev`,
|
|
60
|
+
`shadems`, `ragavi`, `sofia2`, `mosaic-queen`, and all three of `simms`'s
|
|
61
|
+
sub-commands -- `skysim`/`telsim`/classic `simms`) and 14
|
|
62
|
+
`@shinobi.pystep` CASA-task wrappers (`listobs`, `mstransform`, `fixvis`,
|
|
63
|
+
`clearcal`, `initweights`, `flagdata`, `setjy`, `gaincal`, `polcal`,
|
|
64
|
+
`bandpass`, `applycal`, `fluxscale`, `flagmanager`, `plotms`) -- the full
|
|
65
|
+
set a real pipeline ([caracal2](https://github.com/caracal-pipeline/caracal2))
|
|
66
|
+
needs.
|
dosho-0.1.0a1/README.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# dosho
|
|
2
|
+
|
|
3
|
+
*A shinobi's tool bag.*
|
|
4
|
+
|
|
5
|
+
`dosho` is the native cab repository for
|
|
6
|
+
[shinobi](https://github.com/SpheMakh/stimela-ninja) (stimela-ninja,
|
|
7
|
+
Stimela 3.0). Every tool is authored directly in Python -- a
|
|
8
|
+
`shinobi.Cab` object for a real binary, or a `@shinobi.pystep`-produced
|
|
9
|
+
`StepRef` for a Python-package tool with no standalone binary (CASA
|
|
10
|
+
tasks) -- instead of a YAML dialect, so there's no `dynamic_schema`-style
|
|
11
|
+
Python-execution step at cab-load time and no dtype coverage gaps.
|
|
12
|
+
|
|
13
|
+
See [`AGENTS.md`](./AGENTS.md) for the design rationale and
|
|
14
|
+
tool-authoring conventions.
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
Know the tool at write-time? Import it directly:
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from dosho.cabs import wsclean
|
|
22
|
+
from dosho.cabs.casatasks import listobs
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Only know the name at runtime? Use the string-keyed registry:
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import dosho
|
|
29
|
+
|
|
30
|
+
wsclean = dosho.get("wsclean")
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Or, from shinobi itself (any installed `shinobi.cabs`-entry-point
|
|
34
|
+
provider, `dosho` included):
|
|
35
|
+
|
|
36
|
+
```console
|
|
37
|
+
$ ninja cabs list
|
|
38
|
+
$ ninja cabs show wsclean
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Status
|
|
42
|
+
|
|
43
|
+
`dosho` currently hosts 14 real-binary `Cab`s (`wsclean`, `cubical`,
|
|
44
|
+
`quartical`, `aoflagger`, `tricolour`, `crystalball`, `owlcat_plotelev`,
|
|
45
|
+
`shadems`, `ragavi`, `sofia2`, `mosaic-queen`, and all three of `simms`'s
|
|
46
|
+
sub-commands -- `skysim`/`telsim`/classic `simms`) and 14
|
|
47
|
+
`@shinobi.pystep` CASA-task wrappers (`listobs`, `mstransform`, `fixvis`,
|
|
48
|
+
`clearcal`, `initweights`, `flagdata`, `setjy`, `gaincal`, `polcal`,
|
|
49
|
+
`bandpass`, `applycal`, `fluxscale`, `flagmanager`, `plotms`) -- the full
|
|
50
|
+
set a real pipeline ([caracal2](https://github.com/caracal-pipeline/caracal2))
|
|
51
|
+
needs.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dosho"
|
|
7
|
+
version = "0.1.0a1"
|
|
8
|
+
description = "A shinobi's tool bag -- the native shinobi (stimela-ninja) cab repository"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Sphesihle Makhathini", email = "sphemakh@gmail.com" },
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"stimela-ninja>=0.1.0b2",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[dependency-groups]
|
|
20
|
+
dev = [
|
|
21
|
+
"pytest>=8.0",
|
|
22
|
+
"ruff>=0.5",
|
|
23
|
+
]
|
|
24
|
+
# Documentation build (Sphinx + Furo + autodoc). `uv sync --group docs`.
|
|
25
|
+
docs = [
|
|
26
|
+
"sphinx>=7.2",
|
|
27
|
+
"furo>=2024.1",
|
|
28
|
+
"sphinx-copybutton>=0.5",
|
|
29
|
+
"myst-parser>=2.0",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://github.com/SpheMakh/dosho"
|
|
34
|
+
Repository = "https://github.com/SpheMakh/dosho"
|
|
35
|
+
Issues = "https://github.com/SpheMakh/dosho/issues"
|
|
36
|
+
Documentation = "https://dosho.readthedocs.io"
|
|
37
|
+
|
|
38
|
+
[project.entry-points."shinobi.cabs"]
|
|
39
|
+
dosho = "dosho.registry"
|
|
40
|
+
|
|
41
|
+
[tool.hatch.build.targets.wheel]
|
|
42
|
+
packages = ["src/dosho"]
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.sdist]
|
|
45
|
+
include = [
|
|
46
|
+
"/src/dosho",
|
|
47
|
+
"/README.md",
|
|
48
|
+
"/LICENSE",
|
|
49
|
+
"/AGENTS.md",
|
|
50
|
+
"/CONTRIBUTING.md",
|
|
51
|
+
"/pyproject.toml",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
pythonpath = ["src"]
|
|
57
|
+
addopts = "--import-mode=importlib"
|
|
58
|
+
consider_namespace_packages = true
|
|
59
|
+
|
|
60
|
+
[tool.ruff]
|
|
61
|
+
line-length = 100
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""dosho -- a shinobi's tool bag. The native shinobi (stimela-ninja) cab
|
|
2
|
+
repository. See AGENTS.md for the design rationale.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dosho import images
|
|
6
|
+
from dosho._builder import define_cab
|
|
7
|
+
from dosho.registry import get, list_cabs
|
|
8
|
+
|
|
9
|
+
__all__ = ["define_cab", "get", "images", "list_cabs"]
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0a1"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Thin builder for authoring a native `dosho` cab: wraps `shinobi.Cab` +
|
|
2
|
+
`shinobi.loaders.build_model` + `shinobi.loaders.sanitize_unique` to remove
|
|
3
|
+
the boilerplate of hand-transcribing a real tool's parameter list, without
|
|
4
|
+
inventing any new schema surface.
|
|
5
|
+
|
|
6
|
+
Generalises the `cab_from_defaults` helper first proven in
|
|
7
|
+
stimela-ninja's `examples/ninja_selfcal.py`. Deliberately not a
|
|
8
|
+
signature-introspecting decorator (`@cab`) -- that shape was tried once in
|
|
9
|
+
shinobi itself (`ba2bd48`) and removed (`605b6f8`): it can't express
|
|
10
|
+
`nom_de_guerre` (hyphenated CLI names aren't valid Python parameter
|
|
11
|
+
names), `input_patterns`, `Policies`, `positional`, or `implicit`. Raw
|
|
12
|
+
`Cab(...)` stays fully available for anything this helper doesn't cover --
|
|
13
|
+
it's sugar, not a wall.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from shinobi import Cab
|
|
21
|
+
from shinobi.loaders import build_model, sanitize_unique
|
|
22
|
+
from shinobi.steps.schema import ParamMeta, ParamPattern, Policies
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def define_cab(
|
|
26
|
+
name: str,
|
|
27
|
+
command: str,
|
|
28
|
+
image: str,
|
|
29
|
+
fields: dict[str, tuple[str, bool, Any]],
|
|
30
|
+
*,
|
|
31
|
+
outputs: dict[str, tuple[str, bool, Any]] | None = None,
|
|
32
|
+
field_meta: dict[str, ParamMeta] | None = None,
|
|
33
|
+
policies: Policies | None = None,
|
|
34
|
+
input_patterns: list[ParamPattern] | None = None,
|
|
35
|
+
output_patterns: list[ParamPattern] | None = None,
|
|
36
|
+
flavour: str = "binary",
|
|
37
|
+
wranglers: dict[str, list[str]] | None = None,
|
|
38
|
+
info: str | None = None,
|
|
39
|
+
) -> Cab:
|
|
40
|
+
"""Build a `Cab` from a flat `{raw_param_name: (dtype, required,
|
|
41
|
+
default)}` dict -- the shape a tool's own `--help`/docs naturally give
|
|
42
|
+
you (e.g. `"data-ms": ("MS", True, None)`). Hyphenated/dotted raw
|
|
43
|
+
names are sanitised to valid pydantic field names, with the original
|
|
44
|
+
kept as a `nom_de_guerre` so the built cab's `build_argv` still emits
|
|
45
|
+
the tool's real flag.
|
|
46
|
+
|
|
47
|
+
`field_meta` entries the caller passes explicitly (keyed by the
|
|
48
|
+
*sanitised* field name -- e.g. an output field's `implicit` template,
|
|
49
|
+
or a `positional`/`repeat_as_tokens` override) are merged in on top of
|
|
50
|
+
the auto-derived `nom_de_guerre`s; an explicit entry that doesn't set
|
|
51
|
+
its own `nom_de_guerre` keeps the auto-derived one rather than losing
|
|
52
|
+
it.
|
|
53
|
+
"""
|
|
54
|
+
resolved_fields: dict[str, tuple[str, bool, Any]] = {}
|
|
55
|
+
resolved_meta: dict[str, ParamMeta] = {}
|
|
56
|
+
seen: dict[str, str] = {}
|
|
57
|
+
for raw_name, spec in fields.items():
|
|
58
|
+
field = sanitize_unique(raw_name, seen)
|
|
59
|
+
resolved_fields[field] = spec
|
|
60
|
+
if field != raw_name:
|
|
61
|
+
resolved_meta[field] = ParamMeta(nom_de_guerre=raw_name)
|
|
62
|
+
|
|
63
|
+
for field, meta in (field_meta or {}).items():
|
|
64
|
+
auto = resolved_meta.get(field)
|
|
65
|
+
if auto is not None and meta.nom_de_guerre is None:
|
|
66
|
+
meta = meta.model_copy(update={"nom_de_guerre": auto.nom_de_guerre})
|
|
67
|
+
resolved_meta[field] = meta
|
|
68
|
+
|
|
69
|
+
return Cab(
|
|
70
|
+
name=name,
|
|
71
|
+
command=command,
|
|
72
|
+
image=image,
|
|
73
|
+
info=info,
|
|
74
|
+
flavour=flavour,
|
|
75
|
+
inputs_model=build_model(
|
|
76
|
+
f"{name}_Inputs", resolved_fields, allow_extra=bool(input_patterns)
|
|
77
|
+
),
|
|
78
|
+
outputs_model=build_model(f"{name}_Outputs", outputs or {}),
|
|
79
|
+
field_meta=resolved_meta,
|
|
80
|
+
policies=policies or Policies(),
|
|
81
|
+
input_patterns=input_patterns or [],
|
|
82
|
+
output_patterns=output_patterns or [],
|
|
83
|
+
wranglers=wranglers or {},
|
|
84
|
+
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Every tool dosho ports, re-exported by name at package level, so a
|
|
2
|
+
pipeline author who knows what they want at write-time can do
|
|
3
|
+
`from dosho.cabs import wsclean` or `from dosho.cabs.casatasks import
|
|
4
|
+
listobs` directly -- no need to know or care whether a given tool is a
|
|
5
|
+
`Cab` (a real binary, argv-built and shelled out to) or a `StepRef` (a
|
|
6
|
+
Python-package tool with no standalone binary, run via `@shinobi.pystep`
|
|
7
|
+
and `ctx.import_func` inside a container instead -- CASA tasks are the
|
|
8
|
+
only example so far). Both shapes are first-class for `Recipe.add_step`.
|
|
9
|
+
|
|
10
|
+
For *runtime*/string-keyed lookup (the CLI, `shinobi.cabs` entry-point
|
|
11
|
+
discovery), use `dosho.get(name)`/`dosho.registry` instead -- that's a
|
|
12
|
+
second, parallel interface for a caller that doesn't know the name until
|
|
13
|
+
it runs, not a replacement for these direct imports.
|
|
14
|
+
|
|
15
|
+
Importing this module (or any of its submodules -- Python always
|
|
16
|
+
initializes a parent package first) eagerly constructs every `Cab`/
|
|
17
|
+
`StepRef` below. That's a deliberate, accepted tradeoff here: each one is
|
|
18
|
+
cheap (pydantic model + dict construction, no I/O), so the "don't pay for
|
|
19
|
+
what you don't use" laziness `dosho.registry`'s own submodule-path table
|
|
20
|
+
still gives a caller that only wants one specific tool is preserved
|
|
21
|
+
there, not here.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from dosho.cabs.aoflagger import aoflagger
|
|
27
|
+
from dosho.cabs.casaplotms import plotms
|
|
28
|
+
from dosho.cabs.casatasks import (
|
|
29
|
+
applycal,
|
|
30
|
+
bandpass,
|
|
31
|
+
clearcal,
|
|
32
|
+
fixvis,
|
|
33
|
+
flagdata,
|
|
34
|
+
flagmanager,
|
|
35
|
+
fluxscale,
|
|
36
|
+
gaincal,
|
|
37
|
+
initweights,
|
|
38
|
+
listobs,
|
|
39
|
+
mstransform,
|
|
40
|
+
polcal,
|
|
41
|
+
setjy,
|
|
42
|
+
)
|
|
43
|
+
from dosho.cabs.crystalball import crystalball
|
|
44
|
+
from dosho.cabs.cubical import cubical
|
|
45
|
+
from dosho.cabs.mosaic_queen import mosaic_queen
|
|
46
|
+
from dosho.cabs.owlcat_plotelev import owlcat_plotelev
|
|
47
|
+
from dosho.cabs.quartical import quartical
|
|
48
|
+
from dosho.cabs.ragavi import ragavi
|
|
49
|
+
from dosho.cabs.shadems import shadems
|
|
50
|
+
from dosho.cabs.simms import simms_classic, skysim, telsim
|
|
51
|
+
from dosho.cabs.sofia2 import sofia2
|
|
52
|
+
from dosho.cabs.tricolour import tricolour
|
|
53
|
+
from dosho.cabs.wsclean import wsclean
|
|
54
|
+
|
|
55
|
+
__all__ = [
|
|
56
|
+
"aoflagger",
|
|
57
|
+
"applycal",
|
|
58
|
+
"bandpass",
|
|
59
|
+
"clearcal",
|
|
60
|
+
"crystalball",
|
|
61
|
+
"cubical",
|
|
62
|
+
"fixvis",
|
|
63
|
+
"flagdata",
|
|
64
|
+
"flagmanager",
|
|
65
|
+
"fluxscale",
|
|
66
|
+
"gaincal",
|
|
67
|
+
"initweights",
|
|
68
|
+
"listobs",
|
|
69
|
+
"mosaic_queen",
|
|
70
|
+
"mstransform",
|
|
71
|
+
"owlcat_plotelev",
|
|
72
|
+
"plotms",
|
|
73
|
+
"polcal",
|
|
74
|
+
"quartical",
|
|
75
|
+
"ragavi",
|
|
76
|
+
"setjy",
|
|
77
|
+
"shadems",
|
|
78
|
+
"simms_classic",
|
|
79
|
+
"skysim",
|
|
80
|
+
"sofia2",
|
|
81
|
+
"telsim",
|
|
82
|
+
"tricolour",
|
|
83
|
+
"wsclean",
|
|
84
|
+
]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""aoflagger -- automatic RFI flagger (https://aoflagger.readthedocs.io).
|
|
2
|
+
|
|
3
|
+
Ported field-by-field from cult-cargo's aoflagger.yml (a flat, static
|
|
4
|
+
schema -- no dynamic_schema, no package-scoped _include -- so this is a
|
|
5
|
+
mechanical transcription, not a structural fix like wsclean/cubical/
|
|
6
|
+
quartical).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from shinobi.steps.schema import ParamMeta, Policies
|
|
12
|
+
|
|
13
|
+
from dosho import images
|
|
14
|
+
from dosho._builder import define_cab
|
|
15
|
+
|
|
16
|
+
_FIELDS: dict[str, tuple[str, bool, object]] = {
|
|
17
|
+
"msname": ("MS", True, None),
|
|
18
|
+
"verbose": ("bool", False, False),
|
|
19
|
+
"threads": ("int", False, None),
|
|
20
|
+
"strategy": ("File", False, None),
|
|
21
|
+
"indirect_read": ("bool", False, True),
|
|
22
|
+
"memory_read": ("bool", False, False),
|
|
23
|
+
"auto_read_mode": ("bool", False, True),
|
|
24
|
+
"uvw": ("bool", False, False),
|
|
25
|
+
"column": ("str", False, "DATA"),
|
|
26
|
+
"skip_flagged": ("bool", False, False),
|
|
27
|
+
"bands": ("int", False, None),
|
|
28
|
+
"fields": ("int", False, None),
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
_FIELD_META: dict[str, ParamMeta] = {
|
|
32
|
+
"msname": ParamMeta(info="MS name(s) to be flagged"),
|
|
33
|
+
"verbose": ParamMeta(nom_de_guerre="v", info="Produce verbose output"),
|
|
34
|
+
"threads": ParamMeta(
|
|
35
|
+
nom_de_guerre="j",
|
|
36
|
+
info="overrides the number of threads specified in the strategy (default: one thread for each CPU core)",
|
|
37
|
+
),
|
|
38
|
+
"strategy": ParamMeta(info="specifies a possible customized strategy"),
|
|
39
|
+
"indirect_read": ParamMeta(
|
|
40
|
+
nom_de_guerre="indirect-read",
|
|
41
|
+
info="will reorder the measurement set before starting, which is normally faster but requires free disk space to reorder the data to",
|
|
42
|
+
),
|
|
43
|
+
"memory_read": ParamMeta(
|
|
44
|
+
nom_de_guerre="memory-read",
|
|
45
|
+
info="will read the entire measurement set in memory. This is the fastest, but requires much memory.",
|
|
46
|
+
),
|
|
47
|
+
"auto_read_mode": ParamMeta(
|
|
48
|
+
nom_de_guerre="auto-read-mode",
|
|
49
|
+
info="will select either memory or direct mode based on available memory",
|
|
50
|
+
),
|
|
51
|
+
"uvw": ParamMeta(info="reads uvw values (some exotic strategies require these)"),
|
|
52
|
+
"column": ParamMeta(info="specify column to flag"),
|
|
53
|
+
"skip_flagged": ParamMeta(
|
|
54
|
+
nom_de_guerre="skip-flagged",
|
|
55
|
+
info="will skip an ms if it has already been processed by AOFlagger according to its HISTORY table.",
|
|
56
|
+
),
|
|
57
|
+
"bands": ParamMeta(info="comma separated list of (zero-indexed) band ids to process"),
|
|
58
|
+
"fields": ParamMeta(info="Field ID(s). Comma separated string if more than one field"),
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
aoflagger = define_cab(
|
|
62
|
+
"aoflagger",
|
|
63
|
+
"aoflagger",
|
|
64
|
+
images.AOFLAGGER,
|
|
65
|
+
_FIELDS,
|
|
66
|
+
field_meta=_FIELD_META,
|
|
67
|
+
policies=Policies(prefix="-"),
|
|
68
|
+
info="AOFlagger automatic RFI flagger (https://aoflagger.readthedocs.io)",
|
|
69
|
+
)
|