globusiri 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- globusiri-0.0.3/.gitignore +14 -0
- globusiri-0.0.3/PKG-INFO +122 -0
- globusiri-0.0.3/README.md +100 -0
- globusiri-0.0.3/docs/api.md +129 -0
- globusiri-0.0.3/docs/cli.md +458 -0
- globusiri-0.0.3/docs/configuration.md +376 -0
- globusiri-0.0.3/docs/design.md +758 -0
- globusiri-0.0.3/docs/examples.md +338 -0
- globusiri-0.0.3/pyproject.toml +40 -0
- globusiri-0.0.3/src/globusiri/__init__.py +20 -0
- globusiri-0.0.3/src/globusiri/api.py +626 -0
- globusiri-0.0.3/src/globusiri/cleanup.py +92 -0
- globusiri-0.0.3/src/globusiri/cli.py +657 -0
- globusiri-0.0.3/src/globusiri/clients/__init__.py +1 -0
- globusiri-0.0.3/src/globusiri/clients/http.py +82 -0
- globusiri-0.0.3/src/globusiri/clients/https.py +76 -0
- globusiri-0.0.3/src/globusiri/clients/iri.py +181 -0
- globusiri-0.0.3/src/globusiri/clients/transfer.py +53 -0
- globusiri-0.0.3/src/globusiri/config.py +285 -0
- globusiri-0.0.3/src/globusiri/data/jobs/crux_example_job.yaml +62 -0
- globusiri-0.0.3/src/globusiri/data/jobs/crux_hello_job.yaml +31 -0
- globusiri-0.0.3/src/globusiri/data/jobs/crux_mpi_job.yaml +48 -0
- globusiri-0.0.3/src/globusiri/data/jobs/crux_script_job.sh +29 -0
- globusiri-0.0.3/src/globusiri/data/jobs/crux_script_job.yaml +31 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_example_job.yaml +71 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_hello_job.yaml +29 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_mpi_job.yaml +48 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_script_job.sh +14 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_script_job.yaml +29 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_transfer_job.yaml +50 -0
- globusiri-0.0.3/src/globusiri/data/jobs/perlmutter_wordcount_job.yaml +29 -0
- globusiri-0.0.3/src/globusiri/data/sites.example.yaml +185 -0
- globusiri-0.0.3/src/globusiri/data/templates/crux_iri_submit_template.sh +160 -0
- globusiri-0.0.3/src/globusiri/data/templates/perlmutter_iri_submit_template.sh +111 -0
- globusiri-0.0.3/src/globusiri/data/templates/shell_job_template.sh +57 -0
- globusiri-0.0.3/src/globusiri/errors.py +60 -0
- globusiri-0.0.3/src/globusiri/jobfile.py +289 -0
- globusiri-0.0.3/src/globusiri/jobs.py +404 -0
- globusiri-0.0.3/src/globusiri/orm/__init__.py +86 -0
- globusiri-0.0.3/src/globusiri/orm/models.py +107 -0
- globusiri-0.0.3/src/globusiri/orm/session.py +158 -0
- globusiri-0.0.3/src/globusiri/panda_secrets.py +53 -0
- globusiri-0.0.3/src/globusiri/paths.py +151 -0
- globusiri-0.0.3/src/globusiri/runindex.py +215 -0
- globusiri-0.0.3/src/globusiri/runlog.py +101 -0
- globusiri-0.0.3/src/globusiri/secure_io.py +82 -0
- globusiri-0.0.3/src/globusiri/services.py +26 -0
- globusiri-0.0.3/src/globusiri/staging/__init__.py +1 -0
- globusiri-0.0.3/src/globusiri/staging/e2e.py +290 -0
- globusiri-0.0.3/src/globusiri/staging/https.py +77 -0
- globusiri-0.0.3/src/globusiri/tokens/__init__.py +21 -0
- globusiri-0.0.3/src/globusiri/tokens/base.py +216 -0
- globusiri-0.0.3/src/globusiri/tokens/e2e.py +136 -0
- globusiri-0.0.3/src/globusiri/tokens/https.py +133 -0
- globusiri-0.0.3/src/globusiri/tokens/iri.py +149 -0
- globusiri-0.0.3/src/globusiri/validate.py +383 -0
- globusiri-0.0.3/src/globusiri/workflow.py +774 -0
- globusiri-0.0.3/tests/__init__.py +0 -0
- globusiri-0.0.3/tests/conftest.py +83 -0
- globusiri-0.0.3/tests/live/e2e_rucio_chain.sh +33 -0
- globusiri-0.0.3/tests/test_api.py +79 -0
- globusiri-0.0.3/tests/test_async_workflow.py +314 -0
- globusiri-0.0.3/tests/test_clients.py +120 -0
- globusiri-0.0.3/tests/test_config.py +93 -0
- globusiri-0.0.3/tests/test_database.py +139 -0
- globusiri-0.0.3/tests/test_e2e_rucio.py +101 -0
- globusiri-0.0.3/tests/test_e2e_runs.py +119 -0
- globusiri-0.0.3/tests/test_jobs.py +495 -0
- globusiri-0.0.3/tests/test_runindex.py +140 -0
- globusiri-0.0.3/tests/test_tokens.py +259 -0
- globusiri-0.0.3/tests/test_validate_workflow.py +251 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.pyc
|
|
3
|
+
*.egg-info/
|
|
4
|
+
*.DS_Store
|
|
5
|
+
/build/
|
|
6
|
+
/dist/
|
|
7
|
+
/.venv/
|
|
8
|
+
/docs/presentations/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
# a WORK_DIR accidentally created at the repo root: never commit tokens or run state
|
|
11
|
+
/tokens/
|
|
12
|
+
/runs/
|
|
13
|
+
/logs/
|
|
14
|
+
/config/sites.yaml
|
globusiri-0.0.3/PKG-INFO
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: globusiri
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: Submit and monitor HPC jobs through the IRI API and move files with Globus (HTTPS and Transfer)
|
|
5
|
+
Author-email: Wen Guan <wen.guan@cern.ch>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Requires-Dist: globus-sdk<5,>=4
|
|
9
|
+
Requires-Dist: platformdirs>=3
|
|
10
|
+
Requires-Dist: pyyaml>=5.4
|
|
11
|
+
Requires-Dist: requests>=2.25
|
|
12
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
13
|
+
Provides-Extra: mysql
|
|
14
|
+
Requires-Dist: pymysql>=1.0; extra == 'mysql'
|
|
15
|
+
Provides-Extra: panda
|
|
16
|
+
Requires-Dist: panda-client; extra == 'panda'
|
|
17
|
+
Provides-Extra: sqlite
|
|
18
|
+
Requires-Dist: pysqlite3-binary; (platform_system == 'Linux') and extra == 'sqlite'
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# GlobusIRI
|
|
24
|
+
|
|
25
|
+
GlobusIRI submits and monitors jobs on DOE HPC facilities through the **IRI API** and moves
|
|
26
|
+
files with **Globus**:
|
|
27
|
+
|
|
28
|
+
- **HTTPS** carries run scripts up to the HPC and brings logs back.
|
|
29
|
+
- **E2E Transfer** moves bulk input and output data.
|
|
30
|
+
|
|
31
|
+
Supported sites: NERSC Perlmutter and ALCF Crux.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
iri login (once) → iri/https/e2e refresh (cron)
|
|
35
|
+
→ [e2e stagein] → https upload → iri submit → iri monitor → https download logs → [e2e stageout]
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "globusiri[panda]"
|
|
42
|
+
export GLOBUSIRI_WORK_DIR=/path/to/globusiri_work # optional; default: per-user data dir
|
|
43
|
+
globusiri init && globusiri paths
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Config, tokens, logs, and run state all live under `GLOBUSIRI_WORK_DIR`. If the variable is
|
|
47
|
+
not set, they go in the per-user globusiri data directory.
|
|
48
|
+
|
|
49
|
+
Three ways to drive a workflow (same steps, one engine that never blocks):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
globusiri run --job perlmutter_example_job.yaml # blocking: returns when the run ends
|
|
53
|
+
globusiri submit --job perlmutter_example_job.yaml # start a run and return
|
|
54
|
+
globusiri monitor # plugin mode: each call checks every active run and triggers its next step
|
|
55
|
+
globusiri daemon --interval 60 # daemon mode: monitor in a loop
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The job YAML describes the batch job (`resources`, `template_file`), the inputs uploaded over
|
|
59
|
+
HTTPS (`inputs`), and the files moved by E2E (`staging.e2e.stagein/stageout`, as a manifest,
|
|
60
|
+
Rucio DIDs, or an inline `files:` list).
|
|
61
|
+
|
|
62
|
+
> **On Perlmutter, do not call `srun` in the job script.** NERSC's IRI already starts the script
|
|
63
|
+
> under `srun`, once per rank (`n_node × n_process_per_node` copies); a nested `srun` waits for
|
|
64
|
+
> CPUs the ranks hold until the job's time limit. On Crux the script runs once and starts its
|
|
65
|
+
> ranks with `mpiexec`. See [several processes per node](docs/examples.md#several-processes-per-node-mpi-style-launch).
|
|
66
|
+
|
|
67
|
+
Submit as many jobs as needed, including jobs at different sites. One `monitor` call checks
|
|
68
|
+
all active runs in the same WORK_DIR; `daemon` keeps doing that and picks up new submissions:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
globusiri submit --job perlmutter_hello_job.yaml
|
|
72
|
+
globusiri submit --job perlmutter_wordcount_job.yaml # first create WORK_DIR/inputs/words.txt
|
|
73
|
+
globusiri submit --job crux_hello_job.yaml
|
|
74
|
+
globusiri monitor # one pass over all active runs, then exits
|
|
75
|
+
globusiri daemon --interval 60 # continuous monitoring
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Bookkeeping uses `<WORK_DIR>/runs/index.db` (SQLite) by default, or MySQL/MariaDB (for
|
|
79
|
+
example the Harvester database) with `database.url` in `sites.yaml`
|
|
80
|
+
([configuration](docs/configuration.md#database)). Full state is in
|
|
81
|
+
`runs/<run_id>/state.json`. Normal monitoring queries active runs in the index; it does not
|
|
82
|
+
scan the working directory. Concurrent submitters and monitors share file locks and database
|
|
83
|
+
transactions; with SQLite only one process writes at a time. Use a local filesystem with
|
|
84
|
+
POSIX locking and SQLite WAL support. The index
|
|
85
|
+
is rebuilt from saved states when missing, or with `globusiri runs --reindex`.
|
|
86
|
+
|
|
87
|
+
See [job examples](docs/examples.md) for setup, file uploads, E2E transfers, and running many
|
|
88
|
+
jobs with one monitor.
|
|
89
|
+
|
|
90
|
+
## Python API
|
|
91
|
+
|
|
92
|
+
Every command is also a method of `globusiri.GlobusIRI`:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from globusiri import GlobusIRI
|
|
96
|
+
|
|
97
|
+
g = GlobusIRI()
|
|
98
|
+
sub = g.job_submit("perlmutter_example_job.yaml", set=["resources.n_node=2"])
|
|
99
|
+
state, report = g.job_wait("NERSC_Perlmutter", sub["job_id"])
|
|
100
|
+
|
|
101
|
+
r = g.submit("perlmutter_example_job.yaml") # non-blocking workflow
|
|
102
|
+
g.monitor()[r["run_id"]]["plugin_status"] # call each cycle: submitted/running/finished/failed
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Documentation
|
|
106
|
+
|
|
107
|
+
- [docs/design.md](docs/design.md): packaging and local layout, architecture, token model, workflow, and open questions
|
|
108
|
+
- [docs/configuration.md](docs/configuration.md): WORK_DIR and the `sites.yaml` schema
|
|
109
|
+
- [docs/cli.md](docs/cli.md): `globusiri` commands and cron setup
|
|
110
|
+
- [docs/api.md](docs/api.md): the Python API (`globusiri.GlobusIRI`), one method per command
|
|
111
|
+
- [docs/examples.md](docs/examples.md): shell jobs, local inputs, E2E transfers, and many-job monitoring
|
|
112
|
+
- [src/globusiri/data/sites.example.yaml](src/globusiri/data/sites.example.yaml): example site config for `NERSC_Perlmutter` and `ALCF_CRUX` (fixed per site)
|
|
113
|
+
- [src/globusiri/data/jobs/](src/globusiri/data/jobs/): example job files, copied into WORK_DIR by `globusiri init`
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
python -m venv .venv && .venv/bin/pip install -e ".[test]"
|
|
119
|
+
.venv/bin/pytest
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Status: implemented (v0.1.0). The unit tests run without network access. Logins, transfers, and job submission still need to be tried against the real NERSC and ALCF services.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# GlobusIRI
|
|
2
|
+
|
|
3
|
+
GlobusIRI submits and monitors jobs on DOE HPC facilities through the **IRI API** and moves
|
|
4
|
+
files with **Globus**:
|
|
5
|
+
|
|
6
|
+
- **HTTPS** carries run scripts up to the HPC and brings logs back.
|
|
7
|
+
- **E2E Transfer** moves bulk input and output data.
|
|
8
|
+
|
|
9
|
+
Supported sites: NERSC Perlmutter and ALCF Crux.
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
iri login (once) → iri/https/e2e refresh (cron)
|
|
13
|
+
→ [e2e stagein] → https upload → iri submit → iri monitor → https download logs → [e2e stageout]
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install "globusiri[panda]"
|
|
20
|
+
export GLOBUSIRI_WORK_DIR=/path/to/globusiri_work # optional; default: per-user data dir
|
|
21
|
+
globusiri init && globusiri paths
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Config, tokens, logs, and run state all live under `GLOBUSIRI_WORK_DIR`. If the variable is
|
|
25
|
+
not set, they go in the per-user globusiri data directory.
|
|
26
|
+
|
|
27
|
+
Three ways to drive a workflow (same steps, one engine that never blocks):
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
globusiri run --job perlmutter_example_job.yaml # blocking: returns when the run ends
|
|
31
|
+
globusiri submit --job perlmutter_example_job.yaml # start a run and return
|
|
32
|
+
globusiri monitor # plugin mode: each call checks every active run and triggers its next step
|
|
33
|
+
globusiri daemon --interval 60 # daemon mode: monitor in a loop
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The job YAML describes the batch job (`resources`, `template_file`), the inputs uploaded over
|
|
37
|
+
HTTPS (`inputs`), and the files moved by E2E (`staging.e2e.stagein/stageout`, as a manifest,
|
|
38
|
+
Rucio DIDs, or an inline `files:` list).
|
|
39
|
+
|
|
40
|
+
> **On Perlmutter, do not call `srun` in the job script.** NERSC's IRI already starts the script
|
|
41
|
+
> under `srun`, once per rank (`n_node × n_process_per_node` copies); a nested `srun` waits for
|
|
42
|
+
> CPUs the ranks hold until the job's time limit. On Crux the script runs once and starts its
|
|
43
|
+
> ranks with `mpiexec`. See [several processes per node](docs/examples.md#several-processes-per-node-mpi-style-launch).
|
|
44
|
+
|
|
45
|
+
Submit as many jobs as needed, including jobs at different sites. One `monitor` call checks
|
|
46
|
+
all active runs in the same WORK_DIR; `daemon` keeps doing that and picks up new submissions:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
globusiri submit --job perlmutter_hello_job.yaml
|
|
50
|
+
globusiri submit --job perlmutter_wordcount_job.yaml # first create WORK_DIR/inputs/words.txt
|
|
51
|
+
globusiri submit --job crux_hello_job.yaml
|
|
52
|
+
globusiri monitor # one pass over all active runs, then exits
|
|
53
|
+
globusiri daemon --interval 60 # continuous monitoring
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Bookkeeping uses `<WORK_DIR>/runs/index.db` (SQLite) by default, or MySQL/MariaDB (for
|
|
57
|
+
example the Harvester database) with `database.url` in `sites.yaml`
|
|
58
|
+
([configuration](docs/configuration.md#database)). Full state is in
|
|
59
|
+
`runs/<run_id>/state.json`. Normal monitoring queries active runs in the index; it does not
|
|
60
|
+
scan the working directory. Concurrent submitters and monitors share file locks and database
|
|
61
|
+
transactions; with SQLite only one process writes at a time. Use a local filesystem with
|
|
62
|
+
POSIX locking and SQLite WAL support. The index
|
|
63
|
+
is rebuilt from saved states when missing, or with `globusiri runs --reindex`.
|
|
64
|
+
|
|
65
|
+
See [job examples](docs/examples.md) for setup, file uploads, E2E transfers, and running many
|
|
66
|
+
jobs with one monitor.
|
|
67
|
+
|
|
68
|
+
## Python API
|
|
69
|
+
|
|
70
|
+
Every command is also a method of `globusiri.GlobusIRI`:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from globusiri import GlobusIRI
|
|
74
|
+
|
|
75
|
+
g = GlobusIRI()
|
|
76
|
+
sub = g.job_submit("perlmutter_example_job.yaml", set=["resources.n_node=2"])
|
|
77
|
+
state, report = g.job_wait("NERSC_Perlmutter", sub["job_id"])
|
|
78
|
+
|
|
79
|
+
r = g.submit("perlmutter_example_job.yaml") # non-blocking workflow
|
|
80
|
+
g.monitor()[r["run_id"]]["plugin_status"] # call each cycle: submitted/running/finished/failed
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Documentation
|
|
84
|
+
|
|
85
|
+
- [docs/design.md](docs/design.md): packaging and local layout, architecture, token model, workflow, and open questions
|
|
86
|
+
- [docs/configuration.md](docs/configuration.md): WORK_DIR and the `sites.yaml` schema
|
|
87
|
+
- [docs/cli.md](docs/cli.md): `globusiri` commands and cron setup
|
|
88
|
+
- [docs/api.md](docs/api.md): the Python API (`globusiri.GlobusIRI`), one method per command
|
|
89
|
+
- [docs/examples.md](docs/examples.md): shell jobs, local inputs, E2E transfers, and many-job monitoring
|
|
90
|
+
- [src/globusiri/data/sites.example.yaml](src/globusiri/data/sites.example.yaml): example site config for `NERSC_Perlmutter` and `ALCF_CRUX` (fixed per site)
|
|
91
|
+
- [src/globusiri/data/jobs/](src/globusiri/data/jobs/): example job files, copied into WORK_DIR by `globusiri init`
|
|
92
|
+
|
|
93
|
+
## Development
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
python -m venv .venv && .venv/bin/pip install -e ".[test]"
|
|
97
|
+
.venv/bin/pytest
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Status: implemented (v0.1.0). The unit tests run without network access. Logins, transfers, and job submission still need to be tried against the real NERSC and ALCF services.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Python API
|
|
2
|
+
|
|
3
|
+
Every `globusiri` command is also a method of `globusiri.GlobusIRI`, so the same operations
|
|
4
|
+
can be called from Python, for example from a Harvester plugin, a notebook, or another tool.
|
|
5
|
+
The CLI is a thin layer over this API: it parses arguments, calls one method, and prints the
|
|
6
|
+
result.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
from globusiri import GlobusIRI
|
|
10
|
+
|
|
11
|
+
g = GlobusIRI() # WORK_DIR and sites.yaml are found the same way as the CLI
|
|
12
|
+
g.refresh("iri", all_sites=True, min_lifetime="6h")
|
|
13
|
+
|
|
14
|
+
sub = g.job_submit("perlmutter_example_job.yaml", set=["resources.n_node=2"])
|
|
15
|
+
state, report = g.job_wait("NERSC_Perlmutter", sub["job_id"], interval=60)
|
|
16
|
+
g.https_stageout_job_logs("NERSC_Perlmutter", sub["job_id"], to="out/", run_id=sub["run_id"])
|
|
17
|
+
g.job_cleanup("NERSC_Perlmutter", sub["run_id"])
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Conventions
|
|
21
|
+
|
|
22
|
+
- **Construction.** `GlobusIRI(config=None, *, work_dir=None)`. `config` is a path to
|
|
23
|
+
`sites.yaml` or an already loaded `globusiri.config.Config`. `work_dir` overrides
|
|
24
|
+
`$GLOBUSIRI_WORK_DIR`. The config is loaded on first use, so `paths_info()` works before
|
|
25
|
+
`init()`.
|
|
26
|
+
- **Return values.** Methods return plain data (dicts, lists, or the package's dataclasses)
|
|
27
|
+
and never print. Progress is reported through optional callbacks (`on_update`, `echo`).
|
|
28
|
+
- **Errors.** Failures raise `globusiri.errors.GlobusIriError` subclasses: `ConfigError`,
|
|
29
|
+
`LoginRequired`, `PandaError`, `RemoteError`, `JobFailed`, `ValidationFailed`. Each has the
|
|
30
|
+
CLI exit code as `exit_code`, and many have a `fix` command to suggest.
|
|
31
|
+
- **Waiting.** CLI flags such as `--no-wait` and `--no-validate` become keyword arguments
|
|
32
|
+
with positive names: `wait=True` and `validate=True`.
|
|
33
|
+
- **Token commands** act on one or more targets (one per facility for `iri`, one per site for
|
|
34
|
+
`https`, and the single `e2e` token). They return a list of `TargetResult(kind, target,
|
|
35
|
+
result, error)`. By default the first error is raised. With `raise_errors=False`, every
|
|
36
|
+
target is tried and the error is stored in `error`, which is what the CLI does.
|
|
37
|
+
|
|
38
|
+
## Methods
|
|
39
|
+
|
|
40
|
+
| CLI | Python |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `globusiri init [--sites S] [--force]` | `globusiri.api.init(work_dir=None, *, sites=None, force=False)` |
|
|
43
|
+
| `globusiri paths` | `g.paths_info()` → `[(what, path, source)]` |
|
|
44
|
+
| `globusiri {iri,https,e2e} login [SITE]` | `g.login(kind, site=None, *, force, prompt_login, import_file)` |
|
|
45
|
+
| `globusiri {kind} refresh [SITE \| --all]` | `g.refresh(kind, site=None, *, all_sites, min_lifetime, upload_to_panda, validate, raise_errors)` → `[TargetResult(result=RefreshResult)]` |
|
|
46
|
+
| `globusiri {kind} upload_to_panda` | `g.upload_to_panda(kind, site=None, *, all_sites, raise_errors)` |
|
|
47
|
+
| `globusiri {kind} download_from_panda` | `g.download_from_panda(kind, site=None, *, all_sites, raise_errors)` |
|
|
48
|
+
| `globusiri {kind} status` | `g.token_status(kind, site=None, *, all_sites, raise_errors)` → `[TargetResult(result=TokenStatus)]` |
|
|
49
|
+
| `globusiri validate [SITE...]` | `g.validate(sites=None, *, kinds, job, offline, write_test, panda, min_lifetime)` → `[Check]`; use `globusiri.validate.failed(checks, strict)` |
|
|
50
|
+
| `globusiri job submit` | `g.job_submit(job, site=None, *, run_id, set, inputs, dry_run, validate)` → `{"run_id", "job_id", "remote_dir", "stdout_path", ...}`; `ValidationFailed.checks` lists the failed checks |
|
|
51
|
+
| `globusiri job status` | `g.job_status(site, job_id)` → `(state, report)` |
|
|
52
|
+
| `globusiri job status --wait` | `g.job_wait(site, job_id, *, interval, timeout, on_update, raise_on_failure)` |
|
|
53
|
+
| `globusiri job cancel` | `g.job_cancel(site, job_id)` |
|
|
54
|
+
| `globusiri job list` | `g.job_list(site, *, historical, limit)` → `[{"job_id", "name", "iri_state", "state", "report"}]` |
|
|
55
|
+
| `globusiri job cleanup` | `g.job_cleanup(site, run_id)` → `{path: "removed" \| "already gone"}` |
|
|
56
|
+
| `globusiri fs ls/stat/mkdir/chmod/rm` | `g.fs_ls`, `g.fs_stat`, `g.fs_mkdir`, `g.fs_chmod`, `g.fs_rm(site, path, *, resource_id, force)` |
|
|
57
|
+
| `globusiri https stagein` | `g.https_stagein(site, files, to, *, mkdir, executable)` → remote paths |
|
|
58
|
+
| `globusiri https stageout` | `g.https_stageout(site, files, to, *, overwrite)` → `{remote: local}` |
|
|
59
|
+
| `globusiri https stageout --job-id` | `g.https_stageout_job_logs(site, job_id, to, *, run_id, overwrite)` |
|
|
60
|
+
| `globusiri e2e stagein`/`stageout` | `g.e2e_submit(site, direction, pairs=(), *, manifest, rucio, endpoint, sync_level, rucio_deterministic, scope, run_id, validate=True, wait=False, interval, max_attempts=5, echo)` → state dict. Creates a tracked transfer run (`kind: transfer`) and submits it; `monitor()`/`daemon()` check it, and `runs()`/`cancel()`/`reopen()` work on it. `wait=True` polls until it ends |
|
|
61
|
+
| — | `g.e2e_stagein(site, pairs=(), *, manifest, rucio, src_endpoint, sync_level, wait, on_update, label, rucio_deterministic, scope)` → `{"task_ids", "files", "src", "dst"}`; an untracked transfer (no run state) |
|
|
62
|
+
| — | `g.e2e_stageout(site, pairs=(), *, manifest, rucio, dst_endpoint, sync_level, wait, on_update, label, rucio_deterministic, scope)`; untracked, as above |
|
|
63
|
+
| `globusiri e2e cleanup` | `g.e2e_cleanup(site, paths=(), *, manifest, rucio, force, dry_run, wait, on_update, label)` → `{"endpoint", "items", "task_ids"}` |
|
|
64
|
+
| `globusiri e2e task` | `g.e2e_task(task_id, *, failures)`; `g.e2e_wait(task_ids, *, on_update)` |
|
|
65
|
+
| `globusiri e2e cancel` | `g.e2e_cancel(task_id)` |
|
|
66
|
+
| `globusiri run --job` | `g.run(job, site=None, *, set, run_id, stagein_manifest, stageout_manifest, validate, wait, interval, echo)` → `RunState` |
|
|
67
|
+
| `globusiri run --resume` | `g.resume(run_id, *, validate, wait, interval, echo)` → `RunState` |
|
|
68
|
+
| `globusiri submit` | `g.submit(job, site=None, *, set, inputs, overrides, run_id, stagein_manifest, stageout_manifest, validate, advance=True, max_attempts=5, echo)` → state dict; creates the run and makes the first non-blocking pass |
|
|
69
|
+
| `globusiri monitor [RUN_ID...]` | `g.monitor(run_ids=None, *, resume=False, echo)` → `{run_id: state dict or error string}`; one pass over every active run. `resume=True` first reopens the given failed runs |
|
|
70
|
+
| `globusiri submit --resume RUN_ID` | `g.reopen(run_id, *, validate=None)` → state dict, then `g.monitor([run_id])`. `reopen` makes a failed run active again (failed steps → pending, done steps kept) without advancing it |
|
|
71
|
+
| — | `g.advance(run_id, *, echo)` → state dict; one pass over one run (raises `workflow.RunBusy` if another process holds it) |
|
|
72
|
+
| `globusiri daemon` | `g.daemon(*, interval=60, echo, stop, max_passes)`; `monitor()` in a loop until `stop()` is true |
|
|
73
|
+
| `globusiri cancel RUN_ID` | `g.cancel(run_id, *, echo)` → state dict |
|
|
74
|
+
| `globusiri runs [RUN_ID] [--reindex]` | `g.runs(*, status=None, summary=False, reindex=False)` → full state dicts, or index rows with `summary=True`; `g.run_state(run_id)` → one full state |
|
|
75
|
+
| — | `g.run_state(run_id)` → the saved `state.json` as a dict, plus `plugin_status` |
|
|
76
|
+
| — | `g.resolve_job(job, site=None, *, set, inputs)` → `(Job, Site)`; `g.e2e_plan(site, direction, ...)` → transfer plan without submitting |
|
|
77
|
+
|
|
78
|
+
`--rucio` / `rucio=["SCOPE:NAME"]` only builds the Rucio deterministic RSE path; it does not
|
|
79
|
+
contact Rucio (see [cli.md](cli.md#globus-e2e-staging)).
|
|
80
|
+
|
|
81
|
+
## Example: submit, wait and clean up with progress
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from globusiri import GlobusIRI
|
|
85
|
+
from globusiri.errors import GlobusIriError, ValidationFailed
|
|
86
|
+
|
|
87
|
+
g = GlobusIRI()
|
|
88
|
+
try:
|
|
89
|
+
state = g.run("crux_example_job.yaml", set=["resources.queue=debug"], echo=print)
|
|
90
|
+
print(state.data["steps"]["submit"]["job_id"])
|
|
91
|
+
except ValidationFailed as exc:
|
|
92
|
+
print("fix tokens first:", exc)
|
|
93
|
+
except GlobusIriError as exc:
|
|
94
|
+
print(f"failed (exit {exc.exit_code}): {exc}")
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Example: plugin mode (no waiting, no cron)
|
|
98
|
+
|
|
99
|
+
A host program (for example a Harvester plugin) calls `submit` once per job and `monitor`
|
|
100
|
+
on each of its own cycles. Every call checks the run's current step (Globus task, IRI job)
|
|
101
|
+
and starts the next step when the current one is done; it never waits.
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
g = GlobusIRI()
|
|
105
|
+
|
|
106
|
+
# submitter cycle
|
|
107
|
+
r = g.submit("perlmutter_example_job.yaml", run_id=f"worker-{worker_id}",
|
|
108
|
+
overrides={"inputs": {"pandaJobData.out": f"{access_point}/pandaJobData.out"},
|
|
109
|
+
"template_vars": {"workerID": worker_id}})
|
|
110
|
+
|
|
111
|
+
# monitor cycle
|
|
112
|
+
for run_id, r in g.monitor([f"worker-{w}" for w in worker_ids]).items():
|
|
113
|
+
if isinstance(r, str): # busy, or could not be read; try again next cycle
|
|
114
|
+
continue
|
|
115
|
+
status = r["plugin_status"] # submitted | running | finished | failed | cancelled
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`plugin_status` uses the same strings as Harvester's `WorkSpec.ST_*`. It stays `running`
|
|
119
|
+
while the logs and outputs are being staged out after the batch job ends, and becomes
|
|
120
|
+
`finished` only when the whole workflow is done.
|
|
121
|
+
|
|
122
|
+
`g.monitor()` without IDs checks all active runs in WORK_DIR, including runs created by
|
|
123
|
+
other processes. `g.monitor([])` checks none. New submissions are found on the next pass.
|
|
124
|
+
Multiple monitor callers and submitters can share WORK_DIR: a locked run is reported as
|
|
125
|
+
`busy`, while the other runs continue. The index (SQLite, or MySQL/MariaDB with
|
|
126
|
+
[`database.url`](configuration.md#database)) discovers active runs without
|
|
127
|
+
scanning completed run directories. `g.runs(summary=True)` reads the index alone, and
|
|
128
|
+
`g.runs(reindex=True)` rebuilds it from saved states. Use a local filesystem with POSIX
|
|
129
|
+
locks and SQLite WAL support; see [CLI bookkeeping](cli.md#non-blocking-workflow-submit-monitor-daemon).
|