globusiri 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- globusiri/__init__.py +20 -0
- globusiri/api.py +626 -0
- globusiri/cleanup.py +92 -0
- globusiri/cli.py +657 -0
- globusiri/clients/__init__.py +1 -0
- globusiri/clients/http.py +82 -0
- globusiri/clients/https.py +76 -0
- globusiri/clients/iri.py +181 -0
- globusiri/clients/transfer.py +53 -0
- globusiri/config.py +285 -0
- globusiri/data/jobs/crux_example_job.yaml +62 -0
- globusiri/data/jobs/crux_hello_job.yaml +31 -0
- globusiri/data/jobs/crux_mpi_job.yaml +48 -0
- globusiri/data/jobs/crux_script_job.sh +29 -0
- globusiri/data/jobs/crux_script_job.yaml +31 -0
- globusiri/data/jobs/perlmutter_example_job.yaml +71 -0
- globusiri/data/jobs/perlmutter_hello_job.yaml +29 -0
- globusiri/data/jobs/perlmutter_mpi_job.yaml +48 -0
- globusiri/data/jobs/perlmutter_script_job.sh +14 -0
- globusiri/data/jobs/perlmutter_script_job.yaml +29 -0
- globusiri/data/jobs/perlmutter_transfer_job.yaml +50 -0
- globusiri/data/jobs/perlmutter_wordcount_job.yaml +29 -0
- globusiri/data/sites.example.yaml +185 -0
- globusiri/data/templates/crux_iri_submit_template.sh +160 -0
- globusiri/data/templates/perlmutter_iri_submit_template.sh +111 -0
- globusiri/data/templates/shell_job_template.sh +57 -0
- globusiri/errors.py +60 -0
- globusiri/jobfile.py +289 -0
- globusiri/jobs.py +404 -0
- globusiri/orm/__init__.py +86 -0
- globusiri/orm/models.py +107 -0
- globusiri/orm/session.py +158 -0
- globusiri/panda_secrets.py +53 -0
- globusiri/paths.py +151 -0
- globusiri/runindex.py +215 -0
- globusiri/runlog.py +101 -0
- globusiri/secure_io.py +82 -0
- globusiri/services.py +26 -0
- globusiri/staging/__init__.py +1 -0
- globusiri/staging/e2e.py +290 -0
- globusiri/staging/https.py +77 -0
- globusiri/tokens/__init__.py +21 -0
- globusiri/tokens/base.py +216 -0
- globusiri/tokens/e2e.py +136 -0
- globusiri/tokens/https.py +133 -0
- globusiri/tokens/iri.py +149 -0
- globusiri/validate.py +383 -0
- globusiri/workflow.py +774 -0
- globusiri-0.0.3.dist-info/METADATA +122 -0
- globusiri-0.0.3.dist-info/RECORD +52 -0
- globusiri-0.0.3.dist-info/WHEEL +4 -0
- globusiri-0.0.3.dist-info/entry_points.txt +2 -0
globusiri/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""GlobusIRI: run HPC jobs through the IRI API and move files with Globus.
|
|
2
|
+
|
|
3
|
+
Python API (every CLI command is a method)::
|
|
4
|
+
|
|
5
|
+
from globusiri import GlobusIRI
|
|
6
|
+
g = GlobusIRI()
|
|
7
|
+
g.job_submit("perlmutter_example_job.yaml")
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "0.0.3"
|
|
11
|
+
|
|
12
|
+
__all__ = ["GlobusIRI", "__version__"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def __getattr__(name): # lazy, so `import globusiri` stays cheap and cycle-free
|
|
16
|
+
if name == "GlobusIRI":
|
|
17
|
+
from .api import GlobusIRI
|
|
18
|
+
|
|
19
|
+
return GlobusIRI
|
|
20
|
+
raise AttributeError(f"module 'globusiri' has no attribute {name!r}")
|
globusiri/api.py
ADDED
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
"""Python API: every `globusiri` CLI command as a method.
|
|
2
|
+
|
|
3
|
+
from globusiri import GlobusIRI
|
|
4
|
+
|
|
5
|
+
g = GlobusIRI() # WORK_DIR and config as the CLI finds them
|
|
6
|
+
g.refresh("iri", all_sites=True, min_lifetime="6h")
|
|
7
|
+
sub = g.job_submit("perlmutter_example_job.yaml", set=["resources.n_node=2"])
|
|
8
|
+
state, report = g.job_wait("NERSC_Perlmutter", sub["job_id"])
|
|
9
|
+
g.https_stageout_job_logs("NERSC_Perlmutter", sub["job_id"], to="out/", run_id=sub["run_id"])
|
|
10
|
+
g.job_cleanup("NERSC_Perlmutter", sub["run_id"])
|
|
11
|
+
|
|
12
|
+
Methods return plain data (dicts, lists, dataclasses) and never print. Errors are raised
|
|
13
|
+
as `globusiri.errors.GlobusIriError` subclasses; each has the CLI exit code as `exit_code`.
|
|
14
|
+
The CLI (`globusiri.cli`) is a thin layer that parses arguments, calls these methods and
|
|
15
|
+
formats the results.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import tempfile
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import yaml
|
|
25
|
+
|
|
26
|
+
from . import cleanup as cleanup_mod
|
|
27
|
+
from . import jobs, services, workflow
|
|
28
|
+
from . import validate as validation
|
|
29
|
+
from .config import load_config, parse_duration
|
|
30
|
+
from .errors import ConfigError, GlobusIriError, JobFailed, ValidationFailed
|
|
31
|
+
from .jobfile import resolve_job
|
|
32
|
+
from .paths import Paths, package_data_dir, work_dir_source
|
|
33
|
+
from .runlog import run_log_path, run_logging
|
|
34
|
+
from .staging import e2e as e2e_staging
|
|
35
|
+
from .staging import https as https_staging
|
|
36
|
+
from .tokens import KINDS, manager
|
|
37
|
+
|
|
38
|
+
__all__ = ["GlobusIRI", "TargetResult", "init", "KINDS"]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class TargetResult:
|
|
43
|
+
"""Outcome of a token command for one target (a facility, a site, or the e2e token)."""
|
|
44
|
+
|
|
45
|
+
kind: str
|
|
46
|
+
target: str
|
|
47
|
+
result: object = None
|
|
48
|
+
error: GlobusIriError = None
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def ok(self):
|
|
52
|
+
return self.error is None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _target_label(m):
|
|
56
|
+
return "e2e" if m.kind == "e2e" else (f"{m.name} ({m.facility})" if m.kind == "iri" else m.name)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _check_kind(kind):
|
|
60
|
+
if kind not in KINDS:
|
|
61
|
+
raise ConfigError(f"unknown token kind '{kind}' (known: {', '.join(KINDS)})")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------- init (no config needed)
|
|
65
|
+
def init(work_dir=None, *, sites=None, force=False):
|
|
66
|
+
"""Create WORK_DIR, config/sites.yaml and the example job files.
|
|
67
|
+
|
|
68
|
+
sites: optional list (or comma-separated string) of example sites to keep.
|
|
69
|
+
Returns {"work_dir", "source", "written": [paths], "kept": [paths]}.
|
|
70
|
+
"""
|
|
71
|
+
root, source = (Path(work_dir).resolve(), "work_dir argument") if work_dir else work_dir_source()
|
|
72
|
+
paths = Paths(root)
|
|
73
|
+
for parent in [paths.root, *paths.root.parents]:
|
|
74
|
+
if (parent / ".git").exists() and not force:
|
|
75
|
+
raise ConfigError(f"WORK_DIR {paths.root} is inside a git work tree ({parent}); tokens must not be "
|
|
76
|
+
"committed. Set GLOBUSIRI_WORK_DIR elsewhere or pass force=True")
|
|
77
|
+
paths.ensure()
|
|
78
|
+
data = package_data_dir()
|
|
79
|
+
written, kept = [], []
|
|
80
|
+
cfg = paths.default_config
|
|
81
|
+
if cfg.exists() and not force:
|
|
82
|
+
kept.append(cfg)
|
|
83
|
+
else:
|
|
84
|
+
text = (data / "sites.example.yaml").read_text(encoding="utf-8")
|
|
85
|
+
if sites:
|
|
86
|
+
wanted = [s.strip() for s in sites.split(",")] if isinstance(sites, str) else list(sites)
|
|
87
|
+
wanted = [s for s in wanted if s]
|
|
88
|
+
raw = yaml.safe_load(text)
|
|
89
|
+
unknown = [s for s in wanted if s not in raw["sites"]]
|
|
90
|
+
if unknown:
|
|
91
|
+
raise ConfigError(f"sites: unknown in example: {', '.join(unknown)} (known: {', '.join(raw['sites'])})")
|
|
92
|
+
raw["sites"] = {k: v for k, v in raw["sites"].items() if k in wanted}
|
|
93
|
+
text = "# Generated by `globusiri init --sites`; see globusiri/data/sites.example.yaml for comments.\n" + \
|
|
94
|
+
yaml.safe_dump(raw, sort_keys=False)
|
|
95
|
+
cfg.write_text(text, encoding="utf-8")
|
|
96
|
+
written.append(cfg)
|
|
97
|
+
# example job files, and the job scripts that `script_file` examples point to
|
|
98
|
+
for src in sorted([*(data / "jobs").glob("*.yaml"), *(data / "jobs").glob("*.sh")]):
|
|
99
|
+
dst = paths.jobs_dir / src.name
|
|
100
|
+
if dst.exists() and not force:
|
|
101
|
+
kept.append(dst)
|
|
102
|
+
else:
|
|
103
|
+
shutil.copyfile(src, dst)
|
|
104
|
+
written.append(dst)
|
|
105
|
+
return {"work_dir": paths.root, "source": source, "written": written, "kept": kept}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class GlobusIRI:
|
|
109
|
+
"""Entry point of the Python API.
|
|
110
|
+
|
|
111
|
+
config: path to sites.yaml, or a loaded `globusiri.config.Config`. Default: the CLI's
|
|
112
|
+
lookup ($GLOBUSIRI_CONFIG, else <WORK_DIR>/config/sites.yaml).
|
|
113
|
+
work_dir: WORK_DIR override. Default: $GLOBUSIRI_WORK_DIR, else the per-user data dir.
|
|
114
|
+
|
|
115
|
+
The config is loaded on first use, so `paths()` works before `init()`.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
def __init__(self, config=None, *, work_dir=None):
|
|
119
|
+
self.paths = Paths(Path(work_dir).resolve()) if work_dir else Paths()
|
|
120
|
+
if config is None or isinstance(config, (str, Path)):
|
|
121
|
+
self._config_path, self._config = config, None
|
|
122
|
+
else:
|
|
123
|
+
self._config_path, self._config = None, config
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def config(self):
|
|
127
|
+
if self._config is None:
|
|
128
|
+
self._config = load_config(self._config_path, paths=self.paths)
|
|
129
|
+
return self._config
|
|
130
|
+
|
|
131
|
+
def site(self, name):
|
|
132
|
+
return self.config.site(name)
|
|
133
|
+
|
|
134
|
+
# ------------------------------------------------------------------ paths
|
|
135
|
+
def _run_index_row(self):
|
|
136
|
+
"""("run index", database, source); the password of a server URL is hidden."""
|
|
137
|
+
from .orm import ENV_URL, DatabaseConfig
|
|
138
|
+
try:
|
|
139
|
+
db = self.config.database
|
|
140
|
+
except Exception: # noqa: BLE001 - paths works before init and with a broken config
|
|
141
|
+
try:
|
|
142
|
+
db = DatabaseConfig(None, self.paths.runs_dir)
|
|
143
|
+
except Exception as exc: # noqa: BLE001
|
|
144
|
+
return ("run index", f"<{exc}>", ENV_URL)
|
|
145
|
+
if os.environ.get(ENV_URL):
|
|
146
|
+
source = ENV_URL
|
|
147
|
+
elif db.backend == "sqlite" and db.sqlite_path == (self.paths.runs_dir / "index.db").resolve():
|
|
148
|
+
source = "WORK_DIR"
|
|
149
|
+
else:
|
|
150
|
+
source = "sites.yaml database.url"
|
|
151
|
+
where = db.sqlite_path or f"{db.display} (namespace {db.namespace})"
|
|
152
|
+
return ("run index", where, source)
|
|
153
|
+
|
|
154
|
+
def paths_info(self):
|
|
155
|
+
"""Resolved locations as [(what, path, source)]; never fails on a bad config."""
|
|
156
|
+
root, source = work_dir_source()
|
|
157
|
+
if self.paths.root != root:
|
|
158
|
+
source = "work_dir argument"
|
|
159
|
+
cfg_path, cfg_source = self.paths.config_file(self._config_path)
|
|
160
|
+
rows = [
|
|
161
|
+
("WORK_DIR", self.paths.root, source),
|
|
162
|
+
("config", cfg_path, cfg_source + ("" if cfg_path.exists() else " (missing: run globusiri init)")),
|
|
163
|
+
("tokens", self.paths.tokens_dir, "WORK_DIR"),
|
|
164
|
+
("logs", self.paths.log_file, "WORK_DIR"),
|
|
165
|
+
("runs", self.paths.runs_dir, "WORK_DIR"),
|
|
166
|
+
self._run_index_row(),
|
|
167
|
+
("jobs", self.paths.jobs_dir, "WORK_DIR"),
|
|
168
|
+
("templates", f"{self.paths.templates_dir}, then {package_data_dir() / 'templates'}", "WORK_DIR, package"),
|
|
169
|
+
]
|
|
170
|
+
if cfg_path.exists():
|
|
171
|
+
try:
|
|
172
|
+
cfg = self.config
|
|
173
|
+
for fac in cfg.facilities():
|
|
174
|
+
m = manager("iri", cfg, cfg.sites_for_facility(fac)[0])
|
|
175
|
+
rows.append((f"iri {fac}", m.access_file, "exists" if m.access_file.exists() else "missing"))
|
|
176
|
+
for site in cfg.sites.values():
|
|
177
|
+
if site.has_https:
|
|
178
|
+
m = manager("https", cfg, site)
|
|
179
|
+
rows.append((f"https {site.name}", m.access_file, "exists" if m.access_file.exists() else "missing"))
|
|
180
|
+
m = manager("e2e", cfg)
|
|
181
|
+
rows.append(("e2e", m.access_file, "exists" if m.access_file.exists() else "missing"))
|
|
182
|
+
except GlobusIriError as exc:
|
|
183
|
+
rows.append(("config error", str(exc), ""))
|
|
184
|
+
return rows
|
|
185
|
+
|
|
186
|
+
# ------------------------------------------------------------------ tokens
|
|
187
|
+
def token_managers(self, kind, site=None, *, all_sites=False):
|
|
188
|
+
"""Token managers a command acts on. iri: one per facility; https: per site; e2e: one."""
|
|
189
|
+
_check_kind(kind)
|
|
190
|
+
cfg = self.config
|
|
191
|
+
if kind == "e2e":
|
|
192
|
+
return [manager("e2e", cfg)]
|
|
193
|
+
if all_sites:
|
|
194
|
+
sites = list(cfg.sites.values())
|
|
195
|
+
elif site:
|
|
196
|
+
sites = [cfg.site(site)]
|
|
197
|
+
else:
|
|
198
|
+
raise ConfigError("give SITE or --all (all_sites=True)")
|
|
199
|
+
out, seen = [], set()
|
|
200
|
+
for s in sites:
|
|
201
|
+
if kind == "iri":
|
|
202
|
+
if s.facility in seen:
|
|
203
|
+
continue
|
|
204
|
+
seen.add(s.facility)
|
|
205
|
+
if kind == "https" and not s.has_https:
|
|
206
|
+
if all_sites:
|
|
207
|
+
continue
|
|
208
|
+
raise ConfigError(f"site {s.name} has no https section")
|
|
209
|
+
out.append(manager(kind, cfg, s))
|
|
210
|
+
return out
|
|
211
|
+
|
|
212
|
+
def _each(self, kind, site, all_sites, fn, raise_errors):
|
|
213
|
+
results = []
|
|
214
|
+
for m in self.token_managers(kind, site, all_sites=all_sites):
|
|
215
|
+
r = TargetResult(kind, _target_label(m))
|
|
216
|
+
try:
|
|
217
|
+
r.result = fn(m)
|
|
218
|
+
except GlobusIriError as exc:
|
|
219
|
+
if raise_errors:
|
|
220
|
+
raise
|
|
221
|
+
r.error = exc
|
|
222
|
+
results.append(r)
|
|
223
|
+
return results
|
|
224
|
+
|
|
225
|
+
def login(self, kind, site=None, *, force=False, prompt_login=False, import_file=None):
|
|
226
|
+
"""Interactive Globus login (or import a refresh token file). Returns a dict."""
|
|
227
|
+
_check_kind(kind)
|
|
228
|
+
if kind != "e2e" and not site:
|
|
229
|
+
raise ConfigError(f"{kind} login needs a site")
|
|
230
|
+
m = self.token_managers(kind, None if kind == "e2e" else site)[0]
|
|
231
|
+
expires_at = m.login(force=force, prompt_login=prompt_login, import_file=import_file)
|
|
232
|
+
return {"kind": kind, "target": _target_label(m), "refresh_file": m.refresh_file,
|
|
233
|
+
"access_file": m.access_file, "expires_at": expires_at}
|
|
234
|
+
|
|
235
|
+
def refresh(self, kind, site=None, *, all_sites=False, min_lifetime=None, upload_to_panda=False,
|
|
236
|
+
validate=None, raise_errors=True):
|
|
237
|
+
"""Refresh token -> access token (never prompts). Returns [TargetResult(result=RefreshResult)]."""
|
|
238
|
+
min_life = parse_duration(min_lifetime, self.config.refresh_min_lifetime)
|
|
239
|
+
return self._each(kind, site, all_sites,
|
|
240
|
+
lambda m: m.refresh(min_lifetime=min_life, upload_to_panda=upload_to_panda, validate=validate),
|
|
241
|
+
raise_errors)
|
|
242
|
+
|
|
243
|
+
def upload_to_panda(self, kind, site=None, *, all_sites=False, raise_errors=True):
|
|
244
|
+
"""Access token -> PanDA user secret. Returns [TargetResult(result=secret key)]."""
|
|
245
|
+
def fn(m):
|
|
246
|
+
m.upload_to_panda()
|
|
247
|
+
return m.panda_secret_key
|
|
248
|
+
return self._each(kind, site, all_sites, fn, raise_errors)
|
|
249
|
+
|
|
250
|
+
def download_from_panda(self, kind, site=None, *, all_sites=False, raise_errors=True):
|
|
251
|
+
"""PanDA user secret -> access token file. Returns [TargetResult(result=token data)]."""
|
|
252
|
+
def fn(m):
|
|
253
|
+
data = m.download_from_panda()
|
|
254
|
+
return {"access_file": m.access_file, "expires_at": data.get("expires_at")}
|
|
255
|
+
return self._each(kind, site, all_sites, fn, raise_errors)
|
|
256
|
+
|
|
257
|
+
def token_status(self, kind, site=None, *, all_sites=False, raise_errors=True):
|
|
258
|
+
"""Offline: files, permissions, expiry. Returns [TargetResult(result=TokenStatus)]."""
|
|
259
|
+
return self._each(kind, site, all_sites, lambda m: m.status(), raise_errors)
|
|
260
|
+
|
|
261
|
+
# ------------------------------------------------------------------ validate
|
|
262
|
+
def validate(self, sites=None, *, kinds=None, job=None, offline=False, write_test=False, panda=False,
|
|
263
|
+
min_lifetime=None):
|
|
264
|
+
"""Validate tokens (live unless offline). Returns [validate.Check].
|
|
265
|
+
|
|
266
|
+
job: a job file; then only the tokens that job needs are checked, at its site
|
|
267
|
+
(sites may name at most one site, which overrides the job's site).
|
|
268
|
+
Use `globusiri.validate.failed(checks, strict)` to decide pass/fail.
|
|
269
|
+
"""
|
|
270
|
+
kinds = list(kinds or KINDS)
|
|
271
|
+
bad = [k for k in kinds if k not in KINDS]
|
|
272
|
+
if bad:
|
|
273
|
+
raise ConfigError(f"kinds: unknown {', '.join(bad)}")
|
|
274
|
+
if isinstance(sites, str):
|
|
275
|
+
sites = [sites]
|
|
276
|
+
job_obj, site_objs = None, None
|
|
277
|
+
if job:
|
|
278
|
+
if sites and len(sites) > 1:
|
|
279
|
+
raise ConfigError("job validates one site; give at most one site")
|
|
280
|
+
job_obj, _ = resolve_job(self.config, job, site_name=sites[0] if sites else None)
|
|
281
|
+
elif sites:
|
|
282
|
+
site_objs = [self.config.site(s) for s in sites]
|
|
283
|
+
min_life = parse_duration(min_lifetime, self.config.refresh_min_lifetime)
|
|
284
|
+
return validation.validate(self.config, sites=site_objs, kinds=kinds, job=job_obj, offline=offline,
|
|
285
|
+
write_test=write_test, panda=panda, min_lifetime=min_life)
|
|
286
|
+
|
|
287
|
+
# ------------------------------------------------------------------ jobs
|
|
288
|
+
def resolve_job(self, job, site=None, *, set=None, inputs=None):
|
|
289
|
+
"""Merge site job_defaults < job file < overrides. Returns (Job, Site)."""
|
|
290
|
+
overrides = list(set or []) + [f"inputs.{i}" for i in (inputs or [])]
|
|
291
|
+
return resolve_job(self.config, job, site_name=site, overrides=overrides)
|
|
292
|
+
|
|
293
|
+
def job_submit(self, job, site=None, *, run_id=None, set=None, inputs=None, dry_run=False, validate=True):
|
|
294
|
+
"""Render, upload and submit a job file.
|
|
295
|
+
|
|
296
|
+
set: ["key=value", ...] overrides; inputs: ["NAME=LOCAL", ...].
|
|
297
|
+
Returns {"run_id", "job_id", "remote_dir", "stdout_path", "skipped_inputs", "submission"};
|
|
298
|
+
with dry_run: {"run_id", "script", "job_spec", "skipped_inputs"} and nothing is contacted.
|
|
299
|
+
Raises ValidationFailed (with .checks) if the needed tokens fail validation.
|
|
300
|
+
"""
|
|
301
|
+
cfg = self.config
|
|
302
|
+
job_obj, site_obj = self.resolve_job(job, site, set=set, inputs=inputs)
|
|
303
|
+
run_id = run_id or jobs.new_run_id()
|
|
304
|
+
if dry_run:
|
|
305
|
+
with tempfile.TemporaryDirectory() as tmp: # leave no run dir behind
|
|
306
|
+
sub, _ = jobs.prepare(cfg, job_obj, site_obj, run_id, Path(tmp))
|
|
307
|
+
script = Path(sub.script).read_text()
|
|
308
|
+
return {"run_id": run_id, "script": script, "job_spec": sub.job_spec,
|
|
309
|
+
"skipped_inputs": sub.skipped_inputs}
|
|
310
|
+
if validate:
|
|
311
|
+
checks = validation.validate(cfg, job=job_obj,
|
|
312
|
+
kinds=[k for k in job_obj.required_kinds(site_obj) if k != "e2e"])
|
|
313
|
+
if validation.failed(checks):
|
|
314
|
+
exc = ValidationFailed("token validation failed; nothing was uploaded or submitted")
|
|
315
|
+
exc.checks = [c for c in checks if c.status != validation.OK]
|
|
316
|
+
raise exc
|
|
317
|
+
run_dir = cfg.paths.run_dir(run_id)
|
|
318
|
+
with run_logging(run_log_path(cfg, run_id, job_obj), run_id):
|
|
319
|
+
state = workflow.RunState(run_dir, index=cfg.run_index())
|
|
320
|
+
state.data.update({"run_id": run_id, "site": site_obj.name, "created_at": workflow._now()})
|
|
321
|
+
sub, _ = jobs.prepare(cfg, job_obj, site_obj, run_id, run_dir)
|
|
322
|
+
iri = services.iri_client(cfg, site_obj)
|
|
323
|
+
https = services.https_client(cfg, site_obj) if site_obj.upload_method == "globus_https" else None
|
|
324
|
+
jobs.upload_inputs(cfg, site_obj, sub, iri, https)
|
|
325
|
+
state.set("upload", "done", submission=sub.to_dict(), method=site_obj.upload_method)
|
|
326
|
+
job_id = jobs.launch(site_obj, sub, iri)
|
|
327
|
+
state.set("submit", "done", job_id=job_id)
|
|
328
|
+
return {"run_id": run_id, "job_id": job_id, "remote_dir": sub.remote_dir, "stdout_path": sub.stdout_path,
|
|
329
|
+
"skipped_inputs": sub.skipped_inputs, "submission": sub.to_dict()}
|
|
330
|
+
|
|
331
|
+
def _iri(self, site):
|
|
332
|
+
site_obj = self.config.site(site)
|
|
333
|
+
return site_obj, services.iri_client(self.config, site_obj)
|
|
334
|
+
|
|
335
|
+
def job_status(self, site, job_id):
|
|
336
|
+
"""Returns (state, IRI job report). state is one of jobs.TERMINAL or submitted/running."""
|
|
337
|
+
site_obj, iri = self._iri(site)
|
|
338
|
+
return jobs.status(iri, site_obj, job_id)
|
|
339
|
+
|
|
340
|
+
def job_wait(self, site, job_id, *, interval=60, timeout=None, on_update=None, raise_on_failure=False):
|
|
341
|
+
"""Poll until the job ends. on_update(state, report) is called on each state change.
|
|
342
|
+
Returns (state, report); with raise_on_failure, a non-finished end raises JobFailed."""
|
|
343
|
+
site_obj, iri = self._iri(site)
|
|
344
|
+
state, report = jobs.wait(iri, site_obj, job_id, interval=interval, timeout=timeout, on_update=on_update)
|
|
345
|
+
if raise_on_failure and state != jobs.FINISHED:
|
|
346
|
+
raise JobFailed(f"job {job_id} ended {state}")
|
|
347
|
+
return state, report
|
|
348
|
+
|
|
349
|
+
def job_cancel(self, site, job_id):
|
|
350
|
+
site_obj, iri = self._iri(site)
|
|
351
|
+
return jobs.cancel(iri, site_obj, job_id)
|
|
352
|
+
|
|
353
|
+
def job_list(self, site, *, historical=False, limit=None):
|
|
354
|
+
"""Returns [{"job_id", "name", "iri_state", "state", "report"}]."""
|
|
355
|
+
_, iri = self._iri(site)
|
|
356
|
+
result = iri.get_jobs(historical=historical, limit=limit)
|
|
357
|
+
items = result if isinstance(result, list) else result.get("jobs") or result.get("items") or []
|
|
358
|
+
out = []
|
|
359
|
+
for j in items:
|
|
360
|
+
st = j.get("status") or {}
|
|
361
|
+
out.append({"job_id": j.get("id"), "name": (j.get("job_spec") or {}).get("name") or j.get("name"),
|
|
362
|
+
"iri_state": st.get("state"), "state": jobs.map_state(j), "report": j})
|
|
363
|
+
return out
|
|
364
|
+
|
|
365
|
+
def job_cleanup(self, site, run_id):
|
|
366
|
+
"""Remove a run's remote directory (and log directory) via IRI rm.
|
|
367
|
+
Returns {path: "removed" | "already gone"}."""
|
|
368
|
+
site_obj, iri = self._iri(site)
|
|
369
|
+
return cleanup_mod.cleanup_run(iri, site_obj, run_id)
|
|
370
|
+
|
|
371
|
+
# ------------------------------------------------------------------ IRI filesystem
|
|
372
|
+
def _fs(self, site, resource_id):
|
|
373
|
+
site_obj, iri = self._iri(site)
|
|
374
|
+
return site_obj, iri, resource_id or site_obj.work_dir_resource_id
|
|
375
|
+
|
|
376
|
+
def fs_ls(self, site, path, *, resource_id=None, show_hidden=False):
|
|
377
|
+
_, iri, rid = self._fs(site, resource_id)
|
|
378
|
+
return iri.ls(path, resource_id=rid, show_hidden=show_hidden)
|
|
379
|
+
|
|
380
|
+
def fs_stat(self, site, path, *, resource_id=None):
|
|
381
|
+
_, iri, rid = self._fs(site, resource_id)
|
|
382
|
+
return iri.stat(path, resource_id=rid)
|
|
383
|
+
|
|
384
|
+
def fs_mkdir(self, site, path, *, resource_id=None, parents=False):
|
|
385
|
+
_, iri, rid = self._fs(site, resource_id)
|
|
386
|
+
return iri.mkdir(path, resource_id=rid, parents=parents)
|
|
387
|
+
|
|
388
|
+
def fs_chmod(self, site, path, mode, *, resource_id=None):
|
|
389
|
+
_, iri, rid = self._fs(site, resource_id)
|
|
390
|
+
return iri.chmod(path, mode, resource_id=rid)
|
|
391
|
+
|
|
392
|
+
def fs_rm(self, site, path, *, resource_id=None, force=False):
|
|
393
|
+
"""Remove a file or directory. Without force, path must be below the site's
|
|
394
|
+
remote.work_dir, remote.log_dir or e2e.base_path. Returns True if removed,
|
|
395
|
+
False if it was already gone."""
|
|
396
|
+
site_obj, iri, rid = self._fs(site, resource_id)
|
|
397
|
+
path = cleanup_mod.check_under(path, cleanup_mod.site_roots(site_obj), force=force)
|
|
398
|
+
return cleanup_mod.iri_rm(iri, path, resource_id=rid)
|
|
399
|
+
|
|
400
|
+
# ------------------------------------------------------------------ HTTPS staging
|
|
401
|
+
def https_stagein(self, site, files, to, *, mkdir=False, executable=None):
|
|
402
|
+
"""Upload local files into the HPC directory `to`. Returns the remote paths."""
|
|
403
|
+
site_obj = self.config.site(site)
|
|
404
|
+
https = services.https_client(self.config, site_obj)
|
|
405
|
+
iri = services.iri_client(self.config, site_obj) if (mkdir or executable) else None
|
|
406
|
+
return https_staging.stagein(https, files, to, iri=iri, site=site_obj, mkdir=mkdir, executable=executable)
|
|
407
|
+
|
|
408
|
+
def https_stageout(self, site, files, to, *, overwrite=False):
|
|
409
|
+
"""Download remote files into local `to`. Returns {remote: local | "skipped"}."""
|
|
410
|
+
site_obj = self.config.site(site)
|
|
411
|
+
https = services.https_client(self.config, site_obj)
|
|
412
|
+
return https_staging.stageout(https, files, to, skip_existing=not overwrite)
|
|
413
|
+
|
|
414
|
+
def https_stageout_job_logs(self, site, job_id, to, *, run_id=None, overwrite=False):
|
|
415
|
+
"""Download an IRI job's stdout/stderr. Returns {remote: local | "skipped" | "error: ..."}."""
|
|
416
|
+
site_obj, iri = self._iri(site)
|
|
417
|
+
https = services.https_client(self.config, site_obj)
|
|
418
|
+
_, report = jobs.status(iri, site_obj, job_id)
|
|
419
|
+
return https_staging.stageout_job_logs(https, report, site_obj, to, run_id=run_id, skip_existing=not overwrite)
|
|
420
|
+
|
|
421
|
+
# ------------------------------------------------------------------ E2E staging
|
|
422
|
+
def _transfer(self):
|
|
423
|
+
return services.transfer(self.config)
|
|
424
|
+
|
|
425
|
+
def e2e_plan(self, site, direction, *, pairs=(), manifest=None, rucio=(), endpoint=None,
|
|
426
|
+
rucio_deterministic=False, scope=None):
|
|
427
|
+
"""Build (but do not submit) a transfer plan. Returns staging.e2e.Plan.
|
|
428
|
+
|
|
429
|
+
pairs: (SRC, DST), (SRC, DST, SCOPE[, NAME]) or {src, dst[, rucio_deterministic][, scope][, name]}.
|
|
430
|
+
rucio_deterministic/scope are the defaults for pairs and manifest lines. A Rucio-
|
|
431
|
+
deterministic entry's DST is a directory: the file goes to its Rucio deterministic path
|
|
432
|
+
below it, named NAME or SRC's basename."""
|
|
433
|
+
return e2e_staging.plan(self.config, self.config.site(site), direction, manifest=manifest,
|
|
434
|
+
rucio=rucio or [], pairs=list(pairs or []), endpoint_label=endpoint,
|
|
435
|
+
deterministic=rucio_deterministic, scope=scope)
|
|
436
|
+
|
|
437
|
+
def _e2e_transfer(self, site, direction, pairs, manifest, rucio, endpoint, sync_level, wait, on_update, label,
|
|
438
|
+
rucio_deterministic=False, scope=None):
|
|
439
|
+
p = self.e2e_plan(site, direction, pairs=pairs, manifest=manifest, rucio=rucio, endpoint=endpoint,
|
|
440
|
+
rucio_deterministic=rucio_deterministic, scope=scope)
|
|
441
|
+
transfer = self._transfer()
|
|
442
|
+
task_ids = e2e_staging.submit(transfer, self.config, p, label=label, sync_level=sync_level)
|
|
443
|
+
result = {"direction": direction, "src": p.src.label, "dst": p.dst.label, "files": len(p.items),
|
|
444
|
+
"task_ids": task_ids}
|
|
445
|
+
if wait:
|
|
446
|
+
result["tasks"] = self.e2e_wait(task_ids, on_update=on_update, transfer=transfer)
|
|
447
|
+
return result
|
|
448
|
+
|
|
449
|
+
def e2e_stagein(self, site, pairs=(), *, manifest=None, rucio=(), src_endpoint=None, sync_level=None,
|
|
450
|
+
wait=True, on_update=None, label=None, rucio_deterministic=False, scope=None):
|
|
451
|
+
"""Storage endpoint -> HPC. pairs, rucio_deterministic, scope: see e2e_plan; rucio:
|
|
452
|
+
["SCOPE:NAME"] (both sides from the site's Rucio paths; only builds the deterministic RSE
|
|
453
|
+
path, no Rucio lookup). Returns {"task_ids", "files", ...}."""
|
|
454
|
+
return self._e2e_transfer(site, "stagein", pairs, manifest, rucio, src_endpoint, sync_level, wait,
|
|
455
|
+
on_update, label, rucio_deterministic, scope)
|
|
456
|
+
|
|
457
|
+
def e2e_stageout(self, site, pairs=(), *, manifest=None, rucio=(), dst_endpoint=None, sync_level=None,
|
|
458
|
+
wait=True, on_update=None, label=None, rucio_deterministic=False, scope=None):
|
|
459
|
+
"""HPC -> storage endpoint. Same arguments as e2e_stagein."""
|
|
460
|
+
return self._e2e_transfer(site, "stageout", pairs, manifest, rucio, dst_endpoint, sync_level, wait,
|
|
461
|
+
on_update, label, rucio_deterministic, scope)
|
|
462
|
+
|
|
463
|
+
def e2e_submit(self, site, direction, pairs=(), *, manifest=None, rucio=(), endpoint=None, sync_level=None,
|
|
464
|
+
rucio_deterministic=False, scope=None, run_id=None, validate=True, wait=False, interval=None,
|
|
465
|
+
max_attempts=workflow.DEFAULT_MAX_ATTEMPTS, echo=None):
|
|
466
|
+
"""A tracked E2E transfer (the `globusiri e2e stagein/stageout` commands): create a
|
|
467
|
+
transfer run, validate the e2e token (unless validate=False) and submit the Globus task.
|
|
468
|
+
|
|
469
|
+
wait=False returns after that first pass; monitor()/daemon() then check the run like any
|
|
470
|
+
job run, and runs()/cancel()/reopen() work on it. wait=True polls until it ends (every
|
|
471
|
+
`interval` s, default e2e.poll_interval) and raises JobFailed if it failed. Other
|
|
472
|
+
arguments as e2e_stagein. Returns the run's state dict.
|
|
473
|
+
"""
|
|
474
|
+
site_obj = self.config.site(site)
|
|
475
|
+
state = workflow.create_transfer(self.config, site_obj, direction, pairs=pairs, manifest=manifest,
|
|
476
|
+
rucio=rucio, endpoint=endpoint, sync_level=sync_level,
|
|
477
|
+
rucio_deterministic=rucio_deterministic, scope=scope, run_id=run_id,
|
|
478
|
+
do_validate=validate, max_attempts=max_attempts)
|
|
479
|
+
echo = echo or (lambda *a: None)
|
|
480
|
+
if wait:
|
|
481
|
+
st, job_obj, site_obj = workflow.load(self.config, state.run_id)
|
|
482
|
+
workflow.run(self.config, job_obj, site_obj, resume_state=st, echo=echo,
|
|
483
|
+
interval=interval or self.config.e2e.poll_interval)
|
|
484
|
+
else:
|
|
485
|
+
try:
|
|
486
|
+
workflow.advance(self.config, state.run_id, echo=echo)
|
|
487
|
+
except workflow.RunBusy:
|
|
488
|
+
pass # a monitor already picked up the newly published run
|
|
489
|
+
return self.run_state(state.run_id)
|
|
490
|
+
|
|
491
|
+
def e2e_cleanup(self, site, paths=(), *, manifest=None, rucio=(), force=False, dry_run=False, wait=True,
|
|
492
|
+
on_update=None, label=None, rucio_deterministic=False, scope=None):
|
|
493
|
+
"""Delete stage-in files on the HPC endpoint (Globus delete task).
|
|
494
|
+
|
|
495
|
+
manifest: stage-in manifest (its DST column is deleted); rucio: ["SCOPE:NAME"] under
|
|
496
|
+
e2e.base_path; paths: HPC mount paths. Returns {"endpoint", "items", "task_ids"}
|
|
497
|
+
(no task_ids with dry_run).
|
|
498
|
+
"""
|
|
499
|
+
p = e2e_staging.cleanup_plan(self.config.site(site), manifest=manifest, rucio=rucio or [],
|
|
500
|
+
paths=list(paths or []), force=force, deterministic=rucio_deterministic,
|
|
501
|
+
scope=scope)
|
|
502
|
+
result = {"endpoint": p.endpoint.label, "items": list(p.items)}
|
|
503
|
+
if dry_run:
|
|
504
|
+
return result
|
|
505
|
+
transfer = self._transfer()
|
|
506
|
+
result["task_ids"] = e2e_staging.submit_delete(transfer, self.config, p, label=label)
|
|
507
|
+
if wait:
|
|
508
|
+
result["tasks"] = self.e2e_wait(result["task_ids"], on_update=on_update, transfer=transfer)
|
|
509
|
+
return result
|
|
510
|
+
|
|
511
|
+
def e2e_task(self, task_id, *, failures=False):
|
|
512
|
+
"""Task info dict; with failures, adds "failures": [...]."""
|
|
513
|
+
transfer = self._transfer()
|
|
514
|
+
info = e2e_staging.task_info(transfer, task_id)
|
|
515
|
+
if failures:
|
|
516
|
+
info["failures"] = e2e_staging.task_failures(transfer, task_id)
|
|
517
|
+
return info
|
|
518
|
+
|
|
519
|
+
def e2e_wait(self, task_ids, *, on_update=None, transfer=None):
|
|
520
|
+
"""Wait for Globus tasks; raises JobFailed if one fails or e2e.timeout passes."""
|
|
521
|
+
if isinstance(task_ids, str):
|
|
522
|
+
task_ids = [task_ids]
|
|
523
|
+
return e2e_staging.wait(transfer or self._transfer(), task_ids, poll_interval=self.config.e2e.poll_interval,
|
|
524
|
+
timeout=self.config.e2e.timeout, on_update=on_update)
|
|
525
|
+
|
|
526
|
+
def e2e_cancel(self, task_id):
|
|
527
|
+
return dict(e2e_staging.cancel(self._transfer(), task_id))
|
|
528
|
+
|
|
529
|
+
# ------------------------------------------------------------------ end-to-end run
|
|
530
|
+
def run(self, job, site=None, *, set=None, run_id=None, stagein_manifest=None, stageout_manifest=None,
|
|
531
|
+
validate=True, wait=True, interval=60, echo=None):
|
|
532
|
+
"""validate -> e2e stagein -> upload -> submit -> monitor -> https download ->
|
|
533
|
+
e2e stageout -> cleanup. Returns workflow.RunState (state.data is the state.json content).
|
|
534
|
+
echo(msg) receives progress lines (default: silent)."""
|
|
535
|
+
job_obj, site_obj = resolve_job(self.config, job, site_name=site, overrides=set)
|
|
536
|
+
return workflow.run(self.config, job_obj, site_obj, run_id=run_id, stagein_manifest=stagein_manifest,
|
|
537
|
+
stageout_manifest=stageout_manifest, no_wait=not wait, do_validate=validate,
|
|
538
|
+
interval=interval, echo=echo or (lambda *a: None))
|
|
539
|
+
|
|
540
|
+
def resume(self, run_id, *, validate=True, wait=True, interval=60, echo=None):
|
|
541
|
+
"""Continue a run; steps already done are skipped. Returns workflow.RunState."""
|
|
542
|
+
return workflow.resume(self.config, run_id, no_wait=not wait, do_validate=validate, interval=interval,
|
|
543
|
+
echo=echo or (lambda *a: None))
|
|
544
|
+
|
|
545
|
+
def reopen(self, run_id, *, validate=None):
|
|
546
|
+
"""Make a failed (or cancelled) run active again without advancing it: failed steps go
|
|
547
|
+
back to pending, done steps are kept. validate: None keeps the run's setting.
|
|
548
|
+
Returns the state dict; monitor() or daemon() then continue the run."""
|
|
549
|
+
workflow.reopen(self.config, run_id, do_validate=validate)
|
|
550
|
+
return self.run_state(run_id)
|
|
551
|
+
|
|
552
|
+
def run_state(self, run_id):
|
|
553
|
+
"""The saved state.json of a run, as a dict (plus "plugin_status")."""
|
|
554
|
+
data = workflow.RunState.load(self.config.paths.run_dir(run_id)).data
|
|
555
|
+
return {**data, "plugin_status": workflow.plugin_status(data)}
|
|
556
|
+
|
|
557
|
+
# ------------------------------------------------------------------ non-blocking workflow
|
|
558
|
+
# submit() starts a run; every monitor() call checks each run's current step and triggers
|
|
559
|
+
# the next ones. Call monitor() from your own loop (plugin mode, e.g. a Harvester monitor
|
|
560
|
+
# cycle) or let daemon() call it (daemon mode). Neither blocks on a transfer or a job.
|
|
561
|
+
def submit(self, job, site=None, *, set=None, inputs=None, overrides=None, run_id=None, stagein_manifest=None,
|
|
562
|
+
stageout_manifest=None, validate=True, advance=True, max_attempts=workflow.DEFAULT_MAX_ATTEMPTS,
|
|
563
|
+
echo=None):
|
|
564
|
+
"""Create a workflow run and (by default) make its first pass, which goes as far as it
|
|
565
|
+
can without waiting: validate, start the e2e stage-in or, without one, upload and
|
|
566
|
+
submit the IRI job. Returns the run's state dict (see run_state).
|
|
567
|
+
|
|
568
|
+
set: ["key=value", ...]; inputs: ["NAME=LOCAL", ...]; overrides: a nested dict merged
|
|
569
|
+
last (for keys that contain dots, e.g. {"inputs": {"pandaJobData.out": path}}).
|
|
570
|
+
max_attempts: how many passes a step may fail with a transient error before the run fails.
|
|
571
|
+
"""
|
|
572
|
+
job_obj, site_obj = self.resolve_job(job, site, set=set, inputs=inputs)
|
|
573
|
+
if overrides:
|
|
574
|
+
from .jobfile import Job, deep_merge
|
|
575
|
+
|
|
576
|
+
job_obj = Job(deep_merge(job_obj.data, overrides), source=job_obj.source)
|
|
577
|
+
state = workflow.create(self.config, job_obj, site_obj, run_id=run_id, do_validate=validate,
|
|
578
|
+
stagein_manifest=stagein_manifest, stageout_manifest=stageout_manifest,
|
|
579
|
+
max_attempts=max_attempts)
|
|
580
|
+
if advance:
|
|
581
|
+
try:
|
|
582
|
+
workflow.advance(self.config, state.run_id, echo=echo)
|
|
583
|
+
except workflow.RunBusy:
|
|
584
|
+
pass # a monitor already picked up the newly published run
|
|
585
|
+
return self.run_state(state.run_id)
|
|
586
|
+
|
|
587
|
+
def advance(self, run_id, *, echo=None):
|
|
588
|
+
"""One pass over one run: check its current step, trigger the next. Returns the state dict.
|
|
589
|
+
Raises workflow.RunBusy if another process is advancing the same run right now."""
|
|
590
|
+
workflow.advance(self.config, run_id, echo=echo)
|
|
591
|
+
return self.run_state(run_id)
|
|
592
|
+
|
|
593
|
+
def monitor(self, run_ids=None, *, resume=False, echo=None):
|
|
594
|
+
"""One pass over the given runs (default: every active run). Never raises for a single
|
|
595
|
+
run. Returns {run_id: state dict, or an error string if the run could not be advanced}.
|
|
596
|
+
resume=True first reopens the given (failed) runs; it needs explicit run_ids."""
|
|
597
|
+
if resume:
|
|
598
|
+
if not run_ids:
|
|
599
|
+
raise ConfigError("monitor --resume needs RUN_ID(s)")
|
|
600
|
+
for rid in run_ids:
|
|
601
|
+
workflow.reopen(self.config, rid)
|
|
602
|
+
out = {}
|
|
603
|
+
for rid, r in workflow.monitor(self.config, run_ids, echo=echo).items():
|
|
604
|
+
out[rid] = {**r.data, "plugin_status": workflow.plugin_status(r.data)} if isinstance(r, workflow.RunState) else r
|
|
605
|
+
return out
|
|
606
|
+
|
|
607
|
+
def daemon(self, *, interval=60, echo=None, stop=None, max_passes=None):
|
|
608
|
+
"""Run monitor() every `interval` seconds until stop() returns True. One daemon per WORK_DIR."""
|
|
609
|
+
workflow.daemon(self.config, interval=interval, echo=echo, stop=stop, max_passes=max_passes)
|
|
610
|
+
|
|
611
|
+
def cancel(self, run_id, *, echo=None):
|
|
612
|
+
"""Cancel a run: its IRI job and any running Globus transfer. Returns the state dict."""
|
|
613
|
+
workflow.cancel(self.config, run_id, echo=echo)
|
|
614
|
+
return self.run_state(run_id)
|
|
615
|
+
|
|
616
|
+
def runs(self, *, status=None, summary=False, reindex=False):
|
|
617
|
+
"""State dicts of the runs under WORK_DIR/runs, newest first. status: "active",
|
|
618
|
+
"finished", "failed", "cancelled", or a list of them. summary=True returns
|
|
619
|
+
index rows only; reindex=True rebuilds the index from state.json first."""
|
|
620
|
+
index = self.config.run_index()
|
|
621
|
+
if reindex:
|
|
622
|
+
index.rebuild()
|
|
623
|
+
if summary:
|
|
624
|
+
return index.query(status=status)
|
|
625
|
+
return [{**r.data, "plugin_status": workflow.plugin_status(r.data)}
|
|
626
|
+
for r in workflow.list_runs(self.config, status=status)]
|