globusiri 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. globusiri/__init__.py +20 -0
  2. globusiri/api.py +626 -0
  3. globusiri/cleanup.py +92 -0
  4. globusiri/cli.py +657 -0
  5. globusiri/clients/__init__.py +1 -0
  6. globusiri/clients/http.py +82 -0
  7. globusiri/clients/https.py +76 -0
  8. globusiri/clients/iri.py +181 -0
  9. globusiri/clients/transfer.py +53 -0
  10. globusiri/config.py +285 -0
  11. globusiri/data/jobs/crux_example_job.yaml +62 -0
  12. globusiri/data/jobs/crux_hello_job.yaml +31 -0
  13. globusiri/data/jobs/crux_mpi_job.yaml +48 -0
  14. globusiri/data/jobs/crux_script_job.sh +29 -0
  15. globusiri/data/jobs/crux_script_job.yaml +31 -0
  16. globusiri/data/jobs/perlmutter_example_job.yaml +71 -0
  17. globusiri/data/jobs/perlmutter_hello_job.yaml +29 -0
  18. globusiri/data/jobs/perlmutter_mpi_job.yaml +48 -0
  19. globusiri/data/jobs/perlmutter_script_job.sh +14 -0
  20. globusiri/data/jobs/perlmutter_script_job.yaml +29 -0
  21. globusiri/data/jobs/perlmutter_transfer_job.yaml +50 -0
  22. globusiri/data/jobs/perlmutter_wordcount_job.yaml +29 -0
  23. globusiri/data/sites.example.yaml +185 -0
  24. globusiri/data/templates/crux_iri_submit_template.sh +160 -0
  25. globusiri/data/templates/perlmutter_iri_submit_template.sh +111 -0
  26. globusiri/data/templates/shell_job_template.sh +57 -0
  27. globusiri/errors.py +60 -0
  28. globusiri/jobfile.py +289 -0
  29. globusiri/jobs.py +404 -0
  30. globusiri/orm/__init__.py +86 -0
  31. globusiri/orm/models.py +107 -0
  32. globusiri/orm/session.py +158 -0
  33. globusiri/panda_secrets.py +53 -0
  34. globusiri/paths.py +151 -0
  35. globusiri/runindex.py +215 -0
  36. globusiri/runlog.py +101 -0
  37. globusiri/secure_io.py +82 -0
  38. globusiri/services.py +26 -0
  39. globusiri/staging/__init__.py +1 -0
  40. globusiri/staging/e2e.py +290 -0
  41. globusiri/staging/https.py +77 -0
  42. globusiri/tokens/__init__.py +21 -0
  43. globusiri/tokens/base.py +216 -0
  44. globusiri/tokens/e2e.py +136 -0
  45. globusiri/tokens/https.py +133 -0
  46. globusiri/tokens/iri.py +149 -0
  47. globusiri/validate.py +383 -0
  48. globusiri/workflow.py +774 -0
  49. globusiri-0.0.3.dist-info/METADATA +122 -0
  50. globusiri-0.0.3.dist-info/RECORD +52 -0
  51. globusiri-0.0.3.dist-info/WHEEL +4 -0
  52. globusiri-0.0.3.dist-info/entry_points.txt +2 -0
globusiri/__init__.py ADDED
@@ -0,0 +1,20 @@
1
+ """GlobusIRI: run HPC jobs through the IRI API and move files with Globus.
2
+
3
+ Python API (every CLI command is a method)::
4
+
5
+ from globusiri import GlobusIRI
6
+ g = GlobusIRI()
7
+ g.job_submit("perlmutter_example_job.yaml")
8
+ """
9
+
10
+ __version__ = "0.0.3"
11
+
12
+ __all__ = ["GlobusIRI", "__version__"]
13
+
14
+
15
+ def __getattr__(name): # lazy, so `import globusiri` stays cheap and cycle-free
16
+ if name == "GlobusIRI":
17
+ from .api import GlobusIRI
18
+
19
+ return GlobusIRI
20
+ raise AttributeError(f"module 'globusiri' has no attribute {name!r}")
globusiri/api.py ADDED
@@ -0,0 +1,626 @@
1
+ """Python API: every `globusiri` CLI command as a method.
2
+
3
+ from globusiri import GlobusIRI
4
+
5
+ g = GlobusIRI() # WORK_DIR and config as the CLI finds them
6
+ g.refresh("iri", all_sites=True, min_lifetime="6h")
7
+ sub = g.job_submit("perlmutter_example_job.yaml", set=["resources.n_node=2"])
8
+ state, report = g.job_wait("NERSC_Perlmutter", sub["job_id"])
9
+ g.https_stageout_job_logs("NERSC_Perlmutter", sub["job_id"], to="out/", run_id=sub["run_id"])
10
+ g.job_cleanup("NERSC_Perlmutter", sub["run_id"])
11
+
12
+ Methods return plain data (dicts, lists, dataclasses) and never print. Errors are raised
13
+ as `globusiri.errors.GlobusIriError` subclasses; each has the CLI exit code as `exit_code`.
14
+ The CLI (`globusiri.cli`) is a thin layer that parses arguments, calls these methods and
15
+ formats the results.
16
+ """
17
+
18
+ import os
19
+ import shutil
20
+ import tempfile
21
+ from dataclasses import dataclass
22
+ from pathlib import Path
23
+
24
+ import yaml
25
+
26
+ from . import cleanup as cleanup_mod
27
+ from . import jobs, services, workflow
28
+ from . import validate as validation
29
+ from .config import load_config, parse_duration
30
+ from .errors import ConfigError, GlobusIriError, JobFailed, ValidationFailed
31
+ from .jobfile import resolve_job
32
+ from .paths import Paths, package_data_dir, work_dir_source
33
+ from .runlog import run_log_path, run_logging
34
+ from .staging import e2e as e2e_staging
35
+ from .staging import https as https_staging
36
+ from .tokens import KINDS, manager
37
+
38
+ __all__ = ["GlobusIRI", "TargetResult", "init", "KINDS"]
39
+
40
+
41
+ @dataclass
42
+ class TargetResult:
43
+ """Outcome of a token command for one target (a facility, a site, or the e2e token)."""
44
+
45
+ kind: str
46
+ target: str
47
+ result: object = None
48
+ error: GlobusIriError = None
49
+
50
+ @property
51
+ def ok(self):
52
+ return self.error is None
53
+
54
+
55
+ def _target_label(m):
56
+ return "e2e" if m.kind == "e2e" else (f"{m.name} ({m.facility})" if m.kind == "iri" else m.name)
57
+
58
+
59
+ def _check_kind(kind):
60
+ if kind not in KINDS:
61
+ raise ConfigError(f"unknown token kind '{kind}' (known: {', '.join(KINDS)})")
62
+
63
+
64
+ # ---------------------------------------------------------------------- init (no config needed)
65
+ def init(work_dir=None, *, sites=None, force=False):
66
+ """Create WORK_DIR, config/sites.yaml and the example job files.
67
+
68
+ sites: optional list (or comma-separated string) of example sites to keep.
69
+ Returns {"work_dir", "source", "written": [paths], "kept": [paths]}.
70
+ """
71
+ root, source = (Path(work_dir).resolve(), "work_dir argument") if work_dir else work_dir_source()
72
+ paths = Paths(root)
73
+ for parent in [paths.root, *paths.root.parents]:
74
+ if (parent / ".git").exists() and not force:
75
+ raise ConfigError(f"WORK_DIR {paths.root} is inside a git work tree ({parent}); tokens must not be "
76
+ "committed. Set GLOBUSIRI_WORK_DIR elsewhere or pass force=True")
77
+ paths.ensure()
78
+ data = package_data_dir()
79
+ written, kept = [], []
80
+ cfg = paths.default_config
81
+ if cfg.exists() and not force:
82
+ kept.append(cfg)
83
+ else:
84
+ text = (data / "sites.example.yaml").read_text(encoding="utf-8")
85
+ if sites:
86
+ wanted = [s.strip() for s in sites.split(",")] if isinstance(sites, str) else list(sites)
87
+ wanted = [s for s in wanted if s]
88
+ raw = yaml.safe_load(text)
89
+ unknown = [s for s in wanted if s not in raw["sites"]]
90
+ if unknown:
91
+ raise ConfigError(f"sites: unknown in example: {', '.join(unknown)} (known: {', '.join(raw['sites'])})")
92
+ raw["sites"] = {k: v for k, v in raw["sites"].items() if k in wanted}
93
+ text = "# Generated by `globusiri init --sites`; see globusiri/data/sites.example.yaml for comments.\n" + \
94
+ yaml.safe_dump(raw, sort_keys=False)
95
+ cfg.write_text(text, encoding="utf-8")
96
+ written.append(cfg)
97
+ # example job files, and the job scripts that `script_file` examples point to
98
+ for src in sorted([*(data / "jobs").glob("*.yaml"), *(data / "jobs").glob("*.sh")]):
99
+ dst = paths.jobs_dir / src.name
100
+ if dst.exists() and not force:
101
+ kept.append(dst)
102
+ else:
103
+ shutil.copyfile(src, dst)
104
+ written.append(dst)
105
+ return {"work_dir": paths.root, "source": source, "written": written, "kept": kept}
106
+
107
+
108
+ class GlobusIRI:
109
+ """Entry point of the Python API.
110
+
111
+ config: path to sites.yaml, or a loaded `globusiri.config.Config`. Default: the CLI's
112
+ lookup ($GLOBUSIRI_CONFIG, else <WORK_DIR>/config/sites.yaml).
113
+ work_dir: WORK_DIR override. Default: $GLOBUSIRI_WORK_DIR, else the per-user data dir.
114
+
115
+ The config is loaded on first use, so `paths()` works before `init()`.
116
+ """
117
+
118
+ def __init__(self, config=None, *, work_dir=None):
119
+ self.paths = Paths(Path(work_dir).resolve()) if work_dir else Paths()
120
+ if config is None or isinstance(config, (str, Path)):
121
+ self._config_path, self._config = config, None
122
+ else:
123
+ self._config_path, self._config = None, config
124
+
125
+ @property
126
+ def config(self):
127
+ if self._config is None:
128
+ self._config = load_config(self._config_path, paths=self.paths)
129
+ return self._config
130
+
131
+ def site(self, name):
132
+ return self.config.site(name)
133
+
134
+ # ------------------------------------------------------------------ paths
135
+ def _run_index_row(self):
136
+ """("run index", database, source); the password of a server URL is hidden."""
137
+ from .orm import ENV_URL, DatabaseConfig
138
+ try:
139
+ db = self.config.database
140
+ except Exception: # noqa: BLE001 - paths works before init and with a broken config
141
+ try:
142
+ db = DatabaseConfig(None, self.paths.runs_dir)
143
+ except Exception as exc: # noqa: BLE001
144
+ return ("run index", f"<{exc}>", ENV_URL)
145
+ if os.environ.get(ENV_URL):
146
+ source = ENV_URL
147
+ elif db.backend == "sqlite" and db.sqlite_path == (self.paths.runs_dir / "index.db").resolve():
148
+ source = "WORK_DIR"
149
+ else:
150
+ source = "sites.yaml database.url"
151
+ where = db.sqlite_path or f"{db.display} (namespace {db.namespace})"
152
+ return ("run index", where, source)
153
+
154
+ def paths_info(self):
155
+ """Resolved locations as [(what, path, source)]; never fails on a bad config."""
156
+ root, source = work_dir_source()
157
+ if self.paths.root != root:
158
+ source = "work_dir argument"
159
+ cfg_path, cfg_source = self.paths.config_file(self._config_path)
160
+ rows = [
161
+ ("WORK_DIR", self.paths.root, source),
162
+ ("config", cfg_path, cfg_source + ("" if cfg_path.exists() else " (missing: run globusiri init)")),
163
+ ("tokens", self.paths.tokens_dir, "WORK_DIR"),
164
+ ("logs", self.paths.log_file, "WORK_DIR"),
165
+ ("runs", self.paths.runs_dir, "WORK_DIR"),
166
+ self._run_index_row(),
167
+ ("jobs", self.paths.jobs_dir, "WORK_DIR"),
168
+ ("templates", f"{self.paths.templates_dir}, then {package_data_dir() / 'templates'}", "WORK_DIR, package"),
169
+ ]
170
+ if cfg_path.exists():
171
+ try:
172
+ cfg = self.config
173
+ for fac in cfg.facilities():
174
+ m = manager("iri", cfg, cfg.sites_for_facility(fac)[0])
175
+ rows.append((f"iri {fac}", m.access_file, "exists" if m.access_file.exists() else "missing"))
176
+ for site in cfg.sites.values():
177
+ if site.has_https:
178
+ m = manager("https", cfg, site)
179
+ rows.append((f"https {site.name}", m.access_file, "exists" if m.access_file.exists() else "missing"))
180
+ m = manager("e2e", cfg)
181
+ rows.append(("e2e", m.access_file, "exists" if m.access_file.exists() else "missing"))
182
+ except GlobusIriError as exc:
183
+ rows.append(("config error", str(exc), ""))
184
+ return rows
185
+
186
+ # ------------------------------------------------------------------ tokens
187
+ def token_managers(self, kind, site=None, *, all_sites=False):
188
+ """Token managers a command acts on. iri: one per facility; https: per site; e2e: one."""
189
+ _check_kind(kind)
190
+ cfg = self.config
191
+ if kind == "e2e":
192
+ return [manager("e2e", cfg)]
193
+ if all_sites:
194
+ sites = list(cfg.sites.values())
195
+ elif site:
196
+ sites = [cfg.site(site)]
197
+ else:
198
+ raise ConfigError("give SITE or --all (all_sites=True)")
199
+ out, seen = [], set()
200
+ for s in sites:
201
+ if kind == "iri":
202
+ if s.facility in seen:
203
+ continue
204
+ seen.add(s.facility)
205
+ if kind == "https" and not s.has_https:
206
+ if all_sites:
207
+ continue
208
+ raise ConfigError(f"site {s.name} has no https section")
209
+ out.append(manager(kind, cfg, s))
210
+ return out
211
+
212
+ def _each(self, kind, site, all_sites, fn, raise_errors):
213
+ results = []
214
+ for m in self.token_managers(kind, site, all_sites=all_sites):
215
+ r = TargetResult(kind, _target_label(m))
216
+ try:
217
+ r.result = fn(m)
218
+ except GlobusIriError as exc:
219
+ if raise_errors:
220
+ raise
221
+ r.error = exc
222
+ results.append(r)
223
+ return results
224
+
225
+ def login(self, kind, site=None, *, force=False, prompt_login=False, import_file=None):
226
+ """Interactive Globus login (or import a refresh token file). Returns a dict."""
227
+ _check_kind(kind)
228
+ if kind != "e2e" and not site:
229
+ raise ConfigError(f"{kind} login needs a site")
230
+ m = self.token_managers(kind, None if kind == "e2e" else site)[0]
231
+ expires_at = m.login(force=force, prompt_login=prompt_login, import_file=import_file)
232
+ return {"kind": kind, "target": _target_label(m), "refresh_file": m.refresh_file,
233
+ "access_file": m.access_file, "expires_at": expires_at}
234
+
235
+ def refresh(self, kind, site=None, *, all_sites=False, min_lifetime=None, upload_to_panda=False,
236
+ validate=None, raise_errors=True):
237
+ """Refresh token -> access token (never prompts). Returns [TargetResult(result=RefreshResult)]."""
238
+ min_life = parse_duration(min_lifetime, self.config.refresh_min_lifetime)
239
+ return self._each(kind, site, all_sites,
240
+ lambda m: m.refresh(min_lifetime=min_life, upload_to_panda=upload_to_panda, validate=validate),
241
+ raise_errors)
242
+
243
+ def upload_to_panda(self, kind, site=None, *, all_sites=False, raise_errors=True):
244
+ """Access token -> PanDA user secret. Returns [TargetResult(result=secret key)]."""
245
+ def fn(m):
246
+ m.upload_to_panda()
247
+ return m.panda_secret_key
248
+ return self._each(kind, site, all_sites, fn, raise_errors)
249
+
250
+ def download_from_panda(self, kind, site=None, *, all_sites=False, raise_errors=True):
251
+ """PanDA user secret -> access token file. Returns [TargetResult(result=token data)]."""
252
+ def fn(m):
253
+ data = m.download_from_panda()
254
+ return {"access_file": m.access_file, "expires_at": data.get("expires_at")}
255
+ return self._each(kind, site, all_sites, fn, raise_errors)
256
+
257
+ def token_status(self, kind, site=None, *, all_sites=False, raise_errors=True):
258
+ """Offline: files, permissions, expiry. Returns [TargetResult(result=TokenStatus)]."""
259
+ return self._each(kind, site, all_sites, lambda m: m.status(), raise_errors)
260
+
261
+ # ------------------------------------------------------------------ validate
262
+ def validate(self, sites=None, *, kinds=None, job=None, offline=False, write_test=False, panda=False,
263
+ min_lifetime=None):
264
+ """Validate tokens (live unless offline). Returns [validate.Check].
265
+
266
+ job: a job file; then only the tokens that job needs are checked, at its site
267
+ (sites may name at most one site, which overrides the job's site).
268
+ Use `globusiri.validate.failed(checks, strict)` to decide pass/fail.
269
+ """
270
+ kinds = list(kinds or KINDS)
271
+ bad = [k for k in kinds if k not in KINDS]
272
+ if bad:
273
+ raise ConfigError(f"kinds: unknown {', '.join(bad)}")
274
+ if isinstance(sites, str):
275
+ sites = [sites]
276
+ job_obj, site_objs = None, None
277
+ if job:
278
+ if sites and len(sites) > 1:
279
+ raise ConfigError("job validates one site; give at most one site")
280
+ job_obj, _ = resolve_job(self.config, job, site_name=sites[0] if sites else None)
281
+ elif sites:
282
+ site_objs = [self.config.site(s) for s in sites]
283
+ min_life = parse_duration(min_lifetime, self.config.refresh_min_lifetime)
284
+ return validation.validate(self.config, sites=site_objs, kinds=kinds, job=job_obj, offline=offline,
285
+ write_test=write_test, panda=panda, min_lifetime=min_life)
286
+
287
+ # ------------------------------------------------------------------ jobs
288
+ def resolve_job(self, job, site=None, *, set=None, inputs=None):
289
+ """Merge site job_defaults < job file < overrides. Returns (Job, Site)."""
290
+ overrides = list(set or []) + [f"inputs.{i}" for i in (inputs or [])]
291
+ return resolve_job(self.config, job, site_name=site, overrides=overrides)
292
+
293
+ def job_submit(self, job, site=None, *, run_id=None, set=None, inputs=None, dry_run=False, validate=True):
294
+ """Render, upload and submit a job file.
295
+
296
+ set: ["key=value", ...] overrides; inputs: ["NAME=LOCAL", ...].
297
+ Returns {"run_id", "job_id", "remote_dir", "stdout_path", "skipped_inputs", "submission"};
298
+ with dry_run: {"run_id", "script", "job_spec", "skipped_inputs"} and nothing is contacted.
299
+ Raises ValidationFailed (with .checks) if the needed tokens fail validation.
300
+ """
301
+ cfg = self.config
302
+ job_obj, site_obj = self.resolve_job(job, site, set=set, inputs=inputs)
303
+ run_id = run_id or jobs.new_run_id()
304
+ if dry_run:
305
+ with tempfile.TemporaryDirectory() as tmp: # leave no run dir behind
306
+ sub, _ = jobs.prepare(cfg, job_obj, site_obj, run_id, Path(tmp))
307
+ script = Path(sub.script).read_text()
308
+ return {"run_id": run_id, "script": script, "job_spec": sub.job_spec,
309
+ "skipped_inputs": sub.skipped_inputs}
310
+ if validate:
311
+ checks = validation.validate(cfg, job=job_obj,
312
+ kinds=[k for k in job_obj.required_kinds(site_obj) if k != "e2e"])
313
+ if validation.failed(checks):
314
+ exc = ValidationFailed("token validation failed; nothing was uploaded or submitted")
315
+ exc.checks = [c for c in checks if c.status != validation.OK]
316
+ raise exc
317
+ run_dir = cfg.paths.run_dir(run_id)
318
+ with run_logging(run_log_path(cfg, run_id, job_obj), run_id):
319
+ state = workflow.RunState(run_dir, index=cfg.run_index())
320
+ state.data.update({"run_id": run_id, "site": site_obj.name, "created_at": workflow._now()})
321
+ sub, _ = jobs.prepare(cfg, job_obj, site_obj, run_id, run_dir)
322
+ iri = services.iri_client(cfg, site_obj)
323
+ https = services.https_client(cfg, site_obj) if site_obj.upload_method == "globus_https" else None
324
+ jobs.upload_inputs(cfg, site_obj, sub, iri, https)
325
+ state.set("upload", "done", submission=sub.to_dict(), method=site_obj.upload_method)
326
+ job_id = jobs.launch(site_obj, sub, iri)
327
+ state.set("submit", "done", job_id=job_id)
328
+ return {"run_id": run_id, "job_id": job_id, "remote_dir": sub.remote_dir, "stdout_path": sub.stdout_path,
329
+ "skipped_inputs": sub.skipped_inputs, "submission": sub.to_dict()}
330
+
331
+ def _iri(self, site):
332
+ site_obj = self.config.site(site)
333
+ return site_obj, services.iri_client(self.config, site_obj)
334
+
335
+ def job_status(self, site, job_id):
336
+ """Returns (state, IRI job report). state is one of jobs.TERMINAL or submitted/running."""
337
+ site_obj, iri = self._iri(site)
338
+ return jobs.status(iri, site_obj, job_id)
339
+
340
+ def job_wait(self, site, job_id, *, interval=60, timeout=None, on_update=None, raise_on_failure=False):
341
+ """Poll until the job ends. on_update(state, report) is called on each state change.
342
+ Returns (state, report); with raise_on_failure, a non-finished end raises JobFailed."""
343
+ site_obj, iri = self._iri(site)
344
+ state, report = jobs.wait(iri, site_obj, job_id, interval=interval, timeout=timeout, on_update=on_update)
345
+ if raise_on_failure and state != jobs.FINISHED:
346
+ raise JobFailed(f"job {job_id} ended {state}")
347
+ return state, report
348
+
349
+ def job_cancel(self, site, job_id):
350
+ site_obj, iri = self._iri(site)
351
+ return jobs.cancel(iri, site_obj, job_id)
352
+
353
+ def job_list(self, site, *, historical=False, limit=None):
354
+ """Returns [{"job_id", "name", "iri_state", "state", "report"}]."""
355
+ _, iri = self._iri(site)
356
+ result = iri.get_jobs(historical=historical, limit=limit)
357
+ items = result if isinstance(result, list) else result.get("jobs") or result.get("items") or []
358
+ out = []
359
+ for j in items:
360
+ st = j.get("status") or {}
361
+ out.append({"job_id": j.get("id"), "name": (j.get("job_spec") or {}).get("name") or j.get("name"),
362
+ "iri_state": st.get("state"), "state": jobs.map_state(j), "report": j})
363
+ return out
364
+
365
+ def job_cleanup(self, site, run_id):
366
+ """Remove a run's remote directory (and log directory) via IRI rm.
367
+ Returns {path: "removed" | "already gone"}."""
368
+ site_obj, iri = self._iri(site)
369
+ return cleanup_mod.cleanup_run(iri, site_obj, run_id)
370
+
371
+ # ------------------------------------------------------------------ IRI filesystem
372
+ def _fs(self, site, resource_id):
373
+ site_obj, iri = self._iri(site)
374
+ return site_obj, iri, resource_id or site_obj.work_dir_resource_id
375
+
376
+ def fs_ls(self, site, path, *, resource_id=None, show_hidden=False):
377
+ _, iri, rid = self._fs(site, resource_id)
378
+ return iri.ls(path, resource_id=rid, show_hidden=show_hidden)
379
+
380
+ def fs_stat(self, site, path, *, resource_id=None):
381
+ _, iri, rid = self._fs(site, resource_id)
382
+ return iri.stat(path, resource_id=rid)
383
+
384
+ def fs_mkdir(self, site, path, *, resource_id=None, parents=False):
385
+ _, iri, rid = self._fs(site, resource_id)
386
+ return iri.mkdir(path, resource_id=rid, parents=parents)
387
+
388
+ def fs_chmod(self, site, path, mode, *, resource_id=None):
389
+ _, iri, rid = self._fs(site, resource_id)
390
+ return iri.chmod(path, mode, resource_id=rid)
391
+
392
+ def fs_rm(self, site, path, *, resource_id=None, force=False):
393
+ """Remove a file or directory. Without force, path must be below the site's
394
+ remote.work_dir, remote.log_dir or e2e.base_path. Returns True if removed,
395
+ False if it was already gone."""
396
+ site_obj, iri, rid = self._fs(site, resource_id)
397
+ path = cleanup_mod.check_under(path, cleanup_mod.site_roots(site_obj), force=force)
398
+ return cleanup_mod.iri_rm(iri, path, resource_id=rid)
399
+
400
+ # ------------------------------------------------------------------ HTTPS staging
401
+ def https_stagein(self, site, files, to, *, mkdir=False, executable=None):
402
+ """Upload local files into the HPC directory `to`. Returns the remote paths."""
403
+ site_obj = self.config.site(site)
404
+ https = services.https_client(self.config, site_obj)
405
+ iri = services.iri_client(self.config, site_obj) if (mkdir or executable) else None
406
+ return https_staging.stagein(https, files, to, iri=iri, site=site_obj, mkdir=mkdir, executable=executable)
407
+
408
+ def https_stageout(self, site, files, to, *, overwrite=False):
409
+ """Download remote files into local `to`. Returns {remote: local | "skipped"}."""
410
+ site_obj = self.config.site(site)
411
+ https = services.https_client(self.config, site_obj)
412
+ return https_staging.stageout(https, files, to, skip_existing=not overwrite)
413
+
414
+ def https_stageout_job_logs(self, site, job_id, to, *, run_id=None, overwrite=False):
415
+ """Download an IRI job's stdout/stderr. Returns {remote: local | "skipped" | "error: ..."}."""
416
+ site_obj, iri = self._iri(site)
417
+ https = services.https_client(self.config, site_obj)
418
+ _, report = jobs.status(iri, site_obj, job_id)
419
+ return https_staging.stageout_job_logs(https, report, site_obj, to, run_id=run_id, skip_existing=not overwrite)
420
+
421
+ # ------------------------------------------------------------------ E2E staging
422
+ def _transfer(self):
423
+ return services.transfer(self.config)
424
+
425
+ def e2e_plan(self, site, direction, *, pairs=(), manifest=None, rucio=(), endpoint=None,
426
+ rucio_deterministic=False, scope=None):
427
+ """Build (but do not submit) a transfer plan. Returns staging.e2e.Plan.
428
+
429
+ pairs: (SRC, DST), (SRC, DST, SCOPE[, NAME]) or {src, dst[, rucio_deterministic][, scope][, name]}.
430
+ rucio_deterministic/scope are the defaults for pairs and manifest lines. A Rucio-
431
+ deterministic entry's DST is a directory: the file goes to its Rucio deterministic path
432
+ below it, named NAME or SRC's basename."""
433
+ return e2e_staging.plan(self.config, self.config.site(site), direction, manifest=manifest,
434
+ rucio=rucio or [], pairs=list(pairs or []), endpoint_label=endpoint,
435
+ deterministic=rucio_deterministic, scope=scope)
436
+
437
+ def _e2e_transfer(self, site, direction, pairs, manifest, rucio, endpoint, sync_level, wait, on_update, label,
438
+ rucio_deterministic=False, scope=None):
439
+ p = self.e2e_plan(site, direction, pairs=pairs, manifest=manifest, rucio=rucio, endpoint=endpoint,
440
+ rucio_deterministic=rucio_deterministic, scope=scope)
441
+ transfer = self._transfer()
442
+ task_ids = e2e_staging.submit(transfer, self.config, p, label=label, sync_level=sync_level)
443
+ result = {"direction": direction, "src": p.src.label, "dst": p.dst.label, "files": len(p.items),
444
+ "task_ids": task_ids}
445
+ if wait:
446
+ result["tasks"] = self.e2e_wait(task_ids, on_update=on_update, transfer=transfer)
447
+ return result
448
+
449
+ def e2e_stagein(self, site, pairs=(), *, manifest=None, rucio=(), src_endpoint=None, sync_level=None,
450
+ wait=True, on_update=None, label=None, rucio_deterministic=False, scope=None):
451
+ """Storage endpoint -> HPC. pairs, rucio_deterministic, scope: see e2e_plan; rucio:
452
+ ["SCOPE:NAME"] (both sides from the site's Rucio paths; only builds the deterministic RSE
453
+ path, no Rucio lookup). Returns {"task_ids", "files", ...}."""
454
+ return self._e2e_transfer(site, "stagein", pairs, manifest, rucio, src_endpoint, sync_level, wait,
455
+ on_update, label, rucio_deterministic, scope)
456
+
457
+ def e2e_stageout(self, site, pairs=(), *, manifest=None, rucio=(), dst_endpoint=None, sync_level=None,
458
+ wait=True, on_update=None, label=None, rucio_deterministic=False, scope=None):
459
+ """HPC -> storage endpoint. Same arguments as e2e_stagein."""
460
+ return self._e2e_transfer(site, "stageout", pairs, manifest, rucio, dst_endpoint, sync_level, wait,
461
+ on_update, label, rucio_deterministic, scope)
462
+
463
+ def e2e_submit(self, site, direction, pairs=(), *, manifest=None, rucio=(), endpoint=None, sync_level=None,
464
+ rucio_deterministic=False, scope=None, run_id=None, validate=True, wait=False, interval=None,
465
+ max_attempts=workflow.DEFAULT_MAX_ATTEMPTS, echo=None):
466
+ """A tracked E2E transfer (the `globusiri e2e stagein/stageout` commands): create a
467
+ transfer run, validate the e2e token (unless validate=False) and submit the Globus task.
468
+
469
+ wait=False returns after that first pass; monitor()/daemon() then check the run like any
470
+ job run, and runs()/cancel()/reopen() work on it. wait=True polls until it ends (every
471
+ `interval` s, default e2e.poll_interval) and raises JobFailed if it failed. Other
472
+ arguments as e2e_stagein. Returns the run's state dict.
473
+ """
474
+ site_obj = self.config.site(site)
475
+ state = workflow.create_transfer(self.config, site_obj, direction, pairs=pairs, manifest=manifest,
476
+ rucio=rucio, endpoint=endpoint, sync_level=sync_level,
477
+ rucio_deterministic=rucio_deterministic, scope=scope, run_id=run_id,
478
+ do_validate=validate, max_attempts=max_attempts)
479
+ echo = echo or (lambda *a: None)
480
+ if wait:
481
+ st, job_obj, site_obj = workflow.load(self.config, state.run_id)
482
+ workflow.run(self.config, job_obj, site_obj, resume_state=st, echo=echo,
483
+ interval=interval or self.config.e2e.poll_interval)
484
+ else:
485
+ try:
486
+ workflow.advance(self.config, state.run_id, echo=echo)
487
+ except workflow.RunBusy:
488
+ pass # a monitor already picked up the newly published run
489
+ return self.run_state(state.run_id)
490
+
491
+ def e2e_cleanup(self, site, paths=(), *, manifest=None, rucio=(), force=False, dry_run=False, wait=True,
492
+ on_update=None, label=None, rucio_deterministic=False, scope=None):
493
+ """Delete stage-in files on the HPC endpoint (Globus delete task).
494
+
495
+ manifest: stage-in manifest (its DST column is deleted); rucio: ["SCOPE:NAME"] under
496
+ e2e.base_path; paths: HPC mount paths. Returns {"endpoint", "items", "task_ids"}
497
+ (no task_ids with dry_run).
498
+ """
499
+ p = e2e_staging.cleanup_plan(self.config.site(site), manifest=manifest, rucio=rucio or [],
500
+ paths=list(paths or []), force=force, deterministic=rucio_deterministic,
501
+ scope=scope)
502
+ result = {"endpoint": p.endpoint.label, "items": list(p.items)}
503
+ if dry_run:
504
+ return result
505
+ transfer = self._transfer()
506
+ result["task_ids"] = e2e_staging.submit_delete(transfer, self.config, p, label=label)
507
+ if wait:
508
+ result["tasks"] = self.e2e_wait(result["task_ids"], on_update=on_update, transfer=transfer)
509
+ return result
510
+
511
+ def e2e_task(self, task_id, *, failures=False):
512
+ """Task info dict; with failures, adds "failures": [...]."""
513
+ transfer = self._transfer()
514
+ info = e2e_staging.task_info(transfer, task_id)
515
+ if failures:
516
+ info["failures"] = e2e_staging.task_failures(transfer, task_id)
517
+ return info
518
+
519
+ def e2e_wait(self, task_ids, *, on_update=None, transfer=None):
520
+ """Wait for Globus tasks; raises JobFailed if one fails or e2e.timeout passes."""
521
+ if isinstance(task_ids, str):
522
+ task_ids = [task_ids]
523
+ return e2e_staging.wait(transfer or self._transfer(), task_ids, poll_interval=self.config.e2e.poll_interval,
524
+ timeout=self.config.e2e.timeout, on_update=on_update)
525
+
526
+ def e2e_cancel(self, task_id):
527
+ return dict(e2e_staging.cancel(self._transfer(), task_id))
528
+
529
+ # ------------------------------------------------------------------ end-to-end run
530
+ def run(self, job, site=None, *, set=None, run_id=None, stagein_manifest=None, stageout_manifest=None,
531
+ validate=True, wait=True, interval=60, echo=None):
532
+ """validate -> e2e stagein -> upload -> submit -> monitor -> https download ->
533
+ e2e stageout -> cleanup. Returns workflow.RunState (state.data is the state.json content).
534
+ echo(msg) receives progress lines (default: silent)."""
535
+ job_obj, site_obj = resolve_job(self.config, job, site_name=site, overrides=set)
536
+ return workflow.run(self.config, job_obj, site_obj, run_id=run_id, stagein_manifest=stagein_manifest,
537
+ stageout_manifest=stageout_manifest, no_wait=not wait, do_validate=validate,
538
+ interval=interval, echo=echo or (lambda *a: None))
539
+
540
+ def resume(self, run_id, *, validate=True, wait=True, interval=60, echo=None):
541
+ """Continue a run; steps already done are skipped. Returns workflow.RunState."""
542
+ return workflow.resume(self.config, run_id, no_wait=not wait, do_validate=validate, interval=interval,
543
+ echo=echo or (lambda *a: None))
544
+
545
+ def reopen(self, run_id, *, validate=None):
546
+ """Make a failed (or cancelled) run active again without advancing it: failed steps go
547
+ back to pending, done steps are kept. validate: None keeps the run's setting.
548
+ Returns the state dict; monitor() or daemon() then continue the run."""
549
+ workflow.reopen(self.config, run_id, do_validate=validate)
550
+ return self.run_state(run_id)
551
+
552
+ def run_state(self, run_id):
553
+ """The saved state.json of a run, as a dict (plus "plugin_status")."""
554
+ data = workflow.RunState.load(self.config.paths.run_dir(run_id)).data
555
+ return {**data, "plugin_status": workflow.plugin_status(data)}
556
+
557
+ # ------------------------------------------------------------------ non-blocking workflow
558
+ # submit() starts a run; every monitor() call checks each run's current step and triggers
559
+ # the next ones. Call monitor() from your own loop (plugin mode, e.g. a Harvester monitor
560
+ # cycle) or let daemon() call it (daemon mode). Neither blocks on a transfer or a job.
561
+ def submit(self, job, site=None, *, set=None, inputs=None, overrides=None, run_id=None, stagein_manifest=None,
562
+ stageout_manifest=None, validate=True, advance=True, max_attempts=workflow.DEFAULT_MAX_ATTEMPTS,
563
+ echo=None):
564
+ """Create a workflow run and (by default) make its first pass, which goes as far as it
565
+ can without waiting: validate, start the e2e stage-in or, without one, upload and
566
+ submit the IRI job. Returns the run's state dict (see run_state).
567
+
568
+ set: ["key=value", ...]; inputs: ["NAME=LOCAL", ...]; overrides: a nested dict merged
569
+ last (for keys that contain dots, e.g. {"inputs": {"pandaJobData.out": path}}).
570
+ max_attempts: how many passes a step may fail with a transient error before the run fails.
571
+ """
572
+ job_obj, site_obj = self.resolve_job(job, site, set=set, inputs=inputs)
573
+ if overrides:
574
+ from .jobfile import Job, deep_merge
575
+
576
+ job_obj = Job(deep_merge(job_obj.data, overrides), source=job_obj.source)
577
+ state = workflow.create(self.config, job_obj, site_obj, run_id=run_id, do_validate=validate,
578
+ stagein_manifest=stagein_manifest, stageout_manifest=stageout_manifest,
579
+ max_attempts=max_attempts)
580
+ if advance:
581
+ try:
582
+ workflow.advance(self.config, state.run_id, echo=echo)
583
+ except workflow.RunBusy:
584
+ pass # a monitor already picked up the newly published run
585
+ return self.run_state(state.run_id)
586
+
587
+ def advance(self, run_id, *, echo=None):
588
+ """One pass over one run: check its current step, trigger the next. Returns the state dict.
589
+ Raises workflow.RunBusy if another process is advancing the same run right now."""
590
+ workflow.advance(self.config, run_id, echo=echo)
591
+ return self.run_state(run_id)
592
+
593
+ def monitor(self, run_ids=None, *, resume=False, echo=None):
594
+ """One pass over the given runs (default: every active run). Never raises for a single
595
+ run. Returns {run_id: state dict, or an error string if the run could not be advanced}.
596
+ resume=True first reopens the given (failed) runs; it needs explicit run_ids."""
597
+ if resume:
598
+ if not run_ids:
599
+ raise ConfigError("monitor --resume needs RUN_ID(s)")
600
+ for rid in run_ids:
601
+ workflow.reopen(self.config, rid)
602
+ out = {}
603
+ for rid, r in workflow.monitor(self.config, run_ids, echo=echo).items():
604
+ out[rid] = {**r.data, "plugin_status": workflow.plugin_status(r.data)} if isinstance(r, workflow.RunState) else r
605
+ return out
606
+
607
+ def daemon(self, *, interval=60, echo=None, stop=None, max_passes=None):
608
+ """Run monitor() every `interval` seconds until stop() returns True. One daemon per WORK_DIR."""
609
+ workflow.daemon(self.config, interval=interval, echo=echo, stop=stop, max_passes=max_passes)
610
+
611
+ def cancel(self, run_id, *, echo=None):
612
+ """Cancel a run: its IRI job and any running Globus transfer. Returns the state dict."""
613
+ workflow.cancel(self.config, run_id, echo=echo)
614
+ return self.run_state(run_id)
615
+
616
+ def runs(self, *, status=None, summary=False, reindex=False):
617
+ """State dicts of the runs under WORK_DIR/runs, newest first. status: "active",
618
+ "finished", "failed", "cancelled", or a list of them. summary=True returns
619
+ index rows only; reindex=True rebuilds the index from state.json first."""
620
+ index = self.config.run_index()
621
+ if reindex:
622
+ index.rebuild()
623
+ if summary:
624
+ return index.query(status=status)
625
+ return [{**r.data, "plugin_status": workflow.plugin_status(r.data)}
626
+ for r in workflow.list_runs(self.config, status=status)]