datatrail-cli 1.0.2__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datatrail_cli-1.2.0/.github/dependabot.yml +12 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/workflows/continuous-deployment.yml +2 -1
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/workflows/continuous-integration.yml +2 -2
- datatrail_cli-1.2.0/.github/workflows/docs.yml +46 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/workflows/stale.yml +2 -1
- datatrail_cli-1.2.0/.release-please-manifest.json +3 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/CHANGELOG.md +15 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/PKG-INFO +1 -1
- datatrail_cli-1.2.0/docs/workflow.md +55 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/pull.py +2 -0
- datatrail_cli-1.2.0/dtcli/workflow.py +180 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/mkdocs.yml +1 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/pyproject.toml +1 -1
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_pull_reliability.py +20 -0
- datatrail_cli-1.2.0/tests/test_workflow.py +215 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/uv.lock +1 -1
- datatrail_cli-1.0.2/.github/workflows/docs.yml +0 -23
- datatrail_cli-1.0.2/.release-please-manifest.json +0 -3
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.github/workflows/codeql.yml +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.gitignore +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/.pre-commit-config.yaml +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/LICENSE +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/README.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/clear.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/cli.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/commands.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/discovery.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/doctor.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/images/Datatrail-logo.png +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/index.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/initialising.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/install.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/inventory.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/list.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/ps.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/pull-manifest.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/pull.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/scout.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/unregistered.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/user_guide.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/docs/verify.md +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/__init__.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/clear.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/cli.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/config.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/config.yaml +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/doctor.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/inventory.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/locking.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/ls.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/ps.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/pull_manifest.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/scout.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/src/functions.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/unregistered.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/utilities/cadcclient.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/utilities/results.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/utilities/utilities.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/dtcli/verify.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/mkdocs-site.yml +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/release-please-config.json +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/conftest.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_cadc_summary.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_canfar_permissions.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_cli.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_config.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_doctor.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_functions.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_inventory.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_locking.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_pull_manifest.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_pull_size.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_scout.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_utils.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tests/test_verify.py +0 -0
- {datatrail_cli-1.0.2 → datatrail_cli-1.2.0}/tox.ini +0 -0
|
@@ -9,6 +9,7 @@ permissions:
|
|
|
9
9
|
pull-requests: write
|
|
10
10
|
jobs:
|
|
11
11
|
release-please:
|
|
12
|
+
if: github.repository == 'CHIMEFRB/datatrail-cli'
|
|
12
13
|
runs-on: ubuntu-latest
|
|
13
14
|
steps:
|
|
14
15
|
- name: Create release
|
|
@@ -23,7 +24,7 @@ jobs:
|
|
|
23
24
|
|
|
24
25
|
- name: Install uv
|
|
25
26
|
if: ${{ steps.release-please.outputs.release_created }}
|
|
26
|
-
uses: astral-sh/setup-uv@v10.0
|
|
27
|
+
uses: astral-sh/setup-uv@v10.2.0
|
|
27
28
|
with:
|
|
28
29
|
enable-cache: false
|
|
29
30
|
python-version: "3.13"
|
|
@@ -49,7 +49,7 @@ jobs:
|
|
|
49
49
|
- uses: actions/checkout@v7
|
|
50
50
|
|
|
51
51
|
- name: Install uv
|
|
52
|
-
uses: astral-sh/setup-uv@v10.0
|
|
52
|
+
uses: astral-sh/setup-uv@v10.2.0
|
|
53
53
|
with:
|
|
54
54
|
enable-cache: true
|
|
55
55
|
cache-dependency-glob: "uv.lock"
|
|
@@ -100,7 +100,7 @@ jobs:
|
|
|
100
100
|
fail-on-error: false
|
|
101
101
|
|
|
102
102
|
- name: Upload coverage to Codecov
|
|
103
|
-
uses: codecov/codecov-action@
|
|
103
|
+
uses: codecov/codecov-action@v7.1.1
|
|
104
104
|
with:
|
|
105
105
|
fail_ci_if_error: false
|
|
106
106
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: Documentation
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
paths:
|
|
6
|
+
- "docs/**"
|
|
7
|
+
- "mkdocs*.yml"
|
|
8
|
+
- "pyproject.toml"
|
|
9
|
+
- "uv.lock"
|
|
10
|
+
- ".github/workflows/docs.yml"
|
|
11
|
+
pull_request:
|
|
12
|
+
paths:
|
|
13
|
+
- "docs/**"
|
|
14
|
+
- "mkdocs*.yml"
|
|
15
|
+
- "pyproject.toml"
|
|
16
|
+
- "uv.lock"
|
|
17
|
+
- ".github/workflows/docs.yml"
|
|
18
|
+
workflow_dispatch:
|
|
19
|
+
|
|
20
|
+
permissions:
|
|
21
|
+
contents: read
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
build:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v7
|
|
28
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
29
|
+
with:
|
|
30
|
+
python-version: "3.13"
|
|
31
|
+
enable-cache: true
|
|
32
|
+
- run: uv sync --locked --group docs
|
|
33
|
+
- run: uv run --no-sync mkdocs build --strict --config-file mkdocs-site.yml
|
|
34
|
+
|
|
35
|
+
handbook:
|
|
36
|
+
if: github.repository == 'CHIMEFRB/datatrail-cli' && github.event_name == 'push'
|
|
37
|
+
needs: build
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
steps:
|
|
40
|
+
- name: Build the upstream handbook
|
|
41
|
+
uses: peter-evans/repository-dispatch@v4.0.1
|
|
42
|
+
with:
|
|
43
|
+
token: ${{ secrets.BUILD_HANDBOOK_TOKEN }}
|
|
44
|
+
repository: chimefrb/handbook
|
|
45
|
+
event-type: build-handbook
|
|
46
|
+
client-payload: '{"source": "chimefrb/datatrail-cli"}'
|
|
@@ -11,10 +11,11 @@ permissions:
|
|
|
11
11
|
|
|
12
12
|
jobs:
|
|
13
13
|
close_stale_prs:
|
|
14
|
+
if: github.repository == 'CHIMEFRB/datatrail-cli'
|
|
14
15
|
runs-on: ubuntu-latest
|
|
15
16
|
steps:
|
|
16
17
|
- name: Close stale pull requests
|
|
17
|
-
uses: actions/stale@
|
|
18
|
+
uses: actions/stale@v11.0.0
|
|
18
19
|
with:
|
|
19
20
|
days-before-stale: 90
|
|
20
21
|
days-before-close: 7
|
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.2.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.1.0...datatrail-cli-v1.2.0) (2026-11-01)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **workflow:** pull_dataset can pull to a given directory ([7f2da01](https://github.com/CHIMEFRB/datatrail-cli/commit/7f2da01a257668fd3b8efbd916c82d0d4d441340))
|
|
9
|
+
* **workflow:** pull_dataset can pull to a given directory ([ae3bb39](https://github.com/CHIMEFRB/datatrail-cli/commit/ae3bb397ecea15c850fce33b38af6b38f76e44ee))
|
|
10
|
+
|
|
11
|
+
## [1.1.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.0.2...datatrail-cli-v1.1.0) (2026-11-01)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Features
|
|
15
|
+
|
|
16
|
+
* **workflow:** pull and clear as functions for Workflow pipelines ([de80e6d](https://github.com/CHIMEFRB/datatrail-cli/commit/de80e6dfd6f6092457af038002b0dec784a74cf5))
|
|
17
|
+
|
|
3
18
|
## [1.0.2](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.0.1...datatrail-cli-v1.0.2) (2026-09-29)
|
|
4
19
|
|
|
5
20
|
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# 🔁 In Workflow pipelines
|
|
2
|
+
|
|
3
|
+
`dtcli.workflow` has `pull` and `clear` as Python functions for Workflow pipeline
|
|
4
|
+
steps. A function runs in the step's own session, so what the command prints is in
|
|
5
|
+
the step's logs. Each one checks what the command did and raises if it failed, so
|
|
6
|
+
the step fails too: `datatrail pull` reports some failures (no CADC certificate,
|
|
7
|
+
minoc down, no files found) only on the console.
|
|
8
|
+
|
|
9
|
+
They return `(results, products, plots)`, as Workflow expects of a function.
|
|
10
|
+
|
|
11
|
+
## `pull_dataset`
|
|
12
|
+
|
|
13
|
+
Downloads a dataset, then checks that all its files are here.
|
|
14
|
+
|
|
15
|
+
```yaml
|
|
16
|
+
work:
|
|
17
|
+
function: dtcli.workflow.pull_dataset
|
|
18
|
+
parameters:
|
|
19
|
+
scope: chime.event.baseband.raw
|
|
20
|
+
dataset: "123456789"
|
|
21
|
+
cores: 4
|
|
22
|
+
site: canfar
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
- `cores`: parallel fetch processes (default 1).
|
|
26
|
+
- `site`: where it runs. If there's no Datatrail configuration, one is created
|
|
27
|
+
for this site (as `datatrail config init --site <site>`); if there is one for
|
|
28
|
+
another site, the step fails.
|
|
29
|
+
- `verbose`: 1 for INFO (default), 2 for DEBUG.
|
|
30
|
+
- `directory`: where to pull to, created if it isn't there (default: the site's
|
|
31
|
+
root mount). The files go under it at their Datatrail paths, e.g.
|
|
32
|
+
`/scratch/data/chime/baseband/raw/...` for `directory: /scratch`.
|
|
33
|
+
|
|
34
|
+
Results: `scope`, `dataset`, `files` (how many are here) and `directory`.
|
|
35
|
+
|
|
36
|
+
## `clear_dataset`
|
|
37
|
+
|
|
38
|
+
Deletes a dataset's local copy, then checks that it's gone. A dataset with no local
|
|
39
|
+
copy is left as it is, and the step succeeds, so it can clean up after a pull that
|
|
40
|
+
failed.
|
|
41
|
+
|
|
42
|
+
```yaml
|
|
43
|
+
work:
|
|
44
|
+
function: dtcli.workflow.clear_dataset
|
|
45
|
+
parameters:
|
|
46
|
+
scope: chime.event.baseband.raw
|
|
47
|
+
dataset: "123456789"
|
|
48
|
+
site: canfar
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
- `clear_parents`: also delete empty parent directories (default false).
|
|
52
|
+
- `site` and `verbose` as for `pull_dataset`.
|
|
53
|
+
|
|
54
|
+
Results: `scope`, `dataset`, `path` and `cleared` (false if there was nothing to
|
|
55
|
+
clear).
|
|
@@ -98,6 +98,8 @@ def pull( # noqa: C901
|
|
|
98
98
|
if directory is None:
|
|
99
99
|
directory = config["root_mounts"][site]
|
|
100
100
|
logger.info(f"No directory, setting to: {directory}.")
|
|
101
|
+
# The file paths are added to it as they are, and click drops the slash
|
|
102
|
+
directory = path.join(directory, "")
|
|
101
103
|
except Exception:
|
|
102
104
|
logger.exception(
|
|
103
105
|
"Configuration Missing!! Run `datatrail config init`.",
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Datatrail commands as functions for Workflow pipelines.
|
|
2
|
+
|
|
3
|
+
Workflow runs a pipeline step's function in the session's own process, so what the
|
|
4
|
+
command prints is the step's logs. Each function runs the CLI command, then checks
|
|
5
|
+
what it did and raises if it failed: the commands report some failures only on the
|
|
6
|
+
console, which a Workflow step can't see. They return (results, products, plots),
|
|
7
|
+
the shape Workflow expects of a function.
|
|
8
|
+
|
|
9
|
+
Example step:
|
|
10
|
+
|
|
11
|
+
function: dtcli.workflow.pull_dataset
|
|
12
|
+
parameters:
|
|
13
|
+
scope: chime.event.baseband.raw
|
|
14
|
+
dataset: "123456789"
|
|
15
|
+
cores: 4
|
|
16
|
+
site: canfar
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
22
|
+
|
|
23
|
+
import click
|
|
24
|
+
|
|
25
|
+
from dtcli.clear import clear
|
|
26
|
+
from dtcli.config import CONFIG, init, procure
|
|
27
|
+
from dtcli.pull import pull
|
|
28
|
+
from dtcli.src.functions import find_dataset_common_path, find_missing_dataset_files
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger("workflow")
|
|
31
|
+
|
|
32
|
+
Outcome = Tuple[Dict[str, Any], List[str], List[str]]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _configure(site: Optional[str]) -> Dict[str, Any]:
|
|
36
|
+
"""The Datatrail configuration, created for `site` if there's none.
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
RuntimeError: If there's no configuration and no site to create one for,
|
|
40
|
+
or it's set up for another site.
|
|
41
|
+
"""
|
|
42
|
+
if not CONFIG.exists():
|
|
43
|
+
if not site:
|
|
44
|
+
raise RuntimeError(
|
|
45
|
+
f"There's no Datatrail configuration at {CONFIG}; "
|
|
46
|
+
"give a site to create one."
|
|
47
|
+
)
|
|
48
|
+
# init only asks questions when a configuration exists
|
|
49
|
+
init.callback(site=site)
|
|
50
|
+
config = procure()
|
|
51
|
+
if not config:
|
|
52
|
+
raise RuntimeError(f"The Datatrail configuration at {CONFIG} can't be read.")
|
|
53
|
+
if site and config.get("site") != site:
|
|
54
|
+
raise RuntimeError(
|
|
55
|
+
f"The Datatrail configuration at {CONFIG} is for {config.get('site')}, "
|
|
56
|
+
f"not {site}."
|
|
57
|
+
)
|
|
58
|
+
return config
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _run(command: click.Command, args: List[str]) -> None:
|
|
62
|
+
"""Run a CLI command in this process.
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
RuntimeError: If it exits with an error code.
|
|
66
|
+
"""
|
|
67
|
+
code = command.main(args=args, standalone_mode=False)
|
|
68
|
+
# with standalone_mode=False, an exit code is returned rather than raised
|
|
69
|
+
if code:
|
|
70
|
+
raise RuntimeError(f"datatrail {command.name} exited with {code}.")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def pull_dataset(
|
|
74
|
+
scope: str,
|
|
75
|
+
dataset: str,
|
|
76
|
+
cores: int = 1,
|
|
77
|
+
site: Optional[str] = None,
|
|
78
|
+
verbose: int = 1,
|
|
79
|
+
directory: Optional[str] = None,
|
|
80
|
+
) -> Outcome:
|
|
81
|
+
"""Download a dataset, then check that all its files are here.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
scope (str): Scope of the dataset.
|
|
85
|
+
dataset (str): Name of the dataset.
|
|
86
|
+
cores (int): Number of parallel fetch processes. Defaults to 1.
|
|
87
|
+
site (Optional[str]): Site this runs at; the configuration is created for it
|
|
88
|
+
if there's none. Defaults to the configured site.
|
|
89
|
+
verbose (int): Verbosity: 1=INFO, 2=DEBUG. Defaults to 1.
|
|
90
|
+
directory (Optional[str]): Directory to pull to, created if it isn't there;
|
|
91
|
+
the files go under it at their Datatrail paths. Defaults to the site's
|
|
92
|
+
root mount.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
Outcome: Results (scope, dataset, number of files and where they are), no
|
|
96
|
+
products and no plots.
|
|
97
|
+
|
|
98
|
+
Raises:
|
|
99
|
+
RuntimeError: If the dataset can't be found, or any of its files is missing
|
|
100
|
+
after the pull.
|
|
101
|
+
"""
|
|
102
|
+
config = _configure(site)
|
|
103
|
+
dataset = str(dataset)
|
|
104
|
+
args = [scope, dataset, "--force", "--cores", str(cores)] + ["-v"] * verbose
|
|
105
|
+
if directory:
|
|
106
|
+
os.makedirs(directory, exist_ok=True)
|
|
107
|
+
args += ["--directory", directory]
|
|
108
|
+
# the file paths are added to it as they are
|
|
109
|
+
directory = os.path.join(directory, "")
|
|
110
|
+
else:
|
|
111
|
+
directory = config["root_mounts"][config["site"]]
|
|
112
|
+
_run(pull, args)
|
|
113
|
+
|
|
114
|
+
files = find_missing_dataset_files(scope, dataset, directory, verbose)
|
|
115
|
+
if files.get("error"):
|
|
116
|
+
raise RuntimeError(f"Could not check {scope} {dataset}: {files['error']}")
|
|
117
|
+
if files["missing"]:
|
|
118
|
+
raise RuntimeError(
|
|
119
|
+
f"{len(files['missing'])} of {scope} {dataset}'s files weren't pulled, "
|
|
120
|
+
f"e.g. {files['missing'][0]}."
|
|
121
|
+
)
|
|
122
|
+
if not files["existing"]:
|
|
123
|
+
raise RuntimeError(f"Datatrail has no files for {scope} {dataset}.")
|
|
124
|
+
results = {
|
|
125
|
+
"scope": scope,
|
|
126
|
+
"dataset": dataset,
|
|
127
|
+
"files": len(files["existing"]),
|
|
128
|
+
"directory": directory,
|
|
129
|
+
}
|
|
130
|
+
return results, [], []
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def clear_dataset(
|
|
134
|
+
scope: str,
|
|
135
|
+
dataset: str,
|
|
136
|
+
site: Optional[str] = None,
|
|
137
|
+
clear_parents: bool = False,
|
|
138
|
+
verbose: int = 1,
|
|
139
|
+
) -> Outcome:
|
|
140
|
+
"""Delete a dataset's local copy, then check that it's gone.
|
|
141
|
+
|
|
142
|
+
A dataset with no local copy is left as it is, and isn't a failure.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
scope (str): Scope of the dataset.
|
|
146
|
+
dataset (str): Name of the dataset.
|
|
147
|
+
site (Optional[str]): Site this runs at; the configuration is created for it
|
|
148
|
+
if there's none. Defaults to the configured site.
|
|
149
|
+
clear_parents (bool): Also delete empty parent directories. Defaults to False.
|
|
150
|
+
verbose (int): Verbosity: 1=INFO, 2=DEBUG. Defaults to 1.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Outcome: Results (scope, dataset, the path and whether it was cleared), no
|
|
154
|
+
products and no plots.
|
|
155
|
+
|
|
156
|
+
Raises:
|
|
157
|
+
RuntimeError: If the dataset can't be found, or its files are still there.
|
|
158
|
+
"""
|
|
159
|
+
config = _configure(site)
|
|
160
|
+
directory = config["root_mounts"][config["site"]]
|
|
161
|
+
dataset = str(dataset)
|
|
162
|
+
common = find_dataset_common_path(scope, dataset, config["site"], verbose, False)
|
|
163
|
+
# it returns a message rather than a path when the server isn't reachable
|
|
164
|
+
if not common or not common.startswith(("data/", "/")):
|
|
165
|
+
raise RuntimeError(f"Could not find where {scope} {dataset} is: {common}")
|
|
166
|
+
# as `datatrail clear` builds it
|
|
167
|
+
path = (directory + common).replace("//", "/")
|
|
168
|
+
results: Dict[str, Any] = {"scope": scope, "dataset": dataset, "path": path}
|
|
169
|
+
if not os.path.exists(path):
|
|
170
|
+
logger.info(f"{path} doesn't exist; nothing to clear.")
|
|
171
|
+
return {**results, "cleared": False}, [], []
|
|
172
|
+
|
|
173
|
+
args = [scope, dataset, "--force"] + ["-v"] * verbose
|
|
174
|
+
if clear_parents:
|
|
175
|
+
args.append("--clear-parents")
|
|
176
|
+
_run(clear, args)
|
|
177
|
+
|
|
178
|
+
if os.path.exists(path):
|
|
179
|
+
raise RuntimeError(f"{path} is still there after clearing it.")
|
|
180
|
+
return {**results, "cleared": True}, [], []
|
|
@@ -274,6 +274,26 @@ def test_pull_exits_nonzero_when_file_is_still_missing(monkeypatch, tmp_path):
|
|
|
274
274
|
assert "File not downloaded" in result.output
|
|
275
275
|
|
|
276
276
|
|
|
277
|
+
def test_pull_checks_a_given_directory_for_existing_files(monkeypatch, tmp_path):
|
|
278
|
+
"""Files already under --directory are found, though click drops its slash."""
|
|
279
|
+
configure_pull(monkeypatch, tmp_path, lambda *args, **kwargs: [])
|
|
280
|
+
roots = []
|
|
281
|
+
monkeypatch.setattr(
|
|
282
|
+
pull_module,
|
|
283
|
+
"find_missing_dataset_files",
|
|
284
|
+
lambda scope, dataset, root, verbose: roots.append(root)
|
|
285
|
+
or {"missing": [], "existing": ["data/file.dat"]},
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
result = CliRunner().invoke(
|
|
289
|
+
pull_module.pull,
|
|
290
|
+
["scope", "dataset", "--directory", str(tmp_path), "--force"],
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
assert result.exit_code == 0
|
|
294
|
+
assert roots == [str(tmp_path) + "/"]
|
|
295
|
+
|
|
296
|
+
|
|
277
297
|
def test_pull_interruption_exits_nonzero(monkeypatch, tmp_path):
|
|
278
298
|
"""An interrupted pull is not reported as successful."""
|
|
279
299
|
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Tests for the Datatrail functions for Workflow pipelines."""
|
|
2
|
+
|
|
3
|
+
from types import SimpleNamespace
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
import pytest
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from dtcli import workflow
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FakeCommand:
|
|
13
|
+
"""A CLI command that records its arguments and returns an exit code."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, name, code=None, effect=None):
|
|
16
|
+
self.name = name
|
|
17
|
+
self.code = code
|
|
18
|
+
self.effect = effect
|
|
19
|
+
self.calls = []
|
|
20
|
+
|
|
21
|
+
def main(self, args, standalone_mode):
|
|
22
|
+
assert standalone_mode is False
|
|
23
|
+
self.calls.append(args)
|
|
24
|
+
if self.effect:
|
|
25
|
+
self.effect()
|
|
26
|
+
return self.code
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.fixture
|
|
30
|
+
def config(tmp_path, monkeypatch):
|
|
31
|
+
"""A Datatrail configuration for canfar, with its root mount in tmp_path."""
|
|
32
|
+
path = tmp_path / ".datatrail" / "config.yaml"
|
|
33
|
+
root = str(tmp_path / "arc") + "/"
|
|
34
|
+
monkeypatch.setattr(workflow, "CONFIG", path)
|
|
35
|
+
|
|
36
|
+
def write(site="canfar"):
|
|
37
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
path.write_text(yaml.safe_dump({"site": site, "root_mounts": {site: root}}))
|
|
39
|
+
|
|
40
|
+
monkeypatch.setattr(
|
|
41
|
+
workflow, "init", SimpleNamespace(callback=lambda site: write(site))
|
|
42
|
+
)
|
|
43
|
+
monkeypatch.setattr(workflow, "procure", lambda: yaml.safe_load(path.read_text()))
|
|
44
|
+
return SimpleNamespace(path=path, root=root, write=write)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_pull_dataset_returns_workflow_outcome(config, monkeypatch):
|
|
48
|
+
config.write()
|
|
49
|
+
pull = FakeCommand("pull")
|
|
50
|
+
monkeypatch.setattr(workflow, "pull", pull)
|
|
51
|
+
monkeypatch.setattr(
|
|
52
|
+
workflow,
|
|
53
|
+
"find_missing_dataset_files",
|
|
54
|
+
lambda scope, dataset, root, verbose: {"missing": [], "existing": ["a", "b"]},
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
results, products, plots = workflow.pull_dataset(
|
|
58
|
+
"chime.event.baseband.raw", 123, cores=4, site="canfar"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
assert pull.calls == [
|
|
62
|
+
["chime.event.baseband.raw", "123", "--force", "--cores", "4", "-v"]
|
|
63
|
+
]
|
|
64
|
+
assert results == {
|
|
65
|
+
"scope": "chime.event.baseband.raw",
|
|
66
|
+
"dataset": "123",
|
|
67
|
+
"files": 2,
|
|
68
|
+
"directory": config.root,
|
|
69
|
+
}
|
|
70
|
+
assert products == [] and plots == []
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_pull_dataset_pulls_to_a_given_directory(config, tmp_path, monkeypatch):
|
|
74
|
+
config.write()
|
|
75
|
+
pull = FakeCommand("pull")
|
|
76
|
+
monkeypatch.setattr(workflow, "pull", pull)
|
|
77
|
+
checked = []
|
|
78
|
+
monkeypatch.setattr(
|
|
79
|
+
workflow,
|
|
80
|
+
"find_missing_dataset_files",
|
|
81
|
+
lambda scope, dataset, root, verbose: checked.append(root)
|
|
82
|
+
or {"missing": [], "existing": ["a"]},
|
|
83
|
+
)
|
|
84
|
+
scratch = tmp_path / "scratch"
|
|
85
|
+
|
|
86
|
+
results, _, _ = workflow.pull_dataset(
|
|
87
|
+
"scope", "1", site="canfar", directory=str(scratch)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
assert scratch.is_dir()
|
|
91
|
+
assert pull.calls == [
|
|
92
|
+
["scope", "1", "--force", "--cores", "1", "-v", "--directory", str(scratch)]
|
|
93
|
+
]
|
|
94
|
+
# the files' paths are added to it, so it ends with a slash
|
|
95
|
+
assert checked == [str(scratch) + "/"]
|
|
96
|
+
assert results["directory"] == str(scratch) + "/"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_pull_dataset_creates_the_configuration(config, monkeypatch):
|
|
100
|
+
monkeypatch.setattr(workflow, "pull", FakeCommand("pull"))
|
|
101
|
+
monkeypatch.setattr(
|
|
102
|
+
workflow,
|
|
103
|
+
"find_missing_dataset_files",
|
|
104
|
+
lambda *args: {"missing": [], "existing": ["a"]},
|
|
105
|
+
)
|
|
106
|
+
workflow.pull_dataset("scope", "1", site="canfar")
|
|
107
|
+
assert yaml.safe_load(config.path.read_text())["site"] == "canfar"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_pull_dataset_needs_a_site_without_configuration(config):
|
|
111
|
+
with pytest.raises(RuntimeError, match="give a site"):
|
|
112
|
+
workflow.pull_dataset("scope", "1")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_pull_dataset_refuses_another_sites_configuration(config):
|
|
116
|
+
config.write("chime")
|
|
117
|
+
with pytest.raises(RuntimeError, match="is for chime, not canfar"):
|
|
118
|
+
workflow.pull_dataset("scope", "1", site="canfar")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@pytest.mark.parametrize(
|
|
122
|
+
"code, files, message",
|
|
123
|
+
[
|
|
124
|
+
# ctx.exit(1) comes back as a return value
|
|
125
|
+
(1, {"missing": [], "existing": ["a"]}, "exited with 1"),
|
|
126
|
+
# pull reports these only on the console and exits 0
|
|
127
|
+
(None, {"error": "Scope does not exist"}, "Could not check"),
|
|
128
|
+
(None, {"missing": ["a"], "existing": ["b"]}, "1 of scope 1's files"),
|
|
129
|
+
(None, {"missing": [], "existing": []}, "has no files"),
|
|
130
|
+
],
|
|
131
|
+
)
|
|
132
|
+
def test_pull_dataset_raises_when_it_failed(config, monkeypatch, code, files, message):
|
|
133
|
+
config.write()
|
|
134
|
+
monkeypatch.setattr(workflow, "pull", FakeCommand("pull", code=code))
|
|
135
|
+
monkeypatch.setattr(workflow, "find_missing_dataset_files", lambda *args: files)
|
|
136
|
+
with pytest.raises(RuntimeError, match=message):
|
|
137
|
+
workflow.pull_dataset("scope", "1", site="canfar")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_pull_dataset_passes_on_click_errors(config, monkeypatch):
|
|
141
|
+
config.write()
|
|
142
|
+
|
|
143
|
+
def abort():
|
|
144
|
+
raise click.Abort()
|
|
145
|
+
|
|
146
|
+
monkeypatch.setattr(workflow, "pull", FakeCommand("pull", effect=abort))
|
|
147
|
+
with pytest.raises(click.Abort):
|
|
148
|
+
workflow.pull_dataset("scope", "1", site="canfar")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _dataset_dir(config):
|
|
152
|
+
path = config.path.parent.parent / "arc" / "data" / "chime" / "astro_1"
|
|
153
|
+
path.mkdir(parents=True)
|
|
154
|
+
(path / "baseband_1_0.h5").write_text("")
|
|
155
|
+
return path
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_clear_dataset_clears_and_checks(config, monkeypatch):
|
|
159
|
+
config.write()
|
|
160
|
+
path = _dataset_dir(config)
|
|
161
|
+
|
|
162
|
+
def delete():
|
|
163
|
+
for file in path.iterdir():
|
|
164
|
+
file.unlink()
|
|
165
|
+
path.rmdir()
|
|
166
|
+
|
|
167
|
+
clear = FakeCommand("clear", effect=delete)
|
|
168
|
+
monkeypatch.setattr(workflow, "clear", clear)
|
|
169
|
+
monkeypatch.setattr(
|
|
170
|
+
workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_1/"
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
results, products, plots = workflow.clear_dataset("scope", 1, site="canfar")
|
|
174
|
+
|
|
175
|
+
assert clear.calls == [["scope", "1", "--force", "-v"]]
|
|
176
|
+
assert results == {
|
|
177
|
+
"scope": "scope",
|
|
178
|
+
"dataset": "1",
|
|
179
|
+
"path": config.root + "data/chime/astro_1/",
|
|
180
|
+
"cleared": True,
|
|
181
|
+
}
|
|
182
|
+
assert products == [] and plots == []
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_clear_dataset_without_a_local_copy(config, monkeypatch):
|
|
186
|
+
config.write()
|
|
187
|
+
clear = FakeCommand("clear")
|
|
188
|
+
monkeypatch.setattr(workflow, "clear", clear)
|
|
189
|
+
monkeypatch.setattr(
|
|
190
|
+
workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_2/"
|
|
191
|
+
)
|
|
192
|
+
results, _, _ = workflow.clear_dataset("scope", "2", site="canfar")
|
|
193
|
+
assert results["cleared"] is False
|
|
194
|
+
assert clear.calls == []
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def test_clear_dataset_raises_when_files_remain(config, monkeypatch):
|
|
198
|
+
config.write()
|
|
199
|
+
_dataset_dir(config)
|
|
200
|
+
monkeypatch.setattr(workflow, "clear", FakeCommand("clear"))
|
|
201
|
+
monkeypatch.setattr(
|
|
202
|
+
workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_1/"
|
|
203
|
+
)
|
|
204
|
+
with pytest.raises(RuntimeError, match="still there"):
|
|
205
|
+
workflow.clear_dataset("scope", "1", site="canfar")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@pytest.mark.parametrize(
|
|
209
|
+
"common", [None, "The Datatrail Central Server at CHIME at is not reachable!!!"]
|
|
210
|
+
)
|
|
211
|
+
def test_clear_dataset_raises_when_it_cant_find_the_dataset(config, monkeypatch, common):
|
|
212
|
+
config.write()
|
|
213
|
+
monkeypatch.setattr(workflow, "find_dataset_common_path", lambda *args: common)
|
|
214
|
+
with pytest.raises(RuntimeError, match="Could not find where"):
|
|
215
|
+
workflow.clear_dataset("scope", "1", site="canfar")
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
name: Documentation
|
|
2
|
-
on:
|
|
3
|
-
push:
|
|
4
|
-
branches:
|
|
5
|
-
- main
|
|
6
|
-
paths:
|
|
7
|
-
- "docs/**"
|
|
8
|
-
- ".github/workflows/docs.yml"
|
|
9
|
-
|
|
10
|
-
permissions:
|
|
11
|
-
contents: read
|
|
12
|
-
|
|
13
|
-
jobs:
|
|
14
|
-
mkdocs:
|
|
15
|
-
runs-on: ubuntu-latest
|
|
16
|
-
steps:
|
|
17
|
-
- name: Build chimefrb/handbook
|
|
18
|
-
uses: peter-evans/repository-dispatch@v3
|
|
19
|
-
with:
|
|
20
|
-
token: ${{ secrets.BUILD_HANDBOOK_TOKEN }}
|
|
21
|
-
repository: chimefrb/handbook
|
|
22
|
-
event-type: build-handbook
|
|
23
|
-
client-payload: '{"source": "chimefrb/datatrail-cli"}'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|