datatrail-cli 0.11.1__tar.gz → 0.12.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/workflows/continuous-deployment.yml +8 -18
  2. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/workflows/continuous-integration.yml +8 -21
  3. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/CHANGELOG.md +12 -0
  4. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/PKG-INFO +2 -2
  5. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/clear.md +1 -1
  6. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/commands.md +3 -0
  7. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/install.md +3 -3
  8. datatrail_cli-0.12.0/docs/unregistered.md +237 -0
  9. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/src/functions.py +29 -0
  10. datatrail_cli-0.12.0/dtcli/unregistered.py +228 -0
  11. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/mkdocs.yml +1 -0
  12. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/pyproject.toml +16 -24
  13. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/tests/test_cli.py +33 -0
  14. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/tests/test_functions.py +37 -1
  15. datatrail_cli-0.12.0/uv.lock +3431 -0
  16. datatrail_cli-0.11.1/dtcli/unregistered.py +0 -102
  17. datatrail_cli-0.11.1/poetry.lock +0 -3756
  18. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  19. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  20. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  21. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/workflows/docs.yml +0 -0
  22. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.github/workflows/stale.yml +0 -0
  23. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.gitignore +0 -0
  24. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/.pre-commit-config.yaml +0 -0
  25. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/LICENSE +0 -0
  26. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/README.md +0 -0
  27. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/cli.md +0 -0
  28. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/images/Datatrail-logo.png +0 -0
  29. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/index.md +0 -0
  30. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/initialising.md +0 -0
  31. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/list.md +0 -0
  32. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/ps.md +0 -0
  33. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/pull.md +0 -0
  34. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/scout.md +0 -0
  35. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/docs/user_guide.md +0 -0
  36. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/__init__.py +0 -0
  37. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/clear.py +0 -0
  38. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/cli.py +0 -0
  39. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/config.py +0 -0
  40. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/config.yaml +0 -0
  41. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/ls.py +0 -0
  42. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/ps.py +0 -0
  43. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/pull.py +0 -0
  44. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/scout.py +0 -0
  45. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/utilities/cadcclient.py +0 -0
  46. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/dtcli/utilities/utilities.py +0 -0
  47. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/tests/test_utils.py +0 -0
  48. {datatrail_cli-0.11.1 → datatrail_cli-0.12.0}/tox.ini +0 -0
@@ -20,29 +20,19 @@ jobs:
20
20
  with:
21
21
  fetch-depth: 1
22
22
 
23
- - name: Set up Python 3.13
23
+ - name: Install uv
24
24
  if: ${{ steps.release-please.outputs.release_created }}
25
- uses: actions/setup-python@v6
25
+ uses: astral-sh/setup-uv@v8.3.2
26
26
  with:
27
- python-version: 3.13
27
+ enable-cache: false
28
+ python-version: "3.13"
28
29
 
29
- - name: Install poetry
30
+ - name: Build datatrail-cli
30
31
  if: ${{ steps.release-please.outputs.release_created }}
31
- run: |
32
- pip install --upgrade pip
33
- pip install --upgrade poetry
34
- poetry config virtualenvs.create true
35
- poetry config virtualenvs.in-project false
36
-
37
- - name: Install datatrail-cli
38
- if: ${{ steps.release-please.outputs.release_created }}
39
- run: |
40
- poetry install
32
+ run: uv build
41
33
 
42
34
  - name: Publish to pypi
43
35
  if: ${{ steps.release-please.outputs.release_created }}
44
36
  env:
45
- PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
46
- run: |
47
- poetry config pypi-token.pypi $PYPI_TOKEN
48
- poetry publish --build --verbose
37
+ UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
38
+ run: uv publish --verbose
@@ -43,29 +43,16 @@ jobs:
43
43
  steps:
44
44
  - uses: actions/checkout@v6
45
45
 
46
- - name: Set up Python ${{ matrix.python-version }}
47
- uses: actions/setup-python@v6
46
+ - name: Install uv
47
+ uses: astral-sh/setup-uv@v8.3.2
48
48
  with:
49
+ enable-cache: true
50
+ cache-dependency-glob: "uv.lock"
51
+ cache-python: true
49
52
  python-version: ${{ matrix.python-version }}
50
53
 
51
- - name: Load cached venv
52
- id: cached-poetry-dependencies
53
- uses: actions/cache@v5
54
- with:
55
- path: |
56
- ~/.cache/pypoetry
57
- key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
58
-
59
54
  - name: Installation
60
- run: |
61
- pip install --upgrade pip
62
- pip install poetry==2.1.2
63
- pip install vos
64
- poetry config virtualenvs.create true
65
- poetry config virtualenvs.in-project false
66
- poetry config cache-dir ~/.cache/pypoetry
67
- poetry config virtualenvs.path ~/.cache/pypoetry/venv
68
- poetry install --with dev
55
+ run: uv sync
69
56
 
70
57
  - name: Create NETRC file
71
58
  run: |
@@ -76,12 +63,12 @@ jobs:
76
63
  - name: Obtain CADC certificate
77
64
  run: |
78
65
  export CADC_CERT=$HOME"/.ssl/cadcproxy.pem"
79
- cadc-get-cert --netrc-file ${{ env.CANFAR_NETRC_FILE }} --days-valid 1 --cert-filename $CADC_CERT
66
+ uv run --no-sync cadc-get-cert --netrc-file ${{ env.CANFAR_NETRC_FILE }} --days-valid 1 --cert-filename $CADC_CERT
80
67
  echo CADC_CERT=$CADC_CERT >> "$GITHUB_ENV"
81
68
 
82
69
  - name: Run Tests
83
70
  run: |
84
- poetry run pytest tests/ --cov . --cov-report=lcov --cov-report=xml
71
+ uv run --no-sync pytest tests/ --cov . --cov-report=lcov --cov-report=xml
85
72
 
86
73
  - name: Remove NETRC file and CADC certificate
87
74
  run: |
@@ -1,5 +1,17 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.12.0](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.11.1...v0.12.0) (2026-08-24)
4
+
5
+
6
+ ### Features
7
+
8
+ * **unregistered:** command to search for event in unregistered results bucket ([1bb4ca0](https://github.com/CHIMEFRB/datatrail-cli/commit/1bb4ca0c118e042a907f38205af5d7ed6bf9f16d))
9
+
10
+
11
+ ### Documentation
12
+
13
+ * add unregistered command ([d8dd35b](https://github.com/CHIMEFRB/datatrail-cli/commit/d8dd35b22069b69ff7e3e76b6dc935696f8c2840))
14
+
3
15
  ## [0.11.1](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.11.0...v0.11.1) (2026-07-21)
4
16
 
5
17
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: datatrail-cli
3
- Version: 0.11.1
3
+ Version: 0.12.0
4
4
  Summary: CHIME/FRB Datatrail CLI
5
5
  Author: CHIME FRB Project Office
6
6
  Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>
@@ -109,7 +109,7 @@ will ask for confirmation before removing the dataset.
109
109
  PosixPath('dtcli'), PosixPath('pyproject.toml'), PosixPath('tests'),
110
110
  PosixPath('docs'), PosixPath('README.md'), PosixPath('.gitignore'),
111
111
  PosixPath('.github'), PosixPath('tox.ini'),
112
- PosixPath('poetry.lock'), PosixPath('.git')]
112
+ PosixPath('uv.lock'), PosixPath('.git')]
113
113
  DEBUG .: ✗ # (2)!
114
114
  ```
115
115
 
@@ -18,6 +18,9 @@ The commands available to you are:
18
18
  thinks is the current number of files for a given dataset at each storage
19
19
  element, compared to what is observed. If a discrepancy is found at Minoc,
20
20
  the user can choose to create the file replicas missing for Minoc.
21
+ - `unregistered`: This provides insight into datasets which failed to register
22
+ with Datatrail, either summarised across the whole unregistered bucket
23
+ (`summary`) or for a single event (`search`).
21
24
  - `version`: List the CLI and server version.
22
25
 
23
26
  Detailed information on all of the CLI commands can be found on the
@@ -29,12 +29,12 @@ There are two ways to install the Datatrail CLI:
29
29
  pip install .
30
30
  ```
31
31
 
32
- It can also be installed from source using git and poetry:
32
+ It can also be installed from source using git and uv:
33
33
 
34
- !!! example "git+poetry"
34
+ !!! example "git+uv"
35
35
 
36
36
  ``` shell
37
37
  git clone ssh+git://git@github.com/chimefrb/datatrail-cli
38
38
  cd datatrail-cli
39
- poetry install --without docs
39
+ uv sync
40
40
  ```
@@ -0,0 +1,237 @@
1
+ # 🚧 Inspecting failures with `unregistered`
2
+
3
+ <!-- termynal -->
4
+
5
+ ```bash
6
+ ❯ datatrail unregistered --help
7
+ Usage: datatrail unregistered [OPTIONS] COMMAND [ARGS]...
8
+
9
+ Commands related to unregistered datasets.
10
+
11
+ Options:
12
+ --help Show this message and exit.
13
+
14
+ Commands:
15
+ search Check whether an event is an unregistered dataset.
16
+ summary Summarise the reasons for unregistered datasets.
17
+ ```
18
+
19
+ ## Overview
20
+
21
+ When Datatrail cannot register a dataset, the attempt is recorded in the
22
+ unregistered datasets bucket along with the reason it failed. The
23
+ `unregistered` commands let you inspect that bucket, either as a whole with
24
+ `summary`, or for a single event with `search`.
25
+
26
+ This is usually the first place to look when an event you expect to find with
27
+ [`scout`](scout.md) or [`ps`](ps.md) is missing entirely from Datatrail.
28
+
29
+ ## `search`
30
+
31
+ ```bash
32
+ ❯ datatrail unregistered search --help
33
+ Usage: datatrail unregistered search [OPTIONS] EVENT
34
+
35
+ Check whether an event is an unregistered dataset.
36
+
37
+ Options:
38
+ -s, --scope TEXT Only search within this scope.
39
+ -p, --partial Match events containing EVENT.
40
+ -v, --verbose Verbosity: v=INFO, vv=DEBUG.
41
+ -q, --quiet Only errors shown in logs.
42
+ --json Output as JSON.
43
+ --help Show this message and exit.
44
+ ```
45
+
46
+ `search` answers the question "why is this event not in Datatrail?". Each
47
+ record found is shown as a separate block, most recently recorded first,
48
+ detailing the scope and site the registration was attempted for, the parent
49
+ dataset it was to be attached to, and the reason the attempt failed.
50
+
51
+ An event can have more than one record, e.g. one per site, or one per failed
52
+ attempt.
53
+
54
+ === "All scopes"
55
+
56
+ ```bash
57
+ ❯ datatrail unregistered search 1172713191
58
+ ⚠ 1172713191 is an unregistered dataset ⚠
59
+ 4 records found.
60
+ Event 1172713191
61
+ Scope hco.event.baseband.raw
62
+ Site hco
63
+ Parent dataset CHAMPS_Localization.J0408
64
+ Recorded 2026-08-08 19:15 UTC
65
+ Reason Could not attach datasets: 1172713191 and
66
+ CHAMPS_Localization.J0408. ERROR: dataset
67
+ CHAMPS_Localization.J0408, hco.event.baseband.raw not found..
68
+
69
+ Event 1172713191
70
+ Scope gbo.event.baseband.raw
71
+ Site gbo
72
+ Parent dataset CHAMPS_Localization.J0408
73
+ Recorded 2026-08-08 19:09 UTC
74
+ Reason Could not attach datasets: 1172713191 and
75
+ CHAMPS_Localization.J0408. ERROR: dataset
76
+ CHAMPS_Localization.J0408, gbo.event.baseband.raw not found..
77
+
78
+ Event 1172713191
79
+ Scope kko.event.baseband.raw
80
+ Site kko
81
+ Parent dataset CHAMPS_Localization.J0408
82
+ Recorded 2026-08-08 18:54 UTC
83
+ Reason Could not attach datasets: 1172713191 and
84
+ CHAMPS_Localization.J0408. ERROR: dataset
85
+ CHAMPS_Localization.J0408, kko.event.baseband.raw not found..
86
+
87
+ Event 1172713191
88
+ Scope chime.event.intensity.raw
89
+ Site chime
90
+ Parent dataset CHAMPS_Localization.J0408
91
+ Recorded 2026-08-08 14:28 UTC
92
+ Reason Could not attach datasets: 1172713191 and
93
+ CHAMPS_Localization.J0408. ERROR: dataset
94
+ CHAMPS_Localization.J0408, chime.event.intensity.raw not
95
+ found..
96
+ ```
97
+
98
+ === "Filtering by scope"
99
+
100
+ ```bash
101
+ ❯ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw
102
+ ⚠ 1172713191 is an unregistered dataset ⚠
103
+ 1 record found.
104
+ Event 1172713191
105
+ Scope kko.event.baseband.raw
106
+ Site kko
107
+ Parent dataset CHAMPS_Localization.J0408
108
+ Recorded 2026-08-08 18:54 UTC
109
+ Reason Could not attach datasets: 1172713191 and
110
+ CHAMPS_Localization.J0408. ERROR: dataset
111
+ CHAMPS_Localization.J0408, kko.event.baseband.raw not found..
112
+ ```
113
+
114
+ === "No records found"
115
+
116
+ ```bash
117
+ ❯ datatrail unregistered search 382085503
118
+ 382085503 is not an unregistered dataset.
119
+ Use --partial to search for events containing this name.
120
+ ```
121
+
122
+ ### Partial matches
123
+
124
+ By default the event name has to match exactly. With `--partial`, any event
125
+ name _containing_ the given string is returned, which is useful when you only
126
+ know part of the name, or when you want to check a whole family of events such
127
+ as a set of commissioning datasets.
128
+
129
+ ```bash
130
+ ❯ datatrail unregistered search 11728169 --partial
131
+ ⚠ 11728169 is an unregistered dataset ⚠
132
+ 2 records found.
133
+ Event 1172816953
134
+ Scope chime.event.intensity.raw
135
+ Site chime
136
+ Parent dataset CHAMPS_Localization.J0408
137
+ Recorded 2026-08-09 11:14 UTC
138
+ Reason Could not attach datasets: 1172816953 and
139
+ CHAMPS_Localization.J0408. ERROR: dataset
140
+ CHAMPS_Localization.J0408, chime.event.intensity.raw not
141
+ found..
142
+
143
+ Event 1172816944
144
+ Scope chime.event.intensity.raw
145
+ Site chime
146
+ Parent dataset CHAMPS_Localization.J0408
147
+ Recorded 2026-08-09 10:16 UTC
148
+ Reason Could not attach datasets: 1172816944 and
149
+ CHAMPS_Localization.J0408. ERROR: dataset
150
+ CHAMPS_Localization.J0408, chime.event.intensity.raw not
151
+ found..
152
+ ```
153
+
154
+ !!! note "Result limit"
155
+
156
+ A maximum of 100 records are returned. If you hit that limit with
157
+ `--partial`, narrow the search with a longer event name or a `--scope`.
158
+
159
+ ### Machine readable output
160
+
161
+ `--json` prints the full records, including fields not shown in the table,
162
+ which is handy for scripting or for piping into `jq`.
163
+
164
+ ```bash
165
+ ❯ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw --json
166
+ {
167
+ "event": "1172713191",
168
+ "scope": "kko.event.baseband.raw",
169
+ "partial": false,
170
+ "unregistered": [
171
+ {
172
+ "dataset_name": "1172713191",
173
+ "dataset_scope": "kko.event.baseband.raw",
174
+ "storage_name": "kko",
175
+ "storage_element_captured_at": "kko",
176
+ "attach_to_dataset": "CHAMPS_Localization.J0408",
177
+ "reason": "Could not attach datasets: 1172713191 and CHAMPS_Localization.J0408. ERROR: dataset CHAMPS_Localization.J0408, kko.event.baseband.raw not found..",
178
+ "associated_event": {},
179
+ "locked": false
180
+ }
181
+ ]
182
+ }
183
+ ```
184
+
185
+ !!! tip "Errors in JSON mode"
186
+
187
+ With `--json`, a failed query is reported as `{"error": "..."}` on stdout
188
+ and the command exits with status `1`, so failures can be detected in a
189
+ pipeline.
190
+
191
+ ## `summary`
192
+
193
+ ```bash
194
+ ❯ datatrail unregistered summary --help
195
+ Usage: datatrail unregistered summary [OPTIONS]
196
+
197
+ Summarise the reasons for unregistered datasets.
198
+
199
+ Options:
200
+ -v, --verbose Verbosity: v=INFO, vv=DEBUG.
201
+ -q, --quiet Only errors shown in logs.
202
+ --help Show this message and exit.
203
+ ```
204
+
205
+ `summary` groups every record in the bucket by the reason it failed, giving an
206
+ overview of what is currently going wrong. Reasons are grouped into categories,
207
+ e.g. `ATTACH_MISSING` for datasets whose parent dataset does not exist, or
208
+ `POSTGRES` for database errors.
209
+
210
+ ```bash
211
+ ❯ datatrail unregistered summary
212
+ Summary of reasons — 886 unregistered datasets
213
+ ┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━┓
214
+ ┃ Category ┃ Detail ┃ Count ┃ % ┃
215
+ ┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━┩
216
+ │ POSTGRES │ (psycopg2.errors.UniqueViolation) │ 627 │ 70.8% │
217
+ │ │ duplicate key value violates unique │ │ │
218
+ │ │ constraint "name_se_captured_at_pair" │ │ │
219
+ │ │ DETAIL: Key │ │ │
220
+ ├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
221
+ │ STATUS │ (no reason recorded) │ 110 │ 12.4% │
222
+ │ │ event-missing-verification │ 85 │ 9.6% │
223
+ │ │ pending-tsar-classification │ 23 │ 2.6% │
224
+ ├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
225
+ │ ATTACH_MISSING │ CHAMPS_Localization.J0408 → │ 18 │ 2.0% │
226
+ │ │ chime.event.intensity.raw │ │ │
227
+ │ │ outrigger.commissioning.B0834 → │ 12 │ 1.4% │
228
+ │ │ chime.event.baseband.raw │ │ │
229
+ ├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
230
+ │ CREATE_DUPLICATE │ <ID> → chime.event.baseband.raw │ 3 │ 0.3% │
231
+ └──────────────────┴───────────────────────────────────────────┴───────┴───────┘
232
+ ```
233
+
234
+ For `ATTACH_MISSING` and `CREATE_DUPLICATE`, the detail column reads as
235
+ `dataset → scope`. In the `CREATE_DUPLICATE` case the digits of the dataset
236
+ name are replaced with `<ID>`, so that the same failure for many events is
237
+ counted as a single row.
@@ -486,6 +486,35 @@ def get_unregistered_dataset(dataset: str, scope: str) -> Optional[Dict[str, Any
486
486
  return response[0]
487
487
 
488
488
 
489
+ def find_unregistered_datasets(
490
+ event: str,
491
+ scope: Optional[str] = None,
492
+ partial: bool = False,
493
+ limit: int = 100,
494
+ ) -> List[Dict[str, Any]]:
495
+ """Find unregistered datasets recorded for an event.
496
+
497
+ Args:
498
+ event (str): Name of the event, i.e. the dataset name.
499
+ scope (Optional[str]): Only return records for this scope.
500
+ partial (bool): Match events containing `event` rather than exactly.
501
+ limit (int): Maximum number of records to return.
502
+
503
+ Returns:
504
+ List[Dict[str, Any]]: Unregistered dataset records for the event.
505
+ """
506
+ name: Any = {"$regex": re.escape(event)} if partial else event
507
+ query: Dict[str, Any] = {"results.dataset_name": name}
508
+ if scope:
509
+ query["results.dataset_scope"] = scope
510
+ return view_results(
511
+ pipeline="datatrail-unregistered-datasets",
512
+ query=query,
513
+ projection={"results.files": 0},
514
+ limit=limit,
515
+ )
516
+
517
+
489
518
  ATTACH_RE = re.compile(
490
519
  r"Could not attach datasets: .+? ERROR: \"?dataset (.+?), (.+?) not found" # noqa: E501
491
520
  )
@@ -0,0 +1,228 @@
1
+ """Datatrail Unregistered datasets commands."""
2
+
3
+ import json
4
+ import logging
5
+ import re
6
+ from collections import defaultdict
7
+ from datetime import datetime, timezone
8
+ from typing import Any, DefaultDict, Dict, List, Optional, Tuple
9
+
10
+ import click
11
+ from rich.console import Console
12
+ from rich.table import Table
13
+ from rich.text import Text
14
+
15
+ from dtcli.src import functions
16
+ from dtcli.utilities.utilities import set_log_level
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ console = Console()
21
+ error_console = Console(stderr=True, style="bold red")
22
+
23
+ CATEGORY_STYLES = {
24
+ "ATTACH_MISSING": "yellow",
25
+ "CREATE_DUPLICATE": "magenta",
26
+ "POSTGRES": "red",
27
+ "OTHER": "red",
28
+ "STATUS": "cyan",
29
+ }
30
+
31
+
32
+ @click.group(help="Commands related to unregistered datasets.")
33
+ def unregistered():
34
+ """Group of commands related to unregistered datasets."""
35
+ pass
36
+
37
+
38
+ @unregistered.command(help="Summarise the reasons for unregistered datasets.")
39
+ @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
40
+ @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
41
+ @click.pass_context
42
+ def summary(
43
+ ctx: click.Context,
44
+ verbose: int = 0,
45
+ quiet: bool = False,
46
+ ):
47
+ """Show a summary of the unregistered datasets.
48
+
49
+ Args:
50
+ ctx (click.Context): Click context.
51
+ verbose (int): Verbosity: v=INFO, vv=DEBUG.
52
+ quiet (bool): Only errors shown in logs.
53
+ """
54
+ # Set logging level.
55
+ set_log_level(logger, verbose, quiet)
56
+ logger.debug("`summary` called with:")
57
+ logger.debug(f"verbose: {verbose} [{type(verbose)}]")
58
+ logger.debug(f"quiet: {quiet} [{type(quiet)}]")
59
+
60
+ results = functions.summarise_unregistered_datasets()
61
+
62
+ if not results:
63
+ console.print("No unregistered datasets found.")
64
+ return
65
+
66
+ total = sum(results.values())
67
+
68
+ table = Table(
69
+ title=f"Summary of reasons — {total:,} unregistered datasets",
70
+ header_style="magenta",
71
+ title_style="bold magenta",
72
+ row_styles=["none", "dim"],
73
+ )
74
+ table.add_column("Category")
75
+ table.add_column("Detail")
76
+ table.add_column("Count", justify="right")
77
+ table.add_column("%", justify="right")
78
+
79
+ # Group signatures by their category prefix, e.g. "ATTACH_MISSING:...".
80
+ groups: DefaultDict[str, List[Tuple[str, int]]] = defaultdict(list)
81
+ for sig, count in results.items():
82
+ category, _, detail = sig.partition(":")
83
+ groups[category].append((detail, count))
84
+
85
+ ordered = sorted(
86
+ groups.items(),
87
+ key=lambda group: sum(count for _, count in group[1]),
88
+ reverse=True,
89
+ )
90
+ for index, (category, reasons) in enumerate(ordered):
91
+ if index:
92
+ table.add_section()
93
+ style = CATEGORY_STYLES.get(category, "white")
94
+ reasons.sort(key=lambda reason: reason[1], reverse=True)
95
+ for row, (detail, count) in enumerate(reasons):
96
+ if category in ("ATTACH_MISSING", "CREATE_DUPLICATE"):
97
+ detail = detail.replace(":", " → ", 1)
98
+ table.add_row(
99
+ Text(category, style=style) if row == 0 else "",
100
+ Text(detail) if detail else "(no reason recorded)",
101
+ f"{count:,}",
102
+ f"{count / total:.1%}",
103
+ )
104
+
105
+ console.print(table)
106
+
107
+
108
+ @unregistered.command(help="Check whether an event is an unregistered dataset.")
109
+ @click.argument("event", required=True, type=click.STRING, nargs=1)
110
+ @click.option("-s", "--scope", type=click.STRING, help="Only search within this scope.")
111
+ @click.option("-p", "--partial", is_flag=True, help="Match events containing EVENT.")
112
+ @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
113
+ @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
114
+ @click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
115
+ @click.pass_context
116
+ def search(
117
+ ctx: click.Context,
118
+ event: str,
119
+ scope: Optional[str] = None,
120
+ partial: bool = False,
121
+ verbose: int = 0,
122
+ quiet: bool = False,
123
+ output_json: bool = False,
124
+ ):
125
+ """Search the unregistered datasets for an event.
126
+
127
+ Args:
128
+ ctx (click.Context): Click context.
129
+ event (str): Name of the event to search for.
130
+ scope (str): Only search within this scope.
131
+ partial (bool): Match events containing EVENT.
132
+ verbose (int): Verbosity: v=INFO, vv=DEBUG.
133
+ quiet (bool): Only errors shown in logs.
134
+ output_json (bool): Output as JSON.
135
+ """
136
+ # Set logging level.
137
+ set_log_level(logger, verbose, quiet)
138
+ logger.debug("`search` called with:")
139
+ logger.debug(f"event: {event} [{type(event)}]")
140
+ logger.debug(f"scope: {scope} [{type(scope)}]")
141
+ logger.debug(f"partial: {partial} [{type(partial)}]")
142
+ logger.debug(f"verbose: {verbose} [{type(verbose)}]")
143
+ logger.debug(f"quiet: {quiet} [{type(quiet)}]")
144
+
145
+ try:
146
+ results = functions.find_unregistered_datasets(event, scope, partial)
147
+ except Exception as error:
148
+ logger.debug(error)
149
+ if output_json:
150
+ print(json.dumps({"error": str(error)}, indent=2))
151
+ ctx.exit(1)
152
+ error_console.print(error)
153
+ return None
154
+
155
+ if output_json:
156
+ print(
157
+ json.dumps(
158
+ {
159
+ "event": event,
160
+ "scope": scope,
161
+ "partial": partial,
162
+ "unregistered": [result["results"] for result in results],
163
+ },
164
+ indent=2,
165
+ )
166
+ )
167
+ return None
168
+
169
+ if not results:
170
+ within = f" in {scope}" if scope else ""
171
+ console.print(f"{event} is not an unregistered dataset{within}.", style="green")
172
+ if not partial:
173
+ console.print("Use --partial to search for events containing this name.")
174
+ return None
175
+
176
+ console.print(
177
+ f":warning: {event} is an unregistered dataset :warning:",
178
+ style="bold yellow",
179
+ justify="center",
180
+ )
181
+ console.print(
182
+ f"{len(results):,} record{'s' if len(results) > 1 else ''} found.",
183
+ style="bold magenta",
184
+ justify="center",
185
+ )
186
+ results = sorted(
187
+ results, key=lambda result: result.get("creation") or 0, reverse=True
188
+ )
189
+ for index, result in enumerate(results):
190
+ if index:
191
+ console.print()
192
+ console.print(create_record_table(result))
193
+
194
+
195
+ def create_record_table(result: Dict[str, Any]) -> Table:
196
+ """Create a table detailing a single unregistered record.
197
+
198
+ Args:
199
+ result (Dict[str, Any]): Unregistered dataset record.
200
+
201
+ Returns:
202
+ Table: Table of the record details.
203
+ """
204
+ record = result["results"]
205
+ creation = result.get("creation")
206
+ recorded = (
207
+ datetime.fromtimestamp(creation, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
208
+ if creation
209
+ else "unknown"
210
+ )
211
+
212
+ table = Table(box=None, show_header=False, padding=(0, 2, 0, 0))
213
+ table.add_column(style="bold")
214
+ table.add_column(overflow="fold")
215
+ table.add_row("Event", str(record.get("dataset_name")))
216
+ table.add_row("Scope", str(record.get("dataset_scope")))
217
+ table.add_row("Site", str(result.get("site")))
218
+ table.add_row("Parent dataset", str(record.get("attach_to_dataset") or "-"))
219
+ table.add_row("Recorded", recorded)
220
+ table.add_row(
221
+ "Reason",
222
+ Text(
223
+ re.sub(r"\n+", "\n", str(record.get("reason") or "")).strip()
224
+ or "(no reason recorded)",
225
+ style="red",
226
+ ),
227
+ )
228
+ return table
@@ -14,6 +14,7 @@ nav:
14
14
  - ps: ps.md
15
15
  - pull: pull.md
16
16
  - scout: scout.md
17
+ - unregistered: unregistered.md
17
18
  - Command Line Interface:
18
19
  - Commands: commands.md
19
20
  - Reference: cli.md