datatrail-cli 0.11.0__tar.gz → 0.12.0__tar.gz

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/.github/workflows/continuous-deployment.yml

@@ -20,29 +20,19 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.13
+      - name: Install uv
         if: ${{ steps.release-please.outputs.release_created }}
-        uses: actions/setup-python@v6
+        uses: astral-sh/setup-uv@v8.3.2
         with:
-          python-version: 3.13
+          enable-cache: false
+          python-version: "3.13"
 
-      - name: Install poetry
+      - name: Build datatrail-cli
         if: ${{ steps.release-please.outputs.release_created }}
-        run: |
-          pip install --upgrade pip
-          pip install --upgrade poetry
-          poetry config virtualenvs.create true
-          poetry config virtualenvs.in-project false
-
-      - name: Install datatrail-cli
-        if: ${{ steps.release-please.outputs.release_created }}
-        run: |
-          poetry install
+        run: uv build
 
       - name: Publish to pypi
         if: ${{ steps.release-please.outputs.release_created }}
         env:
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          poetry config pypi-token.pypi $PYPI_TOKEN
-          poetry publish --build --verbose
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: uv publish --verbose

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/.github/workflows/continuous-integration.yml

@@ -43,29 +43,16 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.3.2
         with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          cache-python: true
           python-version: ${{ matrix.python-version }}
 
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v5
-        with:
-          path: |
-            ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
-
       - name: Installation
-        run: |
-          pip install --upgrade pip
-          pip install poetry==2.1.2
-          pip install vos
-          poetry config virtualenvs.create true
-          poetry config virtualenvs.in-project false
-          poetry config cache-dir ~/.cache/pypoetry
-          poetry config virtualenvs.path ~/.cache/pypoetry/venv
-          poetry install --with dev
+        run: uv sync
 
       - name: Create NETRC file
         run: |
@@ -76,12 +63,12 @@ jobs:
       - name: Obtain CADC certificate
         run: |
           export CADC_CERT=$HOME"/.ssl/cadcproxy.pem"
-          cadc-get-cert --netrc-file ${{ env.CANFAR_NETRC_FILE }} --days-valid 1 --cert-filename $CADC_CERT
+          uv run --no-sync cadc-get-cert --netrc-file ${{ env.CANFAR_NETRC_FILE }} --days-valid 1 --cert-filename $CADC_CERT
           echo CADC_CERT=$CADC_CERT >> "$GITHUB_ENV"
 
       - name: Run Tests
         run: |
-          poetry run pytest tests/ --cov . --cov-report=lcov --cov-report=xml
+          uv run --no-sync pytest tests/ --cov . --cov-report=lcov --cov-report=xml
 
       - name: Remove NETRC file and CADC certificate
         run: |

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [0.12.0](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.11.1...v0.12.0) (2026-08-24)
+
+
+### Features
+
+* **unregistered:** command to search for event in unregistered results bucket ([1bb4ca0](https://github.com/CHIMEFRB/datatrail-cli/commit/1bb4ca0c118e042a907f38205af5d7ed6bf9f16d))
+
+
+### Documentation
+
+* add unregistered command ([d8dd35b](https://github.com/CHIMEFRB/datatrail-cli/commit/d8dd35b22069b69ff7e3e76b6dc935696f8c2840))
+
+## [0.11.1](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.11.0...v0.11.1) (2026-07-21)
+
+
+### Bug Fixes
+
+* **unregistered:** improve way info displayed ([c5f0fca](https://github.com/CHIMEFRB/datatrail-cli/commit/c5f0fca6f6f53e5004e1f1868517100572d68d1a))
+
 ## [0.11.0](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.10.3...v0.11.0) (2026-07-07)
 
 

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: datatrail-cli
-Version: 0.11.0
+Version: 0.12.0
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>
@@ -13,7 +13,7 @@ Requires-Dist: cadcutils>=1.5.1.1
 Requires-Dist: click-aliasing>=1.0.0
 Requires-Dist: click>=8.1.3
 Requires-Dist: dill>=0.3.6
-Requires-Dist: lxml>=4.9.2
+Requires-Dist: lxml>=6.1.0
 Requires-Dist: mergedeep>=1.3.4
 Requires-Dist: pyaml>=21.10.1
 Requires-Dist: requests>=2.29.0

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/docs/clear.md

@@ -109,7 +109,7 @@ will ask for confirmation before removing the dataset.
                         PosixPath('dtcli'), PosixPath('pyproject.toml'), PosixPath('tests'),
                         PosixPath('docs'), PosixPath('README.md'), PosixPath('.gitignore'),
                         PosixPath('.github'), PosixPath('tox.ini'),
-                        PosixPath('poetry.lock'), PosixPath('.git')]
+                        PosixPath('uv.lock'), PosixPath('.git')]
                DEBUG    .: ✗ # (2)!
     ```
 

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/docs/commands.md

@@ -18,6 +18,9 @@ The commands available to you are:
   thinks is the current number of files for a given dataset at each storage
   element, compared to what is observed. If a discrepancy is found at Minoc,
   the user can choose to create the file replicas missing for Minoc.
+- `unregistered`: This provides insight into datasets which failed to register
+  with Datatrail, either summarised across the whole unregistered bucket
+  (`summary`) or for a single event (`search`).
 - `version`: List the CLI and server version.
 
 Detailed information on all of the CLI commands can be found on the

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/docs/install.md

@@ -29,12 +29,12 @@ There are two ways to install the Datatrail CLI:
         pip install .
         ```
 
-    It can also be installed from source using git and poetry:
+    It can also be installed from source using git and uv:
 
-    !!! example "git+poetry"
+    !!! example "git+uv"
 
         ``` shell
         git clone ssh+git://git@github.com/chimefrb/datatrail-cli
         cd datatrail-cli
-        poetry install --without docs
+        uv sync
         ```

datatrail_cli-0.12.0/docs/unregistered.md

@@ -0,0 +1,237 @@
+# 🚧 Inspecting failures with `unregistered`
+
+<!-- termynal -->
+
+```bash
+❯ datatrail unregistered --help
+Usage: datatrail unregistered [OPTIONS] COMMAND [ARGS]...
+
+  Commands related to unregistered datasets.
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  search   Check whether an event is an unregistered dataset.
+  summary  Summarise the reasons for unregistered datasets.
+```
+
+## Overview
+
+When Datatrail cannot register a dataset, the attempt is recorded in the
+unregistered datasets bucket along with the reason it failed. The
+`unregistered` commands let you inspect that bucket, either as a whole with
+`summary`, or for a single event with `search`.
+
+This is usually the first place to look when an event you expect to find with
+[`scout`](scout.md) or [`ps`](ps.md) is missing entirely from Datatrail.
+
+## `search`
+
+```bash
+❯ datatrail unregistered search --help
+Usage: datatrail unregistered search [OPTIONS] EVENT
+
+  Check whether an event is an unregistered dataset.
+
+Options:
+  -s, --scope TEXT  Only search within this scope.
+  -p, --partial     Match events containing EVENT.
+  -v, --verbose     Verbosity: v=INFO, vv=DEBUG.
+  -q, --quiet       Only errors shown in logs.
+  --json            Output as JSON.
+  --help            Show this message and exit.
+```
+
+`search` answers the question "why is this event not in Datatrail?". Each
+record found is shown as a separate block, most recently recorded first,
+detailing the scope and site the registration was attempted for, the parent
+dataset it was to be attached to, and the reason the attempt failed.
+
+An event can have more than one record, e.g. one per site, or one per failed
+attempt.
+
+=== "All scopes"
+
+    ```bash
+    ❯ datatrail unregistered search 1172713191
+                       ⚠ 1172713191 is an unregistered dataset ⚠
+                                    4 records found.
+    Event           1172713191
+    Scope           hco.event.baseband.raw
+    Site            hco
+    Parent dataset  CHAMPS_Localization.J0408
+    Recorded        2026-08-08 19:15 UTC
+    Reason          Could not attach datasets: 1172713191 and
+                    CHAMPS_Localization.J0408. ERROR: dataset
+                    CHAMPS_Localization.J0408, hco.event.baseband.raw not found..
+
+    Event           1172713191
+    Scope           gbo.event.baseband.raw
+    Site            gbo
+    Parent dataset  CHAMPS_Localization.J0408
+    Recorded        2026-08-08 19:09 UTC
+    Reason          Could not attach datasets: 1172713191 and
+                    CHAMPS_Localization.J0408. ERROR: dataset
+                    CHAMPS_Localization.J0408, gbo.event.baseband.raw not found..
+
+    Event           1172713191
+    Scope           kko.event.baseband.raw
+    Site            kko
+    Parent dataset  CHAMPS_Localization.J0408
+    Recorded        2026-08-08 18:54 UTC
+    Reason          Could not attach datasets: 1172713191 and
+                    CHAMPS_Localization.J0408. ERROR: dataset
+                    CHAMPS_Localization.J0408, kko.event.baseband.raw not found..
+
+    Event           1172713191
+    Scope           chime.event.intensity.raw
+    Site            chime
+    Parent dataset  CHAMPS_Localization.J0408
+    Recorded        2026-08-08 14:28 UTC
+    Reason          Could not attach datasets: 1172713191 and
+                    CHAMPS_Localization.J0408. ERROR: dataset
+                    CHAMPS_Localization.J0408, chime.event.intensity.raw not
+                    found..
+    ```
+
+=== "Filtering by scope"
+
+    ```bash
+    ❯ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw
+                       ⚠ 1172713191 is an unregistered dataset ⚠
+                                    1 record found.
+    Event           1172713191
+    Scope           kko.event.baseband.raw
+    Site            kko
+    Parent dataset  CHAMPS_Localization.J0408
+    Recorded        2026-08-08 18:54 UTC
+    Reason          Could not attach datasets: 1172713191 and
+                    CHAMPS_Localization.J0408. ERROR: dataset
+                    CHAMPS_Localization.J0408, kko.event.baseband.raw not found..
+    ```
+
+=== "No records found"
+
+    ```bash
+    ❯ datatrail unregistered search 382085503
+    382085503 is not an unregistered dataset.
+    Use --partial to search for events containing this name.
+    ```
+
+### Partial matches
+
+By default the event name has to match exactly. With `--partial`, any event
+name _containing_ the given string is returned, which is useful when you only
+know part of the name, or when you want to check a whole family of events such
+as a set of commissioning datasets.
+
+```bash
+❯ datatrail unregistered search 11728169 --partial
+                    ⚠ 11728169 is an unregistered dataset ⚠
+                                2 records found.
+Event           1172816953
+Scope           chime.event.intensity.raw
+Site            chime
+Parent dataset  CHAMPS_Localization.J0408
+Recorded        2026-08-09 11:14 UTC
+Reason          Could not attach datasets: 1172816953 and
+                CHAMPS_Localization.J0408. ERROR: dataset
+                CHAMPS_Localization.J0408, chime.event.intensity.raw not
+                found..
+
+Event           1172816944
+Scope           chime.event.intensity.raw
+Site            chime
+Parent dataset  CHAMPS_Localization.J0408
+Recorded        2026-08-09 10:16 UTC
+Reason          Could not attach datasets: 1172816944 and
+                CHAMPS_Localization.J0408. ERROR: dataset
+                CHAMPS_Localization.J0408, chime.event.intensity.raw not
+                found..
+```
+
+!!! note "Result limit"
+
+    A maximum of 100 records are returned. If you hit that limit with
+    `--partial`, narrow the search with a longer event name or a `--scope`.
+
+### Machine readable output
+
+`--json` prints the full records, including fields not shown in the table,
+which is handy for scripting or for piping into `jq`.
+
+```bash
+❯ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw --json
+{
+  "event": "1172713191",
+  "scope": "kko.event.baseband.raw",
+  "partial": false,
+  "unregistered": [
+    {
+      "dataset_name": "1172713191",
+      "dataset_scope": "kko.event.baseband.raw",
+      "storage_name": "kko",
+      "storage_element_captured_at": "kko",
+      "attach_to_dataset": "CHAMPS_Localization.J0408",
+      "reason": "Could not attach datasets: 1172713191 and CHAMPS_Localization.J0408. ERROR: dataset CHAMPS_Localization.J0408, kko.event.baseband.raw not found..",
+      "associated_event": {},
+      "locked": false
+    }
+  ]
+}
+```
+
+!!! tip "Errors in JSON mode"
+
+    With `--json`, a failed query is reported as `{"error": "..."}` on stdout
+    and the command exits with status `1`, so failures can be detected in a
+    pipeline.
+
+## `summary`
+
+```bash
+❯ datatrail unregistered summary --help
+Usage: datatrail unregistered summary [OPTIONS]
+
+  Summarise the reasons for unregistered datasets.
+
+Options:
+  -v, --verbose  Verbosity: v=INFO, vv=DEBUG.
+  -q, --quiet    Only errors shown in logs.
+  --help         Show this message and exit.
+```
+
+`summary` groups every record in the bucket by the reason it failed, giving an
+overview of what is currently going wrong. Reasons are grouped into categories,
+e.g. `ATTACH_MISSING` for datasets whose parent dataset does not exist, or
+`POSTGRES` for database errors.
+
+```bash
+❯ datatrail unregistered summary
+                 Summary of reasons — 886 unregistered datasets
+┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━┓
+┃ Category         ┃ Detail                                    ┃ Count ┃     % ┃
+┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━┩
+│ POSTGRES         │ (psycopg2.errors.UniqueViolation)         │   627 │ 70.8% │
+│                  │ duplicate key value violates unique       │       │       │
+│                  │ constraint "name_se_captured_at_pair"     │       │       │
+│                  │ DETAIL: Key                               │       │       │
+├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
+│ STATUS           │ (no reason recorded)                      │   110 │ 12.4% │
+│                  │ event-missing-verification                │    85 │  9.6% │
+│                  │ pending-tsar-classification               │    23 │  2.6% │
+├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
+│ ATTACH_MISSING   │ CHAMPS_Localization.J0408 →               │    18 │  2.0% │
+│                  │ chime.event.intensity.raw                 │       │       │
+│                  │ outrigger.commissioning.B0834 →           │    12 │  1.4% │
+│                  │ chime.event.baseband.raw                  │       │       │
+├──────────────────┼───────────────────────────────────────────┼───────┼───────┤
+│ CREATE_DUPLICATE │ <ID> → chime.event.baseband.raw           │     3 │  0.3% │
+└──────────────────┴───────────────────────────────────────────┴───────┴───────┘
+```
+
+For `ATTACH_MISSING` and `CREATE_DUPLICATE`, the detail column reads as
+`dataset → scope`. In the `CREATE_DUPLICATE` case the digits of the dataset
+name are replaced with `<ID>`, so that the same failure for many events is
+counted as a single row.

{datatrail_cli-0.11.0 → datatrail_cli-0.12.0}/dtcli/src/functions.py

@@ -6,7 +6,7 @@ import re
 import shutil
 import subprocess
 import time
-from collections import defaultdict
+from collections import Counter
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -486,25 +486,53 @@ def get_unregistered_dataset(dataset: str, scope: str) -> Optional[Dict[str, Any
         return response[0]
 
 
-def signature(msg: str) -> str:
-    """Create a signature for a reason unregistered message.
+def find_unregistered_datasets(
+    event: str,
+    scope: Optional[str] = None,
+    partial: bool = False,
+    limit: int = 100,
+) -> List[Dict[str, Any]]:
+    """Find unregistered datasets recorded for an event.
 
     Args:
-        msg: Reason message for unregistered dataset.
+        event (str): Name of the event, i.e. the dataset name.
+        scope (Optional[str]): Only return records for this scope.
+        partial (bool): Match events containing `event` rather than exactly.
+        limit (int): Maximum number of records to return.
 
     Returns:
-        str: Signature for error message.
+        List[Dict[str, Any]]: Unregistered dataset records for the event.
     """
-    ATTACH_RE = re.compile(
-        r"Could not attach datasets: .+ ERROR: dataset (.+), (.+) not found"  # noqa: E501
+    name: Any = {"$regex": re.escape(event)} if partial else event
+    query: Dict[str, Any] = {"results.dataset_name": name}
+    if scope:
+        query["results.dataset_scope"] = scope
+    return view_results(
+        pipeline="datatrail-unregistered-datasets",
+        query=query,
+        projection={"results.files": 0},
+        limit=limit,
     )
 
-    CREATE_RE = re.compile(
-        r"Could not create dataset: (.+), scope: (.+)\. .*UniqueViolation"  # noqa: E501
-    )
 
-    POSTGRES_RE = re.compile(r".*psycopg.*")
+ATTACH_RE = re.compile(
+    r"Could not attach datasets: .+? ERROR: \"?dataset (.+?), (.+?) not found"  # noqa: E501
+)
+
+CREATE_RE = re.compile(
+    r"Could not create dataset: (.+?), scope: (.+?)\. .*UniqueViolation"  # noqa: E501
+)
+
 
+def signature(msg: str) -> str:
+    """Create a signature for a reason unregistered message.
+
+    Args:
+        msg: Reason message for unregistered dataset.
+
+    Returns:
+        str: Signature for error message.
+    """
     msg = msg.strip()
 
     # Attach-dataset errors
@@ -517,11 +545,12 @@ def signature(msg: str) -> str:
     m = CREATE_RE.search(msg)
     if m:
         dataset, scope = m.groups()
+        dataset = re.sub(r"\d+", "<ID>", dataset)
         return f"CREATE_DUPLICATE:{dataset}:{scope}"
 
     # PostgreSQL violation
-    m = POSTGRES_RE.search(msg)
-    if m:
+    if "psycopg" in msg:
+        msg = re.sub(r"\s+", " ", msg)
         return f"POSTGRES:{msg[:120]}"
 
     # Short status / token messages
@@ -552,11 +581,4 @@ def summarise_unregistered_datasets() -> Dict[str, int]:
         Dict[str, int]: Dictionary of error message signatures and their counts.
     """
     response = get_all_unregistered_datasets()
-    reason_groups: Dict[str, int] = defaultdict(int)
-    messages = [str(r["results"]["reason"]) for r in response]
-
-    for msg in messages:
-        sig = signature(msg)
-        reason_groups[sig] += 1
-
-    return reason_groups
+    return Counter(signature(str(r["results"]["reason"])) for r in response)