datatrail-cli 0.10.3__tar.gz → 0.11.1__tar.gz

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.11.1](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.11.0...v0.11.1) (2026-07-21)
+
+
+### Bug Fixes
+
+* **unregistered:** improve way info displayed ([c5f0fca](https://github.com/CHIMEFRB/datatrail-cli/commit/c5f0fca6f6f53e5004e1f1868517100572d68d1a))
+
+## [0.11.0](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.10.3...v0.11.0) (2026-07-07)
+
+
+### Features
+
+* **ls & ps:** add --json flag for machine readable output ([c4742a0](https://github.com/CHIMEFRB/datatrail-cli/commit/c4742a0dfe43823ce8e68f21e26e146b90db958c)), closes [#159](https://github.com/CHIMEFRB/datatrail-cli/issues/159)
+
 ## [0.10.3](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.10.2...v0.10.3) (2026-05-19)
 
 

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatrail-cli
-Version: 0.10.3
+Version: 0.11.1
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>
@@ -13,7 +13,7 @@ Requires-Dist: cadcutils>=1.5.1.1
 Requires-Dist: click-aliasing>=1.0.0
 Requires-Dist: click>=8.1.3
 Requires-Dist: dill>=0.3.6
-Requires-Dist: lxml>=4.9.2
+Requires-Dist: lxml>=6.1.0
 Requires-Dist: mergedeep>=1.3.4
 Requires-Dist: pyaml>=21.10.1
 Requires-Dist: requests>=2.29.0

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/docs/list.md

@@ -11,6 +11,7 @@ Options:
   -v, --verbose  Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet    Only errors shown in logs.
   --write        Write the events to file.
+  --json         Output as JSON.
   --help         Show this message and exit.
 
 ```
@@ -100,3 +101,62 @@ Within Datatrail, there are two types of datasets:
 
     Please see the CLI reference page for more information on the `list` command:
     [datatrail list](../cli/#datatrail-list)
+
+## 🤖 Machine-readable JSON output
+
+The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines:
+
+```bash
+# Get all scopes as JSON
+$ datatrail ls --json
+{
+  "scopes": [
+    "chime.event.baseband.raw",
+    "chime.event.intensity.raw",
+    "kko.acquisition.processed",
+    ...
+  ]
+}
+
+# Get larger datasets as JSON
+$ datatrail ls kko.scheduled.baseband.raw --json
+{
+  "scope": "kko.scheduled.baseband.raw",
+  "larger_datasets": [
+    "20230804095251",
+    "scheduled.commissioning.steady"
+  ]
+}
+
+# Get child datasets as JSON
+$ datatrail ls kko.scheduled.baseband.raw scheduled.commissioning.steady --json
+{
+  "datasets": [
+    "20230616150511",
+    "20230604135840",
+    "20230604134842",
+    ...
+  ]
+}
+```
+
+### Usage in scripts
+
+```python
+import json
+import subprocess
+
+# Get all scopes
+result = subprocess.run(["datatrail", "ls", "--json"], capture_output=True, text=True)
+data = json.loads(result.stdout)
+scopes = data["scopes"]
+
+# Get datasets for a scope
+result = subprocess.run(
+    ["datatrail", "ls", "chime.event.baseband.raw", "--json"],
+    capture_output=True,
+    text=True,
+)
+data = json.loads(result.stdout)
+larger_datasets = data["larger_datasets"]
+```

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/docs/ps.md

@@ -11,6 +11,7 @@ Options:
   -s, --show-files  Show file names.
   -v, --verbose     Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet       Set log level to ERROR.
+  --json            Output as JSON.
   --help            Show this message and exit.
 ```
 
@@ -80,3 +81,72 @@ if the `--show-files` or `-s` flag is passed.
     │ - baseband_308892599_111.h5                                     │
     :
     ```
+
+## 🤖 Machine-readable JSON output
+
+The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse dataset information in scripts and pipelines:
+
+```bash
+$ datatrail ps kko.event.baseband.raw 308892599 --json
+{
+  "dataset": "308892599",
+  "scope": "kko.event.baseband.raw",
+  "files": {
+    "contains_datasets": 0,
+    "datasets_contained": [],
+    "file_replica_locations": {
+      "minoc": [
+        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_129.h5",
+        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_1013.h5",
+        ...
+      ]
+    }
+  },
+  "policies": {
+    "replication_policy": {
+      "preferred_storage_elements": ["chime"],
+      "priority": "low",
+      "default": true
+    },
+    "deletion_policy": [
+      {
+        "storage_element": "minoc",
+        "priority": "low",
+        "default": true,
+        "delete_after_days": 36500
+      },
+      ...
+    ],
+    "belongs_to": [
+      {
+        "scope": "kko.event.baseband.raw",
+        "name": "B0531+21.commissioning.pulsar.temp"
+      }
+    ]
+  }
+}
+```
+
+### Usage in scripts
+
+```python
+import json
+import subprocess
+
+# Get dataset information
+result = subprocess.run(
+    ["datatrail", "ps", "kko.event.baseband.raw", "308892599", "--json"],
+    capture_output=True,
+    text=True,
+)
+data = json.loads(result.stdout)
+
+# Access file locations
+file_locations = data["files"]["file_replica_locations"]
+minoc_files = file_locations.get("minoc", [])
+
+# Access policies
+replication_policy = data["policies"]["replication_policy"]
+deletion_policy = data["policies"]["deletion_policy"]
+belongs_to = data["policies"]["belongs_to"]
+```

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/dtcli/ls.py

@@ -34,14 +34,16 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
 @click.option("--write", is_flag=True, help="Write the events to file.")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def list(
+def list(  # noqa: C901
     ctx: click.Context,
     scope: Optional[str] = None,
     datasets: Optional[str] = None,
     verbose: int = 0,
     quiet: bool = False,
     write: bool = False,
+    output_json: bool = False,
 ):
     """List Datatrail Scopes & Datasets.
 
@@ -52,6 +54,7 @@ def list(
         verbose (int): Verbosity: v=INFO, vv=DEBUG.
         quiet (bool): Only errors shown in logs.
         write (bool): Write the events to file.
+        output_json (bool): Output as JSON.
     """
     # Set logging level.
     set_log_level(logger, verbose, quiet)
@@ -73,6 +76,13 @@ def list(
             return None
     results = functions.list(scope, datasets, verbose, quiet)
 
+    # Output JSON if requested.
+    if output_json:
+        print(json.dumps(results, indent=2))
+        if "error" in results:
+            ctx.exit(1)
+        return
+
     # Display scopes.
     if "scopes" in results.keys():
         table = Table(

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/dtcli/ps.py

@@ -26,14 +26,16 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-s", "--show-files", is_flag=True, help="Show file names.")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def ps(
+def ps(  # noqa: C901
     ctx: click.Context,
     scope: str,
     dataset: str,
     show_files: bool,
     verbose: int,
     quiet: bool,
+    output_json: bool,
 ):
     """Detailed status of a dataset.
 
@@ -44,6 +46,7 @@ def ps(
         show_files (bool): Show list of files.
         verbose (int): Verbosity: v=INFO, vv=DUBUG.
         quiet (bool): Set log level to ERROR.
+        output_json (bool): Output as JSON.
 
     Returns:
         None
@@ -73,13 +76,41 @@ def ps(
     try:
         files, policies = functions.ps(scope, dataset, verbose, quiet)
         if isinstance(files, str) or isinstance(policies, str):
+            if output_json:
+                import json
+
+                print(
+                    json.dumps(
+                        {"error": {"files": str(files), "policies": str(policies)}},
+                        indent=2,
+                    )
+                )
+                ctx.exit(1)
             error_console.print("Error: files = ", files)
             error_console.print("Error: policies = ", policies)
             return None
     except Exception as e:
+        if output_json:
+            import json
+
+            print(json.dumps({"error": str(e)}, indent=2))
+            ctx.exit(1)
         error_console.print(e)
         return None
 
+    # Handle JSON output
+    if output_json:
+        import json
+
+        result = {
+            "dataset": dataset,
+            "scope": scope,
+            "files": files,
+            "policies": policies,
+        }
+        print(json.dumps(result, indent=2))
+        return None
+
     if show_files and files:
         # Files table
         file_table = create_files_table(dataset, scope, files)
@@ -234,9 +265,7 @@ def create_files_table(dataset: str, scope: str, files: dict):
                 file_table.add_row(f"Storage Element: [magenta]{se}")
                 file_table.add_row(f"Common Path: {common_path}/", style="bold green")
                 file_table.add_row(f"[green]- {fn}")
-                # file_table.add_row(se, common_path, fn)
             else:
                 file_table.add_row(f"- {fn}", style="green")
-                # file_table.add_row("", "", fn)
         file_table.add_section()
     return file_table

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/dtcli/src/functions.py

@@ -4,8 +4,9 @@ import logging
 import os
 import re
 import shutil
+import subprocess
 import time
-from collections import defaultdict
+from collections import Counter
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -297,8 +298,8 @@ def get_files(
         if site == "canfar":
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)
-                os.system(f"chgrp -R chime-frb-rw {folder}")  # nosec
-                os.system(f"chmod -R g+w {folder}")  # nosec
+                subprocess.run(["chgrp", "-R", "chime-frb-rw", folder])
+                subprocess.run(["chmod", "-R", "g+w", folder])
         else:
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)
@@ -485,6 +486,15 @@ def get_unregistered_dataset(dataset: str, scope: str) -> Optional[Dict[str, Any
         return response[0]
 
 
+ATTACH_RE = re.compile(
+    r"Could not attach datasets: .+? ERROR: \"?dataset (.+?), (.+?) not found"  # noqa: E501
+)
+
+CREATE_RE = re.compile(
+    r"Could not create dataset: (.+?), scope: (.+?)\. .*UniqueViolation"  # noqa: E501
+)
+
+
 def signature(msg: str) -> str:
     """Create a signature for a reason unregistered message.
 
@@ -494,16 +504,6 @@ def signature(msg: str) -> str:
     Returns:
         str: Signature for error message.
     """
-    ATTACH_RE = re.compile(
-        r"Could not attach datasets: .+ ERROR: dataset (.+), (.+) not found"  # noqa: E501
-    )
-
-    CREATE_RE = re.compile(
-        r"Could not create dataset: (.+), scope: (.+)\. .*UniqueViolation"  # noqa: E501
-    )
-
-    POSTGRES_RE = re.compile(r".*psycopg.*")
-
     msg = msg.strip()
 
     # Attach-dataset errors
@@ -516,11 +516,12 @@ def signature(msg: str) -> str:
     m = CREATE_RE.search(msg)
     if m:
         dataset, scope = m.groups()
+        dataset = re.sub(r"\d+", "<ID>", dataset)
         return f"CREATE_DUPLICATE:{dataset}:{scope}"
 
     # PostgreSQL violation
-    m = POSTGRES_RE.search(msg)
-    if m:
+    if "psycopg" in msg:
+        msg = re.sub(r"\s+", " ", msg)
         return f"POSTGRES:{msg[:120]}"
 
     # Short status / token messages
@@ -551,11 +552,4 @@ def summarise_unregistered_datasets() -> Dict[str, int]:
         Dict[str, int]: Dictionary of error message signatures and their counts.
     """
     response = get_all_unregistered_datasets()
-    reason_groups: Dict[str, int] = defaultdict(int)
-    messages = [str(r["results"]["reason"]) for r in response]
-
-    for msg in messages:
-        sig = signature(msg)
-        reason_groups[sig] += 1
-
-    return reason_groups
+    return Counter(signature(str(r["results"]["reason"])) for r in response)

datatrail_cli-0.11.1/dtcli/unregistered.py

@@ -0,0 +1,102 @@
+"""Datatrail Unregistered datasets commands."""
+
+import logging
+from collections import defaultdict
+from typing import DefaultDict, List, Tuple
+
+import click
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from dtcli.src import functions
+from dtcli.utilities.utilities import set_log_level
+
+logger = logging.getLogger(__name__)
+
+console = Console()
+error_console = Console(stderr=True, style="bold red")
+
+CATEGORY_STYLES = {
+    "ATTACH_MISSING": "yellow",
+    "CREATE_DUPLICATE": "magenta",
+    "POSTGRES": "red",
+    "OTHER": "red",
+    "STATUS": "cyan",
+}
+
+
+@click.group(help="Commands related to unregistered datasets.")
+def unregistered():
+    """Group of commands related to unregistered datasets."""
+    pass
+
+
+@unregistered.command(help="Summarise the reasons for unregistered datasets.")
+@click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
+@click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
+@click.pass_context
+def summary(
+    ctx: click.Context,
+    verbose: int = 0,
+    quiet: bool = False,
+):
+    """Show a summary of the unregistered datasets.
+
+    Args:
+        ctx (click.Context): Click context.
+        verbose (int): Verbosity: v=INFO, vv=DEBUG.
+        quiet (bool): Only errors shown in logs.
+    """
+    # Set logging level.
+    set_log_level(logger, verbose, quiet)
+    logger.debug("`summary` called with:")
+    logger.debug(f"verbose: {verbose} [{type(verbose)}]")
+    logger.debug(f"quiet: {quiet} [{type(quiet)}]")
+
+    results = functions.summarise_unregistered_datasets()
+
+    if not results:
+        console.print("No unregistered datasets found.")
+        return
+
+    total = sum(results.values())
+
+    table = Table(
+        title=f"Summary of reasons — {total:,} unregistered datasets",
+        header_style="magenta",
+        title_style="bold magenta",
+        row_styles=["none", "dim"],
+    )
+    table.add_column("Category")
+    table.add_column("Detail")
+    table.add_column("Count", justify="right")
+    table.add_column("%", justify="right")
+
+    # Group signatures by their category prefix, e.g. "ATTACH_MISSING:...".
+    groups: DefaultDict[str, List[Tuple[str, int]]] = defaultdict(list)
+    for sig, count in results.items():
+        category, _, detail = sig.partition(":")
+        groups[category].append((detail, count))
+
+    ordered = sorted(
+        groups.items(),
+        key=lambda group: sum(count for _, count in group[1]),
+        reverse=True,
+    )
+    for index, (category, reasons) in enumerate(ordered):
+        if index:
+            table.add_section()
+        style = CATEGORY_STYLES.get(category, "white")
+        reasons.sort(key=lambda reason: reason[1], reverse=True)
+        for row, (detail, count) in enumerate(reasons):
+            if category in ("ATTACH_MISSING", "CREATE_DUPLICATE"):
+                detail = detail.replace(":", " → ", 1)
+            table.add_row(
+                Text(category, style=style) if row == 0 else "",
+                Text(detail) if detail else "(no reason recorded)",
+                f"{count:,}",
+                f"{count / total:.1%}",
+            )
+
+    console.print(table)

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/dtcli/utilities/cadcclient.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 from multiprocessing import Process  # Use the standard library only
 from typing import Any, Dict, List, Optional, Tuple
@@ -69,10 +70,13 @@ def _connect(
         storage = StorageInventoryClient(cert, resource_id=storage_resource_id)
         query = CadcTapClient(cert, resource_id=query_resource_id)
         return cert, storage, query
-    # TODO: Handle invalid cert
     except ValueError as error:
-        logger.error(error)
-        raise error
+        logger.error(
+            "Authorization failed: The provided CANFAR certificate is "
+            "invalid or expired. Please ensure you have a valid "
+            "certificate and try again."
+        )
+        raise ValueError("Invalid or expired CANFAR certificate.") from error
 
 
 def get(
@@ -410,23 +414,25 @@ def status(
     ]
     if not certfile:
         certfile = procure(key="vospace_certfile")
-    minoc_status = False
-    luskan_status = False
-    for index, url in enumerate(urls):
+
+    def check_url(url: str) -> bool:
         response = requests.get(url, cert=certfile, allow_redirects=True)
         try:
             response.raise_for_status()
             authorised = response.headers.get("x-vo-authenticated")
             if isinstance(authorised, str):
-                if index == 0:
-                    minoc_status = True
-                else:
-                    luskan_status = True
+                return True
             else:
                 raise TypeError
         except HTTPError as error:
             logger.warning(error)
             logger.warning(f"{url.split('/')[3]} is down.")
+            return False
         except TypeError:
             logger.error("Canfar certificate is not valid.")
-    return minoc_status, luskan_status
+            return False
+
+    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
+        results = list(executor.map(check_url, urls))
+
+    return results[0], results[1]

{datatrail_cli-0.10.3 → datatrail_cli-0.11.1}/dtcli/utilities/utilities.py

@@ -70,6 +70,9 @@ def split(data: List[Any], count: int) -> List[List[Any]]:
     Returns:
         List[List[Any]]: List of batches.
     """
+    if count <= 0:
+        raise ValueError("count must be greater than 0")
+
     batch_size = len(data) // count
     remainder = len(data) % count
     batches: List[Any] = []