datatrail-cli 0.10.3__tar.gz → 0.11.0__tar.gz

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.11.0](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.10.3...v0.11.0) (2026-07-07)
+
+
+### Features
+
+* **ls & ps:** add --json flag for machine readable output ([c4742a0](https://github.com/CHIMEFRB/datatrail-cli/commit/c4742a0dfe43823ce8e68f21e26e146b90db958c)), closes [#159](https://github.com/CHIMEFRB/datatrail-cli/issues/159)
+
 ## [0.10.3](https://github.com/CHIMEFRB/datatrail-cli/compare/v0.10.2...v0.10.3) (2026-05-19)
 
 

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatrail-cli
-Version: 0.10.3
+Version: 0.11.0
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/docs/list.md

@@ -11,6 +11,7 @@ Options:
   -v, --verbose  Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet    Only errors shown in logs.
   --write        Write the events to file.
+  --json         Output as JSON.
   --help         Show this message and exit.
 
 ```
@@ -100,3 +101,62 @@ Within Datatrail, there are two types of datasets:
 
     Please see the CLI reference page for more information on the `list` command:
     [datatrail list](../cli/#datatrail-list)
+
+## 🤖 Machine-readable JSON output
+
+The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines:
+
+```bash
+# Get all scopes as JSON
+$ datatrail ls --json
+{
+  "scopes": [
+    "chime.event.baseband.raw",
+    "chime.event.intensity.raw",
+    "kko.acquisition.processed",
+    ...
+  ]
+}
+
+# Get larger datasets as JSON
+$ datatrail ls kko.scheduled.baseband.raw --json
+{
+  "scope": "kko.scheduled.baseband.raw",
+  "larger_datasets": [
+    "20230804095251",
+    "scheduled.commissioning.steady"
+  ]
+}
+
+# Get child datasets as JSON
+$ datatrail ls kko.scheduled.baseband.raw scheduled.commissioning.steady --json
+{
+  "datasets": [
+    "20230616150511",
+    "20230604135840",
+    "20230604134842",
+    ...
+  ]
+}
+```
+
+### Usage in scripts
+
+```python
+import json
+import subprocess
+
+# Get all scopes
+result = subprocess.run(["datatrail", "ls", "--json"], capture_output=True, text=True)
+data = json.loads(result.stdout)
+scopes = data["scopes"]
+
+# Get datasets for a scope
+result = subprocess.run(
+    ["datatrail", "ls", "chime.event.baseband.raw", "--json"],
+    capture_output=True,
+    text=True,
+)
+data = json.loads(result.stdout)
+larger_datasets = data["larger_datasets"]
+```

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/docs/ps.md

@@ -11,6 +11,7 @@ Options:
   -s, --show-files  Show file names.
   -v, --verbose     Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet       Set log level to ERROR.
+  --json            Output as JSON.
   --help            Show this message and exit.
 ```
 
@@ -80,3 +81,72 @@ if the `--show-files` or `-s` flag is passed.
     │ - baseband_308892599_111.h5                                     │
     :
     ```
+
+## 🤖 Machine-readable JSON output
+
+The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse dataset information in scripts and pipelines:
+
+```bash
+$ datatrail ps kko.event.baseband.raw 308892599 --json
+{
+  "dataset": "308892599",
+  "scope": "kko.event.baseband.raw",
+  "files": {
+    "contains_datasets": 0,
+    "datasets_contained": [],
+    "file_replica_locations": {
+      "minoc": [
+        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_129.h5",
+        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_1013.h5",
+        ...
+      ]
+    }
+  },
+  "policies": {
+    "replication_policy": {
+      "preferred_storage_elements": ["chime"],
+      "priority": "low",
+      "default": true
+    },
+    "deletion_policy": [
+      {
+        "storage_element": "minoc",
+        "priority": "low",
+        "default": true,
+        "delete_after_days": 36500
+      },
+      ...
+    ],
+    "belongs_to": [
+      {
+        "scope": "kko.event.baseband.raw",
+        "name": "B0531+21.commissioning.pulsar.temp"
+      }
+    ]
+  }
+}
+```
+
+### Usage in scripts
+
+```python
+import json
+import subprocess
+
+# Get dataset information
+result = subprocess.run(
+    ["datatrail", "ps", "kko.event.baseband.raw", "308892599", "--json"],
+    capture_output=True,
+    text=True,
+)
+data = json.loads(result.stdout)
+
+# Access file locations
+file_locations = data["files"]["file_replica_locations"]
+minoc_files = file_locations.get("minoc", [])
+
+# Access policies
+replication_policy = data["policies"]["replication_policy"]
+deletion_policy = data["policies"]["deletion_policy"]
+belongs_to = data["policies"]["belongs_to"]
+```

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/dtcli/ls.py

@@ -34,14 +34,16 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
 @click.option("--write", is_flag=True, help="Write the events to file.")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def list(
+def list(  # noqa: C901
     ctx: click.Context,
     scope: Optional[str] = None,
     datasets: Optional[str] = None,
     verbose: int = 0,
     quiet: bool = False,
     write: bool = False,
+    output_json: bool = False,
 ):
     """List Datatrail Scopes & Datasets.
 
@@ -52,6 +54,7 @@ def list(
         verbose (int): Verbosity: v=INFO, vv=DEBUG.
         quiet (bool): Only errors shown in logs.
         write (bool): Write the events to file.
+        output_json (bool): Output as JSON.
     """
     # Set logging level.
     set_log_level(logger, verbose, quiet)
@@ -73,6 +76,13 @@ def list(
             return None
     results = functions.list(scope, datasets, verbose, quiet)
 
+    # Output JSON if requested.
+    if output_json:
+        print(json.dumps(results, indent=2))
+        if "error" in results:
+            ctx.exit(1)
+        return
+
     # Display scopes.
     if "scopes" in results.keys():
         table = Table(

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/dtcli/ps.py

@@ -26,14 +26,16 @@ error_console = Console(stderr=True, style="bold red")
 @click.option("-s", "--show-files", is_flag=True, help="Show file names.")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def ps(
+def ps(  # noqa: C901
     ctx: click.Context,
     scope: str,
     dataset: str,
     show_files: bool,
     verbose: int,
     quiet: bool,
+    output_json: bool,
 ):
     """Detailed status of a dataset.
 
@@ -44,6 +46,7 @@ def ps(
         show_files (bool): Show list of files.
         verbose (int): Verbosity: v=INFO, vv=DUBUG.
         quiet (bool): Set log level to ERROR.
+        output_json (bool): Output as JSON.
 
     Returns:
         None
@@ -73,13 +76,41 @@ def ps(
     try:
         files, policies = functions.ps(scope, dataset, verbose, quiet)
         if isinstance(files, str) or isinstance(policies, str):
+            if output_json:
+                import json
+
+                print(
+                    json.dumps(
+                        {"error": {"files": str(files), "policies": str(policies)}},
+                        indent=2,
+                    )
+                )
+                ctx.exit(1)
             error_console.print("Error: files = ", files)
             error_console.print("Error: policies = ", policies)
             return None
     except Exception as e:
+        if output_json:
+            import json
+
+            print(json.dumps({"error": str(e)}, indent=2))
+            ctx.exit(1)
         error_console.print(e)
         return None
 
+    # Handle JSON output
+    if output_json:
+        import json
+
+        result = {
+            "dataset": dataset,
+            "scope": scope,
+            "files": files,
+            "policies": policies,
+        }
+        print(json.dumps(result, indent=2))
+        return None
+
     if show_files and files:
         # Files table
         file_table = create_files_table(dataset, scope, files)
@@ -234,9 +265,7 @@ def create_files_table(dataset: str, scope: str, files: dict):
                 file_table.add_row(f"Storage Element: [magenta]{se}")
                 file_table.add_row(f"Common Path: {common_path}/", style="bold green")
                 file_table.add_row(f"[green]- {fn}")
-                # file_table.add_row(se, common_path, fn)
             else:
                 file_table.add_row(f"- {fn}", style="green")
-                # file_table.add_row("", "", fn)
         file_table.add_section()
     return file_table

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/dtcli/src/functions.py

@@ -4,6 +4,7 @@ import logging
 import os
 import re
 import shutil
+import subprocess
 import time
 from collections import defaultdict
 from pathlib import Path
@@ -297,8 +298,8 @@ def get_files(
         if site == "canfar":
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)
-                os.system(f"chgrp -R chime-frb-rw {folder}")  # nosec
-                os.system(f"chmod -R g+w {folder}")  # nosec
+                subprocess.run(["chgrp", "-R", "chime-frb-rw", folder])
+                subprocess.run(["chmod", "-R", "g+w", folder])
         else:
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/dtcli/utilities/cadcclient.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 from multiprocessing import Process  # Use the standard library only
 from typing import Any, Dict, List, Optional, Tuple
@@ -69,10 +70,13 @@ def _connect(
         storage = StorageInventoryClient(cert, resource_id=storage_resource_id)
         query = CadcTapClient(cert, resource_id=query_resource_id)
         return cert, storage, query
-    # TODO: Handle invalid cert
     except ValueError as error:
-        logger.error(error)
-        raise error
+        logger.error(
+            "Authorization failed: The provided CANFAR certificate is "
+            "invalid or expired. Please ensure you have a valid "
+            "certificate and try again."
+        )
+        raise ValueError("Invalid or expired CANFAR certificate.") from error
 
 
 def get(
@@ -410,23 +414,25 @@ def status(
     ]
     if not certfile:
         certfile = procure(key="vospace_certfile")
-    minoc_status = False
-    luskan_status = False
-    for index, url in enumerate(urls):
+
+    def check_url(url: str) -> bool:
         response = requests.get(url, cert=certfile, allow_redirects=True)
         try:
             response.raise_for_status()
             authorised = response.headers.get("x-vo-authenticated")
             if isinstance(authorised, str):
-                if index == 0:
-                    minoc_status = True
-                else:
-                    luskan_status = True
+                return True
             else:
                 raise TypeError
         except HTTPError as error:
             logger.warning(error)
             logger.warning(f"{url.split('/')[3]} is down.")
+            return False
         except TypeError:
             logger.error("Canfar certificate is not valid.")
-    return minoc_status, luskan_status
+            return False
+
+    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
+        results = list(executor.map(check_url, urls))
+
+    return results[0], results[1]

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/dtcli/utilities/utilities.py

@@ -70,6 +70,9 @@ def split(data: List[Any], count: int) -> List[List[Any]]:
     Returns:
         List[List[Any]]: List of batches.
     """
+    if count <= 0:
+        raise ValueError("count must be greater than 0")
+
     batch_size = len(data) // count
     remainder = len(data) % count
     batches: List[Any] = []

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datatrail-cli"
-version = "0.10.3"
+version = "0.11.0"
 description = "CHIME/FRB Datatrail CLI"
 authors = [
   { name = "CHIME FRB Project Office" },

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/tests/test_cli.py

@@ -97,18 +97,15 @@ def test_cli_list_help(runner: CliRunner) -> None:
         runner (CliRunner) -> None: Click runner.
     """
     result = runner.invoke(datatrail, ["ls", "--help"])
-    expected_response = """Usage: cli list [OPTIONS] [SCOPE] [DATASETS]
-
-  List scopes & datasets
-
-Options:
-  -v, --verbose  Verbosity: v=INFO, vv=DEBUG.
-  -q, --quiet    Only errors shown in logs.
-  --write        Write the events to file.
-  --help         Show this message and exit.
-"""
     assert result.exit_code == 0
-    assert result.output == expected_response
+    # Check that all the expected elements are present
+    assert "Usage: cli list [OPTIONS] [SCOPE] [DATASETS]" in result.output
+    assert "List scopes & datasets" in result.output
+    assert "--verbose" in result.output
+    assert "--quiet" in result.output
+    assert "--write" in result.output
+    assert "--json" in result.output
+    assert "Output as JSON" in result.output
 
 
 def test_cli_ps_help(runner: CliRunner) -> None:
@@ -118,18 +115,15 @@ def test_cli_ps_help(runner: CliRunner) -> None:
         runner (CliRunner) -> None: Click runner.
     """
     result = runner.invoke(datatrail, ["ps", "--help"])
-    expected_response = """Usage: cli ps [OPTIONS] SCOPE DATASET
-
-  Details of a dataset.
-
-Options:
-  -s, --show-files  Show file names.
-  -v, --verbose     Verbosity: v=INFO, vv=DEBUG.
-  -q, --quiet       Set log level to ERROR.
-  --help            Show this message and exit.
-"""
     assert result.exit_code == 0
-    assert result.output == expected_response
+    # Check that all the expected elements are present
+    assert "Usage: cli ps [OPTIONS] SCOPE DATASET" in result.output
+    assert "Details of a dataset" in result.output
+    assert "--show-files" in result.output
+    assert "--verbose" in result.output
+    assert "--quiet" in result.output
+    assert "--json" in result.output
+    assert "Output as JSON" in result.output
 
 
 def test_cli_pull_help(runner: CliRunner) -> None:
@@ -580,3 +574,94 @@ def test_cli_version(runner: CliRunner) -> None:
     result = runner.invoke(datatrail, ["version"])
     assert result.exit_code == 0
     assert "Datatrail Versions" in result.output
+
+
+def test_cli_list_scopes_json(runner: CliRunner) -> None:
+    """Test for CLI list to output scopes as JSON.
+
+    Args:
+        runner (CliRunner): Click runner.
+    """
+    import json
+
+    result = runner.invoke(datatrail, ["ls", "--json"])
+    assert result.exit_code == 0
+    # Extract JSON from output (skip version check message if present)
+    json_start = result.output.find("{")
+    json_output = result.output[json_start:]
+    # Parse the output as JSON
+    output_data = json.loads(json_output)
+    # Should have a 'scopes' key with a list of scopes
+    assert "scopes" in output_data
+    assert isinstance(output_data["scopes"], list)
+    assert "chime.event.intensity.raw" in output_data["scopes"]
+
+
+def test_cli_list_larger_json(runner: CliRunner) -> None:
+    """Test for CLI to list larger datasets as JSON.
+
+    Args:
+        runner (CliRunner): Click runner.
+    """
+    import json
+
+    result = runner.invoke(datatrail, ["ls", "chime.event.baseband.raw", "--json"])
+    assert result.exit_code == 0
+    # Extract JSON from output (skip version check message if present)
+    json_start = result.output.find("{")
+    json_output = result.output[json_start:]
+    # Parse the output as JSON
+    output_data = json.loads(json_output)
+    # Should have a 'larger_datasets' key with a list of datasets
+    assert "larger_datasets" in output_data
+    assert isinstance(output_data["larger_datasets"], list)
+    assert "classified.FRB" in output_data["larger_datasets"]
+
+
+def test_cli_list_children_json(runner: CliRunner) -> None:
+    """Test for CLI list to show child datasets as JSON.
+
+    Args:
+        runner (CliRunner): Click runner.
+    """
+    import json
+
+    result = runner.invoke(
+        datatrail, ["ls", "chime.event.baseband.raw", "classified.FRB", "--json"]
+    )
+    assert result.exit_code == 0
+    # Extract JSON from output (skip version check message if present)
+    json_start = result.output.find("{")
+    json_output = result.output[json_start:]
+    # Parse the output as JSON
+    output_data = json.loads(json_output)
+    # Should have a 'datasets' key with a list of datasets
+    assert "datasets" in output_data
+    assert isinstance(output_data["datasets"], list)
+    assert "289007650" in output_data["datasets"]
+
+
+def test_cli_ps_json(runner: CliRunner) -> None:
+    """Test for CLI ps command with JSON output.
+
+    Args:
+        runner (CliRunner): Click runner.
+    """
+    import json
+
+    result = runner.invoke(
+        datatrail, ["ps", "chime.event.baseband.raw", "289007650", "--json"]
+    )
+    assert result.exit_code == 0
+    # Extract JSON from output (skip version check message if present)
+    json_start = result.output.find("{")
+    json_output = result.output[json_start:]
+    # Parse the output as JSON
+    output_data = json.loads(json_output)
+    # Should have dataset, scope, files, and policies keys
+    assert "dataset" in output_data
+    assert "scope" in output_data
+    assert "files" in output_data
+    assert "policies" in output_data
+    assert output_data["dataset"] == "289007650"
+    assert output_data["scope"] == "chime.event.baseband.raw"

{datatrail_cli-0.10.3 → datatrail_cli-0.11.0}/tests/test_utils.py

@@ -1,3 +1,5 @@
+import pytest
+
 from dtcli.utilities import utilities
 
 
@@ -8,6 +10,20 @@ def test_split():
     assert result == [["a"], ["b"]]
 
 
+def test_split_edges():
+    """Test split() with edge cases like empty list and invalid counts."""
+    # Empty data
+    assert utilities.split([], 2) == []
+
+    # Zero count should raise ValueError
+    with pytest.raises(ValueError, match="count must be greater than 0"):
+        utilities.split([1, 2, 3], 0)
+
+    # Negative count should raise ValueError
+    with pytest.raises(ValueError, match="count must be greater than 0"):
+        utilities.split([1, 2, 3], -1)
+
+
 def test_split_more_batches_than_items():
     """split() must not return more batches than there are items.