datatrail-cli 1.0.2__tar.gz → 1.1.0__tar.gz

datatrail_cli-1.1.0/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: monday
+    open-pull-requests-limit: 5
+    groups:
+      actions-minor-and-patch:
+        patterns: ["*"]
+        update-types: [minor, patch]

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/.github/workflows/continuous-deployment.yml

@@ -9,6 +9,7 @@ permissions:
   pull-requests: write
 jobs:
   release-please:
+    if: github.repository == 'CHIMEFRB/datatrail-cli'
     runs-on: ubuntu-latest
     steps:
       - name: Create release
@@ -23,7 +24,7 @@ jobs:
 
       - name: Install uv
         if: ${{ steps.release-please.outputs.release_created }}
-        uses: astral-sh/setup-uv@v10.0.1
+        uses: astral-sh/setup-uv@v10.2.0
         with:
           enable-cache: false
           python-version: "3.13"

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/.github/workflows/continuous-integration.yml

@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v10.0.1
+        uses: astral-sh/setup-uv@v10.2.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -100,7 +100,7 @@ jobs:
           fail-on-error: false
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v7.1.1
         with:
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}

datatrail_cli-1.1.0/.github/workflows/docs.yml

@@ -0,0 +1,46 @@
+name: Documentation
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "mkdocs*.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/docs.yml"
+  pull_request:
+    paths:
+      - "docs/**"
+      - "mkdocs*.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v10.2.0
+        with:
+          python-version: "3.13"
+          enable-cache: true
+      - run: uv sync --locked --group docs
+      - run: uv run --no-sync mkdocs build --strict --config-file mkdocs-site.yml
+
+  handbook:
+    if: github.repository == 'CHIMEFRB/datatrail-cli' && github.event_name == 'push'
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build the upstream handbook
+        uses: peter-evans/repository-dispatch@v4.0.1
+        with:
+          token: ${{ secrets.BUILD_HANDBOOK_TOKEN }}
+          repository: chimefrb/handbook
+          event-type: build-handbook
+          client-payload: '{"source": "chimefrb/datatrail-cli"}'

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/.github/workflows/stale.yml

@@ -11,10 +11,11 @@ permissions:
 
 jobs:
   close_stale_prs:
+    if: github.repository == 'CHIMEFRB/datatrail-cli'
     runs-on: ubuntu-latest
     steps:
       - name: Close stale pull requests
-        uses: actions/stale@v9
+        uses: actions/stale@v11.0.0
         with:
           days-before-stale: 90
           days-before-close: 7

datatrail_cli-1.1.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "1.1.0"
+}

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [1.1.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.0.2...datatrail-cli-v1.1.0) (2026-11-01)
+
+
+### Features
+
+* **workflow:** pull and clear as functions for Workflow pipelines ([de80e6d](https://github.com/CHIMEFRB/datatrail-cli/commit/de80e6dfd6f6092457af038002b0dec784a74cf5))
+
 ## [1.0.2](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.0.1...datatrail-cli-v1.0.2) (2026-09-29)
 
 

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datatrail-cli
-Version: 1.0.2
+Version: 1.1.0
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>

datatrail_cli-1.1.0/docs/workflow.md

@@ -0,0 +1,52 @@
+# 🔁 In Workflow pipelines
+
+`dtcli.workflow` has `pull` and `clear` as Python functions for Workflow pipeline
+steps. A function runs in the step's own session, so what the command prints is in
+the step's logs. Each one checks what the command did and raises if it failed, so
+the step fails too: `datatrail pull` reports some failures (no CADC certificate,
+minoc down, no files found) only on the console.
+
+They return `(results, products, plots)`, as Workflow expects of a function.
+
+## `pull_dataset`
+
+Downloads a dataset, then checks that all its files are here.
+
+```yaml
+work:
+  function: dtcli.workflow.pull_dataset
+  parameters:
+    scope: chime.event.baseband.raw
+    dataset: "123456789"
+    cores: 4
+    site: canfar
+```
+
+- `cores`: parallel fetch processes (default 1).
+- `site`: where it runs. If there's no Datatrail configuration, one is created
+  for this site (as `datatrail config init --site <site>`); if there is one for
+  another site, the step fails.
+- `verbose`: 1 for INFO (default), 2 for DEBUG.
+
+Results: `scope`, `dataset`, `files` (how many are here) and `directory`.
+
+## `clear_dataset`
+
+Deletes a dataset's local copy, then checks that it's gone. A dataset with no local
+copy is left as it is, and the step succeeds, so it can clean up after a pull that
+failed.
+
+```yaml
+work:
+  function: dtcli.workflow.clear_dataset
+  parameters:
+    scope: chime.event.baseband.raw
+    dataset: "123456789"
+    site: canfar
+```
+
+- `clear_parents`: also delete empty parent directories (default false).
+- `site` and `verbose` as for `pull_dataset`.
+
+Results: `scope`, `dataset`, `path` and `cleared` (false if there was nothing to
+clear).

datatrail_cli-1.1.0/dtcli/workflow.py

@@ -0,0 +1,170 @@
+"""Datatrail commands as functions for Workflow pipelines.
+
+Workflow runs a pipeline step's function in the session's own process, so what the
+command prints is the step's logs. Each function runs the CLI command, then checks
+what it did and raises if it failed: the commands report some failures only on the
+console, which a Workflow step can't see. They return (results, products, plots),
+the shape Workflow expects of a function.
+
+Example step:
+
+    function: dtcli.workflow.pull_dataset
+    parameters:
+      scope: chime.event.baseband.raw
+      dataset: "123456789"
+      cores: 4
+      site: canfar
+"""
+
+import logging
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import click
+
+from dtcli.clear import clear
+from dtcli.config import CONFIG, init, procure
+from dtcli.pull import pull
+from dtcli.src.functions import find_dataset_common_path, find_missing_dataset_files
+
+logger = logging.getLogger("workflow")
+
+Outcome = Tuple[Dict[str, Any], List[str], List[str]]
+
+
+def _configure(site: Optional[str]) -> Dict[str, Any]:
+    """The Datatrail configuration, created for `site` if there's none.
+
+    Raises:
+        RuntimeError: If there's no configuration and no site to create one for,
+            or it's set up for another site.
+    """
+    if not CONFIG.exists():
+        if not site:
+            raise RuntimeError(
+                f"There's no Datatrail configuration at {CONFIG}; "
+                "give a site to create one."
+            )
+        # init only asks questions when a configuration exists
+        init.callback(site=site)
+    config = procure()
+    if not config:
+        raise RuntimeError(f"The Datatrail configuration at {CONFIG} can't be read.")
+    if site and config.get("site") != site:
+        raise RuntimeError(
+            f"The Datatrail configuration at {CONFIG} is for {config.get('site')}, "
+            f"not {site}."
+        )
+    return config
+
+
+def _run(command: click.Command, args: List[str]) -> None:
+    """Run a CLI command in this process.
+
+    Raises:
+        RuntimeError: If it exits with an error code.
+    """
+    code = command.main(args=args, standalone_mode=False)
+    # with standalone_mode=False, an exit code is returned rather than raised
+    if code:
+        raise RuntimeError(f"datatrail {command.name} exited with {code}.")
+
+
+def pull_dataset(
+    scope: str,
+    dataset: str,
+    cores: int = 1,
+    site: Optional[str] = None,
+    verbose: int = 1,
+) -> Outcome:
+    """Download a dataset, then check that all its files are here.
+
+    Args:
+        scope (str): Scope of the dataset.
+        dataset (str): Name of the dataset.
+        cores (int): Number of parallel fetch processes. Defaults to 1.
+        site (Optional[str]): Site this runs at; the configuration is created for it
+            if there's none. Defaults to the configured site.
+        verbose (int): Verbosity: 1=INFO, 2=DEBUG. Defaults to 1.
+
+    Returns:
+        Outcome: Results (scope, dataset, number of files and where they are), no
+            products and no plots.
+
+    Raises:
+        RuntimeError: If the dataset can't be found, or any of its files is missing
+            after the pull.
+    """
+    config = _configure(site)
+    directory = config["root_mounts"][config["site"]]
+    dataset = str(dataset)
+    args = [scope, dataset, "--force", "--cores", str(cores)] + ["-v"] * verbose
+    _run(pull, args)
+
+    files = find_missing_dataset_files(scope, dataset, directory, verbose)
+    if files.get("error"):
+        raise RuntimeError(f"Could not check {scope} {dataset}: {files['error']}")
+    if files["missing"]:
+        raise RuntimeError(
+            f"{len(files['missing'])} of {scope} {dataset}'s files weren't pulled, "
+            f"e.g. {files['missing'][0]}."
+        )
+    if not files["existing"]:
+        raise RuntimeError(f"Datatrail has no files for {scope} {dataset}.")
+    results = {
+        "scope": scope,
+        "dataset": dataset,
+        "files": len(files["existing"]),
+        "directory": directory,
+    }
+    return results, [], []
+
+
+def clear_dataset(
+    scope: str,
+    dataset: str,
+    site: Optional[str] = None,
+    clear_parents: bool = False,
+    verbose: int = 1,
+) -> Outcome:
+    """Delete a dataset's local copy, then check that it's gone.
+
+    A dataset with no local copy is left as it is, and isn't a failure.
+
+    Args:
+        scope (str): Scope of the dataset.
+        dataset (str): Name of the dataset.
+        site (Optional[str]): Site this runs at; the configuration is created for it
+            if there's none. Defaults to the configured site.
+        clear_parents (bool): Also delete empty parent directories. Defaults to False.
+        verbose (int): Verbosity: 1=INFO, 2=DEBUG. Defaults to 1.
+
+    Returns:
+        Outcome: Results (scope, dataset, the path and whether it was cleared), no
+            products and no plots.
+
+    Raises:
+        RuntimeError: If the dataset can't be found, or its files are still there.
+    """
+    config = _configure(site)
+    directory = config["root_mounts"][config["site"]]
+    dataset = str(dataset)
+    common = find_dataset_common_path(scope, dataset, config["site"], verbose, False)
+    # it returns a message rather than a path when the server isn't reachable
+    if not common or not common.startswith(("data/", "/")):
+        raise RuntimeError(f"Could not find where {scope} {dataset} is: {common}")
+    # as `datatrail clear` builds it
+    path = (directory + common).replace("//", "/")
+    results: Dict[str, Any] = {"scope": scope, "dataset": dataset, "path": path}
+    if not os.path.exists(path):
+        logger.info(f"{path} doesn't exist; nothing to clear.")
+        return {**results, "cleared": False}, [], []
+
+    args = [scope, dataset, "--force"] + ["-v"] * verbose
+    if clear_parents:
+        args.append("--clear-parents")
+    _run(clear, args)
+
+    if os.path.exists(path):
+        raise RuntimeError(f"{path} is still there after clearing it.")
+    return {**results, "cleared": True}, [], []

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/mkdocs.yml

@@ -20,6 +20,7 @@ nav:
           - scout: scout.md
           - unregistered: unregistered.md
           - verify: verify.md
+      - In Workflow pipelines: workflow.md
   - Command Line Interface:
       - Commands: commands.md
       - Reference: cli.md

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datatrail-cli"
-version = "1.0.2"
+version = "1.1.0"
 description = "CHIME/FRB Datatrail CLI"
 authors = [
   { name = "CHIME FRB Project Office" },

datatrail_cli-1.1.0/tests/test_workflow.py

@@ -0,0 +1,189 @@
+"""Tests for the Datatrail functions for Workflow pipelines."""
+
+from types import SimpleNamespace
+
+import click
+import pytest
+import yaml
+
+from dtcli import workflow
+
+
+class FakeCommand:
+    """A CLI command that records its arguments and returns an exit code."""
+
+    def __init__(self, name, code=None, effect=None):
+        self.name = name
+        self.code = code
+        self.effect = effect
+        self.calls = []
+
+    def main(self, args, standalone_mode):
+        assert standalone_mode is False
+        self.calls.append(args)
+        if self.effect:
+            self.effect()
+        return self.code
+
+
+@pytest.fixture
+def config(tmp_path, monkeypatch):
+    """A Datatrail configuration for canfar, with its root mount in tmp_path."""
+    path = tmp_path / ".datatrail" / "config.yaml"
+    root = str(tmp_path / "arc") + "/"
+    monkeypatch.setattr(workflow, "CONFIG", path)
+
+    def write(site="canfar"):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(yaml.safe_dump({"site": site, "root_mounts": {site: root}}))
+
+    monkeypatch.setattr(
+        workflow, "init", SimpleNamespace(callback=lambda site: write(site))
+    )
+    monkeypatch.setattr(workflow, "procure", lambda: yaml.safe_load(path.read_text()))
+    return SimpleNamespace(path=path, root=root, write=write)
+
+
+def test_pull_dataset_returns_workflow_outcome(config, monkeypatch):
+    config.write()
+    pull = FakeCommand("pull")
+    monkeypatch.setattr(workflow, "pull", pull)
+    monkeypatch.setattr(
+        workflow,
+        "find_missing_dataset_files",
+        lambda scope, dataset, root, verbose: {"missing": [], "existing": ["a", "b"]},
+    )
+
+    results, products, plots = workflow.pull_dataset(
+        "chime.event.baseband.raw", 123, cores=4, site="canfar"
+    )
+
+    assert pull.calls == [
+        ["chime.event.baseband.raw", "123", "--force", "--cores", "4", "-v"]
+    ]
+    assert results == {
+        "scope": "chime.event.baseband.raw",
+        "dataset": "123",
+        "files": 2,
+        "directory": config.root,
+    }
+    assert products == [] and plots == []
+
+
+def test_pull_dataset_creates_the_configuration(config, monkeypatch):
+    monkeypatch.setattr(workflow, "pull", FakeCommand("pull"))
+    monkeypatch.setattr(
+        workflow,
+        "find_missing_dataset_files",
+        lambda *args: {"missing": [], "existing": ["a"]},
+    )
+    workflow.pull_dataset("scope", "1", site="canfar")
+    assert yaml.safe_load(config.path.read_text())["site"] == "canfar"
+
+
+def test_pull_dataset_needs_a_site_without_configuration(config):
+    with pytest.raises(RuntimeError, match="give a site"):
+        workflow.pull_dataset("scope", "1")
+
+
+def test_pull_dataset_refuses_another_sites_configuration(config):
+    config.write("chime")
+    with pytest.raises(RuntimeError, match="is for chime, not canfar"):
+        workflow.pull_dataset("scope", "1", site="canfar")
+
+
+@pytest.mark.parametrize(
+    "code, files, message",
+    [
+        # ctx.exit(1) comes back as a return value
+        (1, {"missing": [], "existing": ["a"]}, "exited with 1"),
+        # pull reports these only on the console and exits 0
+        (None, {"error": "Scope does not exist"}, "Could not check"),
+        (None, {"missing": ["a"], "existing": ["b"]}, "1 of scope 1's files"),
+        (None, {"missing": [], "existing": []}, "has no files"),
+    ],
+)
+def test_pull_dataset_raises_when_it_failed(config, monkeypatch, code, files, message):
+    config.write()
+    monkeypatch.setattr(workflow, "pull", FakeCommand("pull", code=code))
+    monkeypatch.setattr(workflow, "find_missing_dataset_files", lambda *args: files)
+    with pytest.raises(RuntimeError, match=message):
+        workflow.pull_dataset("scope", "1", site="canfar")
+
+
+def test_pull_dataset_passes_on_click_errors(config, monkeypatch):
+    config.write()
+
+    def abort():
+        raise click.Abort()
+
+    monkeypatch.setattr(workflow, "pull", FakeCommand("pull", effect=abort))
+    with pytest.raises(click.Abort):
+        workflow.pull_dataset("scope", "1", site="canfar")
+
+
+def _dataset_dir(config):
+    path = config.path.parent.parent / "arc" / "data" / "chime" / "astro_1"
+    path.mkdir(parents=True)
+    (path / "baseband_1_0.h5").write_text("")
+    return path
+
+
+def test_clear_dataset_clears_and_checks(config, monkeypatch):
+    config.write()
+    path = _dataset_dir(config)
+
+    def delete():
+        for file in path.iterdir():
+            file.unlink()
+        path.rmdir()
+
+    clear = FakeCommand("clear", effect=delete)
+    monkeypatch.setattr(workflow, "clear", clear)
+    monkeypatch.setattr(
+        workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_1/"
+    )
+
+    results, products, plots = workflow.clear_dataset("scope", 1, site="canfar")
+
+    assert clear.calls == [["scope", "1", "--force", "-v"]]
+    assert results == {
+        "scope": "scope",
+        "dataset": "1",
+        "path": config.root + "data/chime/astro_1/",
+        "cleared": True,
+    }
+    assert products == [] and plots == []
+
+
+def test_clear_dataset_without_a_local_copy(config, monkeypatch):
+    config.write()
+    clear = FakeCommand("clear")
+    monkeypatch.setattr(workflow, "clear", clear)
+    monkeypatch.setattr(
+        workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_2/"
+    )
+    results, _, _ = workflow.clear_dataset("scope", "2", site="canfar")
+    assert results["cleared"] is False
+    assert clear.calls == []
+
+
+def test_clear_dataset_raises_when_files_remain(config, monkeypatch):
+    config.write()
+    _dataset_dir(config)
+    monkeypatch.setattr(workflow, "clear", FakeCommand("clear"))
+    monkeypatch.setattr(
+        workflow, "find_dataset_common_path", lambda *args: "data/chime/astro_1/"
+    )
+    with pytest.raises(RuntimeError, match="still there"):
+        workflow.clear_dataset("scope", "1", site="canfar")
+
+
+@pytest.mark.parametrize(
+    "common", [None, "The Datatrail Central Server at CHIME at is not reachable!!!"]
+)
+def test_clear_dataset_raises_when_it_cant_find_the_dataset(config, monkeypatch, common):
+    config.write()
+    monkeypatch.setattr(workflow, "find_dataset_common_path", lambda *args: common)
+    with pytest.raises(RuntimeError, match="Could not find where"):
+        workflow.clear_dataset("scope", "1", site="canfar")

{datatrail_cli-1.0.2 → datatrail_cli-1.1.0}/uv.lock

@@ -508,7 +508,7 @@ wheels = [
 
 [[package]]
 name = "datatrail-cli"
-version = "1.0.2"
+version = "1.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "cadcdata" },

datatrail_cli-1.0.2/.github/workflows/docs.yml

@@ -1,23 +0,0 @@
-name: Documentation
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - "docs/**"
-      - ".github/workflows/docs.yml"
-
-permissions:
-  contents: read
-
-jobs:
-  mkdocs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Build chimefrb/handbook
-        uses: peter-evans/repository-dispatch@v3
-        with:
-          token: ${{ secrets.BUILD_HANDBOOK_TOKEN }}
-          repository: chimefrb/handbook
-          event-type: build-handbook
-          client-payload: '{"source": "chimefrb/datatrail-cli"}'

datatrail_cli-1.0.2/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-  ".": "1.0.2"
-}