datatrail-cli 1.0.0__tar.gz → 1.0.1__tar.gz

datatrail_cli-1.0.1/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "1.0.1"
+}

{datatrail_cli-1.0.0 → datatrail_cli-1.0.1}/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [1.0.1](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v1.0.0...datatrail-cli-v1.0.1) (2026-09-29)
+
+
+### Bug Fixes
+
+* **pull:** continue when optional size query is unavailable ([#195](https://github.com/CHIMEFRB/datatrail-cli/issues/195)) ([b36bbbf](https://github.com/CHIMEFRB/datatrail-cli/commit/b36bbbf90d47eb22f20eaa7a12cfea8f4ae703bb))
+* **scout:** restore discrepancy detection and confirmed repairs ([#197](https://github.com/CHIMEFRB/datatrail-cli/issues/197)) ([54dfbf4](https://github.com/CHIMEFRB/datatrail-cli/commit/54dfbf4b3abf8ff0f7f9db13e5eb7ccdb28ccce1))
+
+
+### Performance Improvements
+
+* **pull:** apply CANFAR permissions without subprocesses ([#157](https://github.com/CHIMEFRB/datatrail-cli/issues/157)) ([b8c2230](https://github.com/CHIMEFRB/datatrail-cli/commit/b8c22306432a6b24b435b5ddb421ac5cdcd9e1eb))
+
 ## [1.0.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v0.13.0...datatrail-cli-v1.0.0) (2026-09-28)
 
 

{datatrail_cli-1.0.0 → datatrail_cli-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datatrail-cli
-Version: 1.0.0
+Version: 1.0.1
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>

{datatrail_cli-1.0.0 → datatrail_cli-1.0.1}/dtcli/pull.py

@@ -4,7 +4,13 @@ import logging
 from os import cpu_count, path
 
 import click
-from requests.exceptions import ConnectionError, SSLError
+from cadcutils.exceptions import (
+    HttpException,
+    InternalServerException,
+    TransferException,
+    UnexpectedException,
+)
+from requests.exceptions import ConnectionError, SSLError, Timeout
 from rich.console import Console
 from rich.prompt import Confirm
 
@@ -149,6 +155,18 @@ Create one using 'cadc-get-cert -u <USERNAME>'.
 """
             )
             return None
+        except (ConnectionError, Timeout, HttpException) as error:
+            # CADC also uses the base HttpException for failed connections/retries.
+            # Preserve its authentication, certificate, and invalid-request errors.
+            if isinstance(error, HttpException) and type(error) not in (
+                HttpException,
+                InternalServerException,
+                TransferException,
+                UnexpectedException,
+            ):
+                raise
+            logger.debug("Unable to query Luskan for download size", exc_info=True)
+            to_download_size = -1
     elif not luskan_up:
         to_download_size = -1
     else:
@@ -161,7 +179,7 @@ Create one using 'cadc-get-cert -u <USERNAME>'.
         f" - {len(files['missing'])} files can be downloaded from minoc.",
         style="yellow",
     )
-    if luskan_up:
+    if to_download_size >= 0:
         console.print(
             f"     - Size to download: {to_download_size:.2f} GB.\n",
             style="yellow",

datatrail_cli-1.0.1/dtcli/scout.py

@@ -0,0 +1,323 @@
+"""Datatrail Scout Command."""
+
+import logging
+import re
+from pathlib import PurePosixPath
+from typing import Any, Dict, List, Optional
+
+import click
+import requests
+from rich.console import Console
+from rich.prompt import Confirm
+from rich.table import Table
+
+from dtcli.config import procure
+from dtcli.ls import list as ls
+from dtcli.utilities import cadcclient
+from dtcli.utilities.utilities import (
+    REQUEST_TIMEOUT,
+    check_canfar_status,
+    set_log_level,
+    validate_scope,
+)
+
+logger = logging.getLogger("scout")
+
+console = Console()
+error_console = Console(stderr=True, style="bold red")
+
+# Remote roots used by the Datatrail server's scout proxy. These are independent
+# of the client's local root_mounts configuration.
+SCOUT_ROOT_MOUNTS = {
+    "chime": "/",
+    "baseband_buffer": "/data/baseband_buffer/",
+    "kko": "/",
+    "gbo": "/",
+    "hco": "/",
+}
+
+
+@click.command(name="scout", help="Scout a dataset.")
+@click.argument("scopes", required=False, type=click.STRING, nargs=-1)
+@click.argument("dataset", required=True, type=click.STRING, nargs=1)
+@click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
+@click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
+@click.pass_context
+def scout(  # noqa: C901
+    ctx: click.Context,
+    scopes: List[str],
+    dataset: str,
+    verbose: int,
+    quiet: bool,
+):
+    """Scout a dataset.
+
+    Args:
+        ctx (click.Context): Click context.
+        scopes (List[str]): Scopes of dataset.
+        dataset (str): Name of dataset.
+        verbose (int): Verbosity: v=INFO, vv=DUBUG.
+        quiet (bool): Set log level to ERROR.
+
+    Returns:
+        None
+    """
+    # Set logging level.
+    set_log_level(logger, verbose, quiet)
+    logger.debug("`scout` called with:")
+    logger.debug(f"scopes: {scopes} [{type(scopes)}]")
+    logger.debug(f"dataset: {dataset} [{type(dataset)}]")
+    logger.debug(f"verbose: {verbose} [{type(verbose)}]")
+    logger.debug(f"quiet: {quiet} [{type(quiet)}]")
+
+    # Check if scopes are valid.
+    if scopes:
+        logger.debug(f"Scopes limited to: {list(scopes)}")
+        try:
+            valid_scopes = all(validate_scope(scope) for scope in scopes)
+        except Exception:
+            error_console.print("Unable to validate scopes.")
+            ctx.exit(1)
+        if not valid_scopes:
+            error_console.print("A scope is invalid.")
+            console.print("Valid scopes are:")
+            ctx.invoke(ls)
+            ctx.exit(1)
+
+    # Load configuration.
+    try:
+        config = procure()
+        server = config["server"]
+        logger.debug("Configuration loaded successfully.")
+    except Exception:
+        logger.error(
+            "No configuration file found. Create one with `datatrail config init`."
+        )
+        ctx.exit(1)
+
+    # Check Canfar status.
+    check_canfar_status(error_console)
+
+    # Scout dataset.
+    url = server.rstrip("/") + "/query/dataset/scout"
+    params: Dict[str, Any] = {"name": dataset}
+    if scopes:
+        params["scopes"] = scopes
+    logger.debug(f"URL: {url}")
+    try:
+        response = requests.get(url, params=params, timeout=REQUEST_TIMEOUT)
+    except requests.exceptions.Timeout:
+        error_console.print("Error: Datatrail server timed out.")
+        ctx.exit(1)
+    except requests.RequestException:
+        error_console.print("Error: Datatrail scout request failed.")
+        ctx.exit(1)
+    if not 200 <= response.status_code < 300:
+        error_console.print(
+            f"Error: Datatrail server returned HTTP {response.status_code}."
+        )
+        ctx.exit(1)
+    try:
+        data = response.json()
+        logger.debug(f"Data: {data}")
+    except ValueError:
+        error_console.print("Error: Datatrail server returned invalid JSON.")
+        ctx.exit(1)
+    if not _valid_scout_data(data):
+        error_console.print("Error: Datatrail returned no valid scout results.")
+        ctx.exit(1)
+
+    file_discrepancies: List[List] = []
+    failed = False
+
+    for scope in data:
+        basepath = data[scope]["basepath"].replace("'", "''")
+        query = f"select count(*) from inventory.Artifact where uri like 'cadc:CHIMEFRB/{basepath}%'"  # noqa: E501
+        try:
+            count = int(cadcclient.query(query)[0][0])
+            if count < 0:
+                raise ValueError("Invalid file count.")
+        except Exception:
+            error_console.print(f"{scope} - Minoc count query failed.")
+            count = -1
+            failed = True
+        data[scope]["observed"]["minoc"] = count
+        storage_elements = dict.fromkeys(
+            [*data[scope]["observed"], *data[scope]["expected"]]
+        )
+        for se in storage_elements:
+            data[scope]["observed"].setdefault(se, 0)
+            data[scope]["expected"].setdefault(se, 0)
+            if data[scope]["observed"][se] > data[scope]["expected"][se]:
+                file_discrepancies.append([scope, se])
+
+    show_scout_results(dataset, data)
+
+    if file_discrepancies:
+        error_console.print("File discrepancies:")
+    for scope, se in file_discrepancies:
+        error_console.print(f" - {se}: {scope}")
+        if Confirm.ask("\nWould you like to attempt to heal this discrepancy?"):
+            if not _heal(server, dataset, scope, se, data[scope]):
+                failed = True
+    if failed:
+        ctx.exit(1)
+
+
+def _valid_scout_data(data: Any) -> bool:
+    """Require the report fields used to display and select repairs."""
+    if not isinstance(data, dict) or not data or "error" in data:
+        return False
+    for scope, info in data.items():
+        if not isinstance(scope, str) or not scope or not isinstance(info, dict):
+            return False
+        if not isinstance(info.get("basepath"), str) or not info["basepath"]:
+            return False
+        if not isinstance(info.get("filetype"), str):
+            return False
+        for field in ("observed", "expected"):
+            counts = info.get(field)
+            if not isinstance(counts, dict) or not all(
+                isinstance(site, str)
+                and site
+                and isinstance(count, int)
+                and not isinstance(count, bool)
+                and count >= (0 if field == "expected" else -1)
+                for site, count in counts.items()
+            ):
+                return False
+    return True
+
+
+def _validated_checksums(data: Any) -> Optional[Dict[str, str]]:
+    """Accept filename-to-MD5 mappings and normalize the optional MD5 prefix."""
+    if not isinstance(data, dict) or not data or "error" in data:
+        return None
+    checksums = {}
+    for filename, value in data.items():
+        if not isinstance(filename, str) or not filename.strip():
+            return None
+        if not isinstance(value, str):
+            return None
+        checksum = value.strip().lower()
+        if checksum.startswith("md5:"):
+            checksum = checksum[4:]
+        if re.fullmatch(r"[0-9a-f]{32}", checksum) is None:
+            return None
+        checksums[filename] = checksum
+    return checksums
+
+
+def _site_checksums(
+    checksums: Dict[str, str], site: str, basepath: str
+) -> Optional[Dict[str, str]]:
+    """Convert scout filenames to the dataset-relative names used by sync."""
+    if site not in SCOUT_ROOT_MOUNTS:
+        return None
+    root = PurePosixPath(SCOUT_ROOT_MOUNTS[site])
+    base = PurePosixPath(basepath)
+    if base.is_absolute() or ".." in base.parts:
+        return None
+    normalized = {}
+    for filename, checksum in checksums.items():
+        path = PurePosixPath(filename)
+        if ".." in path.parts:
+            return None
+        try:
+            relative = path.relative_to(root) if path.is_absolute() else path
+            suffix = relative.relative_to(base)
+        except ValueError:
+            return None
+        name = relative.as_posix()
+        if not suffix.parts or name in normalized:
+            return None
+        normalized[name] = checksum
+    return normalized
+
+
+def _heal(server: str, dataset: str, scope: str, site: str, info: Dict) -> bool:
+    """Fetch validated checksums and submit one explicitly confirmed repair."""
+    try:
+        if site == "minoc":
+            data = cadcclient.dataset_md5s(info["basepath"])
+        else:
+            response = requests.get(
+                server.rstrip("/") + "/query/dataset/scout/md5sums",
+                params={
+                    "basepath": info["basepath"],
+                    "site": site,
+                    "filetype": info["filetype"],
+                },
+                timeout=REQUEST_TIMEOUT,
+            )
+            if not 200 <= response.status_code < 300:
+                raise ValueError("Checksum request failed.")
+            data = response.json()
+    except Exception:
+        error_console.print(f"{scope} ({site}) - Checksum query failed; skipping.")
+        return False
+    checksums = _validated_checksums(data)
+    if checksums is not None and site != "minoc":
+        checksums = _site_checksums(checksums, site, info["basepath"])
+    if checksums is None:
+        error_console.print(f"{scope} ({site}) - Invalid checksum response; skipping.")
+        return False
+    try:
+        response = requests.post(
+            server.rstrip("/") + "/commit/dataset/scout/sync",
+            params={"name": dataset, "scope": scope, "replicate_to": site},
+            json=checksums,
+            timeout=REQUEST_TIMEOUT,
+        )
+    except requests.RequestException:
+        error_console.print(f"{scope} ({site}) - Healing failed.")
+        return False
+    if not 200 <= response.status_code < 300:
+        error_console.print(f"{scope} ({site}) - Healing failed.")
+        return False
+    console.print(f"{scope} ({site}) - Healing successful.")
+    return True
+
+
+def show_scout_results(dataset: str, data: dict):
+    """Create and display a table with scout results.
+
+    Args:
+        dataset: Name of dataset.
+        data: Data to display.
+    """
+    # Display results.
+    scopes = list(data.keys())
+    storage_elements = dict.fromkeys(
+        site for info in data.values() for site in [*info["observed"], *info["expected"]]
+    )
+    table = Table(
+        title=f"Scout Results for {dataset}",
+        header_style="magenta",
+        title_style="bold magenta",
+    )
+    table.add_column("Scope", style="bold")
+    for se in storage_elements:
+        table.add_column(se, style="bold")
+
+    for scope in scopes:
+        # Observed
+        row = [scope]
+        for se in storage_elements:
+            row.append(str(data[scope]["observed"].get(se, 0)))
+        table.add_row(*row, style="blue")
+
+        # Expected
+        row = [scope]
+        for se in storage_elements:
+            row.append(str(data[scope]["expected"].get(se, 0)))
+        table.add_row(*row, style="yellow", end_section=True)
+
+    console.print(table)
+    console.print("Legend: [blue]Observed[/blue], [yellow]Expected[/yellow]")
+    console.print(
+        "NOTE: In the case where more files are expected at a site other than \
+minoc, that this may be due to the file type filtering when querying each site. This \
+is a known limitation of the current implementation.",
+    )
+    console.print()

{datatrail_cli-1.0.0 → datatrail_cli-1.0.1}/dtcli/src/functions.py

@@ -4,7 +4,7 @@ import logging
 import os
 import re
 import shutil
-import subprocess
+import stat
 import time
 from collections import Counter
 from collections.abc import Sequence
@@ -461,6 +461,63 @@ def find_missing_dataset_files(
     return {"missing": missing_files, "existing": existing_files}
 
 
+def _apply_file_group_permissions(
+    filename: str, directory_fd: int, group_id: Optional[int]
+) -> None:
+    """Update a regular file relative to its directory without following links."""
+    try:
+        metadata = os.stat(filename, dir_fd=directory_fd, follow_symlinks=False)
+        if not stat.S_ISREG(metadata.st_mode):
+            return
+        if group_id is not None:
+            try:
+                os.chown(
+                    filename, -1, group_id, dir_fd=directory_fd, follow_symlinks=False
+                )
+            except OSError as error:
+                logger.warning("Could not update CANFAR ownership: %s", error)
+        metadata = os.stat(filename, dir_fd=directory_fd, follow_symlinks=False)
+        if stat.S_ISREG(metadata.st_mode):
+            os.chmod(
+                filename,
+                stat.S_IMODE(metadata.st_mode) | stat.S_IWGRP,
+                dir_fd=directory_fd,
+                follow_symlinks=False,
+            )
+    except (OSError, ValueError, NotImplementedError) as error:
+        # If a symlink replaces the file, do not follow it. Some platforms
+        # reject no-follow chmod rather than applying it.
+        logger.warning("Could not update CANFAR permissions: %s", error)
+
+
+def _apply_chime_frb_rw_permissions(folder: str, group_id: Optional[int]) -> None:
+    """Set group ownership and write permission without following symlinks."""
+
+    def report_error(error: OSError) -> None:
+        logger.warning("Could not update CANFAR permissions: %s", error)
+
+    try:
+        # fwalk keeps directory access relative to open descriptors and skips
+        # symlinked directories, including a symlink passed as the root folder.
+        for _, _, filenames, directory_fd in os.fwalk(
+            folder, follow_symlinks=False, onerror=report_error
+        ):
+            if group_id is not None:
+                try:
+                    os.fchown(directory_fd, -1, group_id)
+                except OSError as error:
+                    report_error(error)
+            try:
+                mode = stat.S_IMODE(os.fstat(directory_fd).st_mode)
+                os.fchmod(directory_fd, mode | stat.S_IWGRP)
+            except OSError as error:
+                report_error(error)
+            for filename in filenames:
+                _apply_file_group_permissions(filename, directory_fd, group_id)
+    except OSError as error:
+        report_error(error)
+
+
 def get_files(
     files: List[str],
     site: str,
@@ -499,10 +556,16 @@ def get_files(
         # make directory structure if it does not exist.
         folders = {os.path.dirname(path) for path in destinations}
         if site == "canfar":
+            import grp
+
+            try:
+                group_id = grp.getgrnam("chime-frb-rw").gr_gid
+            except (KeyError, OSError) as error:
+                logger.warning("Could not resolve CANFAR group: %s", error)
+                group_id = None
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)
-                subprocess.run(["chgrp", "-R", "chime-frb-rw", folder])
-                subprocess.run(["chmod", "-R", "g+w", folder])
+                _apply_chime_frb_rw_permissions(folder, group_id)
         else:
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)

{datatrail_cli-1.0.0 → datatrail_cli-1.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datatrail-cli"
-version = "1.0.0"
+version = "1.0.1"
 description = "CHIME/FRB Datatrail CLI"
 authors = [
   { name = "CHIME FRB Project Office" },