datatrail-cli 0.13.0__tar.gz → 1.0.0__tar.gz

datatrail_cli-1.0.0/.github/workflows/codeql.yml

@@ -0,0 +1,101 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '24 11 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v7
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - name: Run manual build steps
+      if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:${{matrix.language}}"

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/.github/workflows/continuous-deployment.yml

@@ -3,6 +3,10 @@ on:
     branches:
       - main
 name: Continuous Deployment
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
 jobs:
   release-please:
     runs-on: ubuntu-latest

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/.github/workflows/continuous-integration.yml

@@ -1,5 +1,8 @@
 name: Continuous Integration
 
+permissions:
+  contents: read
+
 on:
   push:
     paths-ignore:
@@ -37,7 +40,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version:
-          - "3.9"
+          - "3.10"
           - "3.11"
           - "3.12"
           - "3.13"

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/.github/workflows/docs.yml

@@ -7,6 +7,9 @@ on:
       - "docs/**"
       - ".github/workflows/docs.yml"
 
+permissions:
+  contents: read
+
 jobs:
   mkdocs:
     runs-on: ubuntu-latest

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/.github/workflows/stale.yml

@@ -4,6 +4,11 @@ on:
   schedule:
     - cron: "0 0 * * *" # Run every day at midnight
 
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
 jobs:
   close_stale_prs:
     runs-on: ubuntu-latest

datatrail_cli-1.0.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "1.0.0"
+}

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## [1.0.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v0.13.0...datatrail-cli-v1.0.0) (2026-09-28)
+
+
+### ⚠ BREAKING CHANGES
+
+* require Python 3.10 for patched dependencies
+
+### Features
+
+* **doctor:** add readiness checks ([ae0565c](https://github.com/CHIMEFRB/datatrail-cli/commit/ae0565cd7963758e465889f312f0fac2d0418c3d))
+* **doctor:** add readiness checks ([6691c31](https://github.com/CHIMEFRB/datatrail-cli/commit/6691c31cb20e5419588af7a21f62810aee1fbc02))
+* **inventory:** add resumable manifests ([c7c4df5](https://github.com/CHIMEFRB/datatrail-cli/commit/c7c4df5d1634e8573f700ba2b1368105f4f67fb7))
+* **inventory:** add resumable manifests ([343a28a](https://github.com/CHIMEFRB/datatrail-cli/commit/343a28a74366f91b8a0bd428250f1576cd4a752f))
+* **ls:** add --match and --expand for dataset discovery ([bf08635](https://github.com/CHIMEFRB/datatrail-cli/commit/bf08635a75f6ec3b1415541ff1b9a1f31e9d443b))
+* **ls:** add match and expand for dataset discovery ([31cccde](https://github.com/CHIMEFRB/datatrail-cli/commit/31cccde47ee8827aec17df7ce13ba5a7ac7b13d3))
+* **ls:** add recursive dataset discovery ([8ea3bc4](https://github.com/CHIMEFRB/datatrail-cli/commit/8ea3bc4e5026fbb9fedb231a0bdb7b78a8f90bdf))
+* **ls:** add recursive dataset discovery ([6f9a85e](https://github.com/CHIMEFRB/datatrail-cli/commit/6f9a85e608eea73a5b3e8b30331203a09908c71c))
+* **ps:** expose common paths in JSON output ([3ee4238](https://github.com/CHIMEFRB/datatrail-cli/commit/3ee423828ee81fca7c9a661051975920e71e17bf))
+* **ps:** expose per-storage-element common paths in --json output ([b748fb4](https://github.com/CHIMEFRB/datatrail-cli/commit/b748fb442589a0259b483774ed6fcf391a8eb391))
+* **pull:** add resumable manifest transfers ([9017c11](https://github.com/CHIMEFRB/datatrail-cli/commit/9017c118450ee98082f72b4142b2c76d77c94f28))
+* **pull:** add resumable manifest transfers ([7272daa](https://github.com/CHIMEFRB/datatrail-cli/commit/7272daa58fc76b3130181fdc1171da7453b01f82))
+* **verify:** verify registered dataset files ([cef2ab7](https://github.com/CHIMEFRB/datatrail-cli/commit/cef2ab708b2d4f26bae4131d18f5613ba86ec0e3))
+* **verify:** verify registered dataset files ([27f3810](https://github.com/CHIMEFRB/datatrail-cli/commit/27f3810dfe78c1f889cb94abae3836a7058ebfd5))
+
+
+### Bug Fixes
+
+* add stable command error metadata ([d1096ff](https://github.com/CHIMEFRB/datatrail-cli/commit/d1096ffb850b6bd1a690370318dcec5be97ce4f3))
+* **deps:** require patched Requests on Python 3.10 and newer ([e0fb559](https://github.com/CHIMEFRB/datatrail-cli/commit/e0fb55951975d60c202181644c703f280602fa5f))
+* **deps:** require patched Requests on Python 3.10+ ([7d1c4ba](https://github.com/CHIMEFRB/datatrail-cli/commit/7d1c4ba3ead33b46cc6c042e554464fca4e78438))
+* **deps:** require patched urllib3 on Python 3.10+ ([5a4ad0b](https://github.com/CHIMEFRB/datatrail-cli/commit/5a4ad0bc1a29dcad305cad8ec739dd47e703ef6a))
+* **deps:** use patched dependencies on Python 3.10 and newer ([69f0806](https://github.com/CHIMEFRB/datatrail-cli/commit/69f080682a6fa11f18c4acbc890b3c043bc22393))
+* **doctor:** handle certificate and authentication failures ([3584b7b](https://github.com/CHIMEFRB/datatrail-cli/commit/3584b7b0d71ee9bc18479bba75e5b10b96ac8ac2))
+* **encoding:** use ASCII status output ([a0646d7](https://github.com/CHIMEFRB/datatrail-cli/commit/a0646d7d17e66b7507a7f6019e1cd48c3faf2f9d))
+* enforce patched urllib3 on Python 3.10 and newer ([ab12a03](https://github.com/CHIMEFRB/datatrail-cli/commit/ab12a03402cb0ecfec471b2c1941bc3513101762))
+* **failure:** add stable command error metadata ([f52a917](https://github.com/CHIMEFRB/datatrail-cli/commit/f52a917acc05a88b4cbe1364ce3d41af8e1af4ae))
+* **inventory:** reconcile recovered discovery checkpoints ([2a0bda5](https://github.com/CHIMEFRB/datatrail-cli/commit/2a0bda5751e849503903c96bba65fe530bb016d2))
+* **ls:** validate discovery filters and response names ([af2de4f](https://github.com/CHIMEFRB/datatrail-cli/commit/af2de4f3526da507e54dc30f68cc139227f3381b))
+* **ps:** preserve paths for duplicate and invalid replicas ([2dae57d](https://github.com/CHIMEFRB/datatrail-cli/commit/2dae57d4654c4a22c81aea17827a5d0f041b977c))
+* **pull:** clean up CADC SDK partial downloads ([1b15bff](https://github.com/CHIMEFRB/datatrail-cli/commit/1b15bffaa32b2685669a74f4d18f9d6bb73ee1ef))
+* **pull:** protect aliased directory ownership paths ([c03f284](https://github.com/CHIMEFRB/datatrail-cli/commit/c03f284d730fdf1dd9bf2f8dbe5ddda8780324ac))
+* **pull:** publish downloads atomically ([b512e5f](https://github.com/CHIMEFRB/datatrail-cli/commit/b512e5f8ba5b6acd3a5a09c68440eb914d9e8d8e))
+* **pull:** publish downloads atomically ([bf7846f](https://github.com/CHIMEFRB/datatrail-cli/commit/bf7846f4efb41ea55d3edd3961b19c5da7d79eb3))
+* **pull:** report lost workers and clean up failed starts ([bfe5d91](https://github.com/CHIMEFRB/datatrail-cli/commit/bfe5d91d71d130230f5a6b232953e09e34fc8219))
+* **pull:** validate manifest paths before creating state ([b0dc320](https://github.com/CHIMEFRB/datatrail-cli/commit/b0dc32064e1859fc90466da5ed2b4a7f62aa8ccd))
+* require Python 3.10 for patched dependencies ([07de5af](https://github.com/CHIMEFRB/datatrail-cli/commit/07de5af18da463d514ca4b05da06a749c5c2b0dd))
+* update TLS dependencies without raising Python minimum ([8658c58](https://github.com/CHIMEFRB/datatrail-cli/commit/8658c5828ce63772ba229ffa6f4ff93b5c9575fe))
+* update TLS dependencies without raising Python minimum ([ec11019](https://github.com/CHIMEFRB/datatrail-cli/commit/ec110197e2f5281cd22261b605de1d1ad47405e3))
+* **verify:** preserve reports when metadata services fail ([8dbeea3](https://github.com/CHIMEFRB/datatrail-cli/commit/8dbeea34caa8a87d94f4f4d0e3418f3bcfb2df9f))
+
 ## [0.13.0](https://github.com/CHIMEFRB/datatrail-cli/compare/datatrail-cli-v0.12.0...datatrail-cli-v0.13.0) (2026-08-25)
 
 

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/PKG-INFO

@@ -1,24 +1,29 @@
 Metadata-Version: 2.5
 Name: datatrail-cli
-Version: 0.13.0
+Version: 1.0.0
 Summary: CHIME/FRB Datatrail CLI
 Author: CHIME FRB Project Office
 Author-email: Tarik Zegmott <tarik.zegmott@mcgill.ca>
 License-Expression: MIT
 License-File: LICENSE
-Requires-Python: !=3.9.0,!=3.9.1,<4.0,>=3.8.1
+Requires-Python: <4.0,>=3.10
 Requires-Dist: cadcdata>=2.5.0
 Requires-Dist: cadctap>=0.9.11
 Requires-Dist: cadcutils>=1.5.1.1
 Requires-Dist: click-aliasing>=1.0.0
 Requires-Dist: click>=8.1.3
+Requires-Dist: cryptography>=50.0.0
 Requires-Dist: dill>=0.3.6
 Requires-Dist: lxml>=6.1.0
 Requires-Dist: mergedeep>=1.3.4
 Requires-Dist: pyaml>=21.10.1
-Requires-Dist: requests>=2.29.0
+Requires-Dist: pygments>=2.20.0
+Requires-Dist: pyopenssl>=26.4.0
+Requires-Dist: requests>=2.33.0
 Requires-Dist: rich>=13.3.5
+Requires-Dist: setuptools>=83.0.0
 Requires-Dist: tenacity>=8.0.0
+Requires-Dist: urllib3>=2.7.0
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -56,6 +61,8 @@ Description-Content-Type: text/markdown
 
 ## 🛠️ Installation
 
+Python 3.10 or newer is required for the supported dependency versions.
+
 ### Install from PYPI
 
 ```shell

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/README.md

@@ -33,6 +33,8 @@
 
 ## 🛠️ Installation
 
+Python 3.10 or newer is required for the supported dependency versions.
+
 ### Install from PYPI
 
 ```shell

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/docs/commands.md

@@ -9,11 +9,14 @@ The commands available to you are:
 - `clear`: This removes all files belonging to the 'scope' and 'dataset', only
   available for the local and canfar sites.
 - `config`: Edit the `.datatrail/config.yaml` configuration file.
+- `inventory`: Recursively discover datasets and write their file replica URIs
+  to a resumable JSON manifest.
 - `list`: This list either the 'scopes' available or all of the datasets
   belonging to the given dataset.
 - `ps`: This provides detailed information for the given 'scope' and 'dataset' combination.
 - `pull`: This allows you to download all files belonging to the 'scope' and
   'dataset' provided.
+- `pull-manifest`: Download Minoc files from a resumable inventory manifest.
 - `scout`: This command provides an overview of what the Datatrail database
   thinks is the current number of files for a given dataset at each storage
   element, compared to what is observed. If a discrepancy is found at Minoc,

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/docs/install.md

@@ -1,5 +1,7 @@
 # 🛠️ Installation
 
+Python 3.10 or newer is required.
+
 There are two ways to install the Datatrail CLI:
 
 === "PYPI"

datatrail_cli-1.0.0/docs/inventory.md

@@ -0,0 +1,80 @@
+# Building a durable inventory
+
+`datatrail inventory` recursively discovers terminal datasets and records each
+file replica URI in a versioned JSON manifest. The manifest can later drive a
+batch download without repeating the archive crawl.
+
+Every run must be bounded in one of these ways:
+
+```shell
+$> datatrail inventory chime.event.baseband.raw
+$> datatrail inventory --match classified,baseband
+$> datatrail inventory gbo.acquisition.processed --parent complex_gains
+```
+
+A scope limits traversal to that scope. `--match` may search across scopes but
+only opens matching larger datasets. `--parent` requires a scope and starts at
+one known dataset. It cannot be combined with `--match`.
+
+Use `--output` to choose the manifest path:
+
+```shell
+$> datatrail inventory chime.event.baseband.raw --match classified \
+     --output baseband-inventory.json
+```
+
+The command writes the manifest atomically after discovery and after every
+dataset query. A rerun with the same selection reuses `ready` and `empty`
+entries, then retries `pending` and `failed` entries. A different selection is
+refused so unrelated inventories cannot be mixed.
+
+Successful discovery replaces obsolete entries with the current terminal
+datasets. If a failed parent lookup recovers, its placeholder is replaced by
+the discovered descendants. A discovery outage preserves earlier checkpoints.
+
+Only one inventory command may own an output at a time. A competing run exits
+with an error before reading or changing the manifest, including when a symlink
+refers to the same output. Ownership is released when the process exits. Leave
+the hidden `.lock` file in place; its presence does not mean a run is active.
+
+An inventory with any failed discovery branch or file query exits nonzero.
+`--allow-incomplete` keeps the incomplete manifest but exits zero when a caller
+wants to inspect or process the available subset.
+
+## Manifest format
+
+The first format is `datatrail.inventory/v1`:
+
+```json
+{
+  "schema": "datatrail.inventory/v1",
+  "selection": {
+    "scope": "gbo.acquisition.processed",
+    "match": [],
+    "parent": "complex_gains"
+  },
+  "complete": true,
+  "discovery_failures": [],
+  "datasets": [
+    {
+      "scope": "gbo.acquisition.processed",
+      "dataset": "20230525",
+      "parent": "complex_gains",
+      "path": ["complex_gains", "20230525"],
+      "status": "ready",
+      "replicas": [
+        {
+          "storage_element": "minoc",
+          "uri": "cadc:CHIMEFRB/example/file.h5"
+        }
+      ]
+    }
+  ]
+}
+```
+
+Replica rows contain only Datatrail information. Size and checksum fields may
+be added in a later schema when they can be obtained without requiring a CADC
+credential during inventory creation. A valid dataset with no replica URIs has
+status `empty`; an unavailable or invalid file response has status `failed`
+and an `error` field.

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/docs/list.md

@@ -12,6 +12,12 @@ Options:
   -q, --quiet    Only errors shown in logs.
   --write        Write the events to file.
   --json         Output as JSON.
+  --match TEXT   Comma-separated, case-insensitive terms a larger dataset must
+                 all contain.
+  --expand       Open each matched larger dataset one level and list its
+                 children.
+  --recursive    Open each matched larger dataset through all descendant
+                 levels.
   --help         Show this message and exit.
 
 ```
@@ -102,6 +108,105 @@ Within Datatrail, there are two types of datasets:
     Please see the CLI reference page for more information on the `list` command:
     [datatrail list](../cli/#datatrail-list)
 
+## Finding datasets with `--match` and `--expand`
+
+Navigating the hierarchy one name at a time gets slow when you do not know
+where a dataset lives. `--match` filters the larger datasets of a scope, or
+of **every** scope when no scope is given, by one or more comma-separated,
+case-insensitive terms, which must all appear in the combined
+`scope dataset` text. At least one nonempty term is required; whitespace and
+commas alone are rejected:
+
+```shell
+$> datatrail ls chime.acquisition.processed --match gains
+      Datatrail: Dataset Map
++-----------------------------+---------------+
+| Scope                       | Dataset       |
++-----------------------------+---------------+
+| chime.acquisition.processed | complex_gains |
++-----------------------------+---------------+
+```
+
+A hit may be a container whose children are the datasets you actually want.
+`--expand` opens each matched larger dataset one level and lists the children
+it finds, recording the parent; a match whose children cannot be listed keeps
+its own row:
+
+```shell
+$> datatrail ls --match gain --expand
+                    Datatrail: Dataset Map
++-----------------------------+---------------+---------------+
+| Scope                       | Dataset       | Parent        |
++-----------------------------+---------------+---------------+
+| chime.acquisition.processed | complex_gains |               |
++-----------------------------+---------------+---------------+
+| gbo.acquisition.processed   | 20230716      | complex_gains |
+| gbo.acquisition.processed   | 20230715      | complex_gains |
+| ...                         | ...           | ...           |
++-----------------------------+---------------+---------------+
+```
+
+Rows reached through a parent resolve directly with
+`datatrail ps <scope> <dataset>`; a row kept for a matched dataset whose
+children could not be listed (or that has none) may still be a container.
+
+!!! warning "Incomplete maps"
+
+    If Datatrail does not answer for a scope or dataset during the walk, the
+    map is reported as **incomplete** and the unanswered queries are listed,
+    rather than silently showing them as empty. With `--json`, those queries
+    appear in the `failed` list. A partial map still exits 0; a map with **no**
+    rows and unanswered queries exits 1, since nothing was determined.
+
+`--recursive` follows every descendant of each matched larger dataset instead
+of stopping after one level. It emits terminal datasets and records the full
+path used to reach each one:
+
+```shell
+$> datatrail ls gbo.acquisition.processed --match gains --recursive
+```
+
+The walk visits each dataset once, so shared descendants are not duplicated
+and hierarchy cycles cannot loop forever. The first path found in sorted order
+is retained. An answered empty child list is a terminal dataset. A branch that
+does not answer is retained as a partial row and also listed under `failed`.
+Like `--expand`, a recursive walk across all scopes requires `--match` to keep
+the request bounded.
+
+With `--json`, the map is emitted as structured rows for scripting; `parent`
+is `null` for rows that were not reached through expansion. Recursive rows
+also include `path`, from the matched larger dataset through the terminal row:
+
+```bash
+$ datatrail ls --match gain --expand --json
+{
+  "results": [
+    {
+      "scope": "gbo.acquisition.processed",
+      "dataset": "20230525",
+      "parent": "complex_gains"
+    },
+    ...
+  ],
+  "failed": []
+}
+```
+
+```bash
+$ datatrail ls gbo.acquisition.processed --match gains --recursive --json
+{
+  "results": [
+    {
+      "scope": "gbo.acquisition.processed",
+      "dataset": "20230525",
+      "parent": "complex_gains",
+      "path": ["complex_gains", "20230525"]
+    }
+  ],
+  "failed": []
+}
+```
+
 ## 🤖 Machine-readable JSON output
 
 The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines:

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/docs/ps.md

@@ -102,6 +102,16 @@ $ datatrail ps kko.event.baseband.raw 308892599 --json
       ]
     }
   },
+  "common_paths": {
+    "minoc": {
+      "common_path": "data/kko/baseband/raw/2023/08/07/astro_308892599",
+      "files": [
+        "baseband_308892599_129.h5",
+        "baseband_308892599_1013.h5",
+        ...
+      ]
+    }
+  },
   "policies": {
     "replication_policy": {
       "preferred_storage_elements": ["chime"],
@@ -127,6 +137,13 @@ $ datatrail ps kko.event.baseband.raw 308892599 --json
 }
 ```
 
+The `common_paths` field gives, per storage element, the deepest common
+directory of its file replicas and the file names relative to it, so scripts
+do not have to re-derive the split. When no common directory exists, it is
+`""` and the original paths are listed. Note that `minoc` paths keep any
+collection prefix (such as `cadc:CHIMEFRB`) exactly as reported in
+`file_replica_locations`.
+
 ### Usage in scripts
 
 ```python
@@ -145,6 +162,13 @@ data = json.loads(result.stdout)
 file_locations = data["files"]["file_replica_locations"]
 minoc_files = file_locations.get("minoc", [])
 
+# Compose full paths from the derived common path
+minoc = data["common_paths"].get("minoc", {})
+full_paths = [
+    f"{minoc['common_path']}/{name}" if minoc["common_path"] else name
+    for name in minoc.get("files", [])
+]
+
 # Access policies
 replication_policy = data["policies"]["replication_policy"]
 deletion_policy = data["policies"]["deletion_policy"]

datatrail_cli-1.0.0/docs/pull-manifest.md

@@ -0,0 +1,69 @@
+# Pulling an inventory
+
+`datatrail pull-manifest` downloads the Minoc files in an inventory and writes
+durable transfer state beside it:
+
+```shell
+$> datatrail pull-manifest baseband-inventory.json --directory ./baseband
+```
+
+The default state path is `<inventory-name>.pull.json`. Use `--state` to put it
+elsewhere. Each completed batch is written atomically, so rerunning the same
+command skips completed files whose recorded size still matches and retries
+pending or failed files.
+
+Minoc replicas may use `cadc:CHIMEFRB/...`, `data/...`, or `/...` paths, matching
+the formats accepted by dataset downloads. Equivalent paths are downloaded
+once. Paths that escape the destination, collide with one another, or replace
+the inventory, transfer state, or lock files are refused before creating state.
+
+Transfers are limited by `--cores`, which defaults to one. A larger value runs
+only that many files at once:
+
+```shell
+$> datatrail pull-manifest baseband-inventory.json --directory ./baseband \
+     --cores 4 --force
+```
+
+Each file is downloaded to a temporary sibling, checked against Minoc's size
+when available, and moved into place only after it completes. An interrupted or
+truncated download does not replace the destination.
+
+The command exits nonzero when any file fails, the inventory is incomplete, or
+a ready dataset has no Minoc replica. Available files are still downloaded and
+checkpointed, so the next run can continue after the service recovers.
+
+Only one transfer command may use a state file or destination directory at a
+time. Competing runs exit before changing transfer state or downloading files,
+even if they use different state files for the same destination. Symlinks to the
+same paths share ownership. The locks are held through confirmation and all
+download batches, and released when the process exits. Leave the hidden `.lock`
+files in place; their presence does not mean a transfer is active.
+Use separate destination trees for concurrent commands; these locks do not
+coordinate a directory with a nested destination used by another command.
+
+## Transfer state
+
+The transfer state uses schema `datatrail.pull/v1`:
+
+```json
+{
+  "schema": "datatrail.pull/v1",
+  "inventory": "/data/baseband-inventory.json",
+  "directory": "/data/baseband",
+  "inventory_complete": true,
+  "unavailable_datasets": [],
+  "complete": true,
+  "files": [
+    {
+      "uri": "cadc:CHIMEFRB/example/file.h5",
+      "path": "example/file.h5",
+      "status": "complete",
+      "bytes": 4096
+    }
+  ]
+}
+```
+
+Keep the state file with its inventory. A state file cannot be reused with a
+different inventory path or destination directory.

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/dtcli/clear.py

@@ -128,9 +128,9 @@ def clear(
         is_delete = True
     else:
         message = (
-            "⚠️  Delete files and empty parent directories?"
+            "WARNING: Delete files and empty parent directories?"
             if clear_parents
-            else "⚠️  Delete files?"
+            else "WARNING: Delete files?"
         )
         is_delete = Confirm.ask(message)
 

{datatrail_cli-0.13.0 → datatrail_cli-1.0.0}/dtcli/cli.py

@@ -6,7 +6,19 @@ import click
 from click_aliasing import ClickAliasedGroup
 from rich import console, pretty
 
-from dtcli import clear, config, ls, ps, pull, scout, unregistered
+from dtcli import (
+    clear,
+    config,
+    doctor,
+    inventory,
+    ls,
+    ps,
+    pull,
+    pull_manifest,
+    scout,
+    unregistered,
+    verify,
+)
 from dtcli.utilities import utilities
 
 pretty.install()
@@ -43,11 +55,15 @@ def version():
 
 cli.add_command(clear.clear)
 cli.add_command(config.config)
+cli.add_command(doctor.doctor)
+cli.add_command(inventory.inventory)
 cli.add_command(ls.list, aliases=["ls"])
 cli.add_command(ps.ps)
 cli.add_command(pull.pull)
+cli.add_command(pull_manifest.pull_manifest)
 cli.add_command(scout.scout)
 cli.add_command(unregistered.unregistered)
+cli.add_command(verify.verify)
 
 
 def check_version() -> None:
@@ -60,7 +76,7 @@ def check_version() -> None:
         current_version = package_version("datatrail-cli")
         latest_version = utilities.get_latest_released_version()
         terminal_stderr.print(
-            f"A new release of datatrail-cli is available: {current_version} → {latest_version}",  # noqa: E501
+            f"A new release of datatrail-cli is available: {current_version} -> {latest_version}",  # noqa: E501
             style="bold yellow",
         )
         terminal_stderr.print()