ebrains-bucket-sync 0.1.0__tar.gz

ebrains_bucket_sync-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Publish
+
+# Publishes to PyPI when a GitHub release is published, through PyPI trusted
+# publishing: PyPI accepts the OIDC token of this workflow, so no API token is
+# stored. The release tag must be "v" followed by the version in pyproject.toml.
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ebrains-bucket-sync
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Check that the tag matches the package version
+        run: |
+          version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+          if [ "$GITHUB_REF_NAME" != "v$version" ]; then
+            echo "::error::Release tag $GITHUB_REF_NAME does not match version $version in pyproject.toml (expected v$version)."
+            exit 1
+          fi
+      - run: uv sync --dev
+      - run: uv run pytest
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1

ebrains_bucket_sync-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,34 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --dev
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest
+
+  # The one check the ruleset requires, so that adding a Python version to the
+  # matrix does not need a ruleset change. It must run even when a test job
+  # fails, because a required check that is skipped counts as passed.
+  tests-passed:
+    if: always()
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - if: needs.test.result != 'success'
+        run: exit 1

ebrains_bucket_sync-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+.DS_Store

ebrains_bucket_sync-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 University of Oslo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ebrains_bucket_sync-0.1.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.5
+Name: ebrains-bucket-sync
+Version: 0.1.0
+Summary: Sync a local folder to an EBRAINS Data Proxy bucket, sending only new and changed files
+Project-URL: Repository, https://github.com/ehennestad/ebrains-bucket-sync
+Author-email: ehennestad <ehennestad@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: EBRAINS,bucket,data-proxy,sync
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: System :: Archiving :: Mirroring
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1
+Requires-Dist: ebrains-drive<0.7.1,>=0.7.0
+Requires-Dist: platformdirs>=3.0
+Requires-Dist: requests>=2.28
+Description-Content-Type: text/markdown
+
+# ebrains-bucket-sync
+
+Sync a local folder to an EBRAINS Data Proxy bucket, the way rsync does: only new and changed files are uploaded, so a sync that is interrupted picks up where it stopped when run again.
+
+The sync rules are shared with the `ebrains.bucket.sync` functions of the [EBRAINS MATLAB toolbox](https://github.com/ehennestad/EBRAINS-MATLAB). Both follow the contract in [spec/README.md](https://github.com/ehennestad/ebrains-bucket-sync/blob/main/spec/README.md) and pass the same fixtures in `spec/fixtures`, so a plan is the same whichever tool makes it.
+
+## Install
+
+```bash
+uv tool install ebrains-bucket-sync
+```
+
+or with pipx:
+
+```bash
+pipx install ebrains-bucket-sync
+```
+
+or, for development:
+
+```bash
+git clone https://github.com/ehennestad/ebrains-bucket-sync
+cd ebrains-bucket-sync
+uv sync --dev
+uv run pytest
+```
+
+## Use
+
+Log in once. A link opens the EBRAINS login in the browser, and the login is kept for later runs:
+
+```bash
+ebrains-bucket-sync login
+```
+
+See what a sync would do, then run it:
+
+```bash
+ebrains-bucket-sync push results my-bucket --prefix results --dry-run
+ebrains-bucket-sync push results my-bucket --prefix results
+```
+
+Object names are the paths relative to the local folder. Files the bucket already has, with the same size and uploaded after the local file was last changed, are not sent again. `--comparison Size` ignores the times, and `--comparison Checksum` compares the MD5 of every file of the same size.
+
+`--delete` also deletes the objects that the local folder does not have, which makes the bucket (or the folder of it) an exact mirror. A sync refuses to empty a bucket from an empty folder, and `--max-delete N` stops it before it deletes more than N objects. Nothing is deleted after an upload failed.
+
+`--exclude PATTERN` leaves out paths that match a wildcard pattern, with the rules of a `.gitignore` file: `*.tmp` in every folder, `.git/` for that folder wherever it is, `raw/scratch` for that path from the root. `--plan-file plan.json` writes the plan and the outcome as JSON.
+
+An upload that fails does not stop the sync. The other files are uploaded, nothing is deleted, the command exits with status 1, and running it again retries the failed files.
+
+From Python:
+
+```python
+from ebrains_bucket_sync import (
+    DeviceFlowAuthenticator,
+    EbrainsDriveStorage,
+    SyncOptions,
+    sync_to_bucket,
+)
+
+storage = EbrainsDriveStorage(DeviceFlowAuthenticator())
+results = sync_to_bucket(
+    "results", "my-bucket", storage, SyncOptions(prefix="results", delete=True)
+)
+for result in results:
+    print(result.path, result.action, result.reason, result.status)
+```
+
+## Authentication
+
+The login uses the OAuth device flow with the same OIDC client as the MATLAB toolbox, so both tools show up as one application in your EBRAINS account. The tokens are kept in the user's configuration folder, in a file only the user can read, and the access token is renewed from the refresh token without a new login for as long as the refresh token lasts. In CI, set `EBRAINS_BUCKET_SYNC_TOKEN` to an access token instead.
+
+## Development
+
+The live tests in `tests/live` run against a real bucket and are skipped unless `EBRAINS_BUCKET_SYNC_TEST_BUCKET` names one. Everything else runs offline against an in-memory bucket.

ebrains_bucket_sync-0.1.0/README.md

@@ -0,0 +1,75 @@
+# ebrains-bucket-sync
+
+Sync a local folder to an EBRAINS Data Proxy bucket, the way rsync does: only new and changed files are uploaded, so a sync that is interrupted picks up where it stopped when run again.
+
+The sync rules are shared with the `ebrains.bucket.sync` functions of the [EBRAINS MATLAB toolbox](https://github.com/ehennestad/EBRAINS-MATLAB). Both follow the contract in [spec/README.md](https://github.com/ehennestad/ebrains-bucket-sync/blob/main/spec/README.md) and pass the same fixtures in `spec/fixtures`, so a plan is the same whichever tool makes it.
+
+## Install
+
+```bash
+uv tool install ebrains-bucket-sync
+```
+
+or with pipx:
+
+```bash
+pipx install ebrains-bucket-sync
+```
+
+or, for development:
+
+```bash
+git clone https://github.com/ehennestad/ebrains-bucket-sync
+cd ebrains-bucket-sync
+uv sync --dev
+uv run pytest
+```
+
+## Use
+
+Log in once. A link opens the EBRAINS login in the browser, and the login is kept for later runs:
+
+```bash
+ebrains-bucket-sync login
+```
+
+See what a sync would do, then run it:
+
+```bash
+ebrains-bucket-sync push results my-bucket --prefix results --dry-run
+ebrains-bucket-sync push results my-bucket --prefix results
+```
+
+Object names are the paths relative to the local folder. Files the bucket already has, with the same size and uploaded after the local file was last changed, are not sent again. `--comparison Size` ignores the times, and `--comparison Checksum` compares the MD5 of every file of the same size.
+
+`--delete` also deletes the objects that the local folder does not have, which makes the bucket (or the folder of it) an exact mirror. A sync refuses to empty a bucket from an empty folder, and `--max-delete N` stops it before it deletes more than N objects. Nothing is deleted after an upload failed.
+
+`--exclude PATTERN` leaves out paths that match a wildcard pattern, with the rules of a `.gitignore` file: `*.tmp` in every folder, `.git/` for that folder wherever it is, `raw/scratch` for that path from the root. `--plan-file plan.json` writes the plan and the outcome as JSON.
+
+An upload that fails does not stop the sync. The other files are uploaded, nothing is deleted, the command exits with status 1, and running it again retries the failed files.
+
+From Python:
+
+```python
+from ebrains_bucket_sync import (
+    DeviceFlowAuthenticator,
+    EbrainsDriveStorage,
+    SyncOptions,
+    sync_to_bucket,
+)
+
+storage = EbrainsDriveStorage(DeviceFlowAuthenticator())
+results = sync_to_bucket(
+    "results", "my-bucket", storage, SyncOptions(prefix="results", delete=True)
+)
+for result in results:
+    print(result.path, result.action, result.reason, result.status)
+```
+
+## Authentication
+
+The login uses the OAuth device flow with the same OIDC client as the MATLAB toolbox, so both tools show up as one application in your EBRAINS account. The tokens are kept in the user's configuration folder, in a file only the user can read, and the access token is renewed from the refresh token without a new login for as long as the refresh token lasts. In CI, set `EBRAINS_BUCKET_SYNC_TOKEN` to an access token instead.
+
+## Development
+
+The live tests in `tests/live` run against a real bucket and are skipped unless `EBRAINS_BUCKET_SYNC_TEST_BUCKET` names one. Everything else runs offline against an in-memory bucket.

ebrains_bucket_sync-0.1.0/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "ebrains-bucket-sync"
+version = "0.1.0"
+description = "Sync a local folder to an EBRAINS Data Proxy bucket, sending only new and changed files"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+authors = [{ name = "ehennestad", email = "ehennestad@gmail.com" }]
+keywords = ["EBRAINS", "data-proxy", "sync", "bucket"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Science/Research",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: System :: Archiving :: Mirroring",
+]
+dependencies = [
+  # storage.object_url_path passes ebrains_drive percent-encoded object names,
+  # which relies on ebrains_drive putting them into URLs as given. A release
+  # that encodes them itself (HumanBrainProject/ebrains-storage#60) would encode
+  # them twice, so newer versions need that encoding removed first.
+  "ebrains-drive>=0.7.0,<0.7.1",
+  "requests>=2.28",
+  "click>=8.1",
+  "platformdirs>=3.0",
+]
+
+[project.scripts]
+ebrains-bucket-sync = "ebrains_bucket_sync.cli:main"
+
+[project.urls]
+Repository = "https://github.com/ehennestad/ebrains-bucket-sync"
+
+[dependency-groups]
+dev = [
+  "pytest>=8.0",
+  "ruff>=0.5",
+  "jsonschema>=4.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ebrains_bucket_sync"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = ["live: needs an EBRAINS login and a test bucket (EBRAINS_BUCKET_SYNC_TEST_BUCKET)"]

ebrains_bucket_sync-0.1.0/spec/README.md

@@ -0,0 +1,63 @@
+# Sync contract
+
+The rules that decide what a sync does, shared by this package and the `ebrains.bucket.sync` functions of the EBRAINS MATLAB toolbox. The Python planner in `src/ebrains_bucket_sync/plan.py` is the reference implementation. Both implementations must pass the fixtures in `fixtures/`, which is what keeps them in step: a change to the rules is a change to the fixtures first.
+
+## File entries
+
+Each side of a sync is a list of file entries:
+
+| field | meaning |
+|---|---|
+| `path` | Path relative to the synced folder, with `/` separators. |
+| `bytes` | Size of the file. |
+| `modified_time` | Time of the last change, in UTC. Unknown where it cannot be read. For an object, the time it was uploaded. |
+| `hash` | Lowercase MD5 checksum. Unknown (`""`) where it is not reported, or where it cannot be the checksum of the content: a multipart upload (`<md5>-<parts>`) or an object above 5 GiB. |
+
+Folder placeholders are not entries: an object whose name ends with `/`, whose content type starts with `application/directory`, or below which other objects are named. An object whose name has an empty, `.` or `..` segment, or a backslash, is left out with a warning.
+
+## Plan
+
+`plan(source, target, comparison, delete)` gives one row per path on either side, sorted by path, with `action` (`copy`, `delete`, `none`), `reason` and `bytes`.
+
+For a path of the source:
+
+1. Not in the target: `copy`, reason `new`.
+2. Sizes differ: `copy`, reason `size`.
+3. `comparison` is `Checksum` and both checksums are known: `copy` with reason `checksum` when they differ, else `none` with reason `unchanged`.
+4. `comparison` is `Size`: `none`, `unchanged`.
+5. Either time unknown: `none`, `unchanged`.
+6. Source time later than target time by more than 2 seconds: `copy`, reason `newer`.
+7. Otherwise `none`, `unchanged`.
+
+For a path only the target has: reason `extraneous`, action `delete` when `delete` is set and `none` otherwise. `bytes` is the size of the source file, or of the target file for an extraneous path.
+
+The 2-second tolerance covers the time resolution of FAT file systems and the fraction of a second the listing times lose.
+
+## Refusal
+
+Before anything is changed, a run with planned deletions stops when:
+
+- the source has no files (`EmptySource`): the sync would empty the target, which is more often a wrong folder or prefix than what is wanted;
+- the plan deletes more than `max_delete` files (`TooManyDeletions`).
+
+A dry run reports the refusal instead of stopping.
+
+## Execution
+
+Copies run first. Nothing is deleted after a copy failed, since a failure may mean the source was listed wrongly or the connection is lost. A failed copy does not stop the other copies, and is reported with its reason. Running the sync again retries it.
+
+## Exclude patterns
+
+Patterns are applied to both sides before planning, so excluded files are neither transferred nor deleted. `*` matches any characters except `/`, `**` any characters, `?` one character except `/`. A pattern without `/` matches a path segment at any depth, and a matched folder excludes everything below it. A pattern with `/` is matched from the root of the synced folder, and also excludes everything below a matched folder. Leading and trailing `/` are ignored.
+
+## Fixtures
+
+`fixtures/plan/*.json`: `policy` (`comparison`, `delete`), `source` and `target` entry lists, and the `expected` plan rows. `modified_time` is ISO 8601 UTC or null; `hash` may be omitted.
+
+`fixtures/exclude/*.json`: `patterns`, `paths`, and `expected_kept`, in order.
+
+`fixtures/refusal/*.json`: a `plan`, `n_source_files`, `max_delete` (null for no limit) and `expected_code` (null for no refusal).
+
+## Plan file
+
+`sync-plan.schema.json` describes the JSON a command writes with `--plan-file`: the policy, and each action with its outcome. `sync-policy.schema.json` describes the policy part on its own.

ebrains_bucket_sync-0.1.0/spec/fixtures/exclude/double-star-crosses-folders.json

@@ -0,0 +1,6 @@
+{
+  "description": "** matches any characters, \"/\" included, where * stops at a folder separator. A ** between two slashes stands for at least one folder, unlike in a .gitignore file, so \"logs/**/*.log\" leaves \"logs/a.log\" in.",
+  "patterns": ["logs/**/*.log"],
+  "paths": ["logs/a.log", "logs/x/y/b.log", "logs/c.txt", "other/logs/d.log"],
+  "expected_kept": ["logs/a.log", "logs/c.txt", "other/logs/d.log"]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/exclude/names-at-any-depth-and-anchored-paths.json

@@ -0,0 +1,6 @@
+{
+  "description": "A pattern without a slash matches a name at any depth and all below it; a pattern with a slash is matched from the root.",
+  "patterns": ["*.tmp", ".git", "/raw/scratch"],
+  "paths": ["a.tmp", "sub/b.tmp", "sub/b.tmpx", ".git/config", "src/.git/HEAD", "raw/scratch/x.dat", "other/raw/scratch/y.dat", "keep.txt"],
+  "expected_kept": ["sub/b.tmpx", "other/raw/scratch/y.dat", "keep.txt"]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/exclude/regular-expression-characters-are-literal.json

@@ -0,0 +1,6 @@
+{
+  "description": "Characters that mean something in a regular expression match themselves.",
+  "patterns": ["a(?).txt", "sub/x.y"],
+  "paths": ["a(1).txt", "a1.txt", "sub/x.y", "sub/xzy"],
+  "expected_kept": ["a1.txt", "sub/xzy"]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/exclude/trailing-slash-like-gitignore.json

@@ -0,0 +1,6 @@
+{
+  "description": "A trailing slash is ignored, so \".git/\" and \"build/\" leave out those folders wherever they are.",
+  "patterns": [".git/", "build/", "/raw/scratch/"],
+  "paths": [".git/config", "src/.git/HEAD", "build/a.o", "raw/scratch/x.dat", "other/raw/scratch/y.dat", "keep.txt"],
+  "expected_kept": ["other/raw/scratch/y.dat", "keep.txt"]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/by-checksum-compares-content-of-same-size.json

@@ -0,0 +1,19 @@
+{
+  "description": "With the Checksum comparison, the checksum decides where it is known on both sides, even against the time, and the time decides where it is not. Checksums compare without regard to case.",
+  "policy": { "comparison": "Checksum", "delete": false },
+  "source": [
+    { "path": "edited.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z", "hash": "aaa" },
+    { "path": "same.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z", "hash": "bbb" },
+    { "path": "unknown.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z", "hash": "" }
+  ],
+  "target": [
+    { "path": "edited.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z", "hash": "ccc" },
+    { "path": "same.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z", "hash": "BBB" },
+    { "path": "unknown.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z", "hash": "ddd" }
+  ],
+  "expected": [
+    { "path": "edited.txt", "action": "copy", "reason": "checksum", "bytes": 1 },
+    { "path": "same.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "unknown.txt", "action": "copy", "reason": "newer", "bytes": 1 }
+  ]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/by-size-ignores-time.json

@@ -0,0 +1,7 @@
+{
+  "description": "With the Size comparison, a newer source file of the same size is unchanged.",
+  "policy": { "comparison": "Size", "delete": false },
+  "source": [{ "path": "a.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z" }],
+  "target": [{ "path": "a.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z" }],
+  "expected": [{ "path": "a.txt", "action": "none", "reason": "unchanged", "bytes": 1 }]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/copies-file-changed-after-target.json

@@ -0,0 +1,19 @@
+{
+  "description": "A source file changed after the target was written is copied. A difference within 2 seconds, or a target newer than the source, is unchanged.",
+  "policy": { "comparison": "SizeAndTime", "delete": false },
+  "source": [
+    { "path": "edited.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z" },
+    { "path": "within-tolerance.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:01Z" },
+    { "path": "target-newer.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z" }
+  ],
+  "target": [
+    { "path": "edited.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z" },
+    { "path": "within-tolerance.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z" },
+    { "path": "target-newer.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z" }
+  ],
+  "expected": [
+    { "path": "edited.txt", "action": "copy", "reason": "newer", "bytes": 1 },
+    { "path": "target-newer.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "within-tolerance.txt", "action": "none", "reason": "unchanged", "bytes": 1 }
+  ]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/copies-new-and-changed-keeps-extraneous.json

@@ -0,0 +1,20 @@
+{
+  "description": "A new file and a file of another size are copied; a file of the same size is unchanged; a file only the target has is kept.",
+  "policy": { "comparison": "SizeAndTime", "delete": false },
+  "source": [
+    { "path": "new.txt", "bytes": 1 },
+    { "path": "same.txt", "bytes": 2 },
+    { "path": "grown.txt", "bytes": 30 }
+  ],
+  "target": [
+    { "path": "same.txt", "bytes": 2 },
+    { "path": "grown.txt", "bytes": 3 },
+    { "path": "extra.txt", "bytes": 4 }
+  ],
+  "expected": [
+    { "path": "extra.txt", "action": "none", "reason": "extraneous", "bytes": 4 },
+    { "path": "grown.txt", "action": "copy", "reason": "size", "bytes": 30 },
+    { "path": "new.txt", "action": "copy", "reason": "new", "bytes": 1 },
+    { "path": "same.txt", "action": "none", "reason": "unchanged", "bytes": 2 }
+  ]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/deletes-extraneous-when-asked.json

@@ -0,0 +1,13 @@
+{
+  "description": "With delete, a file only the target has is deleted.",
+  "policy": { "comparison": "SizeAndTime", "delete": true },
+  "source": [{ "path": "a.txt", "bytes": 1 }],
+  "target": [
+    { "path": "a.txt", "bytes": 1 },
+    { "path": "extra.txt", "bytes": 4 }
+  ],
+  "expected": [
+    { "path": "a.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "extra.txt", "action": "delete", "reason": "extraneous", "bytes": 4 }
+  ]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/empty-sides.json

@@ -0,0 +1,7 @@
+{
+  "description": "Two empty sides give an empty plan.",
+  "policy": { "comparison": "SizeAndTime", "delete": true },
+  "source": [],
+  "target": [],
+  "expected": []
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/leaves-file-of-unknown-time.json

@@ -0,0 +1,16 @@
+{
+  "description": "A file whose time is unknown on either side is left as it is.",
+  "policy": { "comparison": "SizeAndTime", "delete": false },
+  "source": [
+    { "path": "a.txt", "bytes": 1, "modified_time": "2024-01-01T00:00:00Z" },
+    { "path": "b.txt", "bytes": 1, "modified_time": null }
+  ],
+  "target": [
+    { "path": "a.txt", "bytes": 1, "modified_time": null },
+    { "path": "b.txt", "bytes": 1, "modified_time": "2000-01-01T00:00:00Z" }
+  ],
+  "expected": [
+    { "path": "a.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "b.txt", "action": "none", "reason": "unchanged", "bytes": 1 }
+  ]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/plan/size-decides-before-checksum-and-time.json

@@ -0,0 +1,7 @@
+{
+  "description": "A size difference is reported as such, before a checksum or time difference.",
+  "policy": { "comparison": "Checksum", "delete": false },
+  "source": [{ "path": "a.txt", "bytes": 1, "modified_time": "2024-01-02T00:00:00Z", "hash": "aaa" }],
+  "target": [{ "path": "a.txt", "bytes": 2, "modified_time": "2024-01-01T00:00:00Z", "hash": "bbb" }],
+  "expected": [{ "path": "a.txt", "action": "copy", "reason": "size", "bytes": 1 }]
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/refusal/deletions-within-limit.json

@@ -0,0 +1,11 @@
+{
+  "description": "Deletions up to max_delete are allowed.",
+  "plan": [
+    { "path": "keep.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "a.txt", "action": "delete", "reason": "extraneous", "bytes": 1 },
+    { "path": "b.txt", "action": "delete", "reason": "extraneous", "bytes": 1 }
+  ],
+  "n_source_files": 1,
+  "max_delete": 2,
+  "expected_code": null
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/refusal/empty-source-would-empty-target.json

@@ -0,0 +1,10 @@
+{
+  "description": "An empty source with deletions is refused.",
+  "plan": [
+    { "path": "a.txt", "action": "delete", "reason": "extraneous", "bytes": 1 },
+    { "path": "b.txt", "action": "delete", "reason": "extraneous", "bytes": 1 }
+  ],
+  "n_source_files": 0,
+  "max_delete": null,
+  "expected_code": "EmptySource"
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/refusal/more-deletions-than-allowed.json

@@ -0,0 +1,11 @@
+{
+  "description": "More deletions than max_delete are refused; as many are allowed.",
+  "plan": [
+    { "path": "keep.txt", "action": "none", "reason": "unchanged", "bytes": 1 },
+    { "path": "a.txt", "action": "delete", "reason": "extraneous", "bytes": 1 },
+    { "path": "b.txt", "action": "delete", "reason": "extraneous", "bytes": 1 }
+  ],
+  "n_source_files": 1,
+  "max_delete": 1,
+  "expected_code": "TooManyDeletions"
+}

ebrains_bucket_sync-0.1.0/spec/fixtures/refusal/no-deletions-planned.json

@@ -0,0 +1,7 @@
+{
+  "description": "Without deletions there is nothing to refuse, even for an empty source.",
+  "plan": [{ "path": "extra.txt", "action": "none", "reason": "extraneous", "bytes": 1 }],
+  "n_source_files": 0,
+  "max_delete": null,
+  "expected_code": null
+}

ebrains_bucket_sync-0.1.0/spec/sync-plan.schema.json

@@ -0,0 +1,42 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/ehennestad/ebrains-bucket-sync/spec/sync-plan.schema.json",
+  "title": "Sync plan",
+  "description": "The plan of a sync and, after a run, the outcome of each action.",
+  "type": "object",
+  "properties": {
+    "format": { "const": "ebrains-bucket-sync-plan" },
+    "version": { "const": 1 },
+    "direction": { "enum": ["to_bucket", "from_bucket"] },
+    "bucket": { "type": "string" },
+    "prefix": {
+      "description": "Folder of the bucket, ending with \"/\", or \"\" for the root.",
+      "type": "string"
+    },
+    "local_folder": { "type": "string" },
+    "policy": { "$ref": "sync-policy.schema.json" },
+    "dry_run": { "type": "boolean" },
+    "refusal": {
+      "description": "Why the run stopped, or would stop, before changing anything; \"\" otherwise.",
+      "type": "string"
+    },
+    "actions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": { "type": "string" },
+          "action": { "enum": ["upload", "download", "delete", "none"] },
+          "reason": { "enum": ["new", "size", "newer", "checksum", "unchanged", "extraneous"] },
+          "bytes": { "type": "integer", "minimum": 0 },
+          "status": { "enum": ["done", "failed", "skipped", "planned", ""] },
+          "message": { "type": "string" }
+        },
+        "required": ["path", "action", "reason", "bytes", "status", "message"],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": ["format", "version", "direction", "bucket", "prefix", "local_folder", "policy", "dry_run", "refusal", "actions"],
+  "additionalProperties": false
+}