forest-cli 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forest_cli-0.1.1/src/forest_cli.egg-info → forest_cli-0.2.0}/PKG-INFO +7 -2
- {forest_cli-0.1.1 → forest_cli-0.2.0}/README.md +2 -1
- {forest_cli-0.1.1 → forest_cli-0.2.0}/pyproject.toml +7 -1
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/cli.py +1 -1
- {forest_cli-0.1.1 → forest_cli-0.2.0/src/forest_cli.egg-info}/PKG-INFO +7 -2
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli.py +8 -8
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_checkout_adopt.py +2 -2
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_remote_workspace.py +2 -2
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_config.py +16 -16
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_config_migration.py +5 -5
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_rclone.py +12 -9
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_s3_integration.py +8 -9
- {forest_cli-0.1.1 → forest_cli-0.2.0}/LICENSE +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/setup.cfg +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/__init__.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/analytics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/checkout.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/config.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/flags.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/flow.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/fsutil.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/logger.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/manifest.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/metrics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/monitoring.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/paths.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/rclone.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/resilience.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest/sync_state.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest_cli.egg-info/SOURCES.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest_cli.egg-info/dependency_links.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest_cli.egg-info/entry_points.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest_cli.egg-info/requires.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/src/forest_cli.egg-info/top_level.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_analytics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_bare_command_defaults.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_checkout.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_checkout_scaffold.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_checkout_lifecycle.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_cross_area_flows.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_gitignore.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_init.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_resolution.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_stage_binding.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cli_workspace_data_commands.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_cross_backend_parity_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_flags.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_flow.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_forest_agnostic.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_integration.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_legacy_remote_path_safety.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_ls_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_manifest.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_migrate.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_moat_regression_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_paths.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_polish_scrutiny_fixes.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_pull_core_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_pull_resilience_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_push_core_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_push_targeting_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_rclone_gating_errors.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_redaction.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_s3_url_error_hardening.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_scrutiny_fixes.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_sftp_hardening.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_status_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_sync_state.py +0 -0
- {forest_cli-0.1.1 → forest_cli-0.2.0}/tests/test_user_testing_fixes.py +0 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: forest-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
|
|
5
5
|
Author-email: Troy Sincomb <troysincomb@gmail.com>
|
|
6
6
|
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tmsincomb/forest
|
|
8
|
+
Project-URL: Repository, https://github.com/tmsincomb/forest
|
|
9
|
+
Project-URL: Issues, https://github.com/tmsincomb/forest/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/tmsincomb/forest/blob/main/CHANGELOG.md
|
|
7
11
|
Keywords: data-management,sync,cli
|
|
8
12
|
Classifier: Development Status :: 3 - Alpha
|
|
9
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -39,6 +43,7 @@ Dynamic: license-file
|
|
|
39
43
|
[](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
|
|
40
44
|
[](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
|
|
41
45
|
[](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
|
|
46
|
+
[](https://pypi.org/project/forest-cli/)
|
|
42
47
|
[](https://www.python.org/downloads/)
|
|
43
48
|
[](LICENSE)
|
|
44
49
|
[](https://github.com/astral-sh/ruff)
|
|
@@ -193,7 +198,7 @@ set up with exactly the quick-start commands:
|
|
|
193
198
|
```bash
|
|
194
199
|
forest init
|
|
195
200
|
forest checkout demo
|
|
196
|
-
forest remote add origin s3://forest-
|
|
201
|
+
forest remote add origin s3://my-forest-bucket --region us-east-2
|
|
197
202
|
forest add examples ./examples
|
|
198
203
|
forest push
|
|
199
204
|
```
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
[](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
|
|
6
6
|
[](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
|
|
7
7
|
[](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
|
|
8
|
+
[](https://pypi.org/project/forest-cli/)
|
|
8
9
|
[](https://www.python.org/downloads/)
|
|
9
10
|
[](LICENSE)
|
|
10
11
|
[](https://github.com/astral-sh/ruff)
|
|
@@ -159,7 +160,7 @@ set up with exactly the quick-start commands:
|
|
|
159
160
|
```bash
|
|
160
161
|
forest init
|
|
161
162
|
forest checkout demo
|
|
162
|
-
forest remote add origin s3://forest-
|
|
163
|
+
forest remote add origin s3://my-forest-bucket --region us-east-2
|
|
163
164
|
forest add examples ./examples
|
|
164
165
|
forest push
|
|
165
166
|
```
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forest-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -25,6 +25,12 @@ dependencies = [
|
|
|
25
25
|
"pyyaml>=6.0",
|
|
26
26
|
]
|
|
27
27
|
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/tmsincomb/forest"
|
|
30
|
+
Repository = "https://github.com/tmsincomb/forest"
|
|
31
|
+
Issues = "https://github.com/tmsincomb/forest/issues"
|
|
32
|
+
Changelog = "https://github.com/tmsincomb/forest/blob/main/CHANGELOG.md"
|
|
33
|
+
|
|
28
34
|
[project.optional-dependencies]
|
|
29
35
|
# Opt-in error tracking (FOREST_SENTRY_DSN); see docs/runbooks/alerting.md.
|
|
30
36
|
observability = [
|
|
@@ -1048,7 +1048,7 @@ def _validate_remote_add_inputs(
|
|
|
1048
1048
|
@click.argument("url")
|
|
1049
1049
|
@click.option("--region", default=None, help="S3 region (e.g. us-east-2).")
|
|
1050
1050
|
@click.option("--endpoint", default=None, help="S3-compatible endpoint URL (non-AWS S3).")
|
|
1051
|
-
@click.option("--profile", default=None, help="S3 auth profile name (e.g.
|
|
1051
|
+
@click.option("--profile", default=None, help="S3 auth profile name (e.g. myprofile).")
|
|
1052
1052
|
@click.option("--key-file", "key_file", default=None, help="SFTP private key file path.")
|
|
1053
1053
|
@click.option("--known-hosts", "known_hosts", default=None, help="SFTP known_hosts file path.")
|
|
1054
1054
|
@click.pass_context
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: forest-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
|
|
5
5
|
Author-email: Troy Sincomb <troysincomb@gmail.com>
|
|
6
6
|
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tmsincomb/forest
|
|
8
|
+
Project-URL: Repository, https://github.com/tmsincomb/forest
|
|
9
|
+
Project-URL: Issues, https://github.com/tmsincomb/forest/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/tmsincomb/forest/blob/main/CHANGELOG.md
|
|
7
11
|
Keywords: data-management,sync,cli
|
|
8
12
|
Classifier: Development Status :: 3 - Alpha
|
|
9
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -39,6 +43,7 @@ Dynamic: license-file
|
|
|
39
43
|
[](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
|
|
40
44
|
[](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
|
|
41
45
|
[](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
|
|
46
|
+
[](https://pypi.org/project/forest-cli/)
|
|
42
47
|
[](https://www.python.org/downloads/)
|
|
43
48
|
[](LICENSE)
|
|
44
49
|
[](https://github.com/astral-sh/ruff)
|
|
@@ -193,7 +198,7 @@ set up with exactly the quick-start commands:
|
|
|
193
198
|
```bash
|
|
194
199
|
forest init
|
|
195
200
|
forest checkout demo
|
|
196
|
-
forest remote add origin s3://forest-
|
|
201
|
+
forest remote add origin s3://my-forest-bucket --region us-east-2
|
|
197
202
|
forest add examples ./examples
|
|
198
203
|
forest push
|
|
199
204
|
```
|
|
@@ -79,13 +79,13 @@ class TestRemoteAdd:
|
|
|
79
79
|
runner = CliRunner()
|
|
80
80
|
result = _invoke(
|
|
81
81
|
runner,
|
|
82
|
-
["remote", "add", "hpc", "user@hpc.
|
|
82
|
+
["remote", "add", "hpc", "user@hpc.myprofile.edu:/scratch/data/G004"],
|
|
83
83
|
config_path,
|
|
84
84
|
)
|
|
85
85
|
assert result.exit_code == 0, result.output
|
|
86
86
|
data = yaml.safe_load(config_path.read_text())
|
|
87
87
|
assert "hpc" in data["remotes"]
|
|
88
|
-
assert data["remotes"]["hpc"]["url"] == "user@hpc.
|
|
88
|
+
assert data["remotes"]["hpc"]["url"] == "user@hpc.myprofile.edu:/scratch/data/G004"
|
|
89
89
|
|
|
90
90
|
def test_first_remote_auto_active(self, tmp_path):
|
|
91
91
|
"""VAL-REMOTE-004: First remote added becomes active automatically."""
|
|
@@ -177,14 +177,14 @@ class TestRemoteAddNewFields:
|
|
|
177
177
|
"--region",
|
|
178
178
|
"us-east-2",
|
|
179
179
|
"--profile",
|
|
180
|
-
"
|
|
180
|
+
"myprofile",
|
|
181
181
|
],
|
|
182
182
|
config_path,
|
|
183
183
|
)
|
|
184
184
|
assert result.exit_code == 0, result.output
|
|
185
185
|
reloaded = load_config(config_path)
|
|
186
186
|
assert reloaded.remotes["s3prod"].region == "us-east-2"
|
|
187
|
-
assert reloaded.remotes["s3prod"].profile == "
|
|
187
|
+
assert reloaded.remotes["s3prod"].profile == "myprofile"
|
|
188
188
|
|
|
189
189
|
def test_add_s3_with_endpoint(self, tmp_path):
|
|
190
190
|
"""remote add --endpoint persists the non-AWS S3 endpoint."""
|
|
@@ -417,7 +417,7 @@ class TestRemoteShow:
|
|
|
417
417
|
"remotes": {
|
|
418
418
|
"origin": {"url": "s3://bucket/project"},
|
|
419
419
|
"backup": {"url": "/Volumes/rog/G004"},
|
|
420
|
-
"hpc": {"url": "user@hpc.
|
|
420
|
+
"hpc": {"url": "user@hpc.myprofile.edu:/scratch/data/G004"},
|
|
421
421
|
},
|
|
422
422
|
}
|
|
423
423
|
config_path = tmp_path / "forest.yaml"
|
|
@@ -448,7 +448,7 @@ class TestRemoteShow:
|
|
|
448
448
|
runner = CliRunner()
|
|
449
449
|
result = _invoke(runner, ["remote", "show", "hpc"], config_path)
|
|
450
450
|
assert result.exit_code == 0, result.output
|
|
451
|
-
assert "user@hpc.
|
|
451
|
+
assert "user@hpc.myprofile.edu:/scratch/data/G004" in result.output
|
|
452
452
|
assert "sftp" in result.output
|
|
453
453
|
|
|
454
454
|
def test_show_nonexistent_fails(self, tmp_path):
|
|
@@ -468,7 +468,7 @@ class TestRemoteShow:
|
|
|
468
468
|
"url": "s3://bucket/project",
|
|
469
469
|
"region": "us-east-2",
|
|
470
470
|
"endpoint": "https://minio.example",
|
|
471
|
-
"profile": "
|
|
471
|
+
"profile": "myprofile",
|
|
472
472
|
},
|
|
473
473
|
},
|
|
474
474
|
}
|
|
@@ -479,7 +479,7 @@ class TestRemoteShow:
|
|
|
479
479
|
assert result.exit_code == 0
|
|
480
480
|
assert "Region: us-east-2" in result.output
|
|
481
481
|
assert "Endpoint: https://minio.example" in result.output
|
|
482
|
-
assert "Profile:
|
|
482
|
+
assert "Profile: myprofile" in result.output
|
|
483
483
|
|
|
484
484
|
|
|
485
485
|
class TestRemoteNoConfig:
|
|
@@ -59,7 +59,7 @@ def _write_legacy_project(root: Path, *, remote_collision: bool = False, bad_sta
|
|
|
59
59
|
"url": str(remote),
|
|
60
60
|
"region": "us-east-2",
|
|
61
61
|
"endpoint": "https://minio.example",
|
|
62
|
-
"profile": "
|
|
62
|
+
"profile": "myprofile",
|
|
63
63
|
"key_file": "/tmp/no-secret-key-path",
|
|
64
64
|
"known_hosts": "/tmp/no-secret-known-hosts",
|
|
65
65
|
},
|
|
@@ -157,7 +157,7 @@ def test_checkout_adopt_copies_splits_verifies_deletes_and_resolves(git_repo: Pa
|
|
|
157
157
|
"url": str(legacy["remote"]),
|
|
158
158
|
"region": "us-east-2",
|
|
159
159
|
"endpoint": "https://minio.example",
|
|
160
|
-
"profile": "
|
|
160
|
+
"profile": "myprofile",
|
|
161
161
|
"key_file": "/tmp/no-secret-key-path",
|
|
162
162
|
"known_hosts": "/tmp/no-secret-known-hosts",
|
|
163
163
|
}
|
|
@@ -91,7 +91,7 @@ def test_remote_add_workspace_shared_only_no_default_and_local_type(git_repo: Pa
|
|
|
91
91
|
"--endpoint",
|
|
92
92
|
"https://minio.example",
|
|
93
93
|
"--profile",
|
|
94
|
-
"
|
|
94
|
+
"myprofile",
|
|
95
95
|
],
|
|
96
96
|
)
|
|
97
97
|
|
|
@@ -101,7 +101,7 @@ def test_remote_add_workspace_shared_only_no_default_and_local_type(git_repo: Pa
|
|
|
101
101
|
"url": "s3://bucket/p",
|
|
102
102
|
"region": "us-east-2",
|
|
103
103
|
"endpoint": "https://minio.example",
|
|
104
|
-
"profile": "
|
|
104
|
+
"profile": "myprofile",
|
|
105
105
|
}
|
|
106
106
|
assert _read_yaml(_local_path(git_repo)) == {"stage_paths": {"raw": "data/raw"}}
|
|
107
107
|
|
|
@@ -31,7 +31,7 @@ VALID_CONFIG = {
|
|
|
31
31
|
"url": "/Volumes/rog/G004",
|
|
32
32
|
},
|
|
33
33
|
"hpc": {
|
|
34
|
-
"url": "user@hpc.
|
|
34
|
+
"url": "user@hpc.example.edu:/scratch/data/G004",
|
|
35
35
|
},
|
|
36
36
|
},
|
|
37
37
|
"stages": {
|
|
@@ -169,9 +169,9 @@ class TestRemoteConfig:
|
|
|
169
169
|
|
|
170
170
|
def test_sftp_type_inference(self):
|
|
171
171
|
"""Remote type inferred as 'sftp' for user@host: and sftp:// URLs."""
|
|
172
|
-
remote = RemoteConfig(url="user@hpc.
|
|
172
|
+
remote = RemoteConfig(url="user@hpc.example.edu:/scratch/data/G004")
|
|
173
173
|
assert remote.remote_type == "sftp"
|
|
174
|
-
assert RemoteConfig(url="sftp://hpc.
|
|
174
|
+
assert RemoteConfig(url="sftp://hpc.example.edu/scratch/data/G004").remote_type == "sftp"
|
|
175
175
|
|
|
176
176
|
def test_region_optional(self):
|
|
177
177
|
"""Region is optional and defaults to None."""
|
|
@@ -202,8 +202,8 @@ class TestRemoteConfigNewFields:
|
|
|
202
202
|
|
|
203
203
|
def test_profile_stored(self):
|
|
204
204
|
"""profile is stored when provided."""
|
|
205
|
-
remote = RemoteConfig(url="s3://bucket/path", profile="
|
|
206
|
-
assert remote.profile == "
|
|
205
|
+
remote = RemoteConfig(url="s3://bucket/path", profile="myprofile")
|
|
206
|
+
assert remote.profile == "myprofile"
|
|
207
207
|
|
|
208
208
|
def test_key_file_stored(self):
|
|
209
209
|
"""key_file is stored when provided."""
|
|
@@ -221,21 +221,21 @@ class TestRemoteConfigNewFields:
|
|
|
221
221
|
url="s3://bucket/path",
|
|
222
222
|
region="us-east-2",
|
|
223
223
|
endpoint="https://minio.local",
|
|
224
|
-
profile="
|
|
224
|
+
profile="myprofile",
|
|
225
225
|
key_file="/keys/id",
|
|
226
226
|
known_hosts="/keys/kh",
|
|
227
227
|
)
|
|
228
228
|
local = RemoteConfig(
|
|
229
229
|
url="/abs/local/path",
|
|
230
230
|
endpoint="https://minio.local",
|
|
231
|
-
profile="
|
|
231
|
+
profile="myprofile",
|
|
232
232
|
key_file="/keys/id",
|
|
233
233
|
known_hosts="/keys/kh",
|
|
234
234
|
)
|
|
235
235
|
sftp = RemoteConfig(
|
|
236
236
|
url="user@host:/path",
|
|
237
237
|
endpoint="https://minio.local",
|
|
238
|
-
profile="
|
|
238
|
+
profile="myprofile",
|
|
239
239
|
key_file="/keys/id",
|
|
240
240
|
known_hosts="/keys/kh",
|
|
241
241
|
)
|
|
@@ -296,8 +296,8 @@ class TestSaveConfigRemotes:
|
|
|
296
296
|
|
|
297
297
|
def test_profile_roundtrips_individually(self, tmp_path: Path):
|
|
298
298
|
"""VAL-CONFIG-001: profile round-trips on its own."""
|
|
299
|
-
out = self._roundtrip(tmp_path, RemoteConfig(url="s3://b/p", profile="
|
|
300
|
-
assert out.profile == "
|
|
299
|
+
out = self._roundtrip(tmp_path, RemoteConfig(url="s3://b/p", profile="myprofile"))
|
|
300
|
+
assert out.profile == "myprofile"
|
|
301
301
|
assert out.endpoint is None and out.key_file is None and out.known_hosts is None
|
|
302
302
|
|
|
303
303
|
def test_key_file_roundtrips_individually(self, tmp_path: Path):
|
|
@@ -318,7 +318,7 @@ class TestSaveConfigRemotes:
|
|
|
318
318
|
url="s3://b/p",
|
|
319
319
|
region="us-east-2",
|
|
320
320
|
endpoint="https://minio.local",
|
|
321
|
-
profile="
|
|
321
|
+
profile="myprofile",
|
|
322
322
|
key_file="/keys/id",
|
|
323
323
|
known_hosts="/keys/kh",
|
|
324
324
|
)
|
|
@@ -326,7 +326,7 @@ class TestSaveConfigRemotes:
|
|
|
326
326
|
assert out.url == "s3://b/p"
|
|
327
327
|
assert out.region == "us-east-2"
|
|
328
328
|
assert out.endpoint == "https://minio.local"
|
|
329
|
-
assert out.profile == "
|
|
329
|
+
assert out.profile == "myprofile"
|
|
330
330
|
assert out.key_file == "/keys/id"
|
|
331
331
|
assert out.known_hosts == "/keys/kh"
|
|
332
332
|
|
|
@@ -335,7 +335,7 @@ class TestSaveConfigRemotes:
|
|
|
335
335
|
remote = RemoteConfig(
|
|
336
336
|
url="s3://b/p",
|
|
337
337
|
endpoint="https://minio.local",
|
|
338
|
-
profile="
|
|
338
|
+
profile="myprofile",
|
|
339
339
|
key_file="/keys/id",
|
|
340
340
|
known_hosts="/keys/kh",
|
|
341
341
|
)
|
|
@@ -349,7 +349,7 @@ class TestSaveConfigRemotes:
|
|
|
349
349
|
|
|
350
350
|
mapping = yaml.safe_load(config_path.read_text())["remotes"]["r"]
|
|
351
351
|
assert mapping["endpoint"] == "https://minio.local"
|
|
352
|
-
assert mapping["profile"] == "
|
|
352
|
+
assert mapping["profile"] == "myprofile"
|
|
353
353
|
assert mapping["key_file"] == "/keys/id"
|
|
354
354
|
assert mapping["known_hosts"] == "/keys/kh"
|
|
355
355
|
|
|
@@ -655,7 +655,7 @@ class TestLoadConfig:
|
|
|
655
655
|
"origin": {
|
|
656
656
|
"url": "s3://bucket/path",
|
|
657
657
|
"region": "us-west-2",
|
|
658
|
-
"profile": "
|
|
658
|
+
"profile": "myprofile",
|
|
659
659
|
"transport": {
|
|
660
660
|
"backend": "legacy",
|
|
661
661
|
"settings": {"max_concurrency": 10},
|
|
@@ -673,5 +673,5 @@ class TestLoadConfig:
|
|
|
673
673
|
|
|
674
674
|
assert config.remotes["origin"].url == "s3://bucket/path"
|
|
675
675
|
assert config.remotes["origin"].region == "us-west-2"
|
|
676
|
-
assert config.remotes["origin"].profile == "
|
|
676
|
+
assert config.remotes["origin"].profile == "myprofile"
|
|
677
677
|
assert not hasattr(config.remotes["origin"], "transport")
|
|
@@ -40,7 +40,7 @@ def test_load_legacy_transport_warns_once_and_preserves_new_fields(tmp_path, cap
|
|
|
40
40
|
"url": "s3://bucket/prefix",
|
|
41
41
|
"region": "us-east-2",
|
|
42
42
|
"endpoint": "https://minio.example",
|
|
43
|
-
"profile": "
|
|
43
|
+
"profile": "myprofile",
|
|
44
44
|
"key_file": "/keys/id",
|
|
45
45
|
"known_hosts": "/keys/known_hosts",
|
|
46
46
|
"transport": {"backend": "s5cmd"},
|
|
@@ -62,7 +62,7 @@ def test_load_legacy_transport_warns_once_and_preserves_new_fields(tmp_path, cap
|
|
|
62
62
|
assert captured.err.count(LEGACY_TRANSPORT_WARNING) == 1
|
|
63
63
|
assert not hasattr(config.remotes["origin"], "transport")
|
|
64
64
|
assert config.remotes["origin"].endpoint == "https://minio.example"
|
|
65
|
-
assert config.remotes["origin"].profile == "
|
|
65
|
+
assert config.remotes["origin"].profile == "myprofile"
|
|
66
66
|
assert config.remotes["origin"].key_file == "/keys/id"
|
|
67
67
|
assert config.remotes["origin"].known_hosts == "/keys/known_hosts"
|
|
68
68
|
|
|
@@ -73,7 +73,7 @@ def test_load_legacy_transport_warns_once_and_preserves_new_fields(tmp_path, cap
|
|
|
73
73
|
saved = yaml.safe_load(config_path.read_text())
|
|
74
74
|
assert "transport" not in saved["remotes"]["origin"]
|
|
75
75
|
assert saved["remotes"]["origin"]["endpoint"] == "https://minio.example"
|
|
76
|
-
assert saved["remotes"]["origin"]["profile"] == "
|
|
76
|
+
assert saved["remotes"]["origin"]["profile"] == "myprofile"
|
|
77
77
|
assert saved["remotes"]["origin"]["key_file"] == "/keys/id"
|
|
78
78
|
assert saved["remotes"]["origin"]["known_hosts"] == "/keys/known_hosts"
|
|
79
79
|
|
|
@@ -91,7 +91,7 @@ def test_remote_show_displays_new_fields_and_no_transport_tokens(tmp_path):
|
|
|
91
91
|
"url": "s3://bucket/project",
|
|
92
92
|
"region": "us-east-2",
|
|
93
93
|
"endpoint": "https://minio.example",
|
|
94
|
-
"profile": "
|
|
94
|
+
"profile": "myprofile",
|
|
95
95
|
},
|
|
96
96
|
"backup": {"url": "/tmp/forest-backup"},
|
|
97
97
|
"hpc": {
|
|
@@ -111,7 +111,7 @@ def test_remote_show_displays_new_fields_and_no_transport_tokens(tmp_path):
|
|
|
111
111
|
"Type: s3",
|
|
112
112
|
"Region: us-east-2",
|
|
113
113
|
"Endpoint: https://minio.example",
|
|
114
|
-
"Profile:
|
|
114
|
+
"Profile: myprofile",
|
|
115
115
|
],
|
|
116
116
|
"backup": ["URL: /tmp/forest-backup", "Type: local"],
|
|
117
117
|
"hpc": [
|
|
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|
|
10
10
|
|
|
11
11
|
import configparser
|
|
12
12
|
import json
|
|
13
|
+
import os
|
|
13
14
|
import shutil
|
|
14
15
|
import subprocess
|
|
15
16
|
from pathlib import Path
|
|
@@ -80,10 +81,10 @@ class TestIsAvailable:
|
|
|
80
81
|
|
|
81
82
|
class TestBuildRemoteS3:
|
|
82
83
|
def test_full_recipe(self) -> None:
|
|
83
|
-
cfg = RemoteConfig(url="s3://BUCKET/PREFIX", region="us-east-2", profile="
|
|
84
|
+
cfg = RemoteConfig(url="s3://BUCKET/PREFIX", region="us-east-2", profile="myprofile")
|
|
84
85
|
connstr = build_remote(cfg)
|
|
85
86
|
assert connstr == (
|
|
86
|
-
":s3,provider=AWS,env_auth=true,profile=
|
|
87
|
+
":s3,provider=AWS,env_auth=true,profile=myprofile,region=us-east-2,no_check_bucket=true:BUCKET/PREFIX"
|
|
87
88
|
)
|
|
88
89
|
assert "env_auth=true" in connstr
|
|
89
90
|
assert "no_check_bucket=true" in connstr
|
|
@@ -92,7 +93,7 @@ class TestBuildRemoteS3:
|
|
|
92
93
|
assert "s3://" not in connstr
|
|
93
94
|
|
|
94
95
|
def test_omits_region_when_none(self) -> None:
|
|
95
|
-
cfg = RemoteConfig(url="s3://bucket/prefix", region=None, profile="
|
|
96
|
+
cfg = RemoteConfig(url="s3://bucket/prefix", region=None, profile="myprofile")
|
|
96
97
|
connstr = build_remote(cfg)
|
|
97
98
|
assert "region=" not in connstr
|
|
98
99
|
assert "env_auth=true" in connstr
|
|
@@ -426,7 +427,7 @@ class TestLsjson:
|
|
|
426
427
|
assert "._*" in cmd
|
|
427
428
|
|
|
428
429
|
def test_no_exclude_on_s3_connstring(self) -> None:
|
|
429
|
-
connstr = build_remote(RemoteConfig(url="s3://bucket/prefix", region="us-east-2", profile="
|
|
430
|
+
connstr = build_remote(RemoteConfig(url="s3://bucket/prefix", region="us-east-2", profile="myprofile"))
|
|
430
431
|
with patch(
|
|
431
432
|
"forest.rclone.subprocess.run",
|
|
432
433
|
side_effect=lambda cmd, *a, **k: _completed(cmd, 0, stdout="[]"),
|
|
@@ -722,7 +723,7 @@ class TestRcloneTransportPull:
|
|
|
722
723
|
|
|
723
724
|
class TestRcloneTransportMisc:
|
|
724
725
|
def test_ls_delegates_to_lsjson_with_built_root(self) -> None:
|
|
725
|
-
cfg = RemoteConfig(url="s3://bucket/prefix", region="us-east-2", profile="
|
|
726
|
+
cfg = RemoteConfig(url="s3://bucket/prefix", region="us-east-2", profile="myprofile")
|
|
726
727
|
sentinel = [RemoteFile(key="a.txt", size=1)]
|
|
727
728
|
with patch("forest.rclone.lsjson", return_value=sentinel) as m:
|
|
728
729
|
out = rclone.get_transport(cfg).ls(cfg.url, prefix="multi")
|
|
@@ -766,9 +767,11 @@ RCLONE_PRESENT = shutil.which("rclone") is not None
|
|
|
766
767
|
USB_ROOT = Path("/Volumes/trials/G004-rclone")
|
|
767
768
|
USB_PRESENT = Path("/Volumes/trials").is_dir()
|
|
768
769
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
770
|
+
# Real-S3 config comes from the environment (same vars as
|
|
771
|
+
# tests/test_s3_integration.py) so nothing environment-specific lands in git.
|
|
772
|
+
S3_BUCKET = os.environ.get("FOREST_TEST_S3_BUCKET", "")
|
|
773
|
+
S3_REGION = os.environ.get("FOREST_TEST_S3_REGION", "us-east-2")
|
|
774
|
+
S3_PROFILE = os.environ.get("FOREST_TEST_S3_PROFILE", "")
|
|
772
775
|
|
|
773
776
|
|
|
774
777
|
def _aws_profile_present(profile: str) -> bool:
|
|
@@ -788,7 +791,7 @@ def _aws_profile_present(profile: str) -> bool:
|
|
|
788
791
|
|
|
789
792
|
|
|
790
793
|
def _s3_reachable() -> bool:
|
|
791
|
-
if not RCLONE_PRESENT or not _aws_profile_present(S3_PROFILE):
|
|
794
|
+
if not RCLONE_PRESENT or not S3_BUCKET or not _aws_profile_present(S3_PROFILE):
|
|
792
795
|
return False
|
|
793
796
|
connstr = build_remote(RemoteConfig(url=f"s3://{S3_BUCKET}", region=S3_REGION, profile=S3_PROFILE))
|
|
794
797
|
try:
|
|
@@ -7,9 +7,9 @@ catch S3-specific auth, region, or URL bugs.
|
|
|
7
7
|
Opt in via environment, so it skips cleanly for contributors without
|
|
8
8
|
credentials::
|
|
9
9
|
|
|
10
|
-
export FOREST_TEST_S3_BUCKET=forest-test-
|
|
11
|
-
export FOREST_TEST_S3_PROFILE
|
|
12
|
-
export FOREST_TEST_S3_REGION
|
|
10
|
+
export FOREST_TEST_S3_BUCKET=forest-test-<your-suffix>
|
|
11
|
+
export FOREST_TEST_S3_PROFILE=<your-aws-profile>
|
|
12
|
+
export FOREST_TEST_S3_REGION=<your-region>
|
|
13
13
|
pytest tests/test_s3_integration.py -q
|
|
14
14
|
|
|
15
15
|
Every run writes under a unique ``pytest/<uuid>/`` prefix and purges it on
|
|
@@ -34,9 +34,6 @@ from forest.cli import main
|
|
|
34
34
|
from forest.config import RemoteConfig
|
|
35
35
|
from forest.rclone import build_remote
|
|
36
36
|
|
|
37
|
-
EXPECTED_BUCKET = "forest-test-542222635421-us-east-2-an"
|
|
38
|
-
EXPECTED_PROFILE = "scripps"
|
|
39
|
-
EXPECTED_REGION = "us-east-2"
|
|
40
37
|
RUN_PREFIX_RE = re.compile(r"^pytest/[0-9a-f]{32}$")
|
|
41
38
|
|
|
42
39
|
# Prefer the documented FOREST_TEST_S3_* names; fall back to legacy BIOSTORE_TEST_S3_*.
|
|
@@ -52,9 +49,11 @@ pytestmark = pytest.mark.skipif(
|
|
|
52
49
|
|
|
53
50
|
@pytest.fixture(autouse=True)
|
|
54
51
|
def _require_dedicated_s3_env():
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
assert
|
|
52
|
+
# Purge safety: only ever run against a dedicated forest-test-* bucket,
|
|
53
|
+
# never a shared or production one.
|
|
54
|
+
assert BUCKET and BUCKET.startswith("forest-test-"), (
|
|
55
|
+
f"FOREST_TEST_S3_BUCKET must name a dedicated forest-test-* bucket, got {BUCKET!r}"
|
|
56
|
+
)
|
|
58
57
|
|
|
59
58
|
|
|
60
59
|
def _run(runner: CliRunner, root: Path, *args: str):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|