forest-cli 0.1.1__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forest_cli-0.1.1 → forest_cli-1.0.0}/PKG-INFO +97 -30
- {forest_cli-0.1.1 → forest_cli-1.0.0}/README.md +87 -28
- {forest_cli-0.1.1 → forest_cli-1.0.0}/pyproject.toml +25 -3
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/checkout.py +19 -40
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/cli.py +1163 -387
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/config.py +7 -19
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/fsutil.py +12 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/manifest.py +0 -22
- forest_cli-1.0.0/src/forest/output.py +349 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/paths.py +1 -56
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/rclone.py +221 -18
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/resilience.py +1 -1
- forest_cli-1.0.0/src/forest/scenery.py +703 -0
- forest_cli-1.0.0/src/forest/shell_prompt.py +94 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/sync_state.py +40 -7
- forest_cli-1.0.0/src/forest/tui_app.py +283 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/PKG-INFO +97 -30
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/SOURCES.txt +15 -2
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/requires.txt +4 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli.py +27 -13
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_checkout_adopt.py +2 -2
- forest_cli-1.0.0/tests/test_cli_prompt.py +170 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_remote_workspace.py +2 -2
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_config.py +34 -34
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_config_migration.py +5 -5
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cross_backend_parity_lock.py +2 -1
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_ls_lock.py +66 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_manifest.py +0 -101
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_moat_regression_lock.py +2 -0
- forest_cli-1.0.0/tests/test_output.py +213 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_paths.py +0 -151
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_pull_core_lock.py +120 -1
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_pull_resilience_lock.py +3 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_push_core_lock.py +138 -9
- forest_cli-1.0.0/tests/test_python_support_metadata.py +9 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_rclone.py +77 -19
- forest_cli-1.0.0/tests/test_rclone_progress.py +165 -0
- forest_cli-1.0.0/tests/test_rich_render.py +129 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_s3_integration.py +16 -14
- forest_cli-1.0.0/tests/test_scenery.py +360 -0
- forest_cli-1.0.0/tests/test_stage_direction.py +364 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_status_lock.py +206 -0
- forest_cli-1.0.0/tests/test_transfer_lock_scope.py +159 -0
- forest_cli-1.0.0/tests/test_tui_app.py +160 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_user_testing_fixes.py +9 -2
- forest_cli-1.0.0/tests/test_verify_lock.py +151 -0
- forest_cli-0.1.1/tests/test_migrate.py +0 -73
- {forest_cli-0.1.1 → forest_cli-1.0.0}/LICENSE +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/setup.cfg +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/__init__.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/analytics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/flags.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/flow.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/logger.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/metrics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest/monitoring.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/dependency_links.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/entry_points.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/src/forest_cli.egg-info/top_level.txt +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_analytics.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_bare_command_defaults.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_checkout.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_checkout_scaffold.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_checkout_lifecycle.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_cross_area_flows.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_gitignore.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_init.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_resolution.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_stage_binding.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_cli_workspace_data_commands.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_flags.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_flow.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_forest_agnostic.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_integration.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_legacy_remote_path_safety.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_polish_scrutiny_fixes.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_push_targeting_lock.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_rclone_gating_errors.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_redaction.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_s3_url_error_hardening.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_scrutiny_fixes.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_sftp_hardening.py +0 -0
- {forest_cli-0.1.1 → forest_cli-1.0.0}/tests/test_sync_state.py +0 -0
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: forest-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
|
|
5
5
|
Author-email: Troy Sincomb <troysincomb@gmail.com>
|
|
6
6
|
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tmsincomb/forest
|
|
8
|
+
Project-URL: Repository, https://github.com/tmsincomb/forest
|
|
9
|
+
Project-URL: Issues, https://github.com/tmsincomb/forest/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/tmsincomb/forest/blob/main/CHANGELOG.md
|
|
7
11
|
Keywords: data-management,sync,cli
|
|
8
12
|
Classifier: Development Status :: 3 - Alpha
|
|
9
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
14
|
Classifier: Programming Language :: Python :: 3
|
|
11
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10
|
|
12
16
|
Description-Content-Type: text/markdown
|
|
13
17
|
License-File: LICENSE
|
|
14
18
|
Requires-Dist: click>=8.0
|
|
15
19
|
Requires-Dist: polars>=1.0
|
|
16
20
|
Requires-Dist: pydantic>=2.0
|
|
17
21
|
Requires-Dist: pyyaml>=6.0
|
|
22
|
+
Requires-Dist: rich>=14.2
|
|
23
|
+
Requires-Dist: rich-click>=1.9.8
|
|
24
|
+
Requires-Dist: textual>=8
|
|
18
25
|
Provides-Extra: observability
|
|
19
26
|
Requires-Dist: sentry-sdk>=2.0; extra == "observability"
|
|
20
27
|
Provides-Extra: dev
|
|
@@ -23,6 +30,7 @@ Requires-Dist: mypy>=1.14; extra == "dev"
|
|
|
23
30
|
Requires-Dist: pre-commit>=4.0; extra == "dev"
|
|
24
31
|
Requires-Dist: pylint>=3.3; extra == "dev"
|
|
25
32
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
26
34
|
Requires-Dist: pytest-cov>=6.0; extra == "dev"
|
|
27
35
|
Requires-Dist: pytest-randomly>=3.15; extra == "dev"
|
|
28
36
|
Requires-Dist: pytest-rerunfailures>=15.0; extra == "dev"
|
|
@@ -32,15 +40,16 @@ Requires-Dist: types-PyYAML; extra == "dev"
|
|
|
32
40
|
Requires-Dist: vulture>=2.14; extra == "dev"
|
|
33
41
|
Dynamic: license-file
|
|
34
42
|
|
|
35
|
-
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="
|
|
43
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="https://tmsincomb.github.io/forest/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
36
44
|
|
|
37
45
|
**git-like data management for arbitrary data trees.**
|
|
38
46
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
[](https://tmsincomb.github.io/forest/)
|
|
50
|
+
[](https://pypi.org/project/forest-cli/)
|
|
51
|
+
[](https://www.python.org/downloads/)
|
|
52
|
+
[](https://github.com/tmsincomb/forest/blob/main/LICENSE)
|
|
44
53
|
[](https://github.com/astral-sh/ruff)
|
|
45
54
|
[](https://mypy-lang.org/)
|
|
46
55
|
|
|
@@ -55,6 +64,13 @@ knows nothing about what the data means, and depends on no other project. It
|
|
|
55
64
|
moves files and tracks their sync state; it does not validate or interpret their
|
|
56
65
|
contents.
|
|
57
66
|
|
|
67
|
+
## Documentation
|
|
68
|
+
|
|
69
|
+
Full docs live at **[tmsincomb.github.io/forest](https://tmsincomb.github.io/forest/)**.
|
|
70
|
+
Start with the **[hands-on tutorial](https://tmsincomb.github.io/forest/tutorial/)** —
|
|
71
|
+
seven short chapters from `forest init` to a fully synced repo, with every
|
|
72
|
+
output block captured from a real session and no cloud credentials required.
|
|
73
|
+
|
|
58
74
|
## Model
|
|
59
75
|
|
|
60
76
|
Fixed-depth, no arbitrary nesting:
|
|
@@ -73,12 +89,13 @@ workspace → checkout → stage → unit → files
|
|
|
73
89
|
|
|
74
90
|
## Install
|
|
75
91
|
|
|
76
|
-
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
77
|
-
|
|
78
92
|
```bash
|
|
79
|
-
pip install -
|
|
93
|
+
pip install forest-cli
|
|
80
94
|
```
|
|
81
95
|
|
|
96
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers. For a
|
|
97
|
+
development install, clone the repo and `pip install -e ".[dev]"`.
|
|
98
|
+
|
|
82
99
|
## Quick start
|
|
83
100
|
|
|
84
101
|
No config files are hand-written. Onboarding is a few commands:
|
|
@@ -132,13 +149,51 @@ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
|
132
149
|
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
133
150
|
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
134
151
|
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
135
|
-
| `forest
|
|
152
|
+
| `forest tui` (or bare `forest`) | Interactive terminal forest (Textual app): each checkout is a pixel-art tree, canopy bands are stages colored by sync status, and the mist above the treetops is data in the cloud (the active remote). Navigate with keys or the mouse; Enter (or a click) on a stage opens its unit drill-down. |
|
|
153
|
+
| `forest prompt init bash\|zsh` | Print an eval-able snippet that shows the active checkout in your shell prompt. |
|
|
136
154
|
|
|
137
155
|
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
138
156
|
|
|
139
157
|
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
140
158
|
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
141
159
|
|
|
160
|
+
## Terminal output
|
|
161
|
+
|
|
162
|
+
On an interactive terminal forest renders rich output — colored status
|
|
163
|
+
tables, file trees, panels, and live progress bars during `push`/`pull`
|
|
164
|
+
(rclone JSON stats streamed to a bar; needs rclone ≥ 1.49, older versions
|
|
165
|
+
transfer fine without a bar). When stdout is piped or redirected, output
|
|
166
|
+
falls back to the plain, machine-parseable format, so scripts never see
|
|
167
|
+
ANSI codes or box drawing.
|
|
168
|
+
|
|
169
|
+
Overrides: `--color`/`--no-color` per invocation, `FOREST_OUTPUT=rich|plain|auto`
|
|
170
|
+
in the environment, and the [`NO_COLOR`](https://no-color.org)/`FORCE_COLOR`
|
|
171
|
+
conventions. Precedence: flag > `FOREST_OUTPUT` > `NO_COLOR` > `FORCE_COLOR` >
|
|
172
|
+
TTY detection. `--verbose` shows rclone's raw transfer log instead of a
|
|
173
|
+
progress bar; `--quiet` suppresses everything but errors.
|
|
174
|
+
|
|
175
|
+
## Shell prompt
|
|
176
|
+
|
|
177
|
+
Show the active checkout in your prompt (bash and zsh), the way git prompts
|
|
178
|
+
show the current branch:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
# ~/.bashrc
|
|
182
|
+
eval "$(forest prompt init bash)"
|
|
183
|
+
|
|
184
|
+
# ~/.zshrc
|
|
185
|
+
eval "$(forest prompt init zsh)"
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
~/lab/G004 $ # outside a workspace: unchanged
|
|
190
|
+
🌲 G004 ~/lab/G004 $ # inside a workspace with an active checkout
|
|
191
|
+
```
|
|
192
|
+
The snippet is pure shell — it reads `.forest/HEAD` directly and never invokes
|
|
193
|
+
the forest CLI while rendering, so your prompt stays fast. Set
|
|
194
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
195
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
196
|
+
|
|
142
197
|
## Config reference
|
|
143
198
|
|
|
144
199
|
Checkout `forest.yaml` (shared, committed):
|
|
@@ -153,6 +208,8 @@ stages:
|
|
|
153
208
|
raw:
|
|
154
209
|
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
155
210
|
sync_by: subdirectory # subdirectory | directory | file
|
|
211
|
+
direction: both # push | pull | both; bare push/pull skip
|
|
212
|
+
# wrong-direction stages, explicit selection errors
|
|
156
213
|
```
|
|
157
214
|
|
|
158
215
|
Checkout `local.yaml` (per-machine, gitignored):
|
|
@@ -183,44 +240,54 @@ guides live in `docs/runbooks/`.
|
|
|
183
240
|
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
184
241
|
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
185
242
|
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
243
|
+
| `FOREST_OUTPUT` | `auto` | Terminal rendering: `rich`, `plain`, or `auto` (rich on a TTY, plain when piped). |
|
|
244
|
+
| `NO_COLOR` / `FORCE_COLOR` | unset | Standard color conventions; `FOREST_OUTPUT` and `--color/--no-color` take precedence. |
|
|
186
245
|
|
|
187
246
|
## Dogfood: this repo runs forest
|
|
188
247
|
|
|
189
|
-
This repository manages its own `examples/`
|
|
190
|
-
demonstration that `.forest/` and `.git/` coexist without
|
|
191
|
-
set up with exactly the quick-start commands:
|
|
248
|
+
This repository manages its own `examples/` and `tests/data/` trees with
|
|
249
|
+
forest — a live demonstration that `.forest/` and `.git/` coexist without
|
|
250
|
+
overlapping. It was set up with exactly the quick-start commands:
|
|
192
251
|
|
|
193
252
|
```bash
|
|
194
253
|
forest init
|
|
195
254
|
forest checkout demo
|
|
196
|
-
forest remote add origin s3://forest-test
|
|
197
|
-
forest add examples ./examples
|
|
198
|
-
forest
|
|
255
|
+
forest remote add origin s3://forest-cli-test --region us-east-2
|
|
256
|
+
forest add examples ./examples # two-way stage (direction: both)
|
|
257
|
+
forest add data ./tests/data --sync-by directory # two small test files
|
|
258
|
+
forest push # seeds both stages
|
|
199
259
|
```
|
|
200
260
|
|
|
261
|
+
After the seeding push, `data` was flipped to `direction: pull` in the
|
|
262
|
+
checkout's `forest.yaml` — it is a read-only reference mirror now: a bare
|
|
263
|
+
`forest push` skips it with a note, and `forest push --stage data` refuses
|
|
264
|
+
with exit 1 (a new stage can start there directly with
|
|
265
|
+
`forest add ... --direction pull`). The `examples` stage keeps syncing both
|
|
266
|
+
ways.
|
|
267
|
+
|
|
201
268
|
Inspect the result:
|
|
202
269
|
|
|
203
270
|
```bash
|
|
204
271
|
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
205
272
|
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
206
|
-
forest
|
|
273
|
+
forest ls # stage overview: data shows (directory, pull)
|
|
274
|
+
forest status # sync state of both stages
|
|
207
275
|
```
|
|
208
276
|
|
|
209
|
-
A fresh clone bootstraps the local half
|
|
210
|
-
|
|
211
|
-
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
212
|
-
|
|
213
|
-
## Migrating from biostore
|
|
214
|
-
|
|
215
|
-
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
277
|
+
A fresh clone bootstraps the local half (HEAD, bindings, sync state are
|
|
278
|
+
gitignored) with:
|
|
216
279
|
|
|
217
280
|
```bash
|
|
218
|
-
forest
|
|
281
|
+
forest checkout demo # select the checkout; clones have no HEAD
|
|
282
|
+
forest bind examples ./examples
|
|
283
|
+
forest bind data ./tests/data
|
|
284
|
+
forest pull --force # first pull: no local sync state yet, so
|
|
285
|
+
# forest refuses to touch the git-tracked
|
|
286
|
+
# copies without it
|
|
219
287
|
```
|
|
220
288
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
refuses to run if a `.forest/` already exists.
|
|
289
|
+
The single configured remote is used automatically. Pulling needs AWS
|
|
290
|
+
credentials for the bucket; the layout is the demonstration.
|
|
224
291
|
|
|
225
292
|
## Notes
|
|
226
293
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="
|
|
1
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="https://tmsincomb.github.io/forest/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
2
2
|
|
|
3
3
|
**git-like data management for arbitrary data trees.**
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
[](https://tmsincomb.github.io/forest/)
|
|
8
|
+
[](https://pypi.org/project/forest-cli/)
|
|
9
|
+
[](https://www.python.org/downloads/)
|
|
10
|
+
[](https://github.com/tmsincomb/forest/blob/main/LICENSE)
|
|
10
11
|
[](https://github.com/astral-sh/ruff)
|
|
11
12
|
[](https://mypy-lang.org/)
|
|
12
13
|
|
|
@@ -21,6 +22,13 @@ knows nothing about what the data means, and depends on no other project. It
|
|
|
21
22
|
moves files and tracks their sync state; it does not validate or interpret their
|
|
22
23
|
contents.
|
|
23
24
|
|
|
25
|
+
## Documentation
|
|
26
|
+
|
|
27
|
+
Full docs live at **[tmsincomb.github.io/forest](https://tmsincomb.github.io/forest/)**.
|
|
28
|
+
Start with the **[hands-on tutorial](https://tmsincomb.github.io/forest/tutorial/)** —
|
|
29
|
+
seven short chapters from `forest init` to a fully synced repo, with every
|
|
30
|
+
output block captured from a real session and no cloud credentials required.
|
|
31
|
+
|
|
24
32
|
## Model
|
|
25
33
|
|
|
26
34
|
Fixed-depth, no arbitrary nesting:
|
|
@@ -39,12 +47,13 @@ workspace → checkout → stage → unit → files
|
|
|
39
47
|
|
|
40
48
|
## Install
|
|
41
49
|
|
|
42
|
-
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
43
|
-
|
|
44
50
|
```bash
|
|
45
|
-
pip install -
|
|
51
|
+
pip install forest-cli
|
|
46
52
|
```
|
|
47
53
|
|
|
54
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers. For a
|
|
55
|
+
development install, clone the repo and `pip install -e ".[dev]"`.
|
|
56
|
+
|
|
48
57
|
## Quick start
|
|
49
58
|
|
|
50
59
|
No config files are hand-written. Onboarding is a few commands:
|
|
@@ -98,13 +107,51 @@ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
|
98
107
|
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
99
108
|
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
100
109
|
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
101
|
-
| `forest
|
|
110
|
+
| `forest tui` (or bare `forest`) | Interactive terminal forest (Textual app): each checkout is a pixel-art tree, canopy bands are stages colored by sync status, and the mist above the treetops is data in the cloud (the active remote). Navigate with keys or the mouse; Enter (or a click) on a stage opens its unit drill-down. |
|
|
111
|
+
| `forest prompt init bash\|zsh` | Print an eval-able snippet that shows the active checkout in your shell prompt. |
|
|
102
112
|
|
|
103
113
|
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
104
114
|
|
|
105
115
|
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
106
116
|
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
107
117
|
|
|
118
|
+
## Terminal output
|
|
119
|
+
|
|
120
|
+
On an interactive terminal forest renders rich output — colored status
|
|
121
|
+
tables, file trees, panels, and live progress bars during `push`/`pull`
|
|
122
|
+
(rclone JSON stats streamed to a bar; needs rclone ≥ 1.49, older versions
|
|
123
|
+
transfer fine without a bar). When stdout is piped or redirected, output
|
|
124
|
+
falls back to the plain, machine-parseable format, so scripts never see
|
|
125
|
+
ANSI codes or box drawing.
|
|
126
|
+
|
|
127
|
+
Overrides: `--color`/`--no-color` per invocation, `FOREST_OUTPUT=rich|plain|auto`
|
|
128
|
+
in the environment, and the [`NO_COLOR`](https://no-color.org)/`FORCE_COLOR`
|
|
129
|
+
conventions. Precedence: flag > `FOREST_OUTPUT` > `NO_COLOR` > `FORCE_COLOR` >
|
|
130
|
+
TTY detection. `--verbose` shows rclone's raw transfer log instead of a
|
|
131
|
+
progress bar; `--quiet` suppresses everything but errors.
|
|
132
|
+
|
|
133
|
+
## Shell prompt
|
|
134
|
+
|
|
135
|
+
Show the active checkout in your prompt (bash and zsh), the way git prompts
|
|
136
|
+
show the current branch:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
# ~/.bashrc
|
|
140
|
+
eval "$(forest prompt init bash)"
|
|
141
|
+
|
|
142
|
+
# ~/.zshrc
|
|
143
|
+
eval "$(forest prompt init zsh)"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```text
|
|
147
|
+
~/lab/G004 $ # outside a workspace: unchanged
|
|
148
|
+
🌲 G004 ~/lab/G004 $ # inside a workspace with an active checkout
|
|
149
|
+
```
|
|
150
|
+
The snippet is pure shell — it reads `.forest/HEAD` directly and never invokes
|
|
151
|
+
the forest CLI while rendering, so your prompt stays fast. Set
|
|
152
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
153
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
154
|
+
|
|
108
155
|
## Config reference
|
|
109
156
|
|
|
110
157
|
Checkout `forest.yaml` (shared, committed):
|
|
@@ -119,6 +166,8 @@ stages:
|
|
|
119
166
|
raw:
|
|
120
167
|
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
121
168
|
sync_by: subdirectory # subdirectory | directory | file
|
|
169
|
+
direction: both # push | pull | both; bare push/pull skip
|
|
170
|
+
# wrong-direction stages, explicit selection errors
|
|
122
171
|
```
|
|
123
172
|
|
|
124
173
|
Checkout `local.yaml` (per-machine, gitignored):
|
|
@@ -149,44 +198,54 @@ guides live in `docs/runbooks/`.
|
|
|
149
198
|
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
150
199
|
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
151
200
|
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
201
|
+
| `FOREST_OUTPUT` | `auto` | Terminal rendering: `rich`, `plain`, or `auto` (rich on a TTY, plain when piped). |
|
|
202
|
+
| `NO_COLOR` / `FORCE_COLOR` | unset | Standard color conventions; `FOREST_OUTPUT` and `--color/--no-color` take precedence. |
|
|
152
203
|
|
|
153
204
|
## Dogfood: this repo runs forest
|
|
154
205
|
|
|
155
|
-
This repository manages its own `examples/`
|
|
156
|
-
demonstration that `.forest/` and `.git/` coexist without
|
|
157
|
-
set up with exactly the quick-start commands:
|
|
206
|
+
This repository manages its own `examples/` and `tests/data/` trees with
|
|
207
|
+
forest — a live demonstration that `.forest/` and `.git/` coexist without
|
|
208
|
+
overlapping. It was set up with exactly the quick-start commands:
|
|
158
209
|
|
|
159
210
|
```bash
|
|
160
211
|
forest init
|
|
161
212
|
forest checkout demo
|
|
162
|
-
forest remote add origin s3://forest-test
|
|
163
|
-
forest add examples ./examples
|
|
164
|
-
forest
|
|
213
|
+
forest remote add origin s3://forest-cli-test --region us-east-2
|
|
214
|
+
forest add examples ./examples # two-way stage (direction: both)
|
|
215
|
+
forest add data ./tests/data --sync-by directory # two small test files
|
|
216
|
+
forest push # seeds both stages
|
|
165
217
|
```
|
|
166
218
|
|
|
219
|
+
After the seeding push, `data` was flipped to `direction: pull` in the
|
|
220
|
+
checkout's `forest.yaml` — it is a read-only reference mirror now: a bare
|
|
221
|
+
`forest push` skips it with a note, and `forest push --stage data` refuses
|
|
222
|
+
with exit 1 (a new stage can start there directly with
|
|
223
|
+
`forest add ... --direction pull`). The `examples` stage keeps syncing both
|
|
224
|
+
ways.
|
|
225
|
+
|
|
167
226
|
Inspect the result:
|
|
168
227
|
|
|
169
228
|
```bash
|
|
170
229
|
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
171
230
|
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
172
|
-
forest
|
|
231
|
+
forest ls # stage overview: data shows (directory, pull)
|
|
232
|
+
forest status # sync state of both stages
|
|
173
233
|
```
|
|
174
234
|
|
|
175
|
-
A fresh clone bootstraps the local half
|
|
176
|
-
|
|
177
|
-
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
178
|
-
|
|
179
|
-
## Migrating from biostore
|
|
180
|
-
|
|
181
|
-
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
235
|
+
A fresh clone bootstraps the local half (HEAD, bindings, sync state are
|
|
236
|
+
gitignored) with:
|
|
182
237
|
|
|
183
238
|
```bash
|
|
184
|
-
forest
|
|
239
|
+
forest checkout demo # select the checkout; clones have no HEAD
|
|
240
|
+
forest bind examples ./examples
|
|
241
|
+
forest bind data ./tests/data
|
|
242
|
+
forest pull --force # first pull: no local sync state yet, so
|
|
243
|
+
# forest refuses to touch the git-tracked
|
|
244
|
+
# copies without it
|
|
185
245
|
```
|
|
186
246
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
refuses to run if a `.forest/` already exists.
|
|
247
|
+
The single configured remote is used automatically. Pulling needs AWS
|
|
248
|
+
credentials for the bucket; the layout is the demonstration.
|
|
190
249
|
|
|
191
250
|
## Notes
|
|
192
251
|
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forest-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "1.0.0"
|
|
8
8
|
description = "git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
11
|
-
requires-python = ">=3.
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
12
|
authors = [
|
|
13
13
|
{name = "Troy Sincomb", email = "troysincomb@gmail.com"},
|
|
14
14
|
]
|
|
@@ -23,8 +23,19 @@ dependencies = [
|
|
|
23
23
|
"polars>=1.0",
|
|
24
24
|
"pydantic>=2.0",
|
|
25
25
|
"pyyaml>=6.0",
|
|
26
|
+
# rich is a textual dependency, but forest imports it directly (Text.from_ansi).
|
|
27
|
+
"rich>=14.2",
|
|
28
|
+
# rich-click 1.9.8 is the first release compatible with the click 8.4 line.
|
|
29
|
+
"rich-click>=1.9.8",
|
|
30
|
+
"textual>=8",
|
|
26
31
|
]
|
|
27
32
|
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/tmsincomb/forest"
|
|
35
|
+
Repository = "https://github.com/tmsincomb/forest"
|
|
36
|
+
Issues = "https://github.com/tmsincomb/forest/issues"
|
|
37
|
+
Changelog = "https://github.com/tmsincomb/forest/blob/main/CHANGELOG.md"
|
|
38
|
+
|
|
28
39
|
[project.optional-dependencies]
|
|
29
40
|
# Opt-in error tracking (FOREST_SENTRY_DSN); see docs/runbooks/alerting.md.
|
|
30
41
|
observability = [
|
|
@@ -36,6 +47,7 @@ dev = [
|
|
|
36
47
|
"pre-commit>=4.0",
|
|
37
48
|
"pylint>=3.3",
|
|
38
49
|
"pytest>=7.0",
|
|
50
|
+
"pytest-asyncio>=0.24",
|
|
39
51
|
"pytest-cov>=6.0",
|
|
40
52
|
"pytest-randomly>=3.15",
|
|
41
53
|
"pytest-rerunfailures>=15.0",
|
|
@@ -45,6 +57,13 @@ dev = [
|
|
|
45
57
|
"vulture>=2.14",
|
|
46
58
|
]
|
|
47
59
|
|
|
60
|
+
[dependency-groups]
|
|
61
|
+
# Docs-site toolchain (mkdocs.yml at the repo root): uv run --group docs mkdocs serve
|
|
62
|
+
docs = [
|
|
63
|
+
"mkdocs>=1.6,<2",
|
|
64
|
+
"mkdocs-material>=9.5,<10",
|
|
65
|
+
]
|
|
66
|
+
|
|
48
67
|
[project.scripts]
|
|
49
68
|
forest = "forest.cli:main"
|
|
50
69
|
|
|
@@ -54,6 +73,9 @@ where = ["src"]
|
|
|
54
73
|
[tool.pytest.ini_options]
|
|
55
74
|
testpaths = ["tests"]
|
|
56
75
|
doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
|
|
76
|
+
# pytest-asyncio runs in strict mode (the default): only tests marked
|
|
77
|
+
# @pytest.mark.asyncio are collected as async (the Textual app tests).
|
|
78
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
57
79
|
addopts = [
|
|
58
80
|
"--cov=forest",
|
|
59
81
|
"--cov-report=term-missing",
|
|
@@ -71,7 +93,7 @@ addopts = [
|
|
|
71
93
|
]
|
|
72
94
|
|
|
73
95
|
[tool.ruff]
|
|
74
|
-
target-version = "
|
|
96
|
+
target-version = "py310"
|
|
75
97
|
line-length = 120
|
|
76
98
|
src = ["src", "tests"]
|
|
77
99
|
|
|
@@ -137,14 +137,6 @@ def remote_url_has_secret_material(url: str) -> bool:
|
|
|
137
137
|
return SECRET_REMOTE_URL_RE.search(url) is not None
|
|
138
138
|
|
|
139
139
|
|
|
140
|
-
def _is_relative_to(path: Path, parent: Path) -> bool:
|
|
141
|
-
try:
|
|
142
|
-
path.relative_to(parent)
|
|
143
|
-
except ValueError:
|
|
144
|
-
return False
|
|
145
|
-
return True
|
|
146
|
-
|
|
147
|
-
|
|
148
140
|
def normalize_workspace_path(path: str | Path, workspace_root: str | Path, *, cwd: str | Path | None = None) -> Path:
|
|
149
141
|
"""Resolve a path and store inside-workspace paths relative to the workspace root."""
|
|
150
142
|
workspace_root = Path(workspace_root).resolve(strict=False)
|
|
@@ -154,7 +146,7 @@ def normalize_workspace_path(path: str | Path, workspace_root: str | Path, *, cw
|
|
|
154
146
|
candidate = base / candidate
|
|
155
147
|
candidate = candidate.resolve(strict=False)
|
|
156
148
|
|
|
157
|
-
if
|
|
149
|
+
if candidate.is_relative_to(workspace_root):
|
|
158
150
|
return candidate.relative_to(workspace_root)
|
|
159
151
|
return candidate
|
|
160
152
|
|
|
@@ -167,7 +159,7 @@ def validate_workspace_data_path(path: str | Path, workspace_root: str | Path) -
|
|
|
167
159
|
candidate = workspace_root / candidate
|
|
168
160
|
candidate = candidate.resolve(strict=False)
|
|
169
161
|
forest_dir = (workspace_root / ".forest").resolve(strict=False)
|
|
170
|
-
if
|
|
162
|
+
if candidate.is_relative_to(forest_dir):
|
|
171
163
|
raise CheckoutError("Data paths must not point inside .forest metadata.")
|
|
172
164
|
return candidate
|
|
173
165
|
|
|
@@ -325,7 +317,7 @@ def read_head(workspace_root: str | Path) -> str:
|
|
|
325
317
|
except OSError as exc:
|
|
326
318
|
raise CheckoutError(f"Cannot read active checkout {path}: {exc}") from None
|
|
327
319
|
|
|
328
|
-
name = raw
|
|
320
|
+
name = raw.removesuffix("\n")
|
|
329
321
|
if "\n" in name:
|
|
330
322
|
raise CheckoutError(f"Invalid active checkout in {path}: expected exactly one checkout name line.")
|
|
331
323
|
return validate_checkout_name(name)
|
|
@@ -399,11 +391,7 @@ def create_checkout(workspace_root: str | Path, checkout_name: str) -> None:
|
|
|
399
391
|
write_head(workspace_root, checkout_name)
|
|
400
392
|
ensure_workspace_gitignore(workspace_root, checkout_name)
|
|
401
393
|
except Exception:
|
|
402
|
-
|
|
403
|
-
if target_existed:
|
|
404
|
-
_empty_directory(target_dir)
|
|
405
|
-
else:
|
|
406
|
-
shutil.rmtree(target_dir)
|
|
394
|
+
_rollback_target_dir(target_dir, existed=target_existed)
|
|
407
395
|
_restore_optional_text(config_path, config_text)
|
|
408
396
|
_restore_optional_text(head_path, head_text)
|
|
409
397
|
_restore_optional_text(gitignore_path, gitignore_text)
|
|
@@ -476,6 +464,15 @@ def _empty_directory(path: Path) -> None:
|
|
|
476
464
|
child.unlink()
|
|
477
465
|
|
|
478
466
|
|
|
467
|
+
def _rollback_target_dir(target_dir: Path, *, existed: bool) -> None:
|
|
468
|
+
if not target_dir.exists():
|
|
469
|
+
return
|
|
470
|
+
if existed:
|
|
471
|
+
_empty_directory(target_dir)
|
|
472
|
+
else:
|
|
473
|
+
shutil.rmtree(target_dir)
|
|
474
|
+
|
|
475
|
+
|
|
479
476
|
def _git_check_ignore(workspace_root: Path, rel_path: str) -> int:
|
|
480
477
|
try:
|
|
481
478
|
result = subprocess.run(
|
|
@@ -508,8 +505,7 @@ def _is_inside_git_worktree(workspace_root: Path) -> bool:
|
|
|
508
505
|
|
|
509
506
|
|
|
510
507
|
def _managed_gitignore_text(original_text: str, patterns: tuple[str, ...]) -> str:
|
|
511
|
-
|
|
512
|
-
kept = [line for line in lines if line.strip() not in FOREST_OWNED_GITIGNORE_LINES]
|
|
508
|
+
kept = [line for line in original_text.splitlines() if line.strip() not in FOREST_OWNED_GITIGNORE_LINES]
|
|
513
509
|
kept.extend(patterns)
|
|
514
510
|
return "\n".join(kept).rstrip("\n") + "\n"
|
|
515
511
|
|
|
@@ -567,17 +563,6 @@ def ensure_registry_gitignore(workspace_root: str | Path) -> None:
|
|
|
567
563
|
)
|
|
568
564
|
|
|
569
565
|
|
|
570
|
-
def ensure_legacy_gitignore(project_root: str | Path) -> None:
|
|
571
|
-
"""Ensure legacy local state ignores while keeping root forest.yaml trackable."""
|
|
572
|
-
project_root = Path(project_root)
|
|
573
|
-
_ensure_gitignore(
|
|
574
|
-
project_root,
|
|
575
|
-
patterns=LEGACY_GITIGNORE_PATTERNS,
|
|
576
|
-
ignored_paths=LEGACY_GITIGNORE_PATTERNS,
|
|
577
|
-
tracked_paths=("forest.yaml",),
|
|
578
|
-
)
|
|
579
|
-
|
|
580
|
-
|
|
581
566
|
def _start_directory(start: str | Path) -> Path:
|
|
582
567
|
start_path = Path(start).resolve(strict=False)
|
|
583
568
|
return start_path.parent if start_path.is_file() else start_path
|
|
@@ -708,6 +693,8 @@ def _verify_adopted_stages(
|
|
|
708
693
|
raise CheckoutError(f"adopt verify failed: schema changed for stage '{stage_name}'.")
|
|
709
694
|
if merged_stage.sync_by != legacy_stage.sync_by:
|
|
710
695
|
raise CheckoutError(f"adopt verify failed: sync_by changed for stage '{stage_name}'.")
|
|
696
|
+
if merged_stage.direction != legacy_stage.direction:
|
|
697
|
+
raise CheckoutError(f"adopt verify failed: direction changed for stage '{stage_name}'.")
|
|
711
698
|
if legacy_stage.remote_path is not None and merged_stage.remote_path != legacy_stage.remote_path:
|
|
712
699
|
raise CheckoutError(f"adopt verify failed: remote_path changed for stage '{stage_name}'.")
|
|
713
700
|
expected_path = _effective_local_stage_path(workspace_root, legacy_stage.path)
|
|
@@ -725,12 +712,7 @@ def _restore_adopt_staging(
|
|
|
725
712
|
gitignore_text: str | None,
|
|
726
713
|
legacy_texts: dict[Path, str | None],
|
|
727
714
|
) -> None:
|
|
728
|
-
|
|
729
|
-
if target_dir.exists():
|
|
730
|
-
if target_existed:
|
|
731
|
-
_empty_directory(target_dir)
|
|
732
|
-
else:
|
|
733
|
-
shutil.rmtree(target_dir)
|
|
715
|
+
_rollback_target_dir(checkout_dir(workspace_root, checkout_name), existed=target_existed)
|
|
734
716
|
_restore_optional_text(workspace_config_path(workspace_root), workspace_config_text)
|
|
735
717
|
_restore_optional_text(_workspace_dir(workspace_root) / "HEAD", head_text)
|
|
736
718
|
_restore_optional_text(workspace_root / ".gitignore", gitignore_text)
|
|
@@ -769,11 +751,8 @@ def adopt_checkout(start: str | Path, checkout_name: str) -> Path:
|
|
|
769
751
|
if not local_path.exists() or not state_path.exists():
|
|
770
752
|
raise CheckoutError("incomplete legacy metadata: expected .forest/local.yaml and .forest/sync_state.json.")
|
|
771
753
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
legacy_local = _load_checkout_local_config(local_path)
|
|
775
|
-
except CheckoutError:
|
|
776
|
-
raise
|
|
754
|
+
legacy_config = _load_shared_config(root_config_path)
|
|
755
|
+
legacy_local = _load_checkout_local_config(local_path)
|
|
777
756
|
|
|
778
757
|
_validate_legacy_adopt_inputs(legacy_config, legacy_local)
|
|
779
758
|
|