forest-cli 0.2.0__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forest_cli-0.2.0 → forest_cli-1.0.0}/PKG-INFO +92 -30
- {forest_cli-0.2.0 → forest_cli-1.0.0}/README.md +86 -28
- {forest_cli-0.2.0 → forest_cli-1.0.0}/pyproject.toml +19 -3
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/checkout.py +19 -40
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/cli.py +1162 -386
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/config.py +7 -19
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/fsutil.py +12 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/manifest.py +0 -22
- forest_cli-1.0.0/src/forest/output.py +349 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/paths.py +1 -56
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/rclone.py +221 -18
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/resilience.py +1 -1
- forest_cli-1.0.0/src/forest/scenery.py +703 -0
- forest_cli-1.0.0/src/forest/shell_prompt.py +94 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/sync_state.py +40 -7
- forest_cli-1.0.0/src/forest/tui_app.py +283 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/PKG-INFO +92 -30
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/SOURCES.txt +15 -2
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/requires.txt +4 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli.py +19 -5
- forest_cli-1.0.0/tests/test_cli_prompt.py +170 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_config.py +18 -18
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cross_backend_parity_lock.py +2 -1
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_ls_lock.py +66 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_manifest.py +0 -101
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_moat_regression_lock.py +2 -0
- forest_cli-1.0.0/tests/test_output.py +213 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_paths.py +0 -151
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_pull_core_lock.py +120 -1
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_pull_resilience_lock.py +3 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_push_core_lock.py +138 -9
- forest_cli-1.0.0/tests/test_python_support_metadata.py +9 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_rclone.py +68 -13
- forest_cli-1.0.0/tests/test_rclone_progress.py +165 -0
- forest_cli-1.0.0/tests/test_rich_render.py +129 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_s3_integration.py +15 -12
- forest_cli-1.0.0/tests/test_scenery.py +360 -0
- forest_cli-1.0.0/tests/test_stage_direction.py +364 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_status_lock.py +206 -0
- forest_cli-1.0.0/tests/test_transfer_lock_scope.py +159 -0
- forest_cli-1.0.0/tests/test_tui_app.py +160 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_user_testing_fixes.py +9 -2
- forest_cli-1.0.0/tests/test_verify_lock.py +151 -0
- forest_cli-0.2.0/tests/test_migrate.py +0 -73
- {forest_cli-0.2.0 → forest_cli-1.0.0}/LICENSE +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/setup.cfg +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/__init__.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/analytics.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/flags.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/flow.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/logger.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/metrics.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest/monitoring.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/dependency_links.txt +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/entry_points.txt +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/src/forest_cli.egg-info/top_level.txt +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_analytics.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_bare_command_defaults.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_checkout.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_checkout_scaffold.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_checkout_adopt.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_checkout_lifecycle.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_cross_area_flows.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_gitignore.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_init.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_remote_workspace.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_resolution.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_stage_binding.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_cli_workspace_data_commands.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_config_migration.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_flags.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_flow.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_forest_agnostic.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_integration.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_legacy_remote_path_safety.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_polish_scrutiny_fixes.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_push_targeting_lock.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_rclone_gating_errors.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_redaction.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_s3_url_error_hardening.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_scrutiny_fixes.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_sftp_hardening.py +0 -0
- {forest_cli-0.2.0 → forest_cli-1.0.0}/tests/test_sync_state.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: forest-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0
|
|
4
4
|
Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
|
|
5
5
|
Author-email: Troy Sincomb <troysincomb@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -12,13 +12,16 @@ Keywords: data-management,sync,cli
|
|
|
12
12
|
Classifier: Development Status :: 3 - Alpha
|
|
13
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: click>=8.0
|
|
19
19
|
Requires-Dist: polars>=1.0
|
|
20
20
|
Requires-Dist: pydantic>=2.0
|
|
21
21
|
Requires-Dist: pyyaml>=6.0
|
|
22
|
+
Requires-Dist: rich>=14.2
|
|
23
|
+
Requires-Dist: rich-click>=1.9.8
|
|
24
|
+
Requires-Dist: textual>=8
|
|
22
25
|
Provides-Extra: observability
|
|
23
26
|
Requires-Dist: sentry-sdk>=2.0; extra == "observability"
|
|
24
27
|
Provides-Extra: dev
|
|
@@ -27,6 +30,7 @@ Requires-Dist: mypy>=1.14; extra == "dev"
|
|
|
27
30
|
Requires-Dist: pre-commit>=4.0; extra == "dev"
|
|
28
31
|
Requires-Dist: pylint>=3.3; extra == "dev"
|
|
29
32
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
30
34
|
Requires-Dist: pytest-cov>=6.0; extra == "dev"
|
|
31
35
|
Requires-Dist: pytest-randomly>=3.15; extra == "dev"
|
|
32
36
|
Requires-Dist: pytest-rerunfailures>=15.0; extra == "dev"
|
|
@@ -36,16 +40,16 @@ Requires-Dist: types-PyYAML; extra == "dev"
|
|
|
36
40
|
Requires-Dist: vulture>=2.14; extra == "dev"
|
|
37
41
|
Dynamic: license-file
|
|
38
42
|
|
|
39
|
-
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="
|
|
43
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="https://tmsincomb.github.io/forest/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
40
44
|
|
|
41
45
|
**git-like data management for arbitrary data trees.**
|
|
42
46
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
[](https://tmsincomb.github.io/forest/)
|
|
46
50
|
[](https://pypi.org/project/forest-cli/)
|
|
47
|
-
[](LICENSE)
|
|
51
|
+
[](https://www.python.org/downloads/)
|
|
52
|
+
[](https://github.com/tmsincomb/forest/blob/main/LICENSE)
|
|
49
53
|
[](https://github.com/astral-sh/ruff)
|
|
50
54
|
[](https://mypy-lang.org/)
|
|
51
55
|
|
|
@@ -60,6 +64,13 @@ knows nothing about what the data means, and depends on no other project. It
|
|
|
60
64
|
moves files and tracks their sync state; it does not validate or interpret their
|
|
61
65
|
contents.
|
|
62
66
|
|
|
67
|
+
## Documentation
|
|
68
|
+
|
|
69
|
+
Full docs live at **[tmsincomb.github.io/forest](https://tmsincomb.github.io/forest/)**.
|
|
70
|
+
Start with the **[hands-on tutorial](https://tmsincomb.github.io/forest/tutorial/)** —
|
|
71
|
+
seven short chapters from `forest init` to a fully synced repo, with every
|
|
72
|
+
output block captured from a real session and no cloud credentials required.
|
|
73
|
+
|
|
63
74
|
## Model
|
|
64
75
|
|
|
65
76
|
Fixed-depth, no arbitrary nesting:
|
|
@@ -78,12 +89,13 @@ workspace → checkout → stage → unit → files
|
|
|
78
89
|
|
|
79
90
|
## Install
|
|
80
91
|
|
|
81
|
-
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
82
|
-
|
|
83
92
|
```bash
|
|
84
|
-
pip install -
|
|
93
|
+
pip install forest-cli
|
|
85
94
|
```
|
|
86
95
|
|
|
96
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers. For a
|
|
97
|
+
development install, clone the repo and `pip install -e ".[dev]"`.
|
|
98
|
+
|
|
87
99
|
## Quick start
|
|
88
100
|
|
|
89
101
|
No config files are hand-written. Onboarding is a few commands:
|
|
@@ -137,13 +149,51 @@ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
|
137
149
|
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
138
150
|
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
139
151
|
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
140
|
-
| `forest
|
|
152
|
+
| `forest tui` (or bare `forest`) | Interactive terminal forest (Textual app): each checkout is a pixel-art tree, canopy bands are stages colored by sync status, and the mist above the treetops is data in the cloud (the active remote). Navigate with keys or the mouse; Enter (or a click) on a stage opens its unit drill-down. |
|
|
153
|
+
| `forest prompt init bash\|zsh` | Print an eval-able snippet that shows the active checkout in your shell prompt. |
|
|
141
154
|
|
|
142
155
|
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
143
156
|
|
|
144
157
|
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
145
158
|
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
146
159
|
|
|
160
|
+
## Terminal output
|
|
161
|
+
|
|
162
|
+
On an interactive terminal forest renders rich output — colored status
|
|
163
|
+
tables, file trees, panels, and live progress bars during `push`/`pull`
|
|
164
|
+
(rclone JSON stats streamed to a bar; needs rclone ≥ 1.49, older versions
|
|
165
|
+
transfer fine without a bar). When stdout is piped or redirected, output
|
|
166
|
+
falls back to the plain, machine-parseable format, so scripts never see
|
|
167
|
+
ANSI codes or box drawing.
|
|
168
|
+
|
|
169
|
+
Overrides: `--color`/`--no-color` per invocation, `FOREST_OUTPUT=rich|plain|auto`
|
|
170
|
+
in the environment, and the [`NO_COLOR`](https://no-color.org)/`FORCE_COLOR`
|
|
171
|
+
conventions. Precedence: flag > `FOREST_OUTPUT` > `NO_COLOR` > `FORCE_COLOR` >
|
|
172
|
+
TTY detection. `--verbose` shows rclone's raw transfer log instead of a
|
|
173
|
+
progress bar; `--quiet` suppresses everything but errors.
|
|
174
|
+
|
|
175
|
+
## Shell prompt
|
|
176
|
+
|
|
177
|
+
Show the active checkout in your prompt (bash and zsh), the way git prompts
|
|
178
|
+
show the current branch:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
# ~/.bashrc
|
|
182
|
+
eval "$(forest prompt init bash)"
|
|
183
|
+
|
|
184
|
+
# ~/.zshrc
|
|
185
|
+
eval "$(forest prompt init zsh)"
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
~/lab/G004 $ # outside a workspace: unchanged
|
|
190
|
+
🌲 G004 ~/lab/G004 $ # inside a workspace with an active checkout
|
|
191
|
+
```
|
|
192
|
+
The snippet is pure shell — it reads `.forest/HEAD` directly and never invokes
|
|
193
|
+
the forest CLI while rendering, so your prompt stays fast. Set
|
|
194
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
195
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
196
|
+
|
|
147
197
|
## Config reference
|
|
148
198
|
|
|
149
199
|
Checkout `forest.yaml` (shared, committed):
|
|
@@ -158,6 +208,8 @@ stages:
|
|
|
158
208
|
raw:
|
|
159
209
|
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
160
210
|
sync_by: subdirectory # subdirectory | directory | file
|
|
211
|
+
direction: both # push | pull | both; bare push/pull skip
|
|
212
|
+
# wrong-direction stages, explicit selection errors
|
|
161
213
|
```
|
|
162
214
|
|
|
163
215
|
Checkout `local.yaml` (per-machine, gitignored):
|
|
@@ -188,44 +240,54 @@ guides live in `docs/runbooks/`.
|
|
|
188
240
|
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
189
241
|
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
190
242
|
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
243
|
+
| `FOREST_OUTPUT` | `auto` | Terminal rendering: `rich`, `plain`, or `auto` (rich on a TTY, plain when piped). |
|
|
244
|
+
| `NO_COLOR` / `FORCE_COLOR` | unset | Standard color conventions; `FOREST_OUTPUT` and `--color/--no-color` take precedence. |
|
|
191
245
|
|
|
192
246
|
## Dogfood: this repo runs forest
|
|
193
247
|
|
|
194
|
-
This repository manages its own `examples/`
|
|
195
|
-
demonstration that `.forest/` and `.git/` coexist without
|
|
196
|
-
set up with exactly the quick-start commands:
|
|
248
|
+
This repository manages its own `examples/` and `tests/data/` trees with
|
|
249
|
+
forest — a live demonstration that `.forest/` and `.git/` coexist without
|
|
250
|
+
overlapping. It was set up with exactly the quick-start commands:
|
|
197
251
|
|
|
198
252
|
```bash
|
|
199
253
|
forest init
|
|
200
254
|
forest checkout demo
|
|
201
|
-
forest remote add origin s3://
|
|
202
|
-
forest add examples ./examples
|
|
203
|
-
forest
|
|
255
|
+
forest remote add origin s3://forest-cli-test --region us-east-2
|
|
256
|
+
forest add examples ./examples # two-way stage (direction: both)
|
|
257
|
+
forest add data ./tests/data --sync-by directory # two small test files
|
|
258
|
+
forest push # seeds both stages
|
|
204
259
|
```
|
|
205
260
|
|
|
261
|
+
After the seeding push, `data` was flipped to `direction: pull` in the
|
|
262
|
+
checkout's `forest.yaml` — it is a read-only reference mirror now: a bare
|
|
263
|
+
`forest push` skips it with a note, and `forest push --stage data` refuses
|
|
264
|
+
with exit 1 (a new stage can start there directly with
|
|
265
|
+
`forest add ... --direction pull`). The `examples` stage keeps syncing both
|
|
266
|
+
ways.
|
|
267
|
+
|
|
206
268
|
Inspect the result:
|
|
207
269
|
|
|
208
270
|
```bash
|
|
209
271
|
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
210
272
|
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
211
|
-
forest
|
|
273
|
+
forest ls # stage overview: data shows (directory, pull)
|
|
274
|
+
forest status # sync state of both stages
|
|
212
275
|
```
|
|
213
276
|
|
|
214
|
-
A fresh clone bootstraps the local half
|
|
215
|
-
|
|
216
|
-
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
217
|
-
|
|
218
|
-
## Migrating from biostore
|
|
219
|
-
|
|
220
|
-
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
277
|
+
A fresh clone bootstraps the local half (HEAD, bindings, sync state are
|
|
278
|
+
gitignored) with:
|
|
221
279
|
|
|
222
280
|
```bash
|
|
223
|
-
forest
|
|
281
|
+
forest checkout demo # select the checkout; clones have no HEAD
|
|
282
|
+
forest bind examples ./examples
|
|
283
|
+
forest bind data ./tests/data
|
|
284
|
+
forest pull --force # first pull: no local sync state yet, so
|
|
285
|
+
# forest refuses to touch the git-tracked
|
|
286
|
+
# copies without it
|
|
224
287
|
```
|
|
225
288
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
refuses to run if a `.forest/` already exists.
|
|
289
|
+
The single configured remote is used automatically. Pulling needs AWS
|
|
290
|
+
credentials for the bucket; the layout is the demonstration.
|
|
229
291
|
|
|
230
292
|
## Notes
|
|
231
293
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="
|
|
1
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="https://tmsincomb.github.io/forest/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
2
2
|
|
|
3
3
|
**git-like data management for arbitrary data trees.**
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
[](https://tmsincomb.github.io/forest/)
|
|
8
8
|
[](https://pypi.org/project/forest-cli/)
|
|
9
|
-
[](LICENSE)
|
|
9
|
+
[](https://www.python.org/downloads/)
|
|
10
|
+
[](https://github.com/tmsincomb/forest/blob/main/LICENSE)
|
|
11
11
|
[](https://github.com/astral-sh/ruff)
|
|
12
12
|
[](https://mypy-lang.org/)
|
|
13
13
|
|
|
@@ -22,6 +22,13 @@ knows nothing about what the data means, and depends on no other project. It
|
|
|
22
22
|
moves files and tracks their sync state; it does not validate or interpret their
|
|
23
23
|
contents.
|
|
24
24
|
|
|
25
|
+
## Documentation
|
|
26
|
+
|
|
27
|
+
Full docs live at **[tmsincomb.github.io/forest](https://tmsincomb.github.io/forest/)**.
|
|
28
|
+
Start with the **[hands-on tutorial](https://tmsincomb.github.io/forest/tutorial/)** —
|
|
29
|
+
seven short chapters from `forest init` to a fully synced repo, with every
|
|
30
|
+
output block captured from a real session and no cloud credentials required.
|
|
31
|
+
|
|
25
32
|
## Model
|
|
26
33
|
|
|
27
34
|
Fixed-depth, no arbitrary nesting:
|
|
@@ -40,12 +47,13 @@ workspace → checkout → stage → unit → files
|
|
|
40
47
|
|
|
41
48
|
## Install
|
|
42
49
|
|
|
43
|
-
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
44
|
-
|
|
45
50
|
```bash
|
|
46
|
-
pip install -
|
|
51
|
+
pip install forest-cli
|
|
47
52
|
```
|
|
48
53
|
|
|
54
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers. For a
|
|
55
|
+
development install, clone the repo and `pip install -e ".[dev]"`.
|
|
56
|
+
|
|
49
57
|
## Quick start
|
|
50
58
|
|
|
51
59
|
No config files are hand-written. Onboarding is a few commands:
|
|
@@ -99,13 +107,51 @@ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
|
99
107
|
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
100
108
|
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
101
109
|
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
102
|
-
| `forest
|
|
110
|
+
| `forest tui` (or bare `forest`) | Interactive terminal forest (Textual app): each checkout is a pixel-art tree, canopy bands are stages colored by sync status, and the mist above the treetops is data in the cloud (the active remote). Navigate with keys or the mouse; Enter (or a click) on a stage opens its unit drill-down. |
|
|
111
|
+
| `forest prompt init bash\|zsh` | Print an eval-able snippet that shows the active checkout in your shell prompt. |
|
|
103
112
|
|
|
104
113
|
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
105
114
|
|
|
106
115
|
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
107
116
|
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
108
117
|
|
|
118
|
+
## Terminal output
|
|
119
|
+
|
|
120
|
+
On an interactive terminal forest renders rich output — colored status
|
|
121
|
+
tables, file trees, panels, and live progress bars during `push`/`pull`
|
|
122
|
+
(rclone JSON stats streamed to a bar; needs rclone ≥ 1.49, older versions
|
|
123
|
+
transfer fine without a bar). When stdout is piped or redirected, output
|
|
124
|
+
falls back to the plain, machine-parseable format, so scripts never see
|
|
125
|
+
ANSI codes or box drawing.
|
|
126
|
+
|
|
127
|
+
Overrides: `--color`/`--no-color` per invocation, `FOREST_OUTPUT=rich|plain|auto`
|
|
128
|
+
in the environment, and the [`NO_COLOR`](https://no-color.org)/`FORCE_COLOR`
|
|
129
|
+
conventions. Precedence: flag > `FOREST_OUTPUT` > `NO_COLOR` > `FORCE_COLOR` >
|
|
130
|
+
TTY detection. `--verbose` shows rclone's raw transfer log instead of a
|
|
131
|
+
progress bar; `--quiet` suppresses everything but errors.
|
|
132
|
+
|
|
133
|
+
## Shell prompt
|
|
134
|
+
|
|
135
|
+
Show the active checkout in your prompt (bash and zsh), the way git prompts
|
|
136
|
+
show the current branch:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
# ~/.bashrc
|
|
140
|
+
eval "$(forest prompt init bash)"
|
|
141
|
+
|
|
142
|
+
# ~/.zshrc
|
|
143
|
+
eval "$(forest prompt init zsh)"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```text
|
|
147
|
+
~/lab/G004 $ # outside a workspace: unchanged
|
|
148
|
+
🌲 G004 ~/lab/G004 $ # inside a workspace with an active checkout
|
|
149
|
+
```
|
|
150
|
+
The snippet is pure shell — it reads `.forest/HEAD` directly and never invokes
|
|
151
|
+
the forest CLI while rendering, so your prompt stays fast. Set
|
|
152
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
153
|
+
`FOREST_PROMPT_ICON` to replace the 🌲.
|
|
154
|
+
|
|
109
155
|
## Config reference
|
|
110
156
|
|
|
111
157
|
Checkout `forest.yaml` (shared, committed):
|
|
@@ -120,6 +166,8 @@ stages:
|
|
|
120
166
|
raw:
|
|
121
167
|
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
122
168
|
sync_by: subdirectory # subdirectory | directory | file
|
|
169
|
+
direction: both # push | pull | both; bare push/pull skip
|
|
170
|
+
# wrong-direction stages, explicit selection errors
|
|
123
171
|
```
|
|
124
172
|
|
|
125
173
|
Checkout `local.yaml` (per-machine, gitignored):
|
|
@@ -150,44 +198,54 @@ guides live in `docs/runbooks/`.
|
|
|
150
198
|
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
151
199
|
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
152
200
|
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
201
|
+
| `FOREST_OUTPUT` | `auto` | Terminal rendering: `rich`, `plain`, or `auto` (rich on a TTY, plain when piped). |
|
|
202
|
+
| `NO_COLOR` / `FORCE_COLOR` | unset | Standard color conventions; `FOREST_OUTPUT` and `--color/--no-color` take precedence. |
|
|
153
203
|
|
|
154
204
|
## Dogfood: this repo runs forest
|
|
155
205
|
|
|
156
|
-
This repository manages its own `examples/`
|
|
157
|
-
demonstration that `.forest/` and `.git/` coexist without
|
|
158
|
-
set up with exactly the quick-start commands:
|
|
206
|
+
This repository manages its own `examples/` and `tests/data/` trees with
|
|
207
|
+
forest — a live demonstration that `.forest/` and `.git/` coexist without
|
|
208
|
+
overlapping. It was set up with exactly the quick-start commands:
|
|
159
209
|
|
|
160
210
|
```bash
|
|
161
211
|
forest init
|
|
162
212
|
forest checkout demo
|
|
163
|
-
forest remote add origin s3://
|
|
164
|
-
forest add examples ./examples
|
|
165
|
-
forest
|
|
213
|
+
forest remote add origin s3://forest-cli-test --region us-east-2
|
|
214
|
+
forest add examples ./examples # two-way stage (direction: both)
|
|
215
|
+
forest add data ./tests/data --sync-by directory # two small test files
|
|
216
|
+
forest push # seeds both stages
|
|
166
217
|
```
|
|
167
218
|
|
|
219
|
+
After the seeding push, `data` was flipped to `direction: pull` in the
|
|
220
|
+
checkout's `forest.yaml` — it is a read-only reference mirror now: a bare
|
|
221
|
+
`forest push` skips it with a note, and `forest push --stage data` refuses
|
|
222
|
+
with exit 1 (a new stage can start there directly with
|
|
223
|
+
`forest add ... --direction pull`). The `examples` stage keeps syncing both
|
|
224
|
+
ways.
|
|
225
|
+
|
|
168
226
|
Inspect the result:
|
|
169
227
|
|
|
170
228
|
```bash
|
|
171
229
|
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
172
230
|
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
173
|
-
forest
|
|
231
|
+
forest ls # stage overview: data shows (directory, pull)
|
|
232
|
+
forest status # sync state of both stages
|
|
174
233
|
```
|
|
175
234
|
|
|
176
|
-
A fresh clone bootstraps the local half
|
|
177
|
-
|
|
178
|
-
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
179
|
-
|
|
180
|
-
## Migrating from biostore
|
|
181
|
-
|
|
182
|
-
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
235
|
+
A fresh clone bootstraps the local half (HEAD, bindings, sync state are
|
|
236
|
+
gitignored) with:
|
|
183
237
|
|
|
184
238
|
```bash
|
|
185
|
-
forest
|
|
239
|
+
forest checkout demo # select the checkout; clones have no HEAD
|
|
240
|
+
forest bind examples ./examples
|
|
241
|
+
forest bind data ./tests/data
|
|
242
|
+
forest pull --force # first pull: no local sync state yet, so
|
|
243
|
+
# forest refuses to touch the git-tracked
|
|
244
|
+
# copies without it
|
|
186
245
|
```
|
|
187
246
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
refuses to run if a `.forest/` already exists.
|
|
247
|
+
The single configured remote is used automatically. Pulling needs AWS
|
|
248
|
+
credentials for the bucket; the layout is the demonstration.
|
|
191
249
|
|
|
192
250
|
## Notes
|
|
193
251
|
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forest-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "1.0.0"
|
|
8
8
|
description = "git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
11
|
-
requires-python = ">=3.
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
12
|
authors = [
|
|
13
13
|
{name = "Troy Sincomb", email = "troysincomb@gmail.com"},
|
|
14
14
|
]
|
|
@@ -23,6 +23,11 @@ dependencies = [
|
|
|
23
23
|
"polars>=1.0",
|
|
24
24
|
"pydantic>=2.0",
|
|
25
25
|
"pyyaml>=6.0",
|
|
26
|
+
# rich is a textual dependency, but forest imports it directly (Text.from_ansi).
|
|
27
|
+
"rich>=14.2",
|
|
28
|
+
# rich-click 1.9.8 is the first release compatible with the click 8.4 line.
|
|
29
|
+
"rich-click>=1.9.8",
|
|
30
|
+
"textual>=8",
|
|
26
31
|
]
|
|
27
32
|
|
|
28
33
|
[project.urls]
|
|
@@ -42,6 +47,7 @@ dev = [
|
|
|
42
47
|
"pre-commit>=4.0",
|
|
43
48
|
"pylint>=3.3",
|
|
44
49
|
"pytest>=7.0",
|
|
50
|
+
"pytest-asyncio>=0.24",
|
|
45
51
|
"pytest-cov>=6.0",
|
|
46
52
|
"pytest-randomly>=3.15",
|
|
47
53
|
"pytest-rerunfailures>=15.0",
|
|
@@ -51,6 +57,13 @@ dev = [
|
|
|
51
57
|
"vulture>=2.14",
|
|
52
58
|
]
|
|
53
59
|
|
|
60
|
+
[dependency-groups]
|
|
61
|
+
# Docs-site toolchain (mkdocs.yml at the repo root): uv run --group docs mkdocs serve
|
|
62
|
+
docs = [
|
|
63
|
+
"mkdocs>=1.6,<2",
|
|
64
|
+
"mkdocs-material>=9.5,<10",
|
|
65
|
+
]
|
|
66
|
+
|
|
54
67
|
[project.scripts]
|
|
55
68
|
forest = "forest.cli:main"
|
|
56
69
|
|
|
@@ -60,6 +73,9 @@ where = ["src"]
|
|
|
60
73
|
[tool.pytest.ini_options]
|
|
61
74
|
testpaths = ["tests"]
|
|
62
75
|
doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
|
|
76
|
+
# pytest-asyncio runs in strict mode (the default): only tests marked
|
|
77
|
+
# @pytest.mark.asyncio are collected as async (the Textual app tests).
|
|
78
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
63
79
|
addopts = [
|
|
64
80
|
"--cov=forest",
|
|
65
81
|
"--cov-report=term-missing",
|
|
@@ -77,7 +93,7 @@ addopts = [
|
|
|
77
93
|
]
|
|
78
94
|
|
|
79
95
|
[tool.ruff]
|
|
80
|
-
target-version = "
|
|
96
|
+
target-version = "py310"
|
|
81
97
|
line-length = 120
|
|
82
98
|
src = ["src", "tests"]
|
|
83
99
|
|
|
@@ -137,14 +137,6 @@ def remote_url_has_secret_material(url: str) -> bool:
|
|
|
137
137
|
return SECRET_REMOTE_URL_RE.search(url) is not None
|
|
138
138
|
|
|
139
139
|
|
|
140
|
-
def _is_relative_to(path: Path, parent: Path) -> bool:
|
|
141
|
-
try:
|
|
142
|
-
path.relative_to(parent)
|
|
143
|
-
except ValueError:
|
|
144
|
-
return False
|
|
145
|
-
return True
|
|
146
|
-
|
|
147
|
-
|
|
148
140
|
def normalize_workspace_path(path: str | Path, workspace_root: str | Path, *, cwd: str | Path | None = None) -> Path:
|
|
149
141
|
"""Resolve a path and store inside-workspace paths relative to the workspace root."""
|
|
150
142
|
workspace_root = Path(workspace_root).resolve(strict=False)
|
|
@@ -154,7 +146,7 @@ def normalize_workspace_path(path: str | Path, workspace_root: str | Path, *, cw
|
|
|
154
146
|
candidate = base / candidate
|
|
155
147
|
candidate = candidate.resolve(strict=False)
|
|
156
148
|
|
|
157
|
-
if
|
|
149
|
+
if candidate.is_relative_to(workspace_root):
|
|
158
150
|
return candidate.relative_to(workspace_root)
|
|
159
151
|
return candidate
|
|
160
152
|
|
|
@@ -167,7 +159,7 @@ def validate_workspace_data_path(path: str | Path, workspace_root: str | Path) -
|
|
|
167
159
|
candidate = workspace_root / candidate
|
|
168
160
|
candidate = candidate.resolve(strict=False)
|
|
169
161
|
forest_dir = (workspace_root / ".forest").resolve(strict=False)
|
|
170
|
-
if
|
|
162
|
+
if candidate.is_relative_to(forest_dir):
|
|
171
163
|
raise CheckoutError("Data paths must not point inside .forest metadata.")
|
|
172
164
|
return candidate
|
|
173
165
|
|
|
@@ -325,7 +317,7 @@ def read_head(workspace_root: str | Path) -> str:
|
|
|
325
317
|
except OSError as exc:
|
|
326
318
|
raise CheckoutError(f"Cannot read active checkout {path}: {exc}") from None
|
|
327
319
|
|
|
328
|
-
name = raw
|
|
320
|
+
name = raw.removesuffix("\n")
|
|
329
321
|
if "\n" in name:
|
|
330
322
|
raise CheckoutError(f"Invalid active checkout in {path}: expected exactly one checkout name line.")
|
|
331
323
|
return validate_checkout_name(name)
|
|
@@ -399,11 +391,7 @@ def create_checkout(workspace_root: str | Path, checkout_name: str) -> None:
|
|
|
399
391
|
write_head(workspace_root, checkout_name)
|
|
400
392
|
ensure_workspace_gitignore(workspace_root, checkout_name)
|
|
401
393
|
except Exception:
|
|
402
|
-
|
|
403
|
-
if target_existed:
|
|
404
|
-
_empty_directory(target_dir)
|
|
405
|
-
else:
|
|
406
|
-
shutil.rmtree(target_dir)
|
|
394
|
+
_rollback_target_dir(target_dir, existed=target_existed)
|
|
407
395
|
_restore_optional_text(config_path, config_text)
|
|
408
396
|
_restore_optional_text(head_path, head_text)
|
|
409
397
|
_restore_optional_text(gitignore_path, gitignore_text)
|
|
@@ -476,6 +464,15 @@ def _empty_directory(path: Path) -> None:
|
|
|
476
464
|
child.unlink()
|
|
477
465
|
|
|
478
466
|
|
|
467
|
+
def _rollback_target_dir(target_dir: Path, *, existed: bool) -> None:
|
|
468
|
+
if not target_dir.exists():
|
|
469
|
+
return
|
|
470
|
+
if existed:
|
|
471
|
+
_empty_directory(target_dir)
|
|
472
|
+
else:
|
|
473
|
+
shutil.rmtree(target_dir)
|
|
474
|
+
|
|
475
|
+
|
|
479
476
|
def _git_check_ignore(workspace_root: Path, rel_path: str) -> int:
|
|
480
477
|
try:
|
|
481
478
|
result = subprocess.run(
|
|
@@ -508,8 +505,7 @@ def _is_inside_git_worktree(workspace_root: Path) -> bool:
|
|
|
508
505
|
|
|
509
506
|
|
|
510
507
|
def _managed_gitignore_text(original_text: str, patterns: tuple[str, ...]) -> str:
|
|
511
|
-
|
|
512
|
-
kept = [line for line in lines if line.strip() not in FOREST_OWNED_GITIGNORE_LINES]
|
|
508
|
+
kept = [line for line in original_text.splitlines() if line.strip() not in FOREST_OWNED_GITIGNORE_LINES]
|
|
513
509
|
kept.extend(patterns)
|
|
514
510
|
return "\n".join(kept).rstrip("\n") + "\n"
|
|
515
511
|
|
|
@@ -567,17 +563,6 @@ def ensure_registry_gitignore(workspace_root: str | Path) -> None:
|
|
|
567
563
|
)
|
|
568
564
|
|
|
569
565
|
|
|
570
|
-
def ensure_legacy_gitignore(project_root: str | Path) -> None:
|
|
571
|
-
"""Ensure legacy local state ignores while keeping root forest.yaml trackable."""
|
|
572
|
-
project_root = Path(project_root)
|
|
573
|
-
_ensure_gitignore(
|
|
574
|
-
project_root,
|
|
575
|
-
patterns=LEGACY_GITIGNORE_PATTERNS,
|
|
576
|
-
ignored_paths=LEGACY_GITIGNORE_PATTERNS,
|
|
577
|
-
tracked_paths=("forest.yaml",),
|
|
578
|
-
)
|
|
579
|
-
|
|
580
|
-
|
|
581
566
|
def _start_directory(start: str | Path) -> Path:
|
|
582
567
|
start_path = Path(start).resolve(strict=False)
|
|
583
568
|
return start_path.parent if start_path.is_file() else start_path
|
|
@@ -708,6 +693,8 @@ def _verify_adopted_stages(
|
|
|
708
693
|
raise CheckoutError(f"adopt verify failed: schema changed for stage '{stage_name}'.")
|
|
709
694
|
if merged_stage.sync_by != legacy_stage.sync_by:
|
|
710
695
|
raise CheckoutError(f"adopt verify failed: sync_by changed for stage '{stage_name}'.")
|
|
696
|
+
if merged_stage.direction != legacy_stage.direction:
|
|
697
|
+
raise CheckoutError(f"adopt verify failed: direction changed for stage '{stage_name}'.")
|
|
711
698
|
if legacy_stage.remote_path is not None and merged_stage.remote_path != legacy_stage.remote_path:
|
|
712
699
|
raise CheckoutError(f"adopt verify failed: remote_path changed for stage '{stage_name}'.")
|
|
713
700
|
expected_path = _effective_local_stage_path(workspace_root, legacy_stage.path)
|
|
@@ -725,12 +712,7 @@ def _restore_adopt_staging(
|
|
|
725
712
|
gitignore_text: str | None,
|
|
726
713
|
legacy_texts: dict[Path, str | None],
|
|
727
714
|
) -> None:
|
|
728
|
-
|
|
729
|
-
if target_dir.exists():
|
|
730
|
-
if target_existed:
|
|
731
|
-
_empty_directory(target_dir)
|
|
732
|
-
else:
|
|
733
|
-
shutil.rmtree(target_dir)
|
|
715
|
+
_rollback_target_dir(checkout_dir(workspace_root, checkout_name), existed=target_existed)
|
|
734
716
|
_restore_optional_text(workspace_config_path(workspace_root), workspace_config_text)
|
|
735
717
|
_restore_optional_text(_workspace_dir(workspace_root) / "HEAD", head_text)
|
|
736
718
|
_restore_optional_text(workspace_root / ".gitignore", gitignore_text)
|
|
@@ -769,11 +751,8 @@ def adopt_checkout(start: str | Path, checkout_name: str) -> Path:
|
|
|
769
751
|
if not local_path.exists() or not state_path.exists():
|
|
770
752
|
raise CheckoutError("incomplete legacy metadata: expected .forest/local.yaml and .forest/sync_state.json.")
|
|
771
753
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
legacy_local = _load_checkout_local_config(local_path)
|
|
775
|
-
except CheckoutError:
|
|
776
|
-
raise
|
|
754
|
+
legacy_config = _load_shared_config(root_config_path)
|
|
755
|
+
legacy_local = _load_checkout_local_config(local_path)
|
|
777
756
|
|
|
778
757
|
_validate_legacy_adopt_inputs(legacy_config, legacy_local)
|
|
779
758
|
|