forest-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- forest_cli-0.1.0/LICENSE +21 -0
- forest_cli-0.1.0/PKG-INFO +233 -0
- forest_cli-0.1.0/README.md +199 -0
- forest_cli-0.1.0/pyproject.toml +122 -0
- forest_cli-0.1.0/setup.cfg +4 -0
- forest_cli-0.1.0/src/forest/__init__.py +16 -0
- forest_cli-0.1.0/src/forest/analytics.py +29 -0
- forest_cli-0.1.0/src/forest/checkout.py +992 -0
- forest_cli-0.1.0/src/forest/cli.py +2747 -0
- forest_cli-0.1.0/src/forest/config.py +286 -0
- forest_cli-0.1.0/src/forest/flags.py +20 -0
- forest_cli-0.1.0/src/forest/flow.py +157 -0
- forest_cli-0.1.0/src/forest/fsutil.py +36 -0
- forest_cli-0.1.0/src/forest/logger.py +197 -0
- forest_cli-0.1.0/src/forest/manifest.py +112 -0
- forest_cli-0.1.0/src/forest/metrics.py +67 -0
- forest_cli-0.1.0/src/forest/monitoring.py +138 -0
- forest_cli-0.1.0/src/forest/paths.py +192 -0
- forest_cli-0.1.0/src/forest/rclone.py +561 -0
- forest_cli-0.1.0/src/forest/resilience.py +122 -0
- forest_cli-0.1.0/src/forest/sync_state.py +182 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/PKG-INFO +233 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/SOURCES.txt +67 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/dependency_links.txt +1 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/entry_points.txt +2 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/requires.txt +21 -0
- forest_cli-0.1.0/src/forest_cli.egg-info/top_level.txt +1 -0
- forest_cli-0.1.0/tests/test_analytics.py +80 -0
- forest_cli-0.1.0/tests/test_bare_command_defaults.py +316 -0
- forest_cli-0.1.0/tests/test_checkout.py +338 -0
- forest_cli-0.1.0/tests/test_checkout_scaffold.py +138 -0
- forest_cli-0.1.0/tests/test_cli.py +2459 -0
- forest_cli-0.1.0/tests/test_cli_checkout_adopt.py +323 -0
- forest_cli-0.1.0/tests/test_cli_checkout_lifecycle.py +423 -0
- forest_cli-0.1.0/tests/test_cli_cross_area_flows.py +319 -0
- forest_cli-0.1.0/tests/test_cli_gitignore.py +259 -0
- forest_cli-0.1.0/tests/test_cli_init.py +155 -0
- forest_cli-0.1.0/tests/test_cli_remote_workspace.py +363 -0
- forest_cli-0.1.0/tests/test_cli_resolution.py +133 -0
- forest_cli-0.1.0/tests/test_cli_stage_binding.py +342 -0
- forest_cli-0.1.0/tests/test_cli_workspace_data_commands.py +166 -0
- forest_cli-0.1.0/tests/test_config.py +677 -0
- forest_cli-0.1.0/tests/test_config_migration.py +166 -0
- forest_cli-0.1.0/tests/test_cross_backend_parity_lock.py +187 -0
- forest_cli-0.1.0/tests/test_flags.py +28 -0
- forest_cli-0.1.0/tests/test_flow.py +57 -0
- forest_cli-0.1.0/tests/test_forest_agnostic.py +55 -0
- forest_cli-0.1.0/tests/test_integration.py +546 -0
- forest_cli-0.1.0/tests/test_legacy_remote_path_safety.py +136 -0
- forest_cli-0.1.0/tests/test_ls_lock.py +252 -0
- forest_cli-0.1.0/tests/test_manifest.py +426 -0
- forest_cli-0.1.0/tests/test_migrate.py +73 -0
- forest_cli-0.1.0/tests/test_moat_regression_lock.py +165 -0
- forest_cli-0.1.0/tests/test_paths.py +531 -0
- forest_cli-0.1.0/tests/test_polish_scrutiny_fixes.py +288 -0
- forest_cli-0.1.0/tests/test_pull_core_lock.py +644 -0
- forest_cli-0.1.0/tests/test_pull_resilience_lock.py +632 -0
- forest_cli-0.1.0/tests/test_push_core_lock.py +663 -0
- forest_cli-0.1.0/tests/test_push_targeting_lock.py +406 -0
- forest_cli-0.1.0/tests/test_rclone.py +965 -0
- forest_cli-0.1.0/tests/test_rclone_gating_errors.py +147 -0
- forest_cli-0.1.0/tests/test_redaction.py +130 -0
- forest_cli-0.1.0/tests/test_s3_integration.py +235 -0
- forest_cli-0.1.0/tests/test_s3_url_error_hardening.py +227 -0
- forest_cli-0.1.0/tests/test_scrutiny_fixes.py +427 -0
- forest_cli-0.1.0/tests/test_sftp_hardening.py +111 -0
- forest_cli-0.1.0/tests/test_status_lock.py +644 -0
- forest_cli-0.1.0/tests/test_sync_state.py +986 -0
- forest_cli-0.1.0/tests/test_user_testing_fixes.py +374 -0
forest_cli-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Troy Sincomb
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: forest-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
|
|
5
|
+
Author-email: Troy Sincomb <troysincomb@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: data-management,sync,cli
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: click>=8.0
|
|
15
|
+
Requires-Dist: polars>=1.0
|
|
16
|
+
Requires-Dist: pydantic>=2.0
|
|
17
|
+
Requires-Dist: pyyaml>=6.0
|
|
18
|
+
Provides-Extra: observability
|
|
19
|
+
Requires-Dist: sentry-sdk>=2.0; extra == "observability"
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: deptry>=0.23; extra == "dev"
|
|
22
|
+
Requires-Dist: mypy>=1.14; extra == "dev"
|
|
23
|
+
Requires-Dist: pre-commit>=4.0; extra == "dev"
|
|
24
|
+
Requires-Dist: pylint>=3.3; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=6.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-randomly>=3.15; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-rerunfailures>=15.0; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-xdist>=3.6; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
31
|
+
Requires-Dist: types-PyYAML; extra == "dev"
|
|
32
|
+
Requires-Dist: vulture>=2.14; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="docs/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
36
|
+
|
|
37
|
+
**git-like data management for arbitrary data trees.**
|
|
38
|
+
|
|
39
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
|
|
40
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
|
|
41
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
|
|
42
|
+
[](https://www.python.org/downloads/)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
[](https://github.com/astral-sh/ruff)
|
|
45
|
+
[](https://mypy-lang.org/)
|
|
46
|
+
|
|
47
|
+
Forest is the data-side parallel to git's version control. Git tracks code in
|
|
48
|
+
`.git/`; forest tracks large data — local layout plus remote sync — in
|
|
49
|
+
`.forest/`. It borrows git's mental model and verbs (`checkout`, `status`,
|
|
50
|
+
`push`, `pull`, `remote`, a HEAD-style pointer) so your git intuition carries
|
|
51
|
+
over, but the two domains never overlap and neither requires the other.
|
|
52
|
+
|
|
53
|
+
Forest is **domain-agnostic and self-contained**: it manages *any* data trees,
|
|
54
|
+
knows nothing about what the data means, and depends on no other project. It
|
|
55
|
+
moves files and tracks their sync state; it does not validate or interpret their
|
|
56
|
+
contents.
|
|
57
|
+
|
|
58
|
+
## Model
|
|
59
|
+
|
|
60
|
+
Fixed-depth, no arbitrary nesting:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
workspace → checkout → stage → unit → files
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
- **Workspace** — a per-repo `.forest/` control area (registry + active pointer).
|
|
67
|
+
- **Checkout** — a named data view/focus registered in the workspace (like a git
|
|
68
|
+
branch you stay rooted in). Switching is an O(1) pointer rewrite; data never
|
|
69
|
+
moves.
|
|
70
|
+
- **Stage** — a named data category inside a checkout, with a remote layout.
|
|
71
|
+
- **Unit** — one addressable item within a stage (a subdirectory, a directory,
|
|
72
|
+
or a file, per the stage's `sync_by`).
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install -e ./forest
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Quick start
|
|
83
|
+
|
|
84
|
+
No config files are hand-written. Onboarding is a few commands:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
forest init # create the nameless .forest/ workspace container
|
|
88
|
+
forest checkout demo # create if absent, register + activate 'demo'
|
|
89
|
+
forest remote add origin s3://my-bucket/prefix # allowed before any local data exists
|
|
90
|
+
forest add raw ./data/raw # register stage 'raw' and bind it to a local path
|
|
91
|
+
forest push # sync every bound stage to the active remote
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
- `forest init` creates **only** `.forest/config.yaml` (`version: 1`,
|
|
95
|
+
`checkouts: {}`) and a managed `.gitignore`. No root config, no checkout, no
|
|
96
|
+
active pointer.
|
|
97
|
+
- `forest checkout <name>` switches to the checkout, creating, registering, and
|
|
98
|
+
activating it first (with `.forest/checkouts/<name>/forest.yaml`) when the
|
|
99
|
+
name is not registered. `forest checkout create <name>` is the explicit form.
|
|
100
|
+
- Remotes can be added before any local binding — useful when your data is
|
|
101
|
+
remote-only at first.
|
|
102
|
+
- `forest add STAGE PATH` registers a new stage and binds it to a local path in
|
|
103
|
+
one step (use `forest bind` to rebind an existing stage).
|
|
104
|
+
|
|
105
|
+
## Metadata layout
|
|
106
|
+
|
|
107
|
+
Everything forest owns lives under `.forest/`; your data does not.
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
.forest/
|
|
111
|
+
config.yaml # workspace registry: version, checkouts{}
|
|
112
|
+
HEAD # active checkout name (gitignored)
|
|
113
|
+
checkouts/
|
|
114
|
+
demo/
|
|
115
|
+
forest.yaml # shared: stages, remotes, manifest
|
|
116
|
+
local.yaml # user-local: active_remote, stage_paths (gitignored)
|
|
117
|
+
sync_state.json # user-local push/pull state (gitignored)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Shared metadata (`config.yaml`, each `forest.yaml`) is committed so a fresh
|
|
121
|
+
clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
122
|
+
(`HEAD`, `local.yaml`, `sync_state.json`) are gitignored.
|
|
123
|
+
|
|
124
|
+
## Commands
|
|
125
|
+
|
|
126
|
+
| Command | Purpose |
|
|
127
|
+
|---|---|
|
|
128
|
+
| `forest init` | Create the workspace container, or report setup status if it exists. |
|
|
129
|
+
| `forest checkout create/adopt/list/current/remove <name>` | Manage checkouts; bare `forest checkout <name>` switches, creating first if needed. `remove --yes` skips the prompt for scripts. |
|
|
130
|
+
| `forest add STAGE PATH [--sync-by MODE]` | Register a new stage and bind it to a local path; `--sync-by` picks unit discovery (`subdirectory`/`directory`/`file`). |
|
|
131
|
+
| `forest bind [STAGE PATH]` / `forest unbind STAGE` | Manage local stage↔path bindings. |
|
|
132
|
+
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
133
|
+
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
134
|
+
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
135
|
+
| `forest migrate` | Migrate a legacy `biostore` layout in place (see below). |
|
|
136
|
+
|
|
137
|
+
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
138
|
+
|
|
139
|
+
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
140
|
+
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
141
|
+
|
|
142
|
+
## Config reference
|
|
143
|
+
|
|
144
|
+
Checkout `forest.yaml` (shared, committed):
|
|
145
|
+
|
|
146
|
+
```yaml
|
|
147
|
+
project: demo
|
|
148
|
+
remotes:
|
|
149
|
+
origin:
|
|
150
|
+
url: s3://my-bucket/prefix
|
|
151
|
+
region: us-east-2 # optional; also endpoint, profile, key_file, known_hosts
|
|
152
|
+
stages:
|
|
153
|
+
raw:
|
|
154
|
+
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
155
|
+
sync_by: subdirectory # subdirectory | directory | file
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Checkout `local.yaml` (per-machine, gitignored):
|
|
159
|
+
|
|
160
|
+
```yaml
|
|
161
|
+
active_remote: origin
|
|
162
|
+
stage_paths:
|
|
163
|
+
raw: ../data/raw # relative resolves from the workspace root
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Environment variables
|
|
167
|
+
|
|
168
|
+
All optional, all off by default — forest is silent and sends nothing anywhere
|
|
169
|
+
unless configured. Copy `.env.example` for a commented template; operational
|
|
170
|
+
guides live in `docs/runbooks/`.
|
|
171
|
+
|
|
172
|
+
| Variable | Default | Effect |
|
|
173
|
+
|---|---|---|
|
|
174
|
+
| `FOREST_LOG_FILE` | unset | Append structured logs (JSON lines) to this file. |
|
|
175
|
+
| `FOREST_LOG_FORMAT` | `json` | `json` or `text`; set without `FOREST_LOG_FILE` to log to stderr. |
|
|
176
|
+
| `FOREST_LOG_LEVEL` | `INFO` | Standard logging level name. |
|
|
177
|
+
| `FOREST_METRICS_FILE` | unset | Append metric samples as JSON lines for external collectors. |
|
|
178
|
+
| `FOREST_ANALYTICS_FILE` | unset | Opt-in local usage analytics (JSON lines); nothing leaves the machine. |
|
|
179
|
+
| `FOREST_SENTRY_DSN` | unset | Sentry error tracking; needs `pip install "forest-cli[observability]"`. |
|
|
180
|
+
| `FOREST_ALERT_WEBHOOK` | unset | POST failure alerts to this HTTPS endpoint (Slack/Mattermost compatible). |
|
|
181
|
+
| `FOREST_TRANSFER_RETRIES` | `2` | Extra attempts for transient rclone failures; `0` disables. |
|
|
182
|
+
| `FOREST_RETRY_BASE_DELAY` | `0.5` | Initial retry backoff in seconds; doubles per attempt. |
|
|
183
|
+
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
184
|
+
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
185
|
+
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
186
|
+
|
|
187
|
+
## Dogfood: this repo runs forest
|
|
188
|
+
|
|
189
|
+
This repository manages its own `examples/` tree with forest — a live
|
|
190
|
+
demonstration that `.forest/` and `.git/` coexist without overlapping. It was
|
|
191
|
+
set up with exactly the quick-start commands:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
forest init
|
|
195
|
+
forest checkout demo
|
|
196
|
+
forest remote add origin s3://forest-test-542222635421-us-east-2-an --region us-east-2
|
|
197
|
+
forest add examples ./examples
|
|
198
|
+
forest push
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Inspect the result:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
205
|
+
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
206
|
+
forest status # sync state of the examples stage
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
A fresh clone bootstraps the local half with `forest bind examples ./examples`
|
|
210
|
+
followed by `forest pull` (the single configured remote is used automatically).
|
|
211
|
+
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
212
|
+
|
|
213
|
+
## Migrating from biostore
|
|
214
|
+
|
|
215
|
+
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
forest migrate
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
This renames `.biostore/` → `.forest/`, each `biostore.yaml` → `forest.yaml`,
|
|
222
|
+
rewrites the managed `.gitignore` patterns, and verifies the registry parses. It
|
|
223
|
+
refuses to run if a `.forest/` already exists.
|
|
224
|
+
|
|
225
|
+
## Notes
|
|
226
|
+
|
|
227
|
+
- **Single active machine (v1).** `HEAD`/`local.yaml`/`sync_state.json` are
|
|
228
|
+
git-invisible but may be synced by a file-syncing tool; forest assumes one
|
|
229
|
+
active machine and uses atomic writes plus a per-checkout `flock` for
|
|
230
|
+
intra-machine write races.
|
|
231
|
+
- **Real filenames.** Forest stores data under real paths, not a
|
|
232
|
+
content-addressed blob store.
|
|
233
|
+
- See `docs/adr/` for the design decisions behind the workspace/checkout model.
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="docs/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
|
|
2
|
+
|
|
3
|
+
**git-like data management for arbitrary data trees.**
|
|
4
|
+
|
|
5
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
|
|
6
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
|
|
7
|
+
[](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
|
|
8
|
+
[](https://www.python.org/downloads/)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
[](https://github.com/astral-sh/ruff)
|
|
11
|
+
[](https://mypy-lang.org/)
|
|
12
|
+
|
|
13
|
+
Forest is the data-side parallel to git's version control. Git tracks code in
|
|
14
|
+
`.git/`; forest tracks large data — local layout plus remote sync — in
|
|
15
|
+
`.forest/`. It borrows git's mental model and verbs (`checkout`, `status`,
|
|
16
|
+
`push`, `pull`, `remote`, a HEAD-style pointer) so your git intuition carries
|
|
17
|
+
over, but the two domains never overlap and neither requires the other.
|
|
18
|
+
|
|
19
|
+
Forest is **domain-agnostic and self-contained**: it manages *any* data trees,
|
|
20
|
+
knows nothing about what the data means, and depends on no other project. It
|
|
21
|
+
moves files and tracks their sync state; it does not validate or interpret their
|
|
22
|
+
contents.
|
|
23
|
+
|
|
24
|
+
## Model
|
|
25
|
+
|
|
26
|
+
Fixed-depth, no arbitrary nesting:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
workspace → checkout → stage → unit → files
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
- **Workspace** — a per-repo `.forest/` control area (registry + active pointer).
|
|
33
|
+
- **Checkout** — a named data view/focus registered in the workspace (like a git
|
|
34
|
+
branch you stay rooted in). Switching is an O(1) pointer rewrite; data never
|
|
35
|
+
moves.
|
|
36
|
+
- **Stage** — a named data category inside a checkout, with a remote layout.
|
|
37
|
+
- **Unit** — one addressable item within a stage (a subdirectory, a directory,
|
|
38
|
+
or a file, per the stage's `sync_by`).
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install -e ./forest
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Quick start
|
|
49
|
+
|
|
50
|
+
No config files are hand-written. Onboarding is a few commands:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
forest init # create the nameless .forest/ workspace container
|
|
54
|
+
forest checkout demo # create if absent, register + activate 'demo'
|
|
55
|
+
forest remote add origin s3://my-bucket/prefix # allowed before any local data exists
|
|
56
|
+
forest add raw ./data/raw # register stage 'raw' and bind it to a local path
|
|
57
|
+
forest push # sync every bound stage to the active remote
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
- `forest init` creates **only** `.forest/config.yaml` (`version: 1`,
|
|
61
|
+
`checkouts: {}`) and a managed `.gitignore`. No root config, no checkout, no
|
|
62
|
+
active pointer.
|
|
63
|
+
- `forest checkout <name>` switches to the checkout, creating, registering, and
|
|
64
|
+
activating it first (with `.forest/checkouts/<name>/forest.yaml`) when the
|
|
65
|
+
name is not registered. `forest checkout create <name>` is the explicit form.
|
|
66
|
+
- Remotes can be added before any local binding — useful when your data is
|
|
67
|
+
remote-only at first.
|
|
68
|
+
- `forest add STAGE PATH` registers a new stage and binds it to a local path in
|
|
69
|
+
one step (use `forest bind` to rebind an existing stage).
|
|
70
|
+
|
|
71
|
+
## Metadata layout
|
|
72
|
+
|
|
73
|
+
Everything forest owns lives under `.forest/`; your data does not.
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
.forest/
|
|
77
|
+
config.yaml # workspace registry: version, checkouts{}
|
|
78
|
+
HEAD # active checkout name (gitignored)
|
|
79
|
+
checkouts/
|
|
80
|
+
demo/
|
|
81
|
+
forest.yaml # shared: stages, remotes, manifest
|
|
82
|
+
local.yaml # user-local: active_remote, stage_paths (gitignored)
|
|
83
|
+
sync_state.json # user-local push/pull state (gitignored)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Shared metadata (`config.yaml`, each `forest.yaml`) is committed so a fresh
|
|
87
|
+
clone bootstraps with `bind` + `remote use` + `pull`. User-local files
|
|
88
|
+
(`HEAD`, `local.yaml`, `sync_state.json`) are gitignored.
|
|
89
|
+
|
|
90
|
+
## Commands
|
|
91
|
+
|
|
92
|
+
| Command | Purpose |
|
|
93
|
+
|---|---|
|
|
94
|
+
| `forest init` | Create the workspace container, or report setup status if it exists. |
|
|
95
|
+
| `forest checkout create/adopt/list/current/remove <name>` | Manage checkouts; bare `forest checkout <name>` switches, creating first if needed. `remove --yes` skips the prompt for scripts. |
|
|
96
|
+
| `forest add STAGE PATH [--sync-by MODE]` | Register a new stage and bind it to a local path; `--sync-by` picks unit discovery (`subdirectory`/`directory`/`file`). |
|
|
97
|
+
| `forest bind [STAGE PATH]` / `forest unbind STAGE` | Manage local stage↔path bindings. |
|
|
98
|
+
| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
|
|
99
|
+
| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
|
|
100
|
+
| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
|
|
101
|
+
| `forest migrate` | Migrate a legacy `biostore` layout in place (see below). |
|
|
102
|
+
|
|
103
|
+
Run any command with `-C <path>` to operate on another repo without `cd`.
|
|
104
|
+
|
|
105
|
+
Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
|
|
106
|
+
AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
|
|
107
|
+
|
|
108
|
+
## Config reference
|
|
109
|
+
|
|
110
|
+
Checkout `forest.yaml` (shared, committed):
|
|
111
|
+
|
|
112
|
+
```yaml
|
|
113
|
+
project: demo
|
|
114
|
+
remotes:
|
|
115
|
+
origin:
|
|
116
|
+
url: s3://my-bucket/prefix
|
|
117
|
+
region: us-east-2 # optional; also endpoint, profile, key_file, known_hosts
|
|
118
|
+
stages:
|
|
119
|
+
raw:
|
|
120
|
+
remote_path: demo/raw # optional; defaults to <checkout>/<stage>
|
|
121
|
+
sync_by: subdirectory # subdirectory | directory | file
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Checkout `local.yaml` (per-machine, gitignored):
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
active_remote: origin
|
|
128
|
+
stage_paths:
|
|
129
|
+
raw: ../data/raw # relative resolves from the workspace root
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Environment variables
|
|
133
|
+
|
|
134
|
+
All optional, all off by default — forest is silent and sends nothing anywhere
|
|
135
|
+
unless configured. Copy `.env.example` for a commented template; operational
|
|
136
|
+
guides live in `docs/runbooks/`.
|
|
137
|
+
|
|
138
|
+
| Variable | Default | Effect |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| `FOREST_LOG_FILE` | unset | Append structured logs (JSON lines) to this file. |
|
|
141
|
+
| `FOREST_LOG_FORMAT` | `json` | `json` or `text`; set without `FOREST_LOG_FILE` to log to stderr. |
|
|
142
|
+
| `FOREST_LOG_LEVEL` | `INFO` | Standard logging level name. |
|
|
143
|
+
| `FOREST_METRICS_FILE` | unset | Append metric samples as JSON lines for external collectors. |
|
|
144
|
+
| `FOREST_ANALYTICS_FILE` | unset | Opt-in local usage analytics (JSON lines); nothing leaves the machine. |
|
|
145
|
+
| `FOREST_SENTRY_DSN` | unset | Sentry error tracking; needs `pip install "forest-cli[observability]"`. |
|
|
146
|
+
| `FOREST_ALERT_WEBHOOK` | unset | POST failure alerts to this HTTPS endpoint (Slack/Mattermost compatible). |
|
|
147
|
+
| `FOREST_TRANSFER_RETRIES` | `2` | Extra attempts for transient rclone failures; `0` disables. |
|
|
148
|
+
| `FOREST_RETRY_BASE_DELAY` | `0.5` | Initial retry backoff in seconds; doubles per attempt. |
|
|
149
|
+
| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
|
|
150
|
+
| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
|
|
151
|
+
| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
|
|
152
|
+
|
|
153
|
+
## Dogfood: this repo runs forest
|
|
154
|
+
|
|
155
|
+
This repository manages its own `examples/` tree with forest — a live
|
|
156
|
+
demonstration that `.forest/` and `.git/` coexist without overlapping. It was
|
|
157
|
+
set up with exactly the quick-start commands:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
forest init
|
|
161
|
+
forest checkout demo
|
|
162
|
+
forest remote add origin s3://forest-test-542222635421-us-east-2-an --region us-east-2
|
|
163
|
+
forest add examples ./examples
|
|
164
|
+
forest push
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Inspect the result:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
|
|
171
|
+
cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
|
|
172
|
+
forest status # sync state of the examples stage
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
A fresh clone bootstraps the local half with `forest bind examples ./examples`
|
|
176
|
+
followed by `forest pull` (the single configured remote is used automatically).
|
|
177
|
+
Pulling needs AWS credentials for the bucket; the layout is the demonstration.
|
|
178
|
+
|
|
179
|
+
## Migrating from biostore
|
|
180
|
+
|
|
181
|
+
Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
forest migrate
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
This renames `.biostore/` → `.forest/`, each `biostore.yaml` → `forest.yaml`,
|
|
188
|
+
rewrites the managed `.gitignore` patterns, and verifies the registry parses. It
|
|
189
|
+
refuses to run if a `.forest/` already exists.
|
|
190
|
+
|
|
191
|
+
## Notes
|
|
192
|
+
|
|
193
|
+
- **Single active machine (v1).** `HEAD`/`local.yaml`/`sync_state.json` are
|
|
194
|
+
git-invisible but may be synced by a file-syncing tool; forest assumes one
|
|
195
|
+
active machine and uses atomic writes plus a per-checkout `flock` for
|
|
196
|
+
intra-machine write races.
|
|
197
|
+
- **Real filenames.** Forest stores data under real paths, not a
|
|
198
|
+
content-addressed blob store.
|
|
199
|
+
- See `docs/adr/` for the design decisions behind the workspace/checkout model.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "forest-cli"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Troy Sincomb", email = "troysincomb@gmail.com"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["data-management", "sync", "cli"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"click>=8.0",
|
|
23
|
+
"polars>=1.0",
|
|
24
|
+
"pydantic>=2.0",
|
|
25
|
+
"pyyaml>=6.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
# Opt-in error tracking (FOREST_SENTRY_DSN); see docs/runbooks/alerting.md.
|
|
30
|
+
observability = [
|
|
31
|
+
"sentry-sdk>=2.0",
|
|
32
|
+
]
|
|
33
|
+
dev = [
|
|
34
|
+
"deptry>=0.23",
|
|
35
|
+
"mypy>=1.14",
|
|
36
|
+
"pre-commit>=4.0",
|
|
37
|
+
"pylint>=3.3",
|
|
38
|
+
"pytest>=7.0",
|
|
39
|
+
"pytest-cov>=6.0",
|
|
40
|
+
"pytest-randomly>=3.15",
|
|
41
|
+
"pytest-rerunfailures>=15.0",
|
|
42
|
+
"pytest-xdist>=3.6",
|
|
43
|
+
"ruff>=0.4",
|
|
44
|
+
"types-PyYAML",
|
|
45
|
+
"vulture>=2.14",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
forest = "forest.cli:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
|
|
57
|
+
addopts = [
|
|
58
|
+
"--cov=forest",
|
|
59
|
+
"--cov-report=term-missing",
|
|
60
|
+
"--cov-fail-under=70",
|
|
61
|
+
"--durations=15",
|
|
62
|
+
"--durations-min=0.5",
|
|
63
|
+
# Test isolation: run in parallel workers; pytest-randomly shuffles order each
|
|
64
|
+
# run so hidden inter-test dependencies surface immediately.
|
|
65
|
+
"--numprocesses=auto",
|
|
66
|
+
# Flaky-test policy: retry only transient environment errors (never assertion
|
|
67
|
+
# failures); reruns are reported in the summary so instability stays visible.
|
|
68
|
+
"--reruns=1",
|
|
69
|
+
"--reruns-delay=1",
|
|
70
|
+
"--only-rerun=(Timeout|Connection|BrokenPipe)",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
[tool.ruff]
|
|
74
|
+
target-version = "py39"
|
|
75
|
+
line-length = 120
|
|
76
|
+
src = ["src", "tests"]
|
|
77
|
+
|
|
78
|
+
[tool.ruff.lint]
|
|
79
|
+
# N enforces pep8-naming, C90 enforces the cyclomatic complexity budget, and
|
|
80
|
+
# TD/FIX police tech-debt markers: TODOs must carry an author and issue link
|
|
81
|
+
# (TD002/TD003) while FIXME/XXX/HACK never land on main (FIX001/FIX003/FIX004).
|
|
82
|
+
select = ["E", "F", "I", "N", "UP", "B", "SIM", "RUF", "C90", "TD", "FIX"]
|
|
83
|
+
# FIX002 flags every TODO even when well-formed; TD rules already gate those.
|
|
84
|
+
ignore = ["FIX002"]
|
|
85
|
+
|
|
86
|
+
[tool.ruff.lint.mccabe]
|
|
87
|
+
max-complexity = 10
|
|
88
|
+
|
|
89
|
+
[tool.ruff.lint.isort]
|
|
90
|
+
known-first-party = ["forest"]
|
|
91
|
+
|
|
92
|
+
[tool.ruff.format]
|
|
93
|
+
quote-style = "double"
|
|
94
|
+
indent-style = "space"
|
|
95
|
+
line-ending = "auto"
|
|
96
|
+
|
|
97
|
+
[tool.mypy]
|
|
98
|
+
python_version = "3.10"
|
|
99
|
+
files = ["src/forest"]
|
|
100
|
+
strict = true
|
|
101
|
+
warn_unused_configs = true
|
|
102
|
+
|
|
103
|
+
[tool.deptry]
|
|
104
|
+
pep621_dev_dependency_groups = ["dev"]
|
|
105
|
+
|
|
106
|
+
[tool.vulture]
|
|
107
|
+
paths = ["src/forest"]
|
|
108
|
+
min_confidence = 80
|
|
109
|
+
# Pydantic validators receive `cls` per the classmethod protocol even when the
|
|
110
|
+
# body never uses it.
|
|
111
|
+
ignore_names = ["cls"]
|
|
112
|
+
|
|
113
|
+
# Pylint is used solely as the duplicate-code (DRY) detector; ruff owns every
|
|
114
|
+
# other lint concern.
|
|
115
|
+
[tool.pylint.main]
|
|
116
|
+
disable = ["all"]
|
|
117
|
+
enable = ["duplicate-code"]
|
|
118
|
+
|
|
119
|
+
[tool.pylint.similarities]
|
|
120
|
+
min-similarity-lines = 5
|
|
121
|
+
ignore-imports = true
|
|
122
|
+
ignore-signatures = true
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""forest — git-like data management for arbitrary data trees.
|
|
2
|
+
|
|
3
|
+
Forest is a self-contained, domain-agnostic data-sync engine: workspace
|
|
4
|
+
initialization, named checkouts (focus), remotes, local bindings,
|
|
5
|
+
push/pull/status/ls, and sync state. It knows nothing about what the data
|
|
6
|
+
means — it syncs and inspects whatever files live in a unit.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
__version__ = version("forest")
|
|
15
|
+
except PackageNotFoundError:
|
|
16
|
+
__version__ = "0.0.0"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Opt-in local usage analytics for forest commands.
|
|
2
|
+
|
|
3
|
+
No-op unless ``FOREST_ANALYTICS_FILE`` is set; events are appended as JSON
|
|
4
|
+
lines to that local file only — nothing leaves the machine. Eager options
|
|
5
|
+
(``--help``/``--version``) exit during parsing and are not recorded.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
|
|
12
|
+
from forest import __version__
|
|
13
|
+
from forest.logger import get_run_id
|
|
14
|
+
from forest.metrics import append_jsonl
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def record_command(command: str, *, duration_seconds: float, status: str) -> None:
|
|
18
|
+
"""Append one command-usage event. Never raises."""
|
|
19
|
+
append_jsonl(
|
|
20
|
+
"FOREST_ANALYTICS_FILE",
|
|
21
|
+
{
|
|
22
|
+
"ts": datetime.now(tz=timezone.utc).isoformat(),
|
|
23
|
+
"command": command,
|
|
24
|
+
"duration_seconds": round(duration_seconds, 3),
|
|
25
|
+
"status": status, # "success" | "error"
|
|
26
|
+
"run_id": get_run_id(),
|
|
27
|
+
"version": __version__,
|
|
28
|
+
},
|
|
29
|
+
)
|