forest-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. forest_cli-0.1.0/LICENSE +21 -0
  2. forest_cli-0.1.0/PKG-INFO +233 -0
  3. forest_cli-0.1.0/README.md +199 -0
  4. forest_cli-0.1.0/pyproject.toml +122 -0
  5. forest_cli-0.1.0/setup.cfg +4 -0
  6. forest_cli-0.1.0/src/forest/__init__.py +16 -0
  7. forest_cli-0.1.0/src/forest/analytics.py +29 -0
  8. forest_cli-0.1.0/src/forest/checkout.py +992 -0
  9. forest_cli-0.1.0/src/forest/cli.py +2747 -0
  10. forest_cli-0.1.0/src/forest/config.py +286 -0
  11. forest_cli-0.1.0/src/forest/flags.py +20 -0
  12. forest_cli-0.1.0/src/forest/flow.py +157 -0
  13. forest_cli-0.1.0/src/forest/fsutil.py +36 -0
  14. forest_cli-0.1.0/src/forest/logger.py +197 -0
  15. forest_cli-0.1.0/src/forest/manifest.py +112 -0
  16. forest_cli-0.1.0/src/forest/metrics.py +67 -0
  17. forest_cli-0.1.0/src/forest/monitoring.py +138 -0
  18. forest_cli-0.1.0/src/forest/paths.py +192 -0
  19. forest_cli-0.1.0/src/forest/rclone.py +561 -0
  20. forest_cli-0.1.0/src/forest/resilience.py +122 -0
  21. forest_cli-0.1.0/src/forest/sync_state.py +182 -0
  22. forest_cli-0.1.0/src/forest_cli.egg-info/PKG-INFO +233 -0
  23. forest_cli-0.1.0/src/forest_cli.egg-info/SOURCES.txt +67 -0
  24. forest_cli-0.1.0/src/forest_cli.egg-info/dependency_links.txt +1 -0
  25. forest_cli-0.1.0/src/forest_cli.egg-info/entry_points.txt +2 -0
  26. forest_cli-0.1.0/src/forest_cli.egg-info/requires.txt +21 -0
  27. forest_cli-0.1.0/src/forest_cli.egg-info/top_level.txt +1 -0
  28. forest_cli-0.1.0/tests/test_analytics.py +80 -0
  29. forest_cli-0.1.0/tests/test_bare_command_defaults.py +316 -0
  30. forest_cli-0.1.0/tests/test_checkout.py +338 -0
  31. forest_cli-0.1.0/tests/test_checkout_scaffold.py +138 -0
  32. forest_cli-0.1.0/tests/test_cli.py +2459 -0
  33. forest_cli-0.1.0/tests/test_cli_checkout_adopt.py +323 -0
  34. forest_cli-0.1.0/tests/test_cli_checkout_lifecycle.py +423 -0
  35. forest_cli-0.1.0/tests/test_cli_cross_area_flows.py +319 -0
  36. forest_cli-0.1.0/tests/test_cli_gitignore.py +259 -0
  37. forest_cli-0.1.0/tests/test_cli_init.py +155 -0
  38. forest_cli-0.1.0/tests/test_cli_remote_workspace.py +363 -0
  39. forest_cli-0.1.0/tests/test_cli_resolution.py +133 -0
  40. forest_cli-0.1.0/tests/test_cli_stage_binding.py +342 -0
  41. forest_cli-0.1.0/tests/test_cli_workspace_data_commands.py +166 -0
  42. forest_cli-0.1.0/tests/test_config.py +677 -0
  43. forest_cli-0.1.0/tests/test_config_migration.py +166 -0
  44. forest_cli-0.1.0/tests/test_cross_backend_parity_lock.py +187 -0
  45. forest_cli-0.1.0/tests/test_flags.py +28 -0
  46. forest_cli-0.1.0/tests/test_flow.py +57 -0
  47. forest_cli-0.1.0/tests/test_forest_agnostic.py +55 -0
  48. forest_cli-0.1.0/tests/test_integration.py +546 -0
  49. forest_cli-0.1.0/tests/test_legacy_remote_path_safety.py +136 -0
  50. forest_cli-0.1.0/tests/test_ls_lock.py +252 -0
  51. forest_cli-0.1.0/tests/test_manifest.py +426 -0
  52. forest_cli-0.1.0/tests/test_migrate.py +73 -0
  53. forest_cli-0.1.0/tests/test_moat_regression_lock.py +165 -0
  54. forest_cli-0.1.0/tests/test_paths.py +531 -0
  55. forest_cli-0.1.0/tests/test_polish_scrutiny_fixes.py +288 -0
  56. forest_cli-0.1.0/tests/test_pull_core_lock.py +644 -0
  57. forest_cli-0.1.0/tests/test_pull_resilience_lock.py +632 -0
  58. forest_cli-0.1.0/tests/test_push_core_lock.py +663 -0
  59. forest_cli-0.1.0/tests/test_push_targeting_lock.py +406 -0
  60. forest_cli-0.1.0/tests/test_rclone.py +965 -0
  61. forest_cli-0.1.0/tests/test_rclone_gating_errors.py +147 -0
  62. forest_cli-0.1.0/tests/test_redaction.py +130 -0
  63. forest_cli-0.1.0/tests/test_s3_integration.py +235 -0
  64. forest_cli-0.1.0/tests/test_s3_url_error_hardening.py +227 -0
  65. forest_cli-0.1.0/tests/test_scrutiny_fixes.py +427 -0
  66. forest_cli-0.1.0/tests/test_sftp_hardening.py +111 -0
  67. forest_cli-0.1.0/tests/test_status_lock.py +644 -0
  68. forest_cli-0.1.0/tests/test_sync_state.py +986 -0
  69. forest_cli-0.1.0/tests/test_user_testing_fixes.py +374 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Troy Sincomb
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,233 @@
1
+ Metadata-Version: 2.4
2
+ Name: forest-cli
3
+ Version: 0.1.0
4
+ Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
5
+ Author-email: Troy Sincomb <troysincomb@gmail.com>
6
+ License: MIT
7
+ Keywords: data-management,sync,cli
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Requires-Python: >=3.9
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: click>=8.0
15
+ Requires-Dist: polars>=1.0
16
+ Requires-Dist: pydantic>=2.0
17
+ Requires-Dist: pyyaml>=6.0
18
+ Provides-Extra: observability
19
+ Requires-Dist: sentry-sdk>=2.0; extra == "observability"
20
+ Provides-Extra: dev
21
+ Requires-Dist: deptry>=0.23; extra == "dev"
22
+ Requires-Dist: mypy>=1.14; extra == "dev"
23
+ Requires-Dist: pre-commit>=4.0; extra == "dev"
24
+ Requires-Dist: pylint>=3.3; extra == "dev"
25
+ Requires-Dist: pytest>=7.0; extra == "dev"
26
+ Requires-Dist: pytest-cov>=6.0; extra == "dev"
27
+ Requires-Dist: pytest-randomly>=3.15; extra == "dev"
28
+ Requires-Dist: pytest-rerunfailures>=15.0; extra == "dev"
29
+ Requires-Dist: pytest-xdist>=3.6; extra == "dev"
30
+ Requires-Dist: ruff>=0.4; extra == "dev"
31
+ Requires-Dist: types-PyYAML; extra == "dev"
32
+ Requires-Dist: vulture>=2.14; extra == "dev"
33
+ Dynamic: license-file
34
+
35
+ # 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="docs/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
36
+
37
+ **git-like data management for arbitrary data trees.**
38
+
39
+ [![CI](https://github.com/tmsincomb/forest/actions/workflows/ci.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
40
+ [![Docs](https://github.com/tmsincomb/forest/actions/workflows/docs.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
41
+ [![Release](https://github.com/tmsincomb/forest/actions/workflows/release.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
42
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/downloads/)
43
+ [![License: MIT](https://img.shields.io/badge/license-MIT-2ea44f)](LICENSE)
44
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
45
+ [![mypy: strict](https://img.shields.io/badge/mypy-strict-blue)](https://mypy-lang.org/)
46
+
47
+ Forest is the data-side parallel to git's version control. Git tracks code in
48
+ `.git/`; forest tracks large data — local layout plus remote sync — in
49
+ `.forest/`. It borrows git's mental model and verbs (`checkout`, `status`,
50
+ `push`, `pull`, `remote`, a HEAD-style pointer) so your git intuition carries
51
+ over, but the two domains never overlap and neither requires the other.
52
+
53
+ Forest is **domain-agnostic and self-contained**: it manages *any* data trees,
54
+ knows nothing about what the data means, and depends on no other project. It
55
+ moves files and tracks their sync state; it does not validate or interpret their
56
+ contents.
57
+
58
+ ## Model
59
+
60
+ Fixed-depth, no arbitrary nesting:
61
+
62
+ ```
63
+ workspace → checkout → stage → unit → files
64
+ ```
65
+
66
+ - **Workspace** — a per-repo `.forest/` control area (registry + active pointer).
67
+ - **Checkout** — a named data view/focus registered in the workspace (like a git
68
+ branch you stay rooted in). Switching is an O(1) pointer rewrite; data never
69
+ moves.
70
+ - **Stage** — a named data category inside a checkout, with a remote layout.
71
+ - **Unit** — one addressable item within a stage (a subdirectory, a directory,
72
+ or a file, per the stage's `sync_by`).
73
+
74
+ ## Install
75
+
76
+ Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
77
+
78
+ ```bash
79
+ pip install -e ./forest
80
+ ```
81
+
82
+ ## Quick start
83
+
84
+ No config files are hand-written. Onboarding is a few commands:
85
+
86
+ ```bash
87
+ forest init # create the nameless .forest/ workspace container
88
+ forest checkout demo # create if absent, register + activate 'demo'
89
+ forest remote add origin s3://my-bucket/prefix # allowed before any local data exists
90
+ forest add raw ./data/raw # register stage 'raw' and bind it to a local path
91
+ forest push # sync every bound stage to the active remote
92
+ ```
93
+
94
+ - `forest init` creates **only** `.forest/config.yaml` (`version: 1`,
95
+ `checkouts: {}`) and a managed `.gitignore`. No root config, no checkout, no
96
+ active pointer.
97
+ - `forest checkout <name>` switches to the checkout, creating, registering, and
98
+ activating it first (with `.forest/checkouts/<name>/forest.yaml`) when the
99
+ name is not registered. `forest checkout create <name>` is the explicit form.
100
+ - Remotes can be added before any local binding — useful when your data is
101
+ remote-only at first.
102
+ - `forest add STAGE PATH` registers a new stage and binds it to a local path in
103
+ one step (use `forest bind` to rebind an existing stage).
104
+
105
+ ## Metadata layout
106
+
107
+ Everything forest owns lives under `.forest/`; your data does not.
108
+
109
+ ```
110
+ .forest/
111
+ config.yaml # workspace registry: version, checkouts{}
112
+ HEAD # active checkout name (gitignored)
113
+ checkouts/
114
+ demo/
115
+ forest.yaml # shared: stages, remotes, manifest
116
+ local.yaml # user-local: active_remote, stage_paths (gitignored)
117
+ sync_state.json # user-local push/pull state (gitignored)
118
+ ```
119
+
120
+ Shared metadata (`config.yaml`, each `forest.yaml`) is committed so a fresh
121
+ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
122
+ (`HEAD`, `local.yaml`, `sync_state.json`) are gitignored.
123
+
124
+ ## Commands
125
+
126
+ | Command | Purpose |
127
+ |---|---|
128
+ | `forest init` | Create the workspace container, or report setup status if it exists. |
129
+ | `forest checkout create/adopt/list/current/remove <name>` | Manage checkouts; bare `forest checkout <name>` switches, creating first if needed. `remove --yes` skips the prompt for scripts. |
130
+ | `forest add STAGE PATH [--sync-by MODE]` | Register a new stage and bind it to a local path; `--sync-by` picks unit discovery (`subdirectory`/`directory`/`file`). |
131
+ | `forest bind [STAGE PATH]` / `forest unbind STAGE` | Manage local stage↔path bindings. |
132
+ | `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
133
+ | `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
134
+ | `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
135
+ | `forest migrate` | Migrate a legacy `biostore` layout in place (see below). |
136
+
137
+ Run any command with `-C <path>` to operate on another repo without `cd`.
138
+
139
+ Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
140
+ AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
141
+
142
+ ## Config reference
143
+
144
+ Checkout `forest.yaml` (shared, committed):
145
+
146
+ ```yaml
147
+ project: demo
148
+ remotes:
149
+ origin:
150
+ url: s3://my-bucket/prefix
151
+ region: us-east-2 # optional; also endpoint, profile, key_file, known_hosts
152
+ stages:
153
+ raw:
154
+ remote_path: demo/raw # optional; defaults to <checkout>/<stage>
155
+ sync_by: subdirectory # subdirectory | directory | file
156
+ ```
157
+
158
+ Checkout `local.yaml` (per-machine, gitignored):
159
+
160
+ ```yaml
161
+ active_remote: origin
162
+ stage_paths:
163
+ raw: ../data/raw # relative resolves from the workspace root
164
+ ```
165
+
166
+ ## Environment variables
167
+
168
+ All optional, all off by default — forest is silent and sends nothing anywhere
169
+ unless configured. Copy `.env.example` for a commented template; operational
170
+ guides live in `docs/runbooks/`.
171
+
172
+ | Variable | Default | Effect |
173
+ |---|---|---|
174
+ | `FOREST_LOG_FILE` | unset | Append structured logs (JSON lines) to this file. |
175
+ | `FOREST_LOG_FORMAT` | `json` | `json` or `text`; set without `FOREST_LOG_FILE` to log to stderr. |
176
+ | `FOREST_LOG_LEVEL` | `INFO` | Standard logging level name. |
177
+ | `FOREST_METRICS_FILE` | unset | Append metric samples as JSON lines for external collectors. |
178
+ | `FOREST_ANALYTICS_FILE` | unset | Opt-in local usage analytics (JSON lines); nothing leaves the machine. |
179
+ | `FOREST_SENTRY_DSN` | unset | Sentry error tracking; needs `pip install "forest-cli[observability]"`. |
180
+ | `FOREST_ALERT_WEBHOOK` | unset | POST failure alerts to this HTTPS endpoint (Slack/Mattermost compatible). |
181
+ | `FOREST_TRANSFER_RETRIES` | `2` | Extra attempts for transient rclone failures; `0` disables. |
182
+ | `FOREST_RETRY_BASE_DELAY` | `0.5` | Initial retry backoff in seconds; doubles per attempt. |
183
+ | `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
184
+ | `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
185
+ | `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
186
+
187
+ ## Dogfood: this repo runs forest
188
+
189
+ This repository manages its own `examples/` tree with forest — a live
190
+ demonstration that `.forest/` and `.git/` coexist without overlapping. It was
191
+ set up with exactly the quick-start commands:
192
+
193
+ ```bash
194
+ forest init
195
+ forest checkout demo
196
+ forest remote add origin s3://forest-test-542222635421-us-east-2-an --region us-east-2
197
+ forest add examples ./examples
198
+ forest push
199
+ ```
200
+
201
+ Inspect the result:
202
+
203
+ ```bash
204
+ git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
205
+ cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
206
+ forest status # sync state of the examples stage
207
+ ```
208
+
209
+ A fresh clone bootstraps the local half with `forest bind examples ./examples`
210
+ followed by `forest pull` (the single configured remote is used automatically).
211
+ Pulling needs AWS credentials for the bucket; the layout is the demonstration.
212
+
213
+ ## Migrating from biostore
214
+
215
+ Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
216
+
217
+ ```bash
218
+ forest migrate
219
+ ```
220
+
221
+ This renames `.biostore/` → `.forest/`, each `biostore.yaml` → `forest.yaml`,
222
+ rewrites the managed `.gitignore` patterns, and verifies the registry parses. It
223
+ refuses to run if a `.forest/` already exists.
224
+
225
+ ## Notes
226
+
227
+ - **Single active machine (v1).** `HEAD`/`local.yaml`/`sync_state.json` are
228
+ git-invisible but may be synced by a file-syncing tool; forest assumes one
229
+ active machine and uses atomic writes plus a per-checkout `flock` for
230
+ intra-machine write races.
231
+ - **Real filenames.** Forest stores data under real paths, not a
232
+ content-addressed blob store.
233
+ - See `docs/adr/` for the design decisions behind the workspace/checkout model.
@@ -0,0 +1,199 @@
1
+ # 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="docs/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
2
+
3
+ **git-like data management for arbitrary data trees.**
4
+
5
+ [![CI](https://github.com/tmsincomb/forest/actions/workflows/ci.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
6
+ [![Docs](https://github.com/tmsincomb/forest/actions/workflows/docs.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
7
+ [![Release](https://github.com/tmsincomb/forest/actions/workflows/release.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
8
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/downloads/)
9
+ [![License: MIT](https://img.shields.io/badge/license-MIT-2ea44f)](LICENSE)
10
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
11
+ [![mypy: strict](https://img.shields.io/badge/mypy-strict-blue)](https://mypy-lang.org/)
12
+
13
+ Forest is the data-side parallel to git's version control. Git tracks code in
14
+ `.git/`; forest tracks large data — local layout plus remote sync — in
15
+ `.forest/`. It borrows git's mental model and verbs (`checkout`, `status`,
16
+ `push`, `pull`, `remote`, a HEAD-style pointer) so your git intuition carries
17
+ over, but the two domains never overlap and neither requires the other.
18
+
19
+ Forest is **domain-agnostic and self-contained**: it manages *any* data trees,
20
+ knows nothing about what the data means, and depends on no other project. It
21
+ moves files and tracks their sync state; it does not validate or interpret their
22
+ contents.
23
+
24
+ ## Model
25
+
26
+ Fixed-depth, no arbitrary nesting:
27
+
28
+ ```
29
+ workspace → checkout → stage → unit → files
30
+ ```
31
+
32
+ - **Workspace** — a per-repo `.forest/` control area (registry + active pointer).
33
+ - **Checkout** — a named data view/focus registered in the workspace (like a git
34
+ branch you stay rooted in). Switching is an O(1) pointer rewrite; data never
35
+ moves.
36
+ - **Stage** — a named data category inside a checkout, with a remote layout.
37
+ - **Unit** — one addressable item within a stage (a subdirectory, a directory,
38
+ or a file, per the stage's `sync_by`).
39
+
40
+ ## Install
41
+
42
+ Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
43
+
44
+ ```bash
45
+ pip install -e ./forest
46
+ ```
47
+
48
+ ## Quick start
49
+
50
+ No config files are hand-written. Onboarding is a few commands:
51
+
52
+ ```bash
53
+ forest init # create the nameless .forest/ workspace container
54
+ forest checkout demo # create if absent, register + activate 'demo'
55
+ forest remote add origin s3://my-bucket/prefix # allowed before any local data exists
56
+ forest add raw ./data/raw # register stage 'raw' and bind it to a local path
57
+ forest push # sync every bound stage to the active remote
58
+ ```
59
+
60
+ - `forest init` creates **only** `.forest/config.yaml` (`version: 1`,
61
+ `checkouts: {}`) and a managed `.gitignore`. No root config, no checkout, no
62
+ active pointer.
63
+ - `forest checkout <name>` switches to the checkout, creating, registering, and
64
+ activating it first (with `.forest/checkouts/<name>/forest.yaml`) when the
65
+ name is not registered. `forest checkout create <name>` is the explicit form.
66
+ - Remotes can be added before any local binding — useful when your data is
67
+ remote-only at first.
68
+ - `forest add STAGE PATH` registers a new stage and binds it to a local path in
69
+ one step (use `forest bind` to rebind an existing stage).
70
+
71
+ ## Metadata layout
72
+
73
+ Everything forest owns lives under `.forest/`; your data does not.
74
+
75
+ ```
76
+ .forest/
77
+ config.yaml # workspace registry: version, checkouts{}
78
+ HEAD # active checkout name (gitignored)
79
+ checkouts/
80
+ demo/
81
+ forest.yaml # shared: stages, remotes, manifest
82
+ local.yaml # user-local: active_remote, stage_paths (gitignored)
83
+ sync_state.json # user-local push/pull state (gitignored)
84
+ ```
85
+
86
+ Shared metadata (`config.yaml`, each `forest.yaml`) is committed so a fresh
87
+ clone bootstraps with `bind` + `remote use` + `pull`. User-local files
88
+ (`HEAD`, `local.yaml`, `sync_state.json`) are gitignored.
89
+
90
+ ## Commands
91
+
92
+ | Command | Purpose |
93
+ |---|---|
94
+ | `forest init` | Create the workspace container, or report setup status if it exists. |
95
+ | `forest checkout create/adopt/list/current/remove <name>` | Manage checkouts; bare `forest checkout <name>` switches, creating first if needed. `remove --yes` skips the prompt for scripts. |
96
+ | `forest add STAGE PATH [--sync-by MODE]` | Register a new stage and bind it to a local path; `--sync-by` picks unit discovery (`subdirectory`/`directory`/`file`). |
97
+ | `forest bind [STAGE PATH]` / `forest unbind STAGE` | Manage local stage↔path bindings. |
98
+ | `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
99
+ | `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
100
+ | `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
101
+ | `forest migrate` | Migrate a legacy `biostore` layout in place (see below). |
102
+
103
+ Run any command with `-C <path>` to operate on another repo without `cd`.
104
+
105
+ Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
106
+ AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
107
+
108
+ ## Config reference
109
+
110
+ Checkout `forest.yaml` (shared, committed):
111
+
112
+ ```yaml
113
+ project: demo
114
+ remotes:
115
+ origin:
116
+ url: s3://my-bucket/prefix
117
+ region: us-east-2 # optional; also endpoint, profile, key_file, known_hosts
118
+ stages:
119
+ raw:
120
+ remote_path: demo/raw # optional; defaults to <checkout>/<stage>
121
+ sync_by: subdirectory # subdirectory | directory | file
122
+ ```
123
+
124
+ Checkout `local.yaml` (per-machine, gitignored):
125
+
126
+ ```yaml
127
+ active_remote: origin
128
+ stage_paths:
129
+ raw: ../data/raw # relative resolves from the workspace root
130
+ ```
131
+
132
+ ## Environment variables
133
+
134
+ All optional, all off by default — forest is silent and sends nothing anywhere
135
+ unless configured. Copy `.env.example` for a commented template; operational
136
+ guides live in `docs/runbooks/`.
137
+
138
+ | Variable | Default | Effect |
139
+ |---|---|---|
140
+ | `FOREST_LOG_FILE` | unset | Append structured logs (JSON lines) to this file. |
141
+ | `FOREST_LOG_FORMAT` | `json` | `json` or `text`; set without `FOREST_LOG_FILE` to log to stderr. |
142
+ | `FOREST_LOG_LEVEL` | `INFO` | Standard logging level name. |
143
+ | `FOREST_METRICS_FILE` | unset | Append metric samples as JSON lines for external collectors. |
144
+ | `FOREST_ANALYTICS_FILE` | unset | Opt-in local usage analytics (JSON lines); nothing leaves the machine. |
145
+ | `FOREST_SENTRY_DSN` | unset | Sentry error tracking; needs `pip install "forest-cli[observability]"`. |
146
+ | `FOREST_ALERT_WEBHOOK` | unset | POST failure alerts to this HTTPS endpoint (Slack/Mattermost compatible). |
147
+ | `FOREST_TRANSFER_RETRIES` | `2` | Extra attempts for transient rclone failures; `0` disables. |
148
+ | `FOREST_RETRY_BASE_DELAY` | `0.5` | Initial retry backoff in seconds; doubles per attempt. |
149
+ | `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
150
+ | `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
151
+ | `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
152
+
153
+ ## Dogfood: this repo runs forest
154
+
155
+ This repository manages its own `examples/` tree with forest — a live
156
+ demonstration that `.forest/` and `.git/` coexist without overlapping. It was
157
+ set up with exactly the quick-start commands:
158
+
159
+ ```bash
160
+ forest init
161
+ forest checkout demo
162
+ forest remote add origin s3://forest-test-542222635421-us-east-2-an --region us-east-2
163
+ forest add examples ./examples
164
+ forest push
165
+ ```
166
+
167
+ Inspect the result:
168
+
169
+ ```bash
170
+ git ls-files .forest # what a clone gets: config.yaml + checkouts/demo/forest.yaml
171
+ cat .gitignore # forest-managed: HEAD, local.yaml, sync_state.json stay local
172
+ forest status # sync state of the examples stage
173
+ ```
174
+
175
+ A fresh clone bootstraps the local half with `forest bind examples ./examples`
176
+ followed by `forest pull` (the single configured remote is used automatically).
177
+ Pulling needs AWS credentials for the bucket; the layout is the demonstration.
178
+
179
+ ## Migrating from biostore
180
+
181
+ Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
182
+
183
+ ```bash
184
+ forest migrate
185
+ ```
186
+
187
+ This renames `.biostore/` → `.forest/`, each `biostore.yaml` → `forest.yaml`,
188
+ rewrites the managed `.gitignore` patterns, and verifies the registry parses. It
189
+ refuses to run if a `.forest/` already exists.
190
+
191
+ ## Notes
192
+
193
+ - **Single active machine (v1).** `HEAD`/`local.yaml`/`sync_state.json` are
194
+ git-invisible but may be synced by a file-syncing tool; forest assumes one
195
+ active machine and uses atomic writes plus a per-checkout `flock` for
196
+ intra-machine write races.
197
+ - **Real filenames.** Forest stores data under real paths, not a
198
+ content-addressed blob store.
199
+ - See `docs/adr/` for the design decisions behind the workspace/checkout model.
@@ -0,0 +1,122 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "forest-cli"
7
+ version = "0.1.0"
8
+ description = "git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync"
9
+ readme = "README.md"
10
+ license = {text = "MIT"}
11
+ requires-python = ">=3.9"
12
+ authors = [
13
+ {name = "Troy Sincomb", email = "troysincomb@gmail.com"},
14
+ ]
15
+ keywords = ["data-management", "sync", "cli"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "License :: OSI Approved :: MIT License",
19
+ "Programming Language :: Python :: 3",
20
+ ]
21
+ dependencies = [
22
+ "click>=8.0",
23
+ "polars>=1.0",
24
+ "pydantic>=2.0",
25
+ "pyyaml>=6.0",
26
+ ]
27
+
28
+ [project.optional-dependencies]
29
+ # Opt-in error tracking (FOREST_SENTRY_DSN); see docs/runbooks/alerting.md.
30
+ observability = [
31
+ "sentry-sdk>=2.0",
32
+ ]
33
+ dev = [
34
+ "deptry>=0.23",
35
+ "mypy>=1.14",
36
+ "pre-commit>=4.0",
37
+ "pylint>=3.3",
38
+ "pytest>=7.0",
39
+ "pytest-cov>=6.0",
40
+ "pytest-randomly>=3.15",
41
+ "pytest-rerunfailures>=15.0",
42
+ "pytest-xdist>=3.6",
43
+ "ruff>=0.4",
44
+ "types-PyYAML",
45
+ "vulture>=2.14",
46
+ ]
47
+
48
+ [project.scripts]
49
+ forest = "forest.cli:main"
50
+
51
+ [tool.setuptools.packages.find]
52
+ where = ["src"]
53
+
54
+ [tool.pytest.ini_options]
55
+ testpaths = ["tests"]
56
+ doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
57
+ addopts = [
58
+ "--cov=forest",
59
+ "--cov-report=term-missing",
60
+ "--cov-fail-under=70",
61
+ "--durations=15",
62
+ "--durations-min=0.5",
63
+ # Test isolation: run in parallel workers; pytest-randomly shuffles order each
64
+ # run so hidden inter-test dependencies surface immediately.
65
+ "--numprocesses=auto",
66
+ # Flaky-test policy: retry only transient environment errors (never assertion
67
+ # failures); reruns are reported in the summary so instability stays visible.
68
+ "--reruns=1",
69
+ "--reruns-delay=1",
70
+ "--only-rerun=(Timeout|Connection|BrokenPipe)",
71
+ ]
72
+
73
+ [tool.ruff]
74
+ target-version = "py39"
75
+ line-length = 120
76
+ src = ["src", "tests"]
77
+
78
+ [tool.ruff.lint]
79
+ # N enforces pep8-naming, C90 enforces the cyclomatic complexity budget, and
80
+ # TD/FIX police tech-debt markers: TODOs must carry an author and issue link
81
+ # (TD002/TD003) while FIXME/XXX/HACK never land on main (FIX001/FIX003/FIX004).
82
+ select = ["E", "F", "I", "N", "UP", "B", "SIM", "RUF", "C90", "TD", "FIX"]
83
+ # FIX002 flags every TODO even when well-formed; TD rules already gate those.
84
+ ignore = ["FIX002"]
85
+
86
+ [tool.ruff.lint.mccabe]
87
+ max-complexity = 10
88
+
89
+ [tool.ruff.lint.isort]
90
+ known-first-party = ["forest"]
91
+
92
+ [tool.ruff.format]
93
+ quote-style = "double"
94
+ indent-style = "space"
95
+ line-ending = "auto"
96
+
97
+ [tool.mypy]
98
+ python_version = "3.10"
99
+ files = ["src/forest"]
100
+ strict = true
101
+ warn_unused_configs = true
102
+
103
+ [tool.deptry]
104
+ pep621_dev_dependency_groups = ["dev"]
105
+
106
+ [tool.vulture]
107
+ paths = ["src/forest"]
108
+ min_confidence = 80
109
+ # Pydantic validators receive `cls` per the classmethod protocol even when the
110
+ # body never uses it.
111
+ ignore_names = ["cls"]
112
+
113
+ # Pylint is used solely as the duplicate-code (DRY) detector; ruff owns every
114
+ # other lint concern.
115
+ [tool.pylint.main]
116
+ disable = ["all"]
117
+ enable = ["duplicate-code"]
118
+
119
+ [tool.pylint.similarities]
120
+ min-similarity-lines = 5
121
+ ignore-imports = true
122
+ ignore-signatures = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,16 @@
1
+ """forest — git-like data management for arbitrary data trees.
2
+
3
+ Forest is a self-contained, domain-agnostic data-sync engine: workspace
4
+ initialization, named checkouts (focus), remotes, local bindings,
5
+ push/pull/status/ls, and sync state. It knows nothing about what the data
6
+ means — it syncs and inspects whatever files live in a unit.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from importlib.metadata import PackageNotFoundError, version
12
+
13
+ try:
14
+ __version__ = version("forest")
15
+ except PackageNotFoundError:
16
+ __version__ = "0.0.0"
@@ -0,0 +1,29 @@
1
+ """Opt-in local usage analytics for forest commands.
2
+
3
+ No-op unless ``FOREST_ANALYTICS_FILE`` is set; events are appended as JSON
4
+ lines to that local file only — nothing leaves the machine. Eager options
5
+ (``--help``/``--version``) exit during parsing and are not recorded.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from datetime import datetime, timezone
11
+
12
+ from forest import __version__
13
+ from forest.logger import get_run_id
14
+ from forest.metrics import append_jsonl
15
+
16
+
17
+ def record_command(command: str, *, duration_seconds: float, status: str) -> None:
18
+ """Append one command-usage event. Never raises."""
19
+ append_jsonl(
20
+ "FOREST_ANALYTICS_FILE",
21
+ {
22
+ "ts": datetime.now(tz=timezone.utc).isoformat(),
23
+ "command": command,
24
+ "duration_seconds": round(duration_seconds, 3),
25
+ "status": status, # "success" | "error"
26
+ "run_id": get_run_id(),
27
+ "version": __version__,
28
+ },
29
+ )