forest-cli 0.1.0__py3-none-any.whl

forest/sync_state.py

@@ -0,0 +1,182 @@
+"""Sync state models and persistence for tracking push/pull operations."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from forest.fsutil import atomic_write
+
+
+class SyncEntry(BaseModel):
+    """A single sync operation record for a data unit."""
+
+    remote: str
+    stage: str
+    unit_id: str
+    schema_: str | None = Field(default=None, alias="schema")
+    local_path: str | None = None
+    remote_path: str | None = None
+    direction: Literal["push", "pull"]
+    timestamp: str
+    file_count: int
+    total_bytes: int
+    checksum: str
+    schema_version: str | None = None
+    forest_version: str | None = None
+    file_snapshot: dict[str, int] | None = None
+
+    model_config = {"populate_by_name": True}
+
+
+class SyncState(BaseModel):
+    """Top-level sync state tracking all push/pull operations."""
+
+    version: int = 1
+    project: str = ""
+    entries: list[SyncEntry] = Field(default_factory=list)
+
+    model_config = {"populate_by_name": True}
+
+
+def load_state(path: str | Path) -> SyncState:
+    """Load sync state from a JSON file.
+
+    Returns an empty SyncState if the file does not exist.
+    """
+    path = Path(path)
+    if not path.exists():
+        return SyncState()
+
+    data = json.loads(path.read_text())
+    if data == []:
+        return SyncState()
+    return SyncState.model_validate(data)
+
+
+def save_state(state: SyncState, path: str | Path) -> None:
+    """Write sync state to a JSON file with pretty-printed formatting.
+
+    Creates parent directories if they do not exist.
+    """
+    path = Path(path)
+    atomic_write(path, json.dumps(state.model_dump(by_alias=True), indent=2) + "\n")
+
+
+def record_push(state: SyncState, entry_data: dict[str, Any]) -> SyncState:
+    """Record a push operation, returning a new SyncState.
+
+    If an entry with the same (remote, stage, unit_id) already exists,
+    it is replaced. The original state is not mutated.
+    """
+    return _record(state, entry_data, direction="push")
+
+
+def record_pull(state: SyncState, entry_data: dict[str, Any]) -> SyncState:
+    """Record a pull operation, returning a new SyncState.
+
+    If an entry with the same (remote, stage, unit_id) already exists,
+    it is replaced. The original state is not mutated.
+    """
+    return _record(state, entry_data, direction="pull")
+
+
+def _record(state: SyncState, entry_data: dict[str, Any], *, direction: str) -> SyncState:
+    """Internal helper to record a push or pull entry."""
+    remote = entry_data["remote"]
+    stage = entry_data["stage"]
+    unit_id = entry_data["unit_id"]
+
+    entry_data = {
+        **entry_data,
+        "direction": direction,
+        "timestamp": datetime.now(tz=timezone.utc).isoformat(),
+    }
+    new_entry = SyncEntry(**entry_data)
+
+    kept = [
+        e
+        for e in state.entries
+        if not (e.remote == remote and e.stage == stage and e.unit_id == unit_id and e.direction == direction)
+    ]
+    kept.append(new_entry)
+
+    return SyncState(version=state.version, project=state.project, entries=kept)
+
+
+def get_status(
+    state: SyncState,
+    stage: str,
+    unit_id: str,
+    *,
+    unit_path: str | Path | None = None,
+    remote: str | None = None,
+) -> str:
+    """Determine the sync status of a data unit.
+
+    When *unit_path* is not provided, it is derived from the matching
+    ``SyncEntry.local_path`` (if one exists).
+
+    Returns one of:
+    - ``"synced"`` — local checksum matches stored entry
+    - ``"stale"`` — local exists and entry exists but checksums differ
+    - ``"local-only"`` — local dir exists but no sync entry
+    - ``"remote-only"`` — sync entry exists but no local dir
+    """
+    # Find matching entry
+    entry = _find_entry(state, stage, unit_id, remote=remote)
+
+    # Derive unit_path from entry if not explicitly provided
+    if unit_path is None and entry is not None and entry.local_path is not None:
+        unit_path = Path(entry.local_path)
+    elif unit_path is not None:
+        unit_path = Path(unit_path)
+
+    if entry is None:
+        return "local-only"
+
+    if unit_path is None or not unit_path.is_dir():
+        return "remote-only"
+
+    # Both exist — compare checksums
+    local_files = sorted(f for f in unit_path.rglob("*") if f.is_file())
+    current_checksum = compute_checksum(unit_path, local_files)
+
+    if current_checksum == entry.checksum:
+        return "synced"
+
+    return "stale"
+
+
+def _find_entry(
+    state: SyncState,
+    stage: str,
+    unit_id: str,
+    *,
+    remote: str | None = None,
+) -> SyncEntry | None:
+    """Find the most recent matching sync entry."""
+    for entry in reversed(state.entries):
+        if entry.stage == stage and entry.unit_id == unit_id and (remote is None or entry.remote == remote):
+            return entry
+    return None
+
+
+def compute_checksum(unit_path: str | Path, file_list: list[Path]) -> str:
+    """Compute a deterministic SHA-256 checksum from (relative_path, file_size) tuples.
+
+    The checksum is independent of file modification times and content —
+    only the relative path and file size in bytes are used. The list is
+    sorted before hashing to ensure determinism regardless of input order.
+    """
+    unit_path = Path(unit_path)
+    tuples = sorted((str(f.relative_to(unit_path)), f.stat().st_size) for f in file_list)
+    h = hashlib.sha256()
+    for rel_path, size in tuples:
+        h.update(f"{rel_path}:{size}\n".encode())
+    return h.hexdigest()

forest_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,233 @@
+Metadata-Version: 2.4
+Name: forest-cli
+Version: 0.1.0
+Summary: git-like data management for arbitrary data trees: workspaces, checkouts, remotes, and sync
+Author-email: Troy Sincomb <troysincomb@gmail.com>
+License: MIT
+Keywords: data-management,sync,cli
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.0
+Requires-Dist: polars>=1.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: observability
+Requires-Dist: sentry-sdk>=2.0; extra == "observability"
+Provides-Extra: dev
+Requires-Dist: deptry>=0.23; extra == "dev"
+Requires-Dist: mypy>=1.14; extra == "dev"
+Requires-Dist: pre-commit>=4.0; extra == "dev"
+Requires-Dist: pylint>=3.3; extra == "dev"
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=6.0; extra == "dev"
+Requires-Dist: pytest-randomly>=3.15; extra == "dev"
+Requires-Dist: pytest-rerunfailures>=15.0; extra == "dev"
+Requires-Dist: pytest-xdist>=3.6; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: vulture>=2.14; extra == "dev"
+Dynamic: license-file
+
+# 🌲 forest <a href="#-dogfood-this-repo-runs-forest"><img align="right" width="42%" src="docs/assets/hero.svg" alt="Animated isometric forest: data trees on a workspace platform, packets syncing up a git branch to a remote cloud"></a>
+
+**git-like data management for arbitrary data trees.**
+
+[![CI](https://github.com/tmsincomb/forest/actions/workflows/ci.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/ci.yml)
+[![Docs](https://github.com/tmsincomb/forest/actions/workflows/docs.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/docs.yml)
+[![Release](https://github.com/tmsincomb/forest/actions/workflows/release.yml/badge.svg)](https://github.com/tmsincomb/forest/actions/workflows/release.yml)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-2ea44f)](LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![mypy: strict](https://img.shields.io/badge/mypy-strict-blue)](https://mypy-lang.org/)
+
+Forest is the data-side parallel to git's version control. Git tracks code in
+`.git/`; forest tracks large data — local layout plus remote sync — in
+`.forest/`. It borrows git's mental model and verbs (`checkout`, `status`,
+`push`, `pull`, `remote`, a HEAD-style pointer) so your git intuition carries
+over, but the two domains never overlap and neither requires the other.
+
+Forest is **domain-agnostic and self-contained**: it manages *any* data trees,
+knows nothing about what the data means, and depends on no other project. It
+moves files and tracks their sync state; it does not validate or interpret their
+contents.
+
+## Model
+
+Fixed-depth, no arbitrary nesting:
+
+```
+workspace → checkout → stage → unit → files
+```
+
+- **Workspace** — a per-repo `.forest/` control area (registry + active pointer).
+- **Checkout** — a named data view/focus registered in the workspace (like a git
+  branch you stay rooted in). Switching is an O(1) pointer rewrite; data never
+  moves.
+- **Stage** — a named data category inside a checkout, with a remote layout.
+- **Unit** — one addressable item within a stage (a subdirectory, a directory,
+  or a file, per the stage's `sync_by`).
+
+## Install
+
+Requires [`rclone`](https://rclone.org/) on `$PATH` for transfers:
+
+```bash
+pip install -e ./forest
+```
+
+## Quick start
+
+No config files are hand-written. Onboarding is a few commands:
+
+```bash
+forest init                      # create the nameless .forest/ workspace container
+forest checkout demo             # create if absent, register + activate 'demo'
+forest remote add origin s3://my-bucket/prefix   # allowed before any local data exists
+forest add raw ./data/raw        # register stage 'raw' and bind it to a local path
+forest push                      # sync every bound stage to the active remote
+```
+
+- `forest init` creates **only** `.forest/config.yaml` (`version: 1`,
+  `checkouts: {}`) and a managed `.gitignore`. No root config, no checkout, no
+  active pointer.
+- `forest checkout <name>` switches to the checkout, creating, registering, and
+  activating it first (with `.forest/checkouts/<name>/forest.yaml`) when the
+  name is not registered. `forest checkout create <name>` is the explicit form.
+- Remotes can be added before any local binding — useful when your data is
+  remote-only at first.
+- `forest add STAGE PATH` registers a new stage and binds it to a local path in
+  one step (use `forest bind` to rebind an existing stage).
+
+## Metadata layout
+
+Everything forest owns lives under `.forest/`; your data does not.
+
+```
+.forest/
+  config.yaml                     # workspace registry: version, checkouts{}
+  HEAD                            # active checkout name (gitignored)
+  checkouts/
+    demo/
+      forest.yaml                 # shared: stages, remotes, manifest
+      local.yaml                  # user-local: active_remote, stage_paths (gitignored)
+      sync_state.json             # user-local push/pull state (gitignored)
+```
+
+Shared metadata (`config.yaml`, each `forest.yaml`) is committed so a fresh
+clone bootstraps with `bind` + `remote use` + `pull`. User-local files
+(`HEAD`, `local.yaml`, `sync_state.json`) are gitignored.
+
+## Commands
+
+| Command | Purpose |
+|---|---|
+| `forest init` | Create the workspace container, or report setup status if it exists. |
+| `forest checkout create/adopt/list/current/remove <name>` | Manage checkouts; bare `forest checkout <name>` switches, creating first if needed. `remove --yes` skips the prompt for scripts. |
+| `forest add STAGE PATH [--sync-by MODE]` | Register a new stage and bind it to a local path; `--sync-by` picks unit discovery (`subdirectory`/`directory`/`file`). |
+| `forest bind [STAGE PATH]` / `forest unbind STAGE` | Manage local stage↔path bindings. |
+| `forest remote add/remove/list/use/show` | Manage remotes; `use` selects the active remote (optional while only one remote exists). |
+| `forest push / pull / status / diff / ls` | Sync and inspect against the active remote. Bare `push`/`pull`/`status`/`diff` cover every bound stage (unbound stages warn and skip); `--all` requires all stages bound. |
+| `forest flow` | Emit a Mermaid data-flow diagram of the active checkout. |
+| `forest migrate` | Migrate a legacy `biostore` layout in place (see below). |
+
+Run any command with `-C <path>` to operate on another repo without `cd`.
+
+Forest syncs **all** files in a data unit, skipping OS junk (`.DS_Store`,
+AppleDouble `._*`, `*.tmp`). It applies no content-based include/exclude rules.
+
+## Config reference
+
+Checkout `forest.yaml` (shared, committed):
+
+```yaml
+project: demo
+remotes:
+  origin:
+    url: s3://my-bucket/prefix
+    region: us-east-2          # optional; also endpoint, profile, key_file, known_hosts
+stages:
+  raw:
+    remote_path: demo/raw      # optional; defaults to <checkout>/<stage>
+    sync_by: subdirectory      # subdirectory | directory | file
+```
+
+Checkout `local.yaml` (per-machine, gitignored):
+
+```yaml
+active_remote: origin
+stage_paths:
+  raw: ../data/raw             # relative resolves from the workspace root
+```
+
+## Environment variables
+
+All optional, all off by default — forest is silent and sends nothing anywhere
+unless configured. Copy `.env.example` for a commented template; operational
+guides live in `docs/runbooks/`.
+
+| Variable | Default | Effect |
+|---|---|---|
+| `FOREST_LOG_FILE` | unset | Append structured logs (JSON lines) to this file. |
+| `FOREST_LOG_FORMAT` | `json` | `json` or `text`; set without `FOREST_LOG_FILE` to log to stderr. |
+| `FOREST_LOG_LEVEL` | `INFO` | Standard logging level name. |
+| `FOREST_METRICS_FILE` | unset | Append metric samples as JSON lines for external collectors. |
+| `FOREST_ANALYTICS_FILE` | unset | Opt-in local usage analytics (JSON lines); nothing leaves the machine. |
+| `FOREST_SENTRY_DSN` | unset | Sentry error tracking; needs `pip install "forest-cli[observability]"`. |
+| `FOREST_ALERT_WEBHOOK` | unset | POST failure alerts to this HTTPS endpoint (Slack/Mattermost compatible). |
+| `FOREST_TRANSFER_RETRIES` | `2` | Extra attempts for transient rclone failures; `0` disables. |
+| `FOREST_RETRY_BASE_DELAY` | `0.5` | Initial retry backoff in seconds; doubles per attempt. |
+| `FOREST_BREAKER_THRESHOLD` | `5` | Consecutive transfer failures before the circuit opens; `0` disables. |
+| `FOREST_BREAKER_RESET_SECONDS` | `60` | Cool-down before an open circuit allows a probe operation. |
+| `FOREST_FLAGS` | unset | Comma-separated feature flags; `raw-logs` disables log secret-scrubbing. |
+
+## Dogfood: this repo runs forest
+
+This repository manages its own `examples/` tree with forest — a live
+demonstration that `.forest/` and `.git/` coexist without overlapping. It was
+set up with exactly the quick-start commands:
+
+```bash
+forest init
+forest checkout demo
+forest remote add origin s3://forest-test-542222635421-us-east-2-an --region us-east-2
+forest add examples ./examples
+forest push
+```
+
+Inspect the result:
+
+```bash
+git ls-files .forest        # what a clone gets: config.yaml + checkouts/demo/forest.yaml
+cat .gitignore              # forest-managed: HEAD, local.yaml, sync_state.json stay local
+forest status               # sync state of the examples stage
+```
+
+A fresh clone bootstraps the local half with `forest bind examples ./examples`
+followed by `forest pull` (the single configured remote is used automatically).
+Pulling needs AWS credentials for the bucket; the layout is the demonstration.
+
+## Migrating from biostore
+
+Existing biostore repos use `.biostore/` and `biostore.yaml`. Migrate in place:
+
+```bash
+forest migrate
+```
+
+This renames `.biostore/` → `.forest/`, each `biostore.yaml` → `forest.yaml`,
+rewrites the managed `.gitignore` patterns, and verifies the registry parses. It
+refuses to run if a `.forest/` already exists.
+
+## Notes
+
+- **Single active machine (v1).** `HEAD`/`local.yaml`/`sync_state.json` are
+  git-invisible but may be synced by a file-syncing tool; forest assumes one
+  active machine and uses atomic writes plus a per-checkout `flock` for
+  intra-machine write races.
+- **Real filenames.** Forest stores data under real paths, not a
+  content-addressed blob store.
+- See `docs/adr/` for the design decisions behind the workspace/checkout model.

forest_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+forest/__init__.py,sha256=GXkEqeYHPZirXscjOYs5XrAERnyduI-aAuu_gvKbRmA,540
+forest/analytics.py,sha256=RA1GzGAyaETLH1aVxVuILHP56EokzrhVYE8DnA_WKqM,961
+forest/checkout.py,sha256=xwTit-EdSfZxe27LPaEVqGTBE-DJSD3xiChlWOYER-k,38657
+forest/cli.py,sha256=ohDAiYd0OCwh9Y9GdFj1B2Q7XaxcH4EjrFX_jqtUV5c,103533
+forest/config.py,sha256=Dy-pDOGN8F0QnJhH558zB-Xk_PnOzfY8ZyDuDOeSpS8,9271
+forest/flags.py,sha256=4UCA8aSqQT5my9mzqltlWw2kPZ45Ch0PzUii2jw4GWg,641
+forest/flow.py,sha256=ewb1jqw_kI4gyzBpWogSuIdsdhBfufS7N_7dPVlWWS8,5349
+forest/fsutil.py,sha256=BQGOqx5Sa_wBNoOymIcG3bqEhfLsOF5U1mWZ_YXKOGw,1210
+forest/logger.py,sha256=nIYMV2DWjPRYsl7DTvlb1ggQL3Y0EwegZ-d1wsmYeUo,7042
+forest/manifest.py,sha256=hklrMvYU3WolJNTmq42YOvm_uMmhMyS0HBASstdubhg,3260
+forest/metrics.py,sha256=dBz9LNolPi2d4xeSayJMQT7uHd2e8SgMG29rikCJY3w,2494
+forest/monitoring.py,sha256=jaNfdIPfjOV6t3anVIHswf4zc0YLnuOpQl28lHFYcS0,4862
+forest/paths.py,sha256=oc0aYMz9HX4kgaKnRXe2CTk0Hq9BWrrs7UOeA1sYgYE,6818
+forest/rclone.py,sha256=cVsU0NvLTDGH5Rz9M-5dcOCuybbsDU4tXE-a86KL6nA,21359
+forest/resilience.py,sha256=oEHiPOQjCP_voskkE8WZz5zQ0CfN2v8nmthoAYptXL4,4321
+forest/sync_state.py,sha256=h8EmIN3uE3pQOj-20Tlhg6NFq2b5AmHOeHVNfIkvQwI,5542
+forest_cli-0.1.0.dist-info/licenses/LICENSE,sha256=PRvBVRcqmEyg2k5TekU4LBP_BQTdVALzv5F12ofRXbE,1069
+forest_cli-0.1.0.dist-info/METADATA,sha256=QQubDaJYFRltotTWORcGgby76UGZIPMkuQPpGKPxZio,10695
+forest_cli-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+forest_cli-0.1.0.dist-info/entry_points.txt,sha256=ogQn8m-0npXNFnEu1RW14gBIy6Hzttrhp2zcOLcttsM,43
+forest_cli-0.1.0.dist-info/top_level.txt,sha256=Qh1kq9Xt0-pDXxBKCtInwGCWt8k7IEBrKEtwJIQjlKg,7
+forest_cli-0.1.0.dist-info/RECORD,,

forest_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

forest_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+forest = forest.cli:main

forest_cli-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Troy Sincomb
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

forest_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+forest