forest-cli 0.1.0__py3-none-any.whl

forest/manifest.py

@@ -0,0 +1,112 @@
+"""Manifest loading and ID resolution for forest."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import polars as pl
+
+_cache: dict[str, pl.DataFrame] = {}
+
+
+class ManifestError(Exception):
+    """Raised for manifest loading or query errors."""
+
+
+def load_manifest(path: str | Path) -> pl.DataFrame:
+    """Load a manifest CSV into a polars DataFrame.
+
+    The DataFrame is cached by resolved file path so that repeated
+    calls within the same process reuse the same object.
+
+    Raises FileNotFoundError with the path when the file does not exist.
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Manifest file not found: {path}")
+
+    key = str(path.resolve())
+    if key in _cache:
+        return _cache[key]
+
+    df: pl.DataFrame = pl.read_csv(path, infer_schema_length=0)
+
+    # Strip whitespace from column names
+    df = df.rename({col: col.strip() for col in df.columns})
+
+    # Strip whitespace from all string values
+    df = df.with_columns(pl.col(col).str.strip_chars() for col in df.columns)
+
+    _cache[key] = df
+    return df
+
+
+def resolve_id(
+    manifest: pl.DataFrame,
+    id_value: str,
+    id_column: str,
+    stage_columns: dict[str, str],
+    stage_name: str | None = None,
+) -> list[tuple[str, str]]:
+    """Resolve a manifest ID to (stage, unit_id) pairs.
+
+    Filters the manifest by *id_column* == *id_value*, then for each
+    stage in *stage_columns* extracts the unique values from the
+    corresponding column.  When *stage_name* is given only that
+    single stage is returned.
+
+    Raises ManifestError when *id_column* or a stage column is
+    missing from the DataFrame.
+    """
+    _check_column(manifest, id_column)
+
+    columns = stage_columns
+    if stage_name is not None:
+        columns = {stage_name: stage_columns[stage_name]}
+
+    for _stage, col in columns.items():
+        _check_column(manifest, col)
+
+    filtered = manifest.filter(pl.col(id_column) == id_value)
+
+    result: list[tuple[str, str]] = []
+    for stage, col in columns.items():
+        unit_ids = filtered[col].unique().sort().to_list()
+        for uid in unit_ids:
+            if uid is not None and str(uid).strip():
+                result.append((stage, uid))
+
+    return result
+
+
+def list_ids(manifest: pl.DataFrame, id_column: str) -> list[str]:
+    """Return all unique IDs from the given column.
+
+    Raises ManifestError when *id_column* is missing from the DataFrame.
+    """
+    _check_column(manifest, id_column)
+    return manifest[id_column].unique().sort().to_list()
+
+
+def get_metadata(
+    manifest: pl.DataFrame,
+    id_value: str,
+    id_column: str,
+) -> pl.DataFrame:
+    """Return all rows matching *id_value* in *id_column*.
+
+    Raises ManifestError when *id_column* is missing from the DataFrame.
+    """
+    _check_column(manifest, id_column)
+    return manifest.filter(pl.col(id_column) == id_value)
+
+
+def clear_cache() -> None:
+    """Clear the module-level manifest cache."""
+    _cache.clear()
+
+
+def _check_column(df: pl.DataFrame, column: str) -> None:
+    """Raise ManifestError if *column* is not in the DataFrame."""
+    if column not in df.columns:
+        raise ManifestError(f"Column '{column}' not found in manifest. Available columns: {df.columns}")

forest/metrics.py

@@ -0,0 +1,67 @@
+"""Operational metrics for forest transfers and rclone subprocess calls.
+
+Two sinks, both keyed by the invocation ``run_id`` from ``forest.logger``:
+
+- every sample is logged as a structured ``metric`` event on the
+  ``forest.metrics`` logger (visible via ``FOREST_LOG_FILE`` / ``FOREST_LOG_FORMAT``);
+- when ``FOREST_METRICS_FILE`` is set, samples are appended as JSON lines so
+  external collectors (Prometheus textfile exporters, Datadog agents, ad-hoc
+  polars/jq analysis) can ingest them without parsing human output.
+
+Sample shape:
+``{"ts": ..., "metric": "push.duration_seconds", "value": 1.23, "unit": "s",
+"tags": {"stage": "raw", "remote": "origin"}, "run_id": "..."}``
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager, suppress
+from datetime import datetime, timezone
+from typing import Any
+
+from forest.logger import get_logger, get_run_id
+
+_log = get_logger("forest.metrics")
+
+
+def append_jsonl(env_var: str, record: dict[str, Any]) -> None:
+    """Append record as one JSON line to the file named by env var. Never raises."""
+    path = os.environ.get(env_var)
+    if not path:
+        return
+    with suppress(OSError), open(path, "a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record, default=str) + "\n")
+
+
+def emit(metric: str, value: float, *, unit: str = "", tags: dict[str, str] | None = None) -> None:
+    """Record one metric sample. Never raises: telemetry must not break syncs."""
+    sample = {
+        "ts": datetime.now(tz=timezone.utc).isoformat(),
+        "metric": metric,
+        "value": value,
+        "unit": unit,
+        "tags": tags or {},
+        "run_id": get_run_id(),
+    }
+    _log.info("metric", extra={"metric": metric, "value": value, "unit": unit, "tags": sample["tags"]})
+    append_jsonl("FOREST_METRICS_FILE", sample)
+
+
+@contextmanager
+def timed(metric: str, *, tags: dict[str, str] | None = None) -> Iterator[None]:
+    """Measure a block's wall time and emit it as ``<metric>`` in seconds."""
+    start = time.monotonic()
+    try:
+        yield
+    finally:
+        emit(metric, round(time.monotonic() - start, 3), unit="s", tags=tags)
+
+
+def record_transfer(op: str, *, files: int, transferred_bytes: int, tags: dict[str, str] | None = None) -> None:
+    """Emit the standard sample pair for a completed push/pull unit."""
+    emit(f"{op}.files", files, tags=tags)
+    emit(f"{op}.bytes", transferred_bytes, unit="B", tags=tags)

forest/monitoring.py

@@ -0,0 +1,138 @@
+"""Error tracking and failure alerting for forest. Both opt-in, both no-ops
+unless configured, and both best-effort: telemetry must never break a sync.
+
+Error tracking (Sentry)
+    Set ``FOREST_SENTRY_DSN`` and install the ``observability`` extra
+    (``pip install "forest[observability]"``). Events carry the release
+    (``forest@<version>``), the invocation ``run_id`` and command as tags, and
+    breadcrumbs from the structured ``forest.*`` loggers (rclone attempts,
+    retries, transfer results), so a production stack trace arrives with the
+    full operation trail attached.
+
+Alerting (webhook)
+    Set ``FOREST_ALERT_WEBHOOK`` to an HTTPS endpoint (Slack/Mattermost
+    incoming webhook or any JSON collector). Failed commands and failed
+    push/pull runs POST a compact JSON payload; scheduled syncs (cron/CI) then
+    page a channel instead of failing silently. See docs/runbooks/.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import socket
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any
+
+from forest import __version__
+from forest.logger import get_logger, get_run_id, scrub_obj
+
+_log = get_logger("forest.monitoring")
+
+_MAX_DETAIL_CHARS = 500
+_WEBHOOK_TIMEOUT_SECONDS = 5.0
+
+_sentry_enabled = False
+
+
+def _scrub_event(event: Any, _hint: Any) -> Any:
+    """Sentry before_send hook: redact secret-shaped values from events."""
+    return scrub_obj(event)
+
+
+def init_error_tracking(command: str) -> bool:
+    """Initialize Sentry when FOREST_SENTRY_DSN is set and the SDK is present."""
+    global _sentry_enabled
+    _sentry_enabled = False
+    dsn = os.environ.get("FOREST_SENTRY_DSN")
+    if not dsn:
+        return False
+    try:
+        import sentry_sdk
+    except ImportError:
+        _log.warning(
+            "FOREST_SENTRY_DSN is set but sentry-sdk is not installed; "
+            'install with: pip install "forest[observability]"'
+        )
+        return False
+
+    sentry_sdk.init(
+        dsn=dsn,
+        release=f"forest@{__version__}",
+        before_send=_scrub_event,
+        # Default integrations include logging: INFO+ records from forest.*
+        # loggers become breadcrumbs, giving each event its operation trail.
+    )
+    sentry_sdk.set_tag("forest.run_id", get_run_id())
+    sentry_sdk.set_tag("forest.command", command)
+    sentry_sdk.set_context(
+        "forest",
+        {"version": __version__, "python": platform.python_version(), "platform": platform.platform()},
+    )
+    _sentry_enabled = True
+    return True
+
+
+def capture_exception(exc: BaseException) -> None:
+    """Forward an exception to Sentry when error tracking is active."""
+    if not _sentry_enabled:
+        return
+    try:
+        import sentry_sdk
+
+        sentry_sdk.capture_exception(exc)
+        sentry_sdk.flush(timeout=2.0)
+    except Exception:
+        _log.warning("failed to deliver exception to Sentry", exc_info=True)
+
+
+def alert(event: str, detail: str, *, command: str) -> bool:
+    """POST a failure notification to FOREST_ALERT_WEBHOOK. Returns delivery."""
+    url = os.environ.get("FOREST_ALERT_WEBHOOK")
+    if not url:
+        return False
+    if not url.lower().startswith("https://"):
+        _log.warning("FOREST_ALERT_WEBHOOK must be an https:// URL; alert not sent")
+        return False
+
+    detail = (detail or "").strip()
+    if len(detail) > _MAX_DETAIL_CHARS:
+        detail = detail[:_MAX_DETAIL_CHARS] + "..."
+
+    payload: dict[str, Any] = {
+        "source": "forest",
+        "version": __version__,
+        "event": event,
+        "command": command,
+        "detail": detail,
+        "run_id": get_run_id(),
+        "host": socket.gethostname(),
+        "ts": datetime.now(tz=timezone.utc).isoformat(),
+        # Slack/Mattermost-compatible summary line.
+        "text": f"forest {command} failed on {socket.gethostname()}: {event} (run {get_run_id()})",
+    }
+
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(scrub_obj(payload)).encode("utf-8"),
+        headers={"Content-Type": "application/json", "User-Agent": f"forest/{__version__}"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=_WEBHOOK_TIMEOUT_SECONDS):  # nosec B310 - https enforced above
+            pass
+    except Exception as exc:
+        _log.warning("alert webhook delivery failed", extra={"error": str(exc)})
+        return False
+    _log.info("alert delivered", extra={"event": event, "command": command})
+    return True
+
+
+def report_failure(event: str, detail: str, *, command: str, exc: BaseException | None = None) -> None:
+    """One-stop failure hook: structured log + Sentry + webhook alert."""
+    _log.error(event, extra={"command": command, "detail": (detail or "")[:_MAX_DETAIL_CHARS]})
+    if exc is not None:
+        capture_exception(exc)
+    alert(event, detail, command=command)

forest/paths.py

@@ -0,0 +1,192 @@
+"""Path resolution and unit discovery for forest."""
+
+from __future__ import annotations
+
+from pathlib import Path, PurePosixPath
+
+import polars as pl
+
+from forest.config import ProjectConfig, StageConfig
+from forest.manifest import resolve_id
+
+_JUNK_NAMES = {".DS_Store"}
+
+
+def list_sync_files(unit_path: str | Path) -> list[Path]:
+    """Return all files under a data unit, skipping OS junk.
+
+    Schema-free replacement for the old schema-driven sync filter.
+    """
+    unit_path = Path(unit_path)
+    if not unit_path.is_dir():
+        return []
+    files: list[Path] = []
+    for f in unit_path.rglob("*"):
+        if not f.is_file():
+            continue
+        if f.name in _JUNK_NAMES or f.name.startswith("._") or f.suffix == ".tmp":
+            continue
+        files.append(f)
+    return sorted(files)
+
+
+def _required_stage_path(stage_cfg: StageConfig) -> Path:
+    """Return a configured stage path or raise a user-facing value error."""
+    if stage_cfg.path is None:
+        raise ValueError("Stage path is not configured. Bind the stage before using local files.")
+    return stage_cfg.path
+
+
+def _effective_remote_base(stage_cfg: StageConfig) -> Path:
+    """Return the remote base path for a stage.
+
+    When ``remote_path`` is set on the stage, it is used for the remote
+    mapping.  Otherwise the stage ``path`` is used (default behaviour).
+    """
+    if stage_cfg.remote_path is not None:
+        return stage_cfg.remote_path
+    return _required_stage_path(stage_cfg)
+
+
+def resolve_manifest_pairs(
+    config: ProjectConfig,
+    manifest: pl.DataFrame,
+    id_value: str,
+    stage_name: str | None = None,
+) -> list[tuple[str, str]]:
+    """Resolve a manifest ID to (stage, unit_id) pairs using the project config.
+
+    Wraps :func:`forest.manifest.resolve_id` with the ``id_column`` and
+    ``stage_columns`` plumbing so callers never repeat it.
+
+    Raises ``ValueError`` when the project config has no manifest section.
+    """
+    if config.manifest is None:
+        raise ValueError("Manifest configuration is required for ID-based resolution")
+    return resolve_id(manifest, id_value, config.manifest.id_column, config.manifest.stage_columns, stage_name)
+
+
+def resolve_paths(
+    config: ProjectConfig,
+    manifest: pl.DataFrame,
+    id_value: str,
+    stage_name: str | None = None,
+) -> list[tuple[str, str, Path, str]]:
+    """Resolve a manifest ID to (stage, unit_id, local_path, remote_path) tuples.
+
+    Uses the manifest's ``id_column`` and ``stage_columns`` from the project
+    config to map *id_value* to concrete (stage, unit_id) pairs, then
+    computes local and remote paths for each.
+
+    *local_path* is a relative ``Path`` (stage_path / unit_id).
+    *remote_path* is the POSIX string form of the remote-side relative path,
+    which uses ``remote_path`` when set on the stage config.
+
+    Raises ``ValueError`` when the project config has no manifest section.
+    """
+    pairs = resolve_manifest_pairs(config, manifest, id_value, stage_name)
+
+    result: list[tuple[str, str, Path, str]] = []
+    for stage, unit_id in pairs:
+        stage_cfg = config.stages[stage]
+        stage_path = _required_stage_path(stage_cfg)
+        local_path = stage_path if stage_cfg.sync_by == "directory" else stage_path / unit_id
+        remote_base = _effective_remote_base(stage_cfg)
+        remote_rel = remote_base if stage_cfg.sync_by == "directory" else remote_base / unit_id
+        remote_path = PurePosixPath(remote_rel).as_posix()
+        result.append((stage, unit_id, local_path, remote_path))
+
+    return result
+
+
+def resolve_stage_paths(
+    config: ProjectConfig,
+    stage_name: str,
+    unit_ids: list[str],
+) -> list[tuple[str, Path, str]]:
+    """Resolve unit IDs in a stage to (unit_id, local_path, remote_path) tuples.
+
+    *local_path* is a relative ``Path`` (stage_path / unit_id).
+    *remote_path* is the POSIX string form of the remote-side relative path,
+    which uses ``remote_path`` when set on the stage config.
+
+    Raises ``KeyError`` when *stage_name* is not in the config stages.
+    """
+    stage_cfg = config.stages[stage_name]
+    stage_path = _required_stage_path(stage_cfg)
+    remote_base = _effective_remote_base(stage_cfg)
+
+    result: list[tuple[str, Path, str]] = []
+    for unit_id in unit_ids:
+        local_path = stage_path if stage_cfg.sync_by == "directory" else stage_path / unit_id
+        remote_rel = remote_base if stage_cfg.sync_by == "directory" else remote_base / unit_id
+        remote_path = PurePosixPath(remote_rel).as_posix()
+        result.append((unit_id, local_path, remote_path))
+
+    return result
+
+
+def build_remote_path(
+    remote_url: str,
+    local_path: str | Path,
+    project_root: str | Path,
+    *,
+    remote_path_override: str | Path | None = None,
+) -> str:
+    """Build a full remote path from a remote URL and a local path.
+
+    When *remote_path_override* is given, it is used as the relative
+    portion instead of computing from *local_path* / *project_root*.
+
+    Otherwise computes the relative portion of *local_path* with respect
+    to *project_root* and appends it to *remote_url*.
+
+    If *local_path* is already relative, it is used as-is.
+    """
+    if remote_path_override is not None:
+        return remote_url.rstrip("/") + "/" + PurePosixPath(str(remote_path_override)).as_posix()
+
+    local_path = Path(local_path)
+    project_root = Path(project_root)
+
+    relative = local_path.relative_to(project_root) if local_path.is_absolute() else local_path
+
+    return remote_url.rstrip("/") + "/" + PurePosixPath(relative).as_posix()
+
+
+def discover_units(
+    stage_config: StageConfig,
+    project_root: str | Path,
+) -> list[tuple[str, Path]]:
+    """Discover data units in a stage directory on disk.
+
+    Returns a sorted list of ``(unit_id, local_path)`` tuples where
+    *local_path* is the absolute path to the unit.
+
+    Handles ``sync_by`` modes:
+
+    - ``directory`` — the entire stage path is one unit.
+    - ``subdirectory`` — each child directory is a separate unit.
+    - ``file`` — each file in the stage directory is a unit.
+
+    Returns an empty list when the stage directory does not exist.
+    """
+    project_root = Path(project_root)
+    configured_stage_path = _required_stage_path(stage_config)
+    stage_path = project_root / configured_stage_path
+
+    if not stage_path.is_dir():
+        return []
+
+    if stage_config.sync_by == "directory":
+        return [(configured_stage_path.name, stage_path)]
+
+    if stage_config.sync_by == "subdirectory":
+        children = sorted(stage_path.iterdir())
+        return [(child.name, child) for child in children if child.is_dir()]
+
+    if stage_config.sync_by == "file":
+        children = sorted(stage_path.iterdir())
+        return [(child.name, child) for child in children if child.is_file()]
+
+    return []