contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/scratch.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import shutil
+from collections.abc import Mapping
+from pathlib import Path
+from uuid import uuid4
+
+from .env import CTXB_SCRATCH_DIR_ENV_VAR, load_shared_python_settings
+
+
+def _validate_safe_segment(value: str, *, label: str) -> str:
+    normalized = value.strip()
+    if not normalized:
+        raise ValueError(f"{label} cannot be blank.")
+    if normalized in {".", ".."}:
+        raise ValueError(f"{label} cannot be '.' or '..'.")
+    if "/" in normalized or "\\" in normalized or "\x00" in normalized:
+        raise ValueError(f"{label} cannot contain path separators or null bytes.")
+    return normalized
+
+
+def _resolve_scratch_root() -> Path:
+    scratch_root = load_shared_python_settings().ctx_scratch_dir
+    if scratch_root is None:
+        raise RuntimeError(
+            f"{CTXB_SCRATCH_DIR_ENV_VAR} is not set; run via ctxb so scratch paths are configured."
+        )
+    return scratch_root
+
+
+def _parse_relative_dir(relative_dir: str) -> Path:
+    normalized = relative_dir.strip()
+    if not normalized:
+        raise ValueError("relative_dir cannot be blank.")
+
+    relative_path = Path(normalized)
+    if relative_path.is_absolute():
+        raise ValueError("relative_dir must be relative, not absolute.")
+
+    safe_parts = [
+        _validate_safe_segment(part, label="relative_dir segment")
+        for part in relative_path.parts
+    ]
+    return Path(*safe_parts)
+
+
+def _resolve_binding_target_paths(
+    *,
+    binding_id: str,
+    relative_dir: str,
+) -> tuple[Path, Path]:
+    scratch_root = _resolve_scratch_root()
+    safe_binding_id = _validate_safe_segment(binding_id, label="binding_id")
+    safe_relative_dir = _parse_relative_dir(relative_dir)
+
+    binding_root = (scratch_root / safe_binding_id).absolute()
+    target_dir = (binding_root / safe_relative_dir).absolute()
+    if not target_dir.is_relative_to(binding_root):
+        raise ValueError("relative_dir escapes the binding scratch root.")
+    return binding_root, target_dir
+
+
+def _remove_path(path: Path) -> None:
+    if not path.exists():
+        return
+    if path.is_dir():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+
+
+def _build_temp_path(binding_root: Path, *, prefix: str) -> Path:
+    return binding_root / f".{prefix}-{uuid4().hex}"
+
+
+def replace_scratch_dir_files(
+    *,
+    binding_id: str,
+    relative_dir: str,
+    files: Mapping[str, bytes],
+) -> dict[str, str]:
+    binding_root, target_dir = _resolve_binding_target_paths(
+        binding_id=binding_id,
+        relative_dir=relative_dir,
+    )
+    binding_root.mkdir(parents=True, exist_ok=True)
+
+    staging_dir = _build_temp_path(binding_root, prefix="ctx-staging")
+    staging_dir.mkdir(parents=False, exist_ok=False)
+
+    safe_name_by_input_name: dict[str, str] = {}
+    normalized_safe_names: set[str] = set()
+    for file_name, content in files.items():
+        safe_file_name = _validate_safe_segment(file_name, label="file name")
+        if not isinstance(content, bytes):
+            raise TypeError("files values must be bytes.")
+
+        if safe_file_name in normalized_safe_names:
+            raise ValueError(
+                f"files contains duplicate normalized file names: '{safe_file_name}'."
+            )
+        normalized_safe_names.add(safe_file_name)
+        safe_name_by_input_name[file_name] = safe_file_name
+
+        file_path = staging_dir / safe_file_name
+        file_path.write_bytes(content)
+
+    backup_path: Path | None = None
+    try:
+        target_dir.parent.mkdir(parents=True, exist_ok=True)
+        if target_dir.exists():
+            backup_path = _build_temp_path(binding_root, prefix="ctx-backup")
+            target_dir.rename(backup_path)
+        staging_dir.rename(target_dir)
+    except Exception:
+        if backup_path is not None and backup_path.exists() and not target_dir.exists():
+            backup_path.rename(target_dir)
+        raise
+    finally:
+        _remove_path(staging_dir)
+        if backup_path is not None:
+            _remove_path(backup_path)
+
+    return {
+        file_name: str((target_dir / safe_name).absolute())
+        for file_name, safe_name in safe_name_by_input_name.items()
+    }

shared_plugins/sentry.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Callable
+from importlib.metadata import version
+
+import sentry_sdk
+from sentry_sdk.types import Event, Hint
+
+from shared_plugins.values import require_non_negative_int
+
+# Hardcoded write-only ingest key (safe in shipped code). Prod-gated: ctxb (the
+# Node spawn parent) forwards SENTRY_ENABLED=true to the dagster child only from
+# its compiled binary, so the SDK initializes only in shipped runs — never local
+# or CI Dagster, which run from source and never set the flag. Errors only: no
+# tracing, profiling, or replay (internal Sentry spec, pass 1).
+_DSN = "https://96f72b28675ce397ec6cb39a0702e747@o4510556311519232.ingest.us.sentry.io/4511513003229184"
+
+# A UUID contains digit runs, so it must be collapsed BEFORE they are — the
+# reverse order shreds it into fragments and every event keys differently. This
+# mirrors @usecontextlayer/shared/sentry-event-cap on the TypeScript side; the two
+# are duplicated because there is no cross-language home for 30 lines, and they
+# must stay in step.
+_UUID_PATTERN = re.compile(
+    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+    re.IGNORECASE,
+)
+# Six, not three — the rationale lives with the canonical TS copy
+# (@usecontextlayer/shared/sentry-event-cap).
+_LONG_DIGIT_RUN_PATTERN = re.compile(r"\d{6,}")
+
+
+def _parse_event_cap(raw: str | None) -> int | None:
+    """Read the SENTRY_CAP_EVENTS contract: a positive integer enables capping and
+    IS the cap. Absent, blank, or unparseable means capping off."""
+    try:
+        cap = require_non_negative_int(raw)
+    except ValueError:
+        return None
+    # Positivity is this module's policy, not the shared coercer's: zero is a valid
+    # non-negative integer but cannot express a cap.
+    return cap if cap > 0 else None
+
+
+def _normalized_key(event: Event) -> str:
+    # Dagster reports through the logging integration, which writes BOTH the
+    # template (`message`) and the interpolated text (`formatted`). Key on
+    # `formatted`: it is the realized string in either case, so this works whether
+    # or not the caller passed logging params. Fall back to a plain message, then
+    # to the LAST exception value — the same precedence the JS side uses.
+    logentry = event.get("logentry") or {}
+    raw = logentry.get("formatted") or logentry.get("message") or event.get("message")
+    if raw is None:
+        values = (event.get("exception") or {}).get("values") or []
+        raw = values[-1].get("value") if values else ""
+    collapsed = _UUID_PATTERN.sub("<uuid>", raw or "")
+    return _LONG_DIGIT_RUN_PATTERN.sub("<n>", collapsed)
+
+
+def _event_cap_before_send(
+    raw: str | None,
+) -> Callable[[Event, Hint], Event | None] | None:
+    """The composed form init uses: parse the raw env value and build the hook in
+    one step — None (capping off) when the value doesn't parse."""
+    cap = _parse_event_cap(raw)
+    return None if cap is None else _create_capped_before_send(cap)
+
+
+def _create_capped_before_send(cap: int) -> Callable[[Event, Hint], Event | None]:
+    """Admit at most `cap` events per normalized message. The first event of a key
+    never seen before is always admitted, so this understates how often something
+    recurs but can never hide that it happened.
+
+    NOTE the process boundary: core/definitions.py calls init_sentry() at module
+    scope, and that module is loaded by EVERY Dagster process — daemon, per-run
+    subprocess, per-step subprocess. Each therefore gets its own counter. This
+    bounds a burst inside one step; it does NOT bound a run-failure storm across
+    many subprocesses, which is what the July 2026 microsoft_mail_attachment_content
+    cluster (~1058 failed runs, one event each) actually was.
+    """
+    sent_per_key: dict[str, int] = {}
+
+    def before_send(event: Event, hint: Hint) -> Event | None:
+        key = _normalized_key(event)
+        sent = sent_per_key.get(key, 0)
+        if sent >= cap:
+            return None
+        sent_per_key[key] = sent + 1
+        return event
+
+    return before_send
+
+
+def init_sentry() -> None:
+    # Off unless the compiled ctxb forwarded the baked prod-gate (see module note).
+    if os.environ.get("SENTRY_ENABLED") != "true":
+        return
+    # Derive the release from the installed "contextbase-shared-plugins" dist
+    # version (release.ts bumps it in lockstep with the pyproject version),
+    # labeled "contextbase-core@<version>". Computed here, after the gate, so a
+    # disabled/dev/CI run does no metadata lookup, and the dist is guaranteed
+    # installed whenever the gate is on. No literal to keep in sync.
+    release = f"contextbase-core@{version('contextbase-shared-plugins')}"
+    # SENTRY_CAP_EVENTS needs no forwarding of its own: buildDagsterEnv spreads the
+    # raw process.env into the child, so a real env var arrives for free (unlike
+    # SENTRY_ENABLED, which is build-folded out of the compiled ctxb's environment
+    # and therefore has to be re-added explicitly there).
+    sentry_sdk.init(
+        before_send=_event_cap_before_send(os.environ.get("SENTRY_CAP_EVENTS")),
+        dsn=_DSN,
+        environment="production",
+        release=release,
+        send_default_pii=False,
+        # Errors only — performance tracing is deferred to a later pass.
+        traces_sample_rate=0,
+    )

shared_plugins/sqlalchemy_types.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from sqlalchemy.types import Float, Integer, TypeDecorator
+
+
+def _timedelta_total_microseconds(delta: timedelta) -> int:
+    return ((delta.days * 86_400) + delta.seconds) * 1_000_000 + delta.microseconds
+
+
+def _timedelta_total_milliseconds(delta: timedelta) -> int:
+    total_microseconds = _timedelta_total_microseconds(delta)
+    if total_microseconds >= 0:
+        return total_microseconds // 1_000
+    return -((-total_microseconds) // 1_000)
+
+
+APPLE_CORE_DATA_EPOCH = datetime(2001, 1, 1, tzinfo=timezone.utc)
+
+
+class AppleCoreDataTimestamp(TypeDecorator[datetime | None]):
+    """Store Apple Core Data timestamps as aware UTC datetimes."""
+
+    impl = Float
+    cache_ok = True
+
+    def __init__(self, *, null_sentinels: Iterable[float] = ()) -> None:
+        super().__init__()
+        self.null_sentinels = tuple(
+            sorted({float(sentinel) for sentinel in null_sentinels})
+        )
+
+    def process_bind_param(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> float | None:
+        del dialect
+        if value is None:
+            return None
+        if not isinstance(value, datetime):
+            raise TypeError(
+                "AppleCoreDataTimestamp values must be datetime instances or None."
+            )
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise TypeError("AppleCoreDataTimestamp requires an aware datetime.")
+
+        normalized = value.astimezone(timezone.utc)
+        return (normalized - APPLE_CORE_DATA_EPOCH).total_seconds()
+
+    def process_result_value(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> datetime | None:
+        del dialect
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            raise TypeError(
+                "AppleCoreDataTimestamp database values must be numeric or None."
+            )
+
+        numeric_value = float(value)
+        if numeric_value in self.null_sentinels:
+            return None
+
+        return APPLE_CORE_DATA_EPOCH + timedelta(seconds=numeric_value)
+
+
+class AppleMessageTimestamp(TypeDecorator[datetime | None]):
+    """Store iMessage timestamps as aware UTC datetimes.
+
+    iMessage's ``chat.db`` stores timestamps as **nanoseconds** since
+    2001-01-01 UTC starting with iOS 12 / macOS Mojave; pre-iOS 11 databases
+    stored the same columns as **seconds** since the same epoch. Rows from a
+    restored backup can mix both formats in the same column, so each value is
+    classified by magnitude: anything below 10**12 is treated as seconds and
+    everything else as nanoseconds. Sub-microsecond precision is lost on read
+    since ``datetime`` cannot represent it.
+    """
+
+    impl = Integer
+    cache_ok = True
+
+    def __init__(self, *, null_sentinels: Iterable[int] = ()) -> None:
+        super().__init__()
+        self.null_sentinels = tuple(
+            sorted({int(sentinel) for sentinel in null_sentinels})
+        )
+
+    def process_bind_param(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> int | None:
+        del dialect
+        if value is None:
+            return None
+        if not isinstance(value, datetime):
+            raise TypeError(
+                "AppleMessageTimestamp values must be datetime instances or None."
+            )
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise TypeError("AppleMessageTimestamp requires an aware datetime.")
+
+        normalized = value.astimezone(timezone.utc)
+        delta = normalized - APPLE_CORE_DATA_EPOCH
+        return _timedelta_total_microseconds(delta) * 1_000
+
+    def process_result_value(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> datetime | None:
+        del dialect
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, int):
+            raise TypeError(
+                "AppleMessageTimestamp database values must be int or None."
+            )
+
+        if value in self.null_sentinels:
+            return None
+
+        if value >= 10**12:
+            microseconds = value // 1_000
+            return APPLE_CORE_DATA_EPOCH + timedelta(microseconds=microseconds)
+
+        return APPLE_CORE_DATA_EPOCH + timedelta(seconds=value)
+
+
+CHROMIUM_EPOCH = datetime(1601, 1, 1, tzinfo=timezone.utc)
+
+
+class ChromiumTimestamp(TypeDecorator[datetime | None]):
+    """Store Chromium timestamps as aware UTC datetimes.
+
+    Chromium stores timestamps as microseconds since 1601-01-01 00:00:00 UTC.
+    """
+
+    impl = Integer
+    cache_ok = True
+
+    def process_bind_param(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> int | None:
+        del dialect
+        if value is None:
+            return None
+        if not isinstance(value, datetime):
+            raise TypeError(
+                "ChromiumTimestamp values must be datetime instances or None."
+            )
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise TypeError("ChromiumTimestamp requires an aware datetime.")
+
+        normalized = value.astimezone(timezone.utc)
+        delta = normalized - CHROMIUM_EPOCH
+        return _timedelta_total_microseconds(delta)
+
+    def process_result_value(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> datetime | None:
+        del dialect
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            raise TypeError(
+                "ChromiumTimestamp database values must be numeric or None."
+            )
+
+        return CHROMIUM_EPOCH + timedelta(microseconds=int(value))
+
+
+UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
+
+
+class UnixMillisTimestamp(TypeDecorator[datetime | None]):
+    """Store Unix-millisecond timestamps as aware UTC datetimes."""
+
+    impl = Integer
+    cache_ok = True
+
+    def process_bind_param(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> int | None:
+        del dialect
+        if value is None:
+            return None
+        if not isinstance(value, datetime):
+            raise TypeError(
+                "UnixMillisTimestamp values must be datetime instances or None."
+            )
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise TypeError("UnixMillisTimestamp requires an aware datetime.")
+
+        normalized = value.astimezone(timezone.utc)
+        delta = normalized - UNIX_EPOCH
+        return _timedelta_total_milliseconds(delta)
+
+    def process_result_value(
+        self,
+        value: object,
+        dialect: Any,
+    ) -> datetime | None:
+        del dialect
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            raise TypeError(
+                "UnixMillisTimestamp database values must be numeric or None."
+            )
+
+        return UNIX_EPOCH + timedelta(milliseconds=int(value))

shared_plugins/sqlite.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import logging
+import shutil
+import sqlite3
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+from urllib.parse import urlencode
+
+LOGGER = logging.getLogger(__name__)
+
+BACKUP_TIMEOUT_SECONDS = 5.0
+BACKUP_PAGES_PER_STEP = 100
+SQLITE_COPY_SIBLING_SUFFIXES = ("-journal", "-wal", "-shm")
+
+
+class _BackupTimeout(Exception):
+    """Internal sentinel raised from the backup progress callback."""
+
+
+@contextmanager
+def sqlite_snapshot(
+    db_path: Path,
+    *,
+    backup_timeout: float = BACKUP_TIMEOUT_SECONDS,
+) -> Iterator[Path]:
+    """Create a consistent snapshot of a potentially-locked SQLite database.
+
+    Yields a path to a temporary copy that the caller can open with any
+    library (raw sqlite3, SQLAlchemy, etc.). The temporary directory and
+    all files are deleted when the context manager exits.
+
+    Strategy:
+
+    1. Try the SQLite Online Backup API with cooperative timeout. A
+       `progress` callback fires after every backup_step iteration —
+       including SQLITE_BUSY / SQLITE_LOCKED — and aborts the backup
+       by raising when the wall-clock deadline is exceeded. The source
+       connection is opened with `timeout=0` so SQLite's internal busy
+       retry doesn't loop in C; control returns to Python between every
+       step, which is where we check the deadline.
+
+    2. If the backup deadline is exceeded, fall back to copying the
+       database file and any rollback-journal or WAL siblings. Best-
+       effort: not transactionally guaranteed, but works across SQLite
+       journal modes.
+
+    Non-timeout errors (missing file, corrupt DB, etc.) propagate to
+    the caller — they are not part of the "should we fall back?"
+    decision.
+    """
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        snapshot_dir = Path(tmp_dir)
+        snapshot_path = snapshot_dir / db_path.name
+        if _try_backup(db_path, snapshot_path, backup_timeout):
+            LOGGER.debug("sqlite_snapshot.backup_ok path=%s", db_path)
+        else:
+            LOGGER.info(
+                "sqlite_snapshot.backup_timeout path=%s timeout=%.1fs, falling back to file copy",
+                db_path,
+                backup_timeout,
+            )
+            _copy_with_siblings(db_path, snapshot_dir)
+
+        yield snapshot_path
+
+
+def _try_backup(db_path: Path, dest_path: Path, timeout: float) -> bool:
+    """Attempt a backup-API snapshot. Returns True on success, False on timeout."""
+    deadline = time.monotonic() + timeout
+
+    def abort_if_past_deadline(rc: int, remaining: int, total: int) -> None:
+        if time.monotonic() > deadline:
+            raise _BackupTimeout()
+
+    # timeout=0 disables SQLite's internal busy retry so the progress
+    # callback fires on every BUSY/LOCKED step (otherwise SQLite's C
+    # layer would loop on BUSY for up to busy_timeout before returning
+    # to Python, and we'd lose our cooperative cancellation point).
+    source = sqlite3.connect(_sqlite_readonly_uri(db_path), uri=True, timeout=0)
+    try:
+        dest = sqlite3.connect(dest_path)
+        try:
+            source.backup(
+                dest,
+                pages=BACKUP_PAGES_PER_STEP,
+                progress=abort_if_past_deadline,
+            )
+            return True
+        except _BackupTimeout:
+            _cleanup_partial_backup(dest_path)
+            return False
+        finally:
+            dest.close()
+    finally:
+        source.close()
+
+
+def _copy_with_siblings(db_path: Path, dest_dir: Path) -> None:
+    """Copy a database file and any rollback-journal or WAL siblings."""
+    for artifact in _iter_sqlite_artifacts(db_path):
+        if artifact.is_file():
+            shutil.copy2(artifact, dest_dir / artifact.name)
+
+
+def _cleanup_partial_backup(dest_path: Path) -> None:
+    for artifact in _iter_sqlite_artifacts(dest_path):
+        if artifact.exists():
+            artifact.unlink()
+
+
+def _iter_sqlite_artifacts(db_path: Path) -> Iterator[Path]:
+    yield db_path
+    for suffix in SQLITE_COPY_SIBLING_SUFFIXES:
+        yield db_path.parent / f"{db_path.name}{suffix}"
+
+
+def _sqlite_readonly_uri(db_path: Path) -> str:
+    base_uri = db_path.resolve(strict=False).as_uri()
+    return f"{base_uri}?{urlencode({'mode': 'ro'})}"