pythinker-code 0.8.0__py3-none-any.whl

pythinker_code/telemetry/sentry.py

@@ -0,0 +1,212 @@
+"""Sentry / Bugsink integration.
+
+Initializes ``sentry_sdk`` so unhandled exceptions, asyncio task failures, and
+explicit ``capture_exception`` calls flow to Bugsink at ``errors.pythinker.com``.
+The integration is additive: the existing ``crash.py`` excepthook still emits a
+privacy-respecting ``crash`` event for in-process counting.
+
+Privacy posture:
+  - ``send_default_pii=False`` — Sentry will not auto-capture user info from
+    request frames or local variables it considers personal.
+  - ``before_send`` strips file paths to package-relative form so site-packages
+    layouts don't reveal the user's home directory.
+  - Loop integrations are explicitly enabled; everything else inherits Sentry's
+    safe defaults.
+
+Opt out: ``PYTHINKER_DISABLE_TELEMETRY=1``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import re
+from typing import Any, cast
+
+import sentry_sdk
+from sentry_sdk.integrations.asyncio import AsyncioIntegration
+from sentry_sdk.integrations.dedupe import DedupeIntegration
+from sentry_sdk.integrations.excepthook import ExcepthookIntegration
+from sentry_sdk.types import Event, Hint
+
+from pythinker_code.telemetry.config import is_disabled, sentry_dsn
+
+_initialized: bool = False
+
+# File-path scrubbing: collapse anything before "site-packages/" or
+# "pythinker_code/" to "<env>" so Sentry stack frames don't expose home dirs.
+_PATH_SCRUB = re.compile(
+    r"^(.*?)(site-packages|pythinker_code|src/pythinker_code)/",
+)
+
+
+def _scrub_path(path: str) -> str:
+    return _PATH_SCRUB.sub(r"<env>/\2/", path)
+
+
+def _frame_path(frame: dict[str, Any]) -> str:
+    for key in ("abs_path", "filename"):
+        value = frame.get(key)
+        if isinstance(value, str):
+            return value.replace("\\", "/")
+    return ""
+
+
+def _event_has_test_frame(values: list[Any]) -> bool:
+    for exception_raw in values:
+        if not isinstance(exception_raw, dict):
+            continue
+        exception = cast(dict[str, Any], exception_raw)
+        stacktrace_raw = exception.get("stacktrace")
+        if not isinstance(stacktrace_raw, dict):
+            continue
+        stacktrace = cast(dict[str, Any], stacktrace_raw)
+        frames_raw = stacktrace.get("frames")
+        if not isinstance(frames_raw, list):
+            continue
+        frames = cast(list[Any], frames_raw)
+        for frame_raw in frames:
+            if not isinstance(frame_raw, dict):
+                continue
+            path = _frame_path(cast(dict[str, Any], frame_raw))
+            filename = path.rsplit("/", 1)[-1]
+            if "/tests/" in path or filename.startswith("test_"):
+                return True
+    return False
+
+
+def _event_is_normal_queue_shutdown(values: list[Any]) -> bool:
+    for exception_raw in values:
+        if not isinstance(exception_raw, dict):
+            continue
+        exception = cast(dict[str, Any], exception_raw)
+        if exception.get("module") == "asyncio.queues" and exception.get("type") == "QueueShutDown":
+            return True
+    return False
+
+
+def _before_send(event: Event, hint: Hint) -> Event | None:
+    """Sentry hook applied to every outgoing event.
+
+    Drops absolute file paths (replaces the prefix above ``site-packages/`` or
+    ``pythinker_code/`` with ``<env>``) and removes a couple of fields known to
+    leak local context.
+    """
+    # Strip server name (usually the user's hostname).
+    event.pop("server_name", None)
+
+    exception_data = event.get("exception")
+    if exception_data is None:
+        return event
+    values = exception_data.get("values")
+    if not isinstance(values, list):
+        return event
+
+    typed_values = cast(list[Any], values)
+    if _event_has_test_frame(typed_values) or _event_is_normal_queue_shutdown(typed_values):
+        return None
+
+    for exception_raw in typed_values:
+        if not isinstance(exception_raw, dict):
+            continue
+        exception = cast(dict[str, Any], exception_raw)
+        stacktrace_raw = exception.get("stacktrace")
+        if not isinstance(stacktrace_raw, dict):
+            continue
+        stacktrace = cast(dict[str, Any], stacktrace_raw)
+        frames = stacktrace.get("frames")
+        if not isinstance(frames, list):
+            continue
+        for frame_raw in cast(list[Any], frames):
+            if not isinstance(frame_raw, dict):
+                continue
+            frame = cast(dict[str, Any], frame_raw)
+            abs_path = frame.get("abs_path")
+            if isinstance(abs_path, str):
+                frame["abs_path"] = _scrub_path(abs_path)
+            filename = frame.get("filename")
+            if isinstance(filename, str):
+                frame["filename"] = _scrub_path(filename)
+    return event
+
+
+def init(
+    *,
+    version: str,
+    environment: str | None = None,
+    device_id: str | None = None,
+    extra_tags: dict[str, str] | None = None,
+) -> bool:
+    """Initialize ``sentry_sdk`` once per process. Returns True if active."""
+    global _initialized
+    if _initialized:
+        return True
+    if is_disabled():
+        return False
+
+    dsn = sentry_dsn()
+    if not dsn:
+        return False
+
+    env = environment or os.environ.get("PYTHINKER_ENV") or "production"
+
+    sentry_sdk.init(
+        dsn=dsn,
+        release=f"pythinker-code@{version}" if version else None,
+        environment=env,
+        # Dev-mode defaults: capture every error, no traces (OTel handles
+        # those). Tune later when distributing to users.
+        traces_sample_rate=0.0,
+        profiles_sample_rate=0.0,
+        send_default_pii=False,
+        attach_stacktrace=True,
+        max_breadcrumbs=50,
+        # Only the integrations that catch unhandled errors. Skip stdlib
+        # integrations (logging, atexit) so we don't double-emit alongside
+        # OTel logs.
+        default_integrations=False,
+        integrations=[
+            ExcepthookIntegration(always_run=False),
+            AsyncioIntegration(),
+            DedupeIntegration(),
+        ],
+        before_send=_before_send,
+    )
+
+    if device_id:
+        sentry_sdk.set_user({"id": device_id})
+    if extra_tags:
+        for k, v in extra_tags.items():
+            sentry_sdk.set_tag(k, v)
+
+    _initialized = True
+    return True
+
+
+def capture_exception(exc: BaseException) -> None:
+    """Forward an exception to Sentry/Bugsink. Safe to call when uninitialized
+    (becomes a no-op). Telemetry must never raise into the host program."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        sentry_sdk.capture_exception(exc)
+
+
+def add_breadcrumb(
+    *, category: str, message: str, level: str = "info", data: dict[str, Any] | None = None
+) -> None:
+    """Append a breadcrumb to the Sentry scope (visible in the next captured event)."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        sentry_sdk.add_breadcrumb(category=category, message=message, level=level, data=data or {})
+
+
+def flush(timeout_seconds: float = 2.0) -> None:
+    """Block until pending events are sent or the timeout expires."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        client = sentry_sdk.get_client()
+        if client:
+            client.flush(timeout=timeout_seconds)

pythinker_code/telemetry/sink.py

@@ -0,0 +1,189 @@
+"""EventSink: opt-out check, context enrichment, buffer management, timed flush.
+
+Forwards every accepted event to OpenTelemetry as a structured log record.
+The earlier custom-HTTP transport that posted to ``telemetry-logs.pythinker.com``
+was retired in the SigNoz migration — the OTel exporter inside ``otel.py`` now
+handles batching, retries, and disk-spool semantics on its own.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import locale
+import os
+import platform
+import threading
+from typing import Any, cast
+
+from pythinker_code.utils.logging import logger
+
+
+def _assert_primitive(scope: str, key: str, value: Any) -> None:
+    """Telemetry attribute values must be primitives. Catches accidental
+    nested dicts/lists before they reach the OTel SDK serializer."""
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return
+    raise TypeError(f"telemetry {scope}.{key} must be primitive, got {type(value).__name__}")
+
+
+def _flatten_event(event: dict[str, Any]) -> dict[str, Any]:
+    """Expand ``properties``/``context`` sub-dicts into flat ``property.*`` /
+    ``context.*`` keys for OTel attributes. Top-level fields pass through.
+
+    Raises ``TypeError`` on nested values inside properties/context.
+    """
+    out: dict[str, Any] = {}
+    for key, value in event.items():
+        if key == "properties":
+            properties = cast(dict[str, Any], value) if isinstance(value, dict) else {}
+            for pk, pv in properties.items():
+                _assert_primitive("property", pk, pv)
+                out[f"property.{pk}"] = pv
+        elif key == "context":
+            context = cast(dict[str, Any], value) if isinstance(value, dict) else {}
+            for ck, cv in context.items():
+                _assert_primitive("context", ck, cv)
+                out[f"context.{ck}"] = cv
+        else:
+            out[key] = value
+    return out
+
+
+class EventSink:
+    """Buffers telemetry events and flushes them in batches to OTel logs."""
+
+    FLUSH_INTERVAL_S = 30.0
+    FLUSH_THRESHOLD = 50
+
+    def __init__(
+        self,
+        *,
+        version: str = "",
+        model: str = "",
+        ui_mode: str = "shell",
+    ) -> None:
+        self._buffer: list[dict[str, Any]] = []
+        self._lock = threading.Lock()
+        self._flush_task: asyncio.Task[None] | None = None
+        # Static context enrichment
+        self._context: dict[str, Any] = {
+            "version": version,
+            "runtime": "python",
+            "platform": platform.system().lower(),
+            "arch": platform.machine(),
+            "python_version": platform.python_version(),
+            "os_version": platform.release(),
+            "ci": bool(os.environ.get("CI")),
+            "locale": locale.getlocale()[0] or "",
+            "terminal": os.environ.get("TERM_PROGRAM", ""),
+        }
+        self._model = model
+        self._ui_mode = ui_mode
+
+    def accept(self, event: dict[str, Any]) -> None:
+        """Accept an event into the buffer. Non-blocking, thread-safe."""
+        # Enrich with static context (copy to avoid mutating the caller's dict)
+        ctx = {**self._context, "ui_mode": self._ui_mode}
+        if self._model:
+            ctx["model"] = self._model
+        enriched = {**event, "context": ctx}
+
+        with self._lock:
+            self._buffer.append(enriched)
+            should_flush = len(self._buffer) >= self.FLUSH_THRESHOLD
+
+        if should_flush:
+            self._schedule_async_flush()
+
+    def start_periodic_flush(self, loop: asyncio.AbstractEventLoop | None = None) -> None:
+        """Start a background task that flushes every FLUSH_INTERVAL_S seconds."""
+        if self._flush_task is not None:
+            return
+
+        async def _periodic() -> None:
+            try:
+                while True:
+                    await asyncio.sleep(self.FLUSH_INTERVAL_S)
+                    await self._flush_async()
+            except asyncio.CancelledError:
+                pass
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+        self._flush_task = loop.create_task(_periodic())
+
+    async def retry_disk_events(self) -> None:
+        """Compatibility shim — disk retries now happen inside the OTel
+        exporter. No-op kept so existing callers don't break."""
+
+    def clear_buffer(self) -> None:
+        """Discard all buffered events without sending them."""
+        with self._lock:
+            self._buffer.clear()
+
+    def stop_periodic_flush(self) -> None:
+        """Cancel the periodic flush task."""
+        if self._flush_task is not None:
+            self._flush_task.cancel()
+            self._flush_task = None
+
+    async def flush(self) -> None:
+        """Async flush: send all buffered events."""
+        await self._flush_async()
+
+    def flush_sync(self) -> None:
+        """Synchronous flush for atexit / signal handlers.
+
+        Drops the in-process buffer into the OTel pipeline. Network I/O is
+        scheduled by the BatchLogRecordProcessor; the OTel SDK's own shutdown
+        (called from ``otel.shutdown``) waits for that batch to drain.
+        """
+        with self._lock:
+            if not self._buffer:
+                return
+            events = list(self._buffer)
+            self._buffer.clear()
+        self._emit_to_otel(events)
+
+    async def _flush_async(self) -> None:
+        """Take all buffered events and forward them to OTel logs."""
+        with self._lock:
+            if not self._buffer:
+                return
+            events = list(self._buffer)
+            self._buffer.clear()
+        self._emit_to_otel(events)
+
+    def _emit_to_otel(self, events: list[dict[str, Any]]) -> None:
+        if not events:
+            return
+        try:
+            from pythinker_code.telemetry import otel as _otel
+
+            for event in events:
+                ts = event.get("timestamp")
+                ts_ns = int(ts * 1_000_000_000) if isinstance(ts, (int, float)) else None
+                try:
+                    attrs = _flatten_event(event)
+                except TypeError as exc:
+                    # Schema violation — drop, never retry.
+                    logger.debug("Telemetry event dropped (non-primitive attr): {err}", err=exc)
+                    continue
+                attrs.pop("event", None)
+                attrs.pop("timestamp", None)
+                _otel.emit_log(
+                    name=str(event.get("event") or "event"),
+                    attributes=attrs,
+                    timestamp_ns=ts_ns,
+                )
+        except Exception:
+            logger.debug("OTel flush failed; events dropped")
+
+    def _schedule_async_flush(self) -> None:
+        """Schedule an async flush from any thread."""
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(self._flush_async())
+        except RuntimeError:
+            # No running event loop — will be flushed by periodic task or on exit
+            pass

pythinker_code/tools/AGENTS.md

@@ -0,0 +1,6 @@
+# Pythinker CLI Tools
+
+## Guidelines
+
+- Tools should not refer to types in `pythinker_code/wire/` unless they are explicitly implementing a UI / runtime bridge. When importing things like `ToolReturnValue` or `DisplayBlock`, prefer `pythinker_core.tooling`.
+- Current bridge exceptions include `ask_user`, `plan`, and `file` media/display helpers; keep new tool logic in `pythinker_core.tooling` unless it must emit wire-facing requests or display blocks.

pythinker_code/tools/__init__.py

@@ -0,0 +1,105 @@
+import json
+from typing import cast
+
+import streamingjson  # type: ignore[reportMissingTypeStubs]
+from pythinker_core.utils.typing import JsonType
+from pythinker_host.path import HostPath
+
+from pythinker_code.utils.string import shorten_middle
+
+
+class SkipThisTool(Exception):
+    """Raised when a tool decides to skip itself from the loading process."""
+
+    pass
+
+
+def extract_key_argument(json_content: str | streamingjson.Lexer, tool_name: str) -> str | None:
+    if isinstance(json_content, streamingjson.Lexer):
+        json_str = json_content.complete_json()
+    else:
+        json_str = json_content
+    try:
+        curr_args: JsonType = json.loads(json_str, strict=False)
+    except json.JSONDecodeError:
+        return None
+    if not curr_args:
+        return None
+    key_argument: str = ""
+    match tool_name:
+        case "Agent":
+            if not isinstance(curr_args, dict) or not curr_args.get("description"):
+                return None
+            key_argument = str(curr_args["description"])
+        case "SendDMail":
+            return None
+        case "Think":
+            if not isinstance(curr_args, dict) or not curr_args.get("thought"):
+                return None
+            key_argument = str(curr_args["thought"])
+        case "SetTodoList":
+            return None
+        case "Bash" | "Shell":
+            if not isinstance(curr_args, dict) or not curr_args.get("command"):
+                return None
+            key_argument = str(curr_args["command"])
+        case "TaskOutput":
+            if not isinstance(curr_args, dict) or not curr_args.get("task_id"):
+                return None
+            key_argument = str(curr_args["task_id"])
+        case "TaskList":
+            if not isinstance(curr_args, dict):
+                return None
+            key_argument = "active" if curr_args.get("active_only", True) else "all"
+        case "TaskStop":
+            if not isinstance(curr_args, dict) or not curr_args.get("task_id"):
+                return None
+            key_argument = str(curr_args["task_id"])
+        case "ReadFile":
+            if not isinstance(curr_args, dict) or not curr_args.get("path"):
+                return None
+            key_argument = _normalize_path(str(curr_args["path"]))
+        case "ReadMediaFile":
+            if not isinstance(curr_args, dict) or not curr_args.get("path"):
+                return None
+            key_argument = _normalize_path(str(curr_args["path"]))
+        case "Glob":
+            if not isinstance(curr_args, dict) or not curr_args.get("pattern"):
+                return None
+            key_argument = str(curr_args["pattern"])
+        case "Grep":
+            if not isinstance(curr_args, dict) or not curr_args.get("pattern"):
+                return None
+            key_argument = str(curr_args["pattern"])
+        case "WriteFile":
+            if not isinstance(curr_args, dict) or not curr_args.get("path"):
+                return None
+            key_argument = _normalize_path(str(curr_args["path"]))
+        case "StrReplaceFile":
+            if not isinstance(curr_args, dict) or not curr_args.get("path"):
+                return None
+            key_argument = _normalize_path(str(curr_args["path"]))
+        case "SearchWeb":
+            if not isinstance(curr_args, dict) or not curr_args.get("query"):
+                return None
+            key_argument = str(curr_args["query"])
+        case "FetchURL":
+            if not isinstance(curr_args, dict) or not curr_args.get("url"):
+                return None
+            key_argument = str(curr_args["url"])
+        case _:
+            if isinstance(json_content, streamingjson.Lexer):
+                # lexer.json_content is list[str] based on streamingjson source code
+                content: list[str] = cast(list[str], json_content.json_content)  # type: ignore[reportUnknownMemberType]
+                key_argument = "".join(content)
+            else:
+                key_argument = json_content
+    key_argument = shorten_middle(key_argument, width=50)
+    return key_argument
+
+
+def _normalize_path(path: str) -> str:
+    cwd = str(HostPath.cwd().canonical())
+    if path.startswith(cwd):
+        path = path[len(cwd) :].lstrip("/\\")
+    return path