pythinker-code 2.0.0__py3-none-any.whl

pythinker_code/telemetry/otel.py

@@ -0,0 +1,240 @@
+"""OpenTelemetry (OTLP HTTP) integration.
+
+Sends traces, metrics, and logs to the pythinker edge OTel collector at
+``otel.pythinker.com``. The collector forwards to SigNoz internally; clients
+authenticate with a bearer token (embedded as a default, env-overridable).
+
+Public API:
+  - ``init(version, ui_mode, ...)`` — wire up Tracer/Meter/LoggerProvider.
+  - ``get_tracer()`` — get a tracer for span creation.
+  - ``get_meter()`` — get the meter for instrument creation.
+  - ``emit_log(name, attributes, severity)`` — push a structured log event.
+  - ``shutdown()`` — flush exporters at process exit.
+
+Uninitialized state is safe: ``get_tracer`` / ``get_meter`` return no-op
+instances and ``emit_log`` becomes a no-op when init wasn't called or the kill
+switch is on.
+"""
+
+from __future__ import annotations
+
+import logging
+import platform
+from typing import Any
+
+from opentelemetry import metrics, trace
+from opentelemetry._logs import SeverityNumber, set_logger_provider
+from opentelemetry._logs._internal import LogRecord
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.metrics import Meter
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace import Tracer
+
+from pythinker_code.telemetry.config import is_disabled, otel_endpoint, otel_ingest_token
+
+_TRACER_NAME = "pythinker-code"
+_initialized: bool = False
+_tracer: Tracer | None = None
+_meter: Meter | None = None
+_meter_provider: MeterProvider | None = None
+_logger_provider: LoggerProvider | None = None
+_log = logging.getLogger(__name__)
+
+
+def _bearer_headers() -> dict[str, str]:
+    return {"Authorization": f"Bearer {otel_ingest_token()}"}
+
+
+def _resource(*, version: str, ui_mode: str, device_id: str | None) -> Resource:
+    """Build the static resource attributes for every emitted span/log."""
+    attrs: dict[str, Any] = {
+        "service.name": "pythinker-code",
+        "service.version": version or "unknown",
+        "deployment.environment": "production",
+        "ui.mode": ui_mode or "shell",
+        "host.arch": platform.machine(),
+        "os.type": platform.system().lower(),
+        "process.runtime.name": "python",
+        "process.runtime.version": platform.python_version(),
+    }
+    if device_id:
+        # Treat device_id as a stable user-pseudonym (same role as Sentry user.id).
+        attrs["user.id"] = device_id
+    return Resource.create(attrs)
+
+
+def init(
+    *,
+    version: str,
+    ui_mode: str,
+    device_id: str | None = None,
+) -> bool:
+    """Idempotently install the OTel SDK with OTLP HTTP exporters."""
+    global _initialized, _tracer, _meter, _meter_provider, _logger_provider
+    if _initialized:
+        return True
+    if is_disabled():
+        return False
+
+    endpoint = otel_endpoint()
+    if not endpoint:
+        return False
+
+    resource = _resource(version=version, ui_mode=ui_mode, device_id=device_id)
+    headers = _bearer_headers()
+
+    # --- Traces ---
+    span_exporter = OTLPSpanExporter(
+        endpoint=f"{endpoint}/v1/traces",
+        headers=headers,
+        timeout=10,
+    )
+    tracer_provider = TracerProvider(resource=resource)
+    tracer_provider.add_span_processor(BatchSpanProcessor(span_exporter))
+    trace.set_tracer_provider(tracer_provider)
+    _tracer = tracer_provider.get_tracer(_TRACER_NAME, version)
+
+    # --- Metrics ---
+    metric_exporter = OTLPMetricExporter(
+        endpoint=f"{endpoint}/v1/metrics",
+        headers=headers,
+        timeout=10,
+    )
+    # 30s export interval keeps backend volume low for a CLI that's typically
+    # invoked for short bursts; final flush at process exit catches the tail.
+    metric_reader = PeriodicExportingMetricReader(metric_exporter, export_interval_millis=30_000)
+    _meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+    metrics.set_meter_provider(_meter_provider)
+    _meter = _meter_provider.get_meter(_TRACER_NAME, version)
+
+    # --- Logs ---
+    log_exporter = OTLPLogExporter(
+        endpoint=f"{endpoint}/v1/logs",
+        headers=headers,
+        timeout=10,
+    )
+    _logger_provider = LoggerProvider(resource=resource)
+    _logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter))
+    set_logger_provider(_logger_provider)
+
+    # Pre-create the metric instruments so call sites pay nothing on the hot
+    # path. Imported here to avoid a circular dependency at module load time.
+    from pythinker_code.telemetry import metrics as _m
+
+    _m.bind(_meter)
+
+    _initialized = True
+    _log.debug("OTel SDK initialized at %s", endpoint)
+    return True
+
+
+def get_tracer() -> Tracer:
+    """Return the active tracer, or the global no-op tracer when uninitialized."""
+    if _tracer is not None:
+        return _tracer
+    return trace.get_tracer(_TRACER_NAME)
+
+
+def get_meter() -> Meter:
+    """Return the active meter, or the global no-op meter when uninitialized."""
+    if _meter is not None:
+        return _meter
+    return metrics.get_meter(_TRACER_NAME)
+
+
+def start_span(name: str, attributes: dict[str, Any] | None = None) -> Any:
+    """Convenience: ``with start_span("pythinker.turn", {...}) as span:``.
+
+    When OTel is uninitialized this returns a no-op span (the global tracer's
+    no-op behaviour) — call sites stay clean and pay nothing in the disabled
+    case. The span is the *current* span inside the with block, so child spans
+    automatically nest.
+    """
+    return get_tracer().start_as_current_span(name, attributes=attributes or {})
+
+
+# ---------------------------------------------------------------------------
+# Log emission
+# ---------------------------------------------------------------------------
+
+_SEVERITY_BY_NAME = {
+    "debug": (SeverityNumber.DEBUG, "DEBUG"),
+    "info": (SeverityNumber.INFO, "INFO"),
+    "warn": (SeverityNumber.WARN, "WARN"),
+    "warning": (SeverityNumber.WARN, "WARN"),
+    "error": (SeverityNumber.ERROR, "ERROR"),
+}
+
+
+def emit_log(
+    *,
+    name: str,
+    attributes: dict[str, Any] | None = None,
+    severity: str = "info",
+    timestamp_ns: int | None = None,
+) -> None:
+    """Emit a structured OTel LogRecord. Used as the OTLP backend for ``track()``.
+
+    The ``name`` becomes the log body; properties + context become attributes.
+    Safe to call before or after init: pre-init calls are dropped silently.
+    """
+    if not _initialized or _logger_provider is None:
+        return
+    try:
+        sev_number, sev_text = _SEVERITY_BY_NAME.get(
+            severity.lower(), (SeverityNumber.INFO, "INFO")
+        )
+        record = LogRecord(
+            timestamp=timestamp_ns,
+            observed_timestamp=timestamp_ns,
+            severity_number=sev_number,
+            severity_text=sev_text,
+            body=name,
+            attributes=dict(attributes or {}),
+        )
+        # Use the SDK's logger directly so we don't need to bridge through the
+        # stdlib logging module (which has its own handlers we don't want to
+        # collide with).
+        otel_logger = _logger_provider.get_logger(_TRACER_NAME)
+        otel_logger.emit(record)
+    except Exception:
+        # Never propagate.
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Shutdown
+# ---------------------------------------------------------------------------
+
+
+def shutdown(timeout_millis: int = 2000) -> None:
+    """Flush all three providers. Called at process exit."""
+    if not _initialized:
+        return
+    try:
+        tp = trace.get_tracer_provider()
+        # The global provider may be the no-op when init failed; guard with
+        # hasattr instead of an isinstance check that pulls in private SDK
+        # types.
+        if hasattr(tp, "shutdown"):
+            tp.shutdown()  # type: ignore[attr-defined]
+    except Exception:
+        pass
+    try:
+        if _meter_provider is not None:
+            _meter_provider.shutdown()
+    except Exception:
+        pass
+    try:
+        if _logger_provider is not None:
+            _logger_provider.shutdown()
+    except Exception:
+        pass

pythinker_code/telemetry/sentry.py

@@ -0,0 +1,167 @@
+"""Sentry / Bugsink integration.
+
+Initializes ``sentry_sdk`` so unhandled exceptions, asyncio task failures, and
+explicit ``capture_exception`` calls flow to Bugsink at ``errors.pythinker.com``.
+The integration is additive: the existing ``crash.py`` excepthook still emits a
+privacy-respecting ``crash`` event for in-process counting.
+
+Privacy posture:
+  - ``send_default_pii=False`` — Sentry will not auto-capture user info from
+    request frames or local variables it considers personal.
+  - ``before_send`` strips file paths to package-relative form so site-packages
+    layouts don't reveal the user's home directory.
+  - Loop integrations are explicitly enabled; everything else inherits Sentry's
+    safe defaults.
+
+Opt out: ``PYTHINKER_DISABLE_TELEMETRY=1``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import re
+from typing import Any, cast
+
+import sentry_sdk
+from sentry_sdk.integrations.asyncio import AsyncioIntegration
+from sentry_sdk.integrations.dedupe import DedupeIntegration
+from sentry_sdk.integrations.excepthook import ExcepthookIntegration
+from sentry_sdk.types import Event, Hint
+
+from pythinker_code.telemetry.config import is_disabled, sentry_dsn
+
+_initialized: bool = False
+
+# File-path scrubbing: collapse anything before "site-packages/" or
+# "pythinker_code/" to "<env>" so Sentry stack frames don't expose home dirs.
+_PATH_SCRUB = re.compile(
+    r"^(.*?)(site-packages|pythinker_code|src/pythinker_code)/",
+)
+
+
+def _scrub_path(path: str) -> str:
+    return _PATH_SCRUB.sub(r"<env>/\2/", path)
+
+
+def _before_send(event: Event, hint: Hint) -> Event | None:
+    """Sentry hook applied to every outgoing event.
+
+    Drops absolute file paths (replaces the prefix above ``site-packages/`` or
+    ``pythinker_code/`` with ``<env>``) and removes a couple of fields known to
+    leak local context.
+    """
+    # Strip server name (usually the user's hostname).
+    event.pop("server_name", None)
+
+    exception_data = event.get("exception")
+    if exception_data is None:
+        return event
+    values = exception_data.get("values")
+    if not isinstance(values, list):
+        return event
+
+    for exception_raw in cast(list[Any], values):
+        if not isinstance(exception_raw, dict):
+            continue
+        exception = cast(dict[str, Any], exception_raw)
+        stacktrace_raw = exception.get("stacktrace")
+        if not isinstance(stacktrace_raw, dict):
+            continue
+        stacktrace = cast(dict[str, Any], stacktrace_raw)
+        frames = stacktrace.get("frames")
+        if not isinstance(frames, list):
+            continue
+        for frame_raw in cast(list[Any], frames):
+            if not isinstance(frame_raw, dict):
+                continue
+            frame = cast(dict[str, Any], frame_raw)
+            abs_path = frame.get("abs_path")
+            if isinstance(abs_path, str):
+                frame["abs_path"] = _scrub_path(abs_path)
+            filename = frame.get("filename")
+            if isinstance(filename, str):
+                frame["filename"] = _scrub_path(filename)
+    return event
+
+
+def init(
+    *,
+    version: str,
+    environment: str | None = None,
+    device_id: str | None = None,
+    extra_tags: dict[str, str] | None = None,
+) -> bool:
+    """Initialize ``sentry_sdk`` once per process. Returns True if active."""
+    global _initialized
+    if _initialized:
+        return True
+    if is_disabled():
+        return False
+
+    dsn = sentry_dsn()
+    if not dsn:
+        return False
+
+    env = environment or os.environ.get("PYTHINKER_ENV") or "production"
+
+    sentry_sdk.init(
+        dsn=dsn,
+        release=f"pythinker-code@{version}" if version else None,
+        environment=env,
+        # Dev-mode defaults: capture every error, no traces (OTel handles
+        # those). Tune later when distributing to users.
+        traces_sample_rate=0.0,
+        profiles_sample_rate=0.0,
+        send_default_pii=False,
+        attach_stacktrace=True,
+        max_breadcrumbs=50,
+        # Only the integrations that catch unhandled errors. Skip stdlib
+        # integrations (logging, atexit) so we don't double-emit alongside
+        # OTel logs.
+        default_integrations=False,
+        integrations=[
+            ExcepthookIntegration(always_run=False),
+            AsyncioIntegration(),
+            DedupeIntegration(),
+        ],
+        before_send=_before_send,
+    )
+
+    if device_id:
+        sentry_sdk.set_user({"id": device_id})
+    if extra_tags:
+        for k, v in extra_tags.items():
+            sentry_sdk.set_tag(k, v)
+
+    _initialized = True
+    return True
+
+
+def capture_exception(exc: BaseException) -> None:
+    """Forward an exception to Sentry/Bugsink. Safe to call when uninitialized
+    (becomes a no-op). Telemetry must never raise into the host program."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        sentry_sdk.capture_exception(exc)
+
+
+def add_breadcrumb(
+    *, category: str, message: str, level: str = "info", data: dict[str, Any] | None = None
+) -> None:
+    """Append a breadcrumb to the Sentry scope (visible in the next captured event)."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        sentry_sdk.add_breadcrumb(category=category, message=message, level=level, data=data or {})
+
+
+def flush(timeout_seconds: float = 2.0) -> None:
+    """Block until pending events are sent or the timeout expires."""
+    if not _initialized:
+        return
+    with contextlib.suppress(Exception):
+        client = sentry_sdk.get_client()
+        if client:
+            client.flush(timeout=timeout_seconds)

pythinker_code/telemetry/sink.py

@@ -0,0 +1,189 @@
+"""EventSink: opt-out check, context enrichment, buffer management, timed flush.
+
+Forwards every accepted event to OpenTelemetry as a structured log record.
+The earlier custom-HTTP transport that posted to ``telemetry-logs.pythinker.com``
+was retired in the SigNoz migration — the OTel exporter inside ``otel.py`` now
+handles batching, retries, and disk-spool semantics on its own.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import locale
+import os
+import platform
+import threading
+from typing import Any, cast
+
+from pythinker_code.utils.logging import logger
+
+
+def _assert_primitive(scope: str, key: str, value: Any) -> None:
+    """Telemetry attribute values must be primitives. Catches accidental
+    nested dicts/lists before they reach the OTel SDK serializer."""
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return
+    raise TypeError(f"telemetry {scope}.{key} must be primitive, got {type(value).__name__}")
+
+
+def _flatten_event(event: dict[str, Any]) -> dict[str, Any]:
+    """Expand ``properties``/``context`` sub-dicts into flat ``property.*`` /
+    ``context.*`` keys for OTel attributes. Top-level fields pass through.
+
+    Raises ``TypeError`` on nested values inside properties/context.
+    """
+    out: dict[str, Any] = {}
+    for key, value in event.items():
+        if key == "properties":
+            properties = cast(dict[str, Any], value) if isinstance(value, dict) else {}
+            for pk, pv in properties.items():
+                _assert_primitive("property", pk, pv)
+                out[f"property.{pk}"] = pv
+        elif key == "context":
+            context = cast(dict[str, Any], value) if isinstance(value, dict) else {}
+            for ck, cv in context.items():
+                _assert_primitive("context", ck, cv)
+                out[f"context.{ck}"] = cv
+        else:
+            out[key] = value
+    return out
+
+
+class EventSink:
+    """Buffers telemetry events and flushes them in batches to OTel logs."""
+
+    FLUSH_INTERVAL_S = 30.0
+    FLUSH_THRESHOLD = 50
+
+    def __init__(
+        self,
+        *,
+        version: str = "",
+        model: str = "",
+        ui_mode: str = "shell",
+    ) -> None:
+        self._buffer: list[dict[str, Any]] = []
+        self._lock = threading.Lock()
+        self._flush_task: asyncio.Task[None] | None = None
+        # Static context enrichment
+        self._context: dict[str, Any] = {
+            "version": version,
+            "runtime": "python",
+            "platform": platform.system().lower(),
+            "arch": platform.machine(),
+            "python_version": platform.python_version(),
+            "os_version": platform.release(),
+            "ci": bool(os.environ.get("CI")),
+            "locale": locale.getlocale()[0] or "",
+            "terminal": os.environ.get("TERM_PROGRAM", ""),
+        }
+        self._model = model
+        self._ui_mode = ui_mode
+
+    def accept(self, event: dict[str, Any]) -> None:
+        """Accept an event into the buffer. Non-blocking, thread-safe."""
+        # Enrich with static context (copy to avoid mutating the caller's dict)
+        ctx = {**self._context, "ui_mode": self._ui_mode}
+        if self._model:
+            ctx["model"] = self._model
+        enriched = {**event, "context": ctx}
+
+        with self._lock:
+            self._buffer.append(enriched)
+            should_flush = len(self._buffer) >= self.FLUSH_THRESHOLD
+
+        if should_flush:
+            self._schedule_async_flush()
+
+    def start_periodic_flush(self, loop: asyncio.AbstractEventLoop | None = None) -> None:
+        """Start a background task that flushes every FLUSH_INTERVAL_S seconds."""
+        if self._flush_task is not None:
+            return
+
+        async def _periodic() -> None:
+            try:
+                while True:
+                    await asyncio.sleep(self.FLUSH_INTERVAL_S)
+                    await self._flush_async()
+            except asyncio.CancelledError:
+                pass
+
+        if loop is None:
+            loop = asyncio.get_running_loop()
+        self._flush_task = loop.create_task(_periodic())
+
+    async def retry_disk_events(self) -> None:
+        """Compatibility shim — disk retries now happen inside the OTel
+        exporter. No-op kept so existing callers don't break."""
+
+    def clear_buffer(self) -> None:
+        """Discard all buffered events without sending them."""
+        with self._lock:
+            self._buffer.clear()
+
+    def stop_periodic_flush(self) -> None:
+        """Cancel the periodic flush task."""
+        if self._flush_task is not None:
+            self._flush_task.cancel()
+            self._flush_task = None
+
+    async def flush(self) -> None:
+        """Async flush: send all buffered events."""
+        await self._flush_async()
+
+    def flush_sync(self) -> None:
+        """Synchronous flush for atexit / signal handlers.
+
+        Drops the in-process buffer into the OTel pipeline. Network I/O is
+        scheduled by the BatchLogRecordProcessor; the OTel SDK's own shutdown
+        (called from ``otel.shutdown``) waits for that batch to drain.
+        """
+        with self._lock:
+            if not self._buffer:
+                return
+            events = list(self._buffer)
+            self._buffer.clear()
+        self._emit_to_otel(events)
+
+    async def _flush_async(self) -> None:
+        """Take all buffered events and forward them to OTel logs."""
+        with self._lock:
+            if not self._buffer:
+                return
+            events = list(self._buffer)
+            self._buffer.clear()
+        self._emit_to_otel(events)
+
+    def _emit_to_otel(self, events: list[dict[str, Any]]) -> None:
+        if not events:
+            return
+        try:
+            from pythinker_code.telemetry import otel as _otel
+
+            for event in events:
+                ts = event.get("timestamp")
+                ts_ns = int(ts * 1_000_000_000) if isinstance(ts, (int, float)) else None
+                try:
+                    attrs = _flatten_event(event)
+                except TypeError as exc:
+                    # Schema violation — drop, never retry.
+                    logger.debug("Telemetry event dropped (non-primitive attr): {err}", err=exc)
+                    continue
+                attrs.pop("event", None)
+                attrs.pop("timestamp", None)
+                _otel.emit_log(
+                    name=str(event.get("event") or "event"),
+                    attributes=attrs,
+                    timestamp_ns=ts_ns,
+                )
+        except Exception:
+            logger.debug("OTel flush failed; events dropped")
+
+    def _schedule_async_flush(self) -> None:
+        """Schedule an async flush from any thread."""
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(self._flush_async())
+        except RuntimeError:
+            # No running event loop — will be flushed by periodic task or on exit
+            pass

pythinker_code/tools/AGENTS.md

@@ -0,0 +1,6 @@
+# Pythinker CLI Tools
+
+## Guidelines
+
+- Tools should not refer to types in `pythinker_code/wire/` unless they are explicitly implementing a UI / runtime bridge. When importing things like `ToolReturnValue` or `DisplayBlock`, prefer `pythinker_core.tooling`.
+- Current bridge exceptions include `ask_user`, `plan`, and `file` media/display helpers; keep new tool logic in `pythinker_core.tooling` unless it must emit wire-facing requests or display blocks.