visionary-sdk 0.4.0__py3-none-any.whl

visionary/__init__.py

@@ -0,0 +1,29 @@
+"""Visionary collector SDK.
+
+Logs one row per HTTP request into ``api_logs``, the database and outbound
+HTTP calls each request makes into ``spans``, unhandled and reported
+exceptions into ``errors``, and samples host, process and runtime metrics
+into ``system_metrics``, from background threads.
+
+    import visionary
+    agent = visionary.init()            # from VISIONARY_* / D1_* env vars
+
+Then add the integration for your framework (``visionary.integrations.django``,
+``.asgi`` or ``.flask``) and call ``visionary.set_user(id)`` once the user is
+known. See README.md and SPEC.md.
+"""
+
+from ._agent import Agent, get_agent, init
+from ._api import capture_exception, instrument_sqlalchemy, span
+from ._config import Config, from_env
+from ._context import set_route, set_user
+from ._db import instrument_connection, instrument_cursor
+from ._http import instrument_http
+
+__version__ = "0.4.0"
+
+__all__ = [
+    "Agent", "Config", "capture_exception", "from_env", "get_agent", "init", "instrument_connection",
+    "instrument_cursor", "instrument_http", "instrument_sqlalchemy", "set_route", "set_user", "span",
+    "__version__",
+]

visionary/_agent.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+import socket
+import threading
+import time
+import weakref
+from typing import Callable, Optional
+
+from . import _errors, _spans
+from ._config import Config, from_env
+from ._context import RequestInfo
+from ._entry import Entry
+from ._storage import Backend, open_backend
+from ._system import HttpCounters, Recorder
+from ._writer import Release, RequestWriter, Retention
+
+logger = logging.getLogger("visionary")
+
+
+class Agent:
+    """Runs request logging, telemetry (spans, errors, releases) and system
+    metrics for one process.
+
+    With no database configured the agent is a no-op: it logs one warning
+    and every method does nothing. It never raises into the host app once
+    constructed.
+    """
+
+    def __init__(self, config: Optional[Config] = None, *, backend: Optional[Backend] = None) -> None:
+        cfg = config if config is not None else from_env()
+        self.config = cfg
+        self.http = HttpCounters()
+        self._backend: Optional[Backend] = None
+        self._requests: Optional[RequestWriter] = None
+        self._system: Optional[Recorder] = None
+        self._retention: Optional[Retention] = None
+        self._error_limit = _errors.RateLimiter()
+        self.spans_on = False
+        self.errors_on = False
+        self._started = False
+        self._closed = False
+        self._lock = threading.Lock()
+
+        try:
+            self._backend = backend if backend is not None else open_backend(cfg)
+        except Exception as e:
+            logger.warning("%s; collection is off", e)
+            return
+        if self._backend is None:
+            logger.warning("visionary: no database configured (set VISIONARY_DATABASE_URL or D1_*); collection is off")
+            return
+
+        instance = cfg.instance or socket.gethostname()
+        self.instance = instance
+        if cfg.log_requests or cfg.spans or cfg.errors or cfg.version:
+            # One writer for api_logs and the telemetry tables.
+            self._requests = RequestWriter(cfg, self._backend, logger, instance, _errors.runtime())
+            self.spans_on = cfg.spans
+            self.errors_on = cfg.errors
+        if cfg.system_metrics and cfg.system_interval > 0:
+            self._system = Recorder(cfg, instance, self._backend, self.http, logger)
+        retention = {}
+        if self._requests is not None and cfg.log_requests:
+            retention["requests"] = cfg.log_retention_days
+        if self.spans_on:
+            retention["spans"] = cfg.log_retention_days
+        if self.errors_on:
+            retention["errors"] = cfg.log_retention_days
+        if self._system is not None:
+            retention["system"] = cfg.system_retention_days
+        self._retention = Retention(self._backend, logger, retention)
+
+    @property
+    def enabled(self) -> bool:
+        """True when anything is collected (a database and at least one feature)."""
+        return self._requests is not None or self._system is not None
+
+    @property
+    def logging_requests(self) -> bool:
+        return self._requests is not None and self.config.log_requests
+
+    def should_log(self, path: str) -> bool:
+        """Whether a request to path gets an api_logs row (integration API)."""
+        return self.logging_requests and path.startswith(self.config.path_prefix)
+
+    def record(self, entry: Entry) -> None:
+        """Queue a captured request (integration API). Never blocks."""
+        if self.logging_requests and not self._closed:
+            self._requests.log(entry)
+
+    def new_trace(self, request_id: str, t0: float) -> Optional[_spans.Trace]:
+        """The span collector for a logged request (integration API); t0 is
+        its time.perf_counter() start."""
+        if not self.spans_on:
+            return None
+        return _spans.Trace(request_id, t0, self.config.span_max)
+
+    def finish_trace(self, entry: Entry, trace: Optional[_spans.Trace]) -> None:
+        """Copy call totals and, for slow or failed requests, the spans onto
+        the entry (integration API, after status and duration are set)."""
+        if trace is None:
+            return
+        slow = entry.duration * 1000.0 >= self.config.span_threshold_ms or entry.status >= 500
+        entry.spans = trace.close(slow)
+        if _spans.db_visible:
+            entry.db_count, entry.db_ms = trace.db_count, trace.db_ms
+        if _spans.http_visible:
+            entry.http_count, entry.http_ms = trace.http_count, trace.http_ms
+
+    def record_span(self, span: _spans.Span) -> None:
+        """Queue a span recorded outside a request. Never blocks."""
+        if self.spans_on and not self._closed:
+            self._requests.log(span)
+
+    def capture_error(self, exc: BaseException, handled: bool, info: Optional[RequestInfo] = None) -> None:
+        """Queue an errors row for exc, rate limited per fingerprint. Never
+        raises."""
+        if not self.errors_on or self._closed or exc is None:
+            return
+        try:
+            ev = _errors.build(exc, handled, info)
+            if self._error_limit.allow(ev.fingerprint):
+                self._requests.log(ev)
+        except Exception:
+            logger.exception("visionary: capturing an error failed")
+
+    def default_db_stats(self, fn: Callable[[], Optional[dict]]) -> None:
+        """Fill the db_* columns from fn unless the app set config.db_stats
+        (integration API). Works before or after start()."""
+        if self._system is not None and self.config.db_stats is None:
+            self._system.set_default_db_stats(fn)
+
+    def start(self) -> "Agent":
+        """Launch the background threads. Tables are created on them, so a slow
+        or unreachable database never delays startup. Safe to call twice."""
+        with self._lock:
+            if self._started or self._closed or not self.enabled:
+                return self
+            self._started = True
+        if self._requests is not None:
+            self._requests.start()
+            if self.config.version:
+                self._requests.log(Release(time.time(), _sdk_version(), _errors.runtime()))
+            if self.logging_requests:
+                logger.info("visionary: logging requests (path_prefix=%r, bodies=%s, retention_days=%d)",
+                            self.config.path_prefix, self.config.capture_bodies, self.config.log_retention_days)
+            if self.spans_on or self.errors_on:
+                _spans.set_active(self)
+            if self.spans_on and self.config.instrument_http:
+                from ._http import instrument_http
+
+                instrument_http()
+        if self._system is not None:
+            self._system.start()
+            logger.info("visionary: recording system metrics every %ss", self.config.system_interval)
+        if self._retention is not None and self._retention.needed:
+            self._retention.start()
+        _track_for_fork(self)
+        return self
+
+    def close(self, timeout: float = 10) -> None:
+        """Flush queued requests and buffered samples, waiting at most timeout
+        seconds in total."""
+        with self._lock:
+            if self._closed:
+                return
+            self._closed = True
+            started = self._started
+        _spans.clear_active(self)
+        idle = True
+        if started:
+            deadline = _Deadline(timeout)
+            if self._retention is not None:
+                self._retention.close()
+            if self._requests is not None:
+                idle = self._requests.close(deadline.left()) and idle
+            if self._system is not None:
+                idle = self._system.close(deadline.left()) and idle
+        # A worker still busy (e.g. stuck connecting) keeps its connection;
+        # it is a daemon thread and goes away with the process.
+        if idle and self._backend is not None:
+            try:
+                self._backend.close()
+            except Exception:
+                pass
+
+    def _after_fork(self) -> None:
+        # Threads don't survive fork (e.g. gunicorn --preload): restart them
+        # in the child with fresh queues and connections.
+        if not self._started or self._closed:
+            return
+        self._lock = threading.Lock()
+        self.http = HttpCounters()
+        if self._backend is not None:
+            self._backend.after_fork()
+        if self._system is not None:
+            self._system._http = self.http
+            self._system.restart_after_fork()
+        if self._requests is not None:
+            self._requests.restart_after_fork()
+        if self._retention is not None and self._retention.needed:
+            self._retention.restart_after_fork()
+
+    def __enter__(self) -> "Agent":
+        return self.start()
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+def _sdk_version() -> str:
+    from . import __version__
+
+    return __version__
+
+
+class _Deadline:
+    def __init__(self, seconds: float) -> None:
+        self._end = time.monotonic() + max(0.0, seconds)
+
+    def left(self) -> float:
+        return max(0.0, self._end - time.monotonic())
+
+
+_fork_agents: "weakref.WeakSet[Agent]" = weakref.WeakSet()
+_fork_hooked = False
+
+
+def _track_for_fork(agent: Agent) -> None:
+    global _fork_hooked
+    _fork_agents.add(agent)
+    if not _fork_hooked and hasattr(os, "register_at_fork"):
+        os.register_at_fork(after_in_child=lambda: [a._after_fork() for a in list(_fork_agents)])
+        _fork_hooked = True
+
+
+_default: Optional[Agent] = None
+_default_lock = threading.Lock()
+
+
+def init(config: Optional[Config] = None) -> Agent:
+    """Build the process-wide agent (from the environment unless config is
+    given), start it and flush it at exit. Returns the existing agent when
+    called again. Never raises: on any problem it returns a no-op agent."""
+    global _default
+    with _default_lock:
+        if _default is not None:
+            return _default
+        try:
+            agent = Agent(config)
+            agent.start()
+        except Exception:
+            logger.exception("visionary: init failed; collection is off")
+            agent = Agent(Config(log_requests=False, system_metrics=False, spans=False, errors=False),
+                          backend=_NoBackend())
+        atexit.register(agent.close)
+        _default = agent
+        return agent
+
+
+def get_agent() -> Optional[Agent]:
+    """The agent set up by init(), if any."""
+    return _default
+
+
+class _NoBackend(Backend):
+    """Placeholder for the disabled agent built after a failed init."""

visionary/_api.py

@@ -0,0 +1,70 @@
+"""Public telemetry API: manual spans and handled errors."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any, Optional
+
+from . import _agent, _spans
+
+
+class span:
+    """Time a block as a span (sync or async)::
+
+        with visionary.span("cache", "GET session") as s:
+            value = redis.get(key)
+            s.set_row_count(1)
+
+    ``kind`` is ``db``, ``http``, ``cache``, ``queue`` or ``custom`` (anything
+    else becomes ``custom``). An exception leaving the block marks the span as
+    failed and propagates unchanged. Does nothing when spans are off.
+    """
+
+    __slots__ = ("_args", "_span")
+
+    def __init__(self, kind: str, name: str, target: Optional[str] = None, detail: Optional[str] = None) -> None:
+        self._args = (kind, name, target, detail)
+        self._span: Optional[_spans.Span] = None
+
+    def set_error(self, error: Any) -> None:
+        """Mark the span failed without raising (e.g. a cache miss you treat as an error)."""
+        if self._span is not None:
+            self._span.set_error(error)
+
+    def set_row_count(self, n: int) -> None:
+        if self._span is not None and n is not None and n >= 0:
+            self._span.row_count = int(n)
+
+    def __enter__(self) -> "span":
+        self._span = _spans.start(*self._args)
+        return self
+
+    def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        if self._span is not None:
+            self._span.finish(error=exc)
+
+    async def __aenter__(self) -> "span":
+        return self.__enter__()
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        self.__exit__(exc_type, exc, tb)
+
+
+def capture_exception(exc: Optional[BaseException] = None) -> None:
+    """Record a handled exception (``handled = 1``): ``exc``, or the one being
+    handled when called inside ``except``. Inside a request it is linked to
+    that request. Never raises."""
+    if exc is None:
+        exc = sys.exc_info()[1]
+    if exc is None:
+        return
+    agent = _spans.active() or _agent.get_agent()
+    if agent is not None:
+        agent.capture_error(exc, True)
+
+
+def instrument_sqlalchemy(engine: Any) -> Any:
+    """Record the statements of an SQLAlchemy ``Engine``/``AsyncEngine`` as spans."""
+    from ._db import instrument_sqlalchemy as instrument
+
+    return instrument(engine)

visionary/_config.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+LookupUser = Callable[[str], Optional[Tuple[Optional[str], Optional[str]]]]
+DBStatsFunc = Callable[[], Dict[str, Any]]
+
+
+@dataclass
+class Config:
+    """Everything the agent needs. ``from_env()`` fills it from ``VISIONARY_*``.
+
+    Storage is chosen in this order: ``connection_factory`` + ``dialect``,
+    then ``database_url``, then the ``d1_*`` credentials.
+    """
+
+    database_url: str = ""
+    d1_account_id: str = ""
+    d1_database_id: str = ""
+    d1_api_token: str = ""
+    d1_api_base: str = ""
+    # A zero-argument callable returning a DB-API connection the SDK may keep
+    # for itself (it reconnects through it after errors and closes it at
+    # shutdown). ``dialect`` is "sqlite", "postgres" or "mysql".
+    connection_factory: Optional[Callable[[], Any]] = None
+    dialect: str = ""
+
+    instance: str = ""
+
+    log_requests: bool = True
+    path_prefix: str = ""
+    capture_bodies: bool = True
+    max_body_bytes: int = 8192
+    queue_size: int = 5000
+    batch_size: int = 50
+    flush_interval: float = 3.0
+    log_retention_days: int = 30
+    redact_headers: List[str] = field(default_factory=list)
+    redact_fields: List[str] = field(default_factory=list)
+    redact_request_fields: List[str] = field(default_factory=list)
+    error_code_field: str = "error.code"
+    error_key_field: str = "error.key"
+    # Runs on the writer thread, once per distinct user per batch, for users
+    # missing a handle or role. Return (handle, role) or None.
+    lookup_user: Optional[LookupUser] = None
+
+    system_metrics: bool = True
+    system_interval: float = 15.0
+    system_retention_days: int = 30
+    # Returns pool stats: total, idle, in_use, max, acquires, empty_acquires,
+    # acquire_wait_ms (the last three cumulative).
+    db_stats: Optional[DBStatsFunc] = None
+    # Filesystem reported in disk_* columns (default: the working directory).
+    disk_path: str = ""
+
+    # Telemetry (SPEC "Telemetry"). service and version label every row;
+    # version defaults to the platform's commit variable (see from_env).
+    service: str = ""
+    version: str = ""
+    spans: bool = True
+    span_threshold_ms: float = 100.0
+    span_max: int = 100
+    errors: bool = True
+    # Patch requests, httpx and urllib3 at start (same as instrument_http()).
+    instrument_http: bool = False
+
+
+def from_env(environ: Optional[Dict[str, str]] = None) -> Config:
+    """Build a Config from environment variables (see SPEC.md "Environment")."""
+    env = os.environ if environ is None else environ
+
+    def s(key: str, default: str = "") -> str:
+        v = env.get(key, "").strip()
+        return v or default
+
+    return Config(
+        database_url=s("VISIONARY_DATABASE_URL"),
+        d1_account_id=s("D1_ACCOUNT_ID"),
+        d1_database_id=s("D1_DATABASE_ID"),
+        d1_api_token=s("D1_API_TOKEN"),
+        d1_api_base=s("D1_API_BASE"),
+        instance=s("VISIONARY_INSTANCE"),
+        log_requests=parse_bool(s("VISIONARY_LOG_REQUESTS"), True),
+        path_prefix=s("VISIONARY_LOG_PATH_PREFIX"),
+        capture_bodies=parse_bool(s("VISIONARY_LOG_BODIES"), True),
+        max_body_bytes=parse_int(s("VISIONARY_LOG_MAX_BODY_BYTES"), 8192),
+        log_retention_days=parse_int(s("VISIONARY_LOG_RETENTION_DAYS"), 30),
+        redact_headers=parse_list(s("VISIONARY_LOG_REDACT_HEADERS")),
+        redact_fields=parse_list(s("VISIONARY_LOG_REDACT_FIELDS")),
+        redact_request_fields=parse_list(s("VISIONARY_LOG_REDACT_REQUEST_FIELDS")),
+        error_code_field=s("VISIONARY_LOG_ERROR_CODE_FIELD", "error.code"),
+        error_key_field=s("VISIONARY_LOG_ERROR_KEY_FIELD", "error.key"),
+        system_metrics=parse_bool(s("VISIONARY_SYSTEM_METRICS"), True),
+        system_interval=parse_duration(s("VISIONARY_SYSTEM_INTERVAL"), 15.0),
+        system_retention_days=parse_int(s("VISIONARY_SYSTEM_RETENTION_DAYS"), 30),
+        service=s("VISIONARY_SERVICE"),
+        version=release_from_env(env),
+        spans=parse_bool(s("VISIONARY_SPANS"), True),
+        span_threshold_ms=float(parse_int(s("VISIONARY_SPAN_THRESHOLD_MS"), 100)),
+        span_max=parse_int(s("VISIONARY_SPAN_MAX"), 100),
+        errors=parse_bool(s("VISIONARY_ERRORS"), True),
+        instrument_http=parse_bool(s("VISIONARY_INSTRUMENT_HTTP"), False),
+    )
+
+
+# Commit variables set by common platforms, in the order SPEC.md lists them.
+_COMMIT_VARS = (
+    "RAILWAY_GIT_COMMIT_SHA", "RENDER_GIT_COMMIT", "VERCEL_GIT_COMMIT_SHA", "HEROKU_SLUG_COMMIT",
+    "SOURCE_VERSION", "GIT_COMMIT", "GITHUB_SHA", "K_REVISION",
+)
+_COMMIT_HASH = re.compile(r"^[0-9a-fA-F]{13,64}$")
+
+
+def release_from_env(env: Dict[str, str]) -> str:
+    """VISIONARY_RELEASE, else the first platform commit variable set; commit
+    hashes are shortened to 12 characters."""
+    for key in ("VISIONARY_RELEASE",) + _COMMIT_VARS:
+        v = env.get(key, "").strip()
+        if v:
+            return v[:12] if _COMMIT_HASH.match(v) else v
+    return ""
+
+
+_TRUE = {"true", "1", "yes"}
+_FALSE = {"false", "0", "no"}
+
+
+def parse_bool(value: str, default: bool) -> bool:
+    v = value.strip().lower()
+    if v in _TRUE:
+        return True
+    if v in _FALSE:
+        return False
+    return default
+
+
+def parse_int(value: str, default: int) -> int:
+    try:
+        n = int(value.strip())
+    except ValueError:
+        return default
+    return n if n >= 0 else default
+
+
+def parse_list(value: str) -> List[str]:
+    return [p.strip() for p in value.split(",") if p.strip()]
+
+
+_DURATION_PART = re.compile(r"(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)")
+_UNITS = {"ns": 1e-9, "us": 1e-6, "µs": 1e-6, "ms": 1e-3, "s": 1.0, "m": 60.0, "h": 3600.0}
+
+
+def parse_duration(value: str, default: float) -> float:
+    """Seconds from "15", "15s", "500ms", "1m30s" (Go duration syntax)."""
+    v = value.strip()
+    if not v:
+        return default
+    try:
+        return float(int(v))
+    except ValueError:
+        pass
+    pos, total = 0, 0.0
+    for m in _DURATION_PART.finditer(v):
+        if m.start() != pos:
+            return default
+        total += float(m.group(1)) * _UNITS[m.group(2)]
+        pos = m.end()
+    if pos != len(v) or pos == 0:
+        return default
+    return total

visionary/_context.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import threading
+from contextvars import ContextVar, Token
+from typing import Any, Callable, Optional
+
+
+class Annotations:
+    """What app code says about the current request.
+
+    Integrations put one instance in a contextvar at request start. Because
+    copied contexts (thread pools, asyncio tasks) share the same object,
+    set_user works from wherever the request's code runs.
+    """
+
+    __slots__ = ("_lock", "user_id", "user_handle", "user_role", "route", "trace", "request")
+
+    def __init__(self, trace: Any = None, request: Optional["RequestInfo"] = None) -> None:
+        self._lock = threading.Lock()
+        self.user_id = ""
+        self.user_handle = ""
+        self.user_role = ""
+        self.route = ""
+        self.trace = trace  # _spans.Trace, or None when spans are off
+        self.request = request
+
+    def set_user(self, id: Any, handle: Optional[str], role: Optional[str]) -> None:
+        with self._lock:
+            self.user_id = "" if id is None else str(id)
+            self.user_handle = handle or ""
+            self.user_role = role or ""
+
+    def user(self) -> tuple:
+        with self._lock:
+            return self.user_id, self.user_handle, self.user_role
+
+
+class RequestInfo:
+    """What an error row says about the request it happened in. The route is
+    resolved lazily, since routers only match it during the call."""
+
+    __slots__ = ("request_id", "method", "path", "_route", "_user")
+
+    def __init__(self, request_id: Optional[str], method: str, path: str,
+                 route: Optional[Callable[[], str]] = None, user: Optional[Callable[[], str]] = None) -> None:
+        self.request_id = request_id
+        self.method = method
+        self.path = path
+        self._route = route
+        self._user = user
+
+    def route(self, ann: Optional[Annotations] = None) -> str:
+        if ann is not None and ann.route:
+            return ann.route
+        try:
+            return self._route() if self._route is not None else ""
+        except Exception:
+            return ""
+
+    def user_id(self, ann: Optional[Annotations] = None) -> str:
+        if ann is not None and ann.user_id:
+            return ann.user_id
+        try:
+            return self._user() if self._user is not None else ""
+        except Exception:
+            return ""
+
+
+_current: ContextVar[Optional[Annotations]] = ContextVar("visionary_request", default=None)
+
+
+def begin(trace: Any = None, request: Optional[RequestInfo] = None) -> tuple:
+    ann = Annotations(trace, request)
+    return ann, _current.set(ann)
+
+
+def end(token: Token) -> None:
+    try:
+        _current.reset(token)
+    except ValueError:
+        # Reset from a different context than the one that set it (some
+        # frameworks run hooks in copies); clearing is equivalent here.
+        _current.set(None)
+
+
+def current() -> Optional[Annotations]:
+    return _current.get()
+
+
+def set_user(id: Any, handle: Optional[str] = None, role: Optional[str] = None) -> None:
+    """Attach the signed-in user to the request being logged.
+
+    Call it from auth middleware or a view once the user is known. Only
+    ``id`` is required. Outside a request handled by a Visionary integration
+    this does nothing.
+    """
+    ann = _current.get()
+    if ann is not None:
+        ann.set_user(id, handle, role)
+
+
+def set_route(pattern: str) -> None:
+    """Record the route pattern (e.g. ``/users/{id}``) for routers the
+    integration can't see. No-op outside a request."""
+    ann = _current.get()
+    if ann is not None:
+        ann.route = pattern or ""