slowpoke-python 0.1.3__py3-none-any.whl

slowpoke/__init__.py

@@ -0,0 +1,174 @@
+"""Tells Slowpoke which line of your Python app ran each query.
+
+One trace per HTTP request (Django, Flask, FastAPI/Starlette) or job, with the route template, the status
+and every query with its file:line, sent as OTLP/JSON to the Slowpoke agent on the same machine.
+"""
+
+import functools
+import inspect
+import os
+import threading
+
+from ._config import Config
+from ._origin import OriginFinder
+from ._sender import BackgroundSender, HttpSender
+from ._tracer import VERSION, Tracer, current
+
+__version__ = VERSION
+__all__ = ["configure", "get_tracer", "set_defaults", "reset", "flush", "job", "command", "current"]
+
+_lock = threading.Lock()
+_state = {"built": False, "tracer": None, "explicit": {}, "defaults": {}}
+
+
+def configure(**overrides):
+    """Builds the tracer now from SLOWPOKE_* variables, with these keyword arguments on top: any Config
+    field (enabled, endpoint, timeout, service, max_queries, max_sql_length, backtrace_limit, code_root,
+    queue_size), plus `sender` (an object with send(bytes) -> bool) and `background` (False sends inline,
+    for tests). Returns None when Slowpoke is disabled."""
+    with _lock:
+        _state["explicit"] = dict(overrides)
+        return _build()
+
+
+def get_tracer():
+    """The tracer the integrations use, built from the environment on first use; None when disabled."""
+    if _state["built"]:
+        return _state["tracer"]
+    with _lock:
+        return _state["tracer"] if _state["built"] else _build()
+
+
+def set_defaults(**defaults):
+    """Framework defaults (code_root, service) that apply only where the environment is silent."""
+    with _lock:
+        merged = dict(_state["defaults"], **{k: v for k, v in defaults.items() if v})
+        if merged != _state["defaults"]:
+            _state["defaults"] = merged
+            _state["built"] = False
+
+
+def reset():
+    with _lock:
+        _state.update(built=False, tracer=None, explicit={}, defaults={})
+
+
+def flush(timeout=1.0):
+    """Waits up to `timeout` seconds for queued traces to reach the agent. Never needed in a web app."""
+    tracer = get_tracer()
+    background = getattr(tracer, "background", None)
+    return background.drain(timeout) if background is not None else True
+
+
+def _build():
+    _state["built"] = True
+    _state["tracer"] = None
+    try:
+        explicit = _state["explicit"]
+        defaults = _state["defaults"]
+        cfg = Config.from_env()
+        for key, value in explicit.items():
+            if hasattr(cfg, key):
+                setattr(cfg, key, value)
+        if not cfg.enabled:
+            return None
+        sender = explicit.get("sender") or HttpSender.from_url(cfg.endpoint, cfg.timeout)
+        if sender is None:
+            return None  # not a local or private endpoint: nothing may be sent, so nothing is recorded
+        code_root = os.path.abspath(str(cfg.code_root or defaults.get("code_root") or os.getcwd()))
+        service = cfg.service or defaults.get("service") or os.path.basename(code_root.rstrip(os.sep)) or "app"
+        tracer = Tracer(
+            origin=OriginFinder(code_root, cfg.backtrace_limit),
+            submit=None,
+            service=service,
+            max_queries=cfg.max_queries,
+            max_sql_length=cfg.max_sql_length,
+        )
+        if explicit.get("background", True):
+            tracer.background = BackgroundSender(sender, encode=tracer.encode_json, maxsize=cfg.queue_size)
+            tracer.submit = tracer.background.submit
+        else:
+            tracer.submit = lambda trace: sender.send(tracer.encode_json(trace))
+        _state["tracer"] = tracer
+        return tracer
+    except Exception:
+        return None  # a broken configuration disables Slowpoke, it never stops the app from booting
+
+
+class job:
+    """Traces work outside HTTP requests: a script, a cron command, a custom worker loop.
+
+        with slowpoke.job("import_orders", queue="nightly"):
+            ...
+
+        @slowpoke.job()
+        def send_invoices(): ...
+
+    Inside a request or another job it does nothing: the queries already belong to that trace."""
+
+    def __init__(self, name=None, queue=None):
+        self.name = name
+        self.queue = queue
+        self._tracer = None
+        self._trace = None
+
+    def _start(self, tracer):
+        return tracer.start_job(self.name or "job", self.queue)
+
+    def __enter__(self):
+        try:
+            self._tracer = get_tracer()
+            if self._tracer is not None:
+                self._trace = self._start(self._tracer)
+        except Exception:
+            self._trace = None
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        if self._trace is not None:
+            self._tracer.finish_job(self._trace, failed=exc_type is not None)
+            self._trace = None
+        return False
+
+    async def __aenter__(self):
+        return self.__enter__()
+
+    async def __aexit__(self, exc_type, exc, tb):
+        return self.__exit__(exc_type, exc, tb)
+
+    def __call__(self, fn):
+        name = self.name or "%s.%s" % (fn.__module__, fn.__qualname__)
+        queue = self.queue
+        again = type(self)
+
+        if inspect.iscoroutinefunction(fn):
+            @functools.wraps(fn)
+            async def async_wrapper(*args, **kwargs):
+                with again(name, queue):
+                    return await fn(*args, **kwargs)
+            return async_wrapper
+
+        @functools.wraps(fn)
+        def wrapper(*args, **kwargs):
+            with again(name, queue):
+                return fn(*args, **kwargs)
+        return wrapper
+
+
+class command(job):
+    """Traces a command run by cron, or any script nobody is waiting for.
+
+        with slowpoke.command("import_orders"):
+            ...
+
+        @slowpoke.command()
+        def import_orders(): ...
+
+    Same trace as a job, filed under its own kind: on the Jobs page a nightly command and a queued
+    job are different things. Inside a request or another job it does nothing."""
+
+    def __init__(self, name=None, queue=None):
+        super().__init__(name, None)  # a command has no queue: nothing dispatched it
+
+    def _start(self, tracer):
+        return tracer.start_command(self.name or "command")

slowpoke/_config.py

@@ -0,0 +1,45 @@
+import os
+
+DEFAULT_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
+
+
+def _number(raw, default, cast, minimum):
+    try:
+        value = cast(raw)
+    except (TypeError, ValueError):
+        return default
+    return value if value >= minimum else default
+
+
+class Config:
+    """Settings from SLOWPOKE_* environment variables, like the Laravel package. Nonsense values fall
+    back to the defaults: a typo in an env file must never stop the app from booting."""
+
+    def __init__(self, enabled=True, endpoint=DEFAULT_ENDPOINT, timeout=0.1, service=None, max_queries=500,
+                 max_sql_length=10000, backtrace_limit=100, code_root=None, queue_size=256):
+        self.enabled = enabled
+        self.endpoint = endpoint
+        self.timeout = timeout
+        self.service = service
+        self.max_queries = max_queries
+        self.max_sql_length = max_sql_length
+        self.backtrace_limit = backtrace_limit
+        self.code_root = code_root
+        self.queue_size = queue_size
+
+    @classmethod
+    def from_env(cls, environ=None):
+        env = os.environ if environ is None else environ
+        d = cls()
+        enabled = env.get("SLOWPOKE_ENABLED", "").strip().lower()
+        return cls(
+            enabled=enabled not in ("0", "false", "no", "off"),
+            endpoint=env.get("SLOWPOKE_OTLP_ENDPOINT", "").strip() or d.endpoint,
+            timeout=_number(env.get("SLOWPOKE_TIMEOUT"), d.timeout, float, 0.001),
+            service=env.get("SLOWPOKE_SERVICE", "").strip() or None,
+            max_queries=_number(env.get("SLOWPOKE_MAX_QUERIES"), d.max_queries, int, 0),
+            max_sql_length=_number(env.get("SLOWPOKE_MAX_SQL_LENGTH"), d.max_sql_length, int, 1),
+            backtrace_limit=_number(env.get("SLOWPOKE_BACKTRACE_LIMIT"), d.backtrace_limit, int, 1),
+            code_root=env.get("SLOWPOKE_CODE_ROOT", "").strip() or None,
+            queue_size=_number(env.get("SLOWPOKE_QUEUE_SIZE"), d.queue_size, int, 1),
+        )

slowpoke/_origin.py

@@ -0,0 +1,136 @@
+import os
+import sys
+import sysconfig
+import time
+
+_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)) + os.sep
+_THIRD_PARTY_PARTS = ("/site-packages/", "/dist-packages/", "/node_modules/", "/vendor/")
+
+
+def _stdlib_dirs():
+    dirs = set()
+    for key in ("stdlib", "platstdlib"):
+        try:
+            path = sysconfig.get_paths().get(key)
+        except Exception:
+            path = None
+        if path:
+            dirs.add(os.path.join(os.path.realpath(path), ""))
+            dirs.add(os.path.join(os.path.abspath(path), ""))
+    return tuple(dirs)
+
+
+class OriginFinder:
+    """Finds the application line that issued a query. OpenTelemetry's instrumentations point into
+    Django or SQLAlchemy, which tells nobody what to fix: the answer is the first frame of the app's own
+    code, outside site-packages, the standard library and this package."""
+
+    def __init__(self, code_root, limit=100, cache_size=2048):
+        self.root = os.path.join(os.path.abspath(code_root), "")
+        self.limit = max(1, int(limit))
+        self._skip = _stdlib_dirs() + (_PACKAGE_DIR,)
+        self._cache = {}
+        self._cache_size = cache_size
+
+    def find(self, task=None):
+        """(relative file, line) of the innermost application frame, or None. `task` is the asyncio task
+        that opened the trace, for statements run on a worker thread on its behalf."""
+        frame = sys._getframe(1)
+        origin, budget = self._walk(frame, self.limit)
+        if origin is None and budget > 0:
+            # SQLAlchemy's asyncio layer runs the statement in a greenlet: the coroutine that awaited
+            # it is on the stack of the parent greenlet, suspended where it switched.
+            parent = _greenlet_parent_frame()
+            if parent is not None:
+                origin, budget = self._walk(parent, budget)
+        if origin is None and budget > 0 and task is not None:
+            # Django's async ORM and sync_to_async run the statement in a thread: the view is a
+            # coroutine suspended on "await", reachable from the task that opened the trace.
+            origin = self._from_task(task, budget)
+        return origin
+
+    def _from_task(self, task, budget):
+        try:
+            frames = []
+            awaitable = task.get_coro()
+            if getattr(awaitable, "cr_running", False):
+                # Running, not suspended: its await chain is empty and would name the outermost
+                # coroutine, a wrong answer. On the loop's own thread it cannot be suspended while we
+                # run; on a worker thread it is a few instructions away from its "await": give it at
+                # most a couple of milliseconds, then no origin rather than a misleading one.
+                asyncio = sys.modules.get("asyncio")
+                if asyncio is None or asyncio._get_running_loop() is not None:
+                    return None
+                deadline = time.monotonic() + 0.002
+                while awaitable.cr_running:
+                    if time.monotonic() > deadline:
+                        return None
+                    time.sleep(0)
+            while awaitable is not None and len(frames) < budget:
+                frame = None
+                for attr in ("cr_frame", "ag_frame", "gi_frame"):
+                    frame = getattr(awaitable, attr, None)
+                    if frame is not None:
+                        break
+                if frame is None:
+                    break
+                frames.append(frame)
+                awaitable = (getattr(awaitable, "cr_await", None) or getattr(awaitable, "ag_await", None)
+                             or getattr(awaitable, "gi_yieldfrom", None))
+            for frame in reversed(frames):  # innermost first
+                relative = self.classify(frame.f_code.co_filename)
+                if relative is not None:
+                    return relative, frame.f_lineno
+        except Exception:
+            pass
+        return None
+
+    def from_frame(self, frame):
+        return self._walk(frame, self.limit)[0]
+
+    def _walk(self, frame, budget):
+        while frame is not None and budget > 0:
+            budget -= 1
+            code = frame.f_code
+            relative = self.classify(code.co_filename)
+            if relative is not None:
+                return (relative, frame.f_lineno), budget
+            frame = frame.f_back
+        return None, budget
+
+    def classify(self, filename):
+        """The path to report for a file, or None for code that is not the application's. Cached per
+        file name: a request runs hundreds of queries through the same few dozen files."""
+        try:
+            return self._cache[filename]
+        except KeyError:
+            pass
+        result = self._classify(filename)
+        if len(self._cache) >= self._cache_size:
+            self._cache.clear()
+        self._cache[filename] = result
+        return result
+
+    def _classify(self, filename):
+        if not filename or filename.startswith("<"):
+            return None  # <frozen ...>, <string>, <stdin>
+        path = filename.replace("\\", "/")
+        if any(part in path for part in _THIRD_PARTY_PARTS) or path.startswith(("vendor/", "node_modules/")):
+            return None
+        absolute = os.path.abspath(filename)
+        if absolute.startswith(self._skip):
+            return None
+        if absolute.startswith(self.root):
+            return absolute[len(self.root):].replace("\\", "/")
+        return path
+
+
+def _greenlet_parent_frame():
+    greenlet = sys.modules.get("greenlet")
+    if greenlet is None:
+        return None
+    try:
+        parent = greenlet.getcurrent().parent
+        return parent.gr_frame if parent is not None else None
+    except Exception:
+        return None

slowpoke/_sender.py

@@ -0,0 +1,145 @@
+import atexit
+import http.client
+import ipaddress
+import os
+import queue
+import threading
+import time
+import weakref
+from urllib.parse import urlsplit
+
+_PRIVATE_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
+
+
+def _private_host(host):
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        host = host.lower()
+        # single-label names are Docker services or /etc/hosts entries
+        return host == "localhost" or "." not in host or host.endswith(_PRIVATE_SUFFIXES)
+    return ip.is_private or ip.is_loopback or ip.is_link_local
+
+
+class HttpSender:
+    """Posts OTLP/JSON to the Slowpoke agent on this machine or the private network, with a hard time
+    budget shared by connect, write and read. Only the background worker calls it."""
+
+    def __init__(self, host, port, path, timeout):
+        self.host = host
+        self.port = port
+        self.path = path
+        self.timeout = timeout
+
+    @classmethod
+    def from_url(cls, url, timeout):
+        """None unless the URL is plain http to a local or private host: queries must not travel the internet."""
+        try:
+            parts = urlsplit(url or "")
+            if parts.scheme != "http" or not parts.hostname or not _private_host(parts.hostname):
+                return None
+            port = parts.port or 80
+        except ValueError:
+            return None
+        path = parts.path if parts.path not in ("", "/") else "/v1/traces"
+        return cls(parts.hostname, port, path, max(0.001, float(timeout)))
+
+    def send(self, body):
+        deadline = time.monotonic() + self.timeout
+        conn = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
+        try:
+            conn.connect()
+            conn.sock.settimeout(max(0.001, deadline - time.monotonic()))
+            conn.request("POST", self.path, body, {
+                "Content-Type": "application/json",
+                "Connection": "close",
+                "User-Agent": "slowpoke-python",
+            })
+            left = deadline - time.monotonic()
+            if left <= 0:
+                return False
+            conn.sock.settimeout(left)
+            response = conn.getresponse()
+            return 200 <= response.status < 300
+        except Exception:
+            return False
+        finally:
+            try:
+                conn.close()
+            except Exception:
+                pass
+
+
+_instances = weakref.WeakSet()
+
+
+class BackgroundSender:
+    """A bounded queue and one daemon thread: the request only pays for put_nowait. A full queue means
+    the agent is slow or gone, and the trace is dropped instead of piling up in the app's memory."""
+
+    def __init__(self, sender, encode, maxsize=256):
+        self.sender = sender
+        self.encode = encode
+        self.maxsize = maxsize
+        self._lock = threading.Lock()
+        self._after_fork()
+        _instances.add(self)
+
+    def _after_fork(self):
+        # A forked worker (gunicorn --preload, Celery prefork) inherits the object but not the thread.
+        self._queue = queue.Queue(self.maxsize)
+        self._thread = None
+        self._pid = os.getpid()
+
+    def submit(self, item):
+        try:
+            if self._thread is None or self._pid != os.getpid():
+                self._start()
+            self._queue.put_nowait(item)
+            return True
+        except Exception:
+            return False
+
+    def _start(self):
+        with self._lock:
+            if self._pid != os.getpid():
+                self._after_fork()
+            if self._thread is None:
+                thread = threading.Thread(target=self._run, args=(self._queue,), name="slowpoke-sender", daemon=True)
+                thread.start()
+                self._thread = thread
+
+    def _run(self, q):
+        while True:
+            item = q.get()
+            try:
+                self.sender.send(self.encode(item))
+            except Exception:
+                pass  # the agent is optional: a broken one costs a trace, nothing else
+            finally:
+                q.task_done()
+
+    def drain(self, timeout):
+        """Waits until every queued trace was handed to the agent (or given up). For tests and exit."""
+        deadline = time.monotonic() + timeout
+        q = self._queue
+        with q.all_tasks_done:
+            while q.unfinished_tasks:
+                left = deadline - time.monotonic()
+                if left <= 0:
+                    return False
+                q.all_tasks_done.wait(left)
+        return True
+
+
+@atexit.register
+def _drain_at_exit():
+    # Scripts and short jobs end right after their last trace: give it a moment, never more.
+    deadline = time.monotonic() + 0.5
+    for instance in list(_instances):
+        if instance._thread is not None and instance._pid == os.getpid():
+            instance.drain(max(0.0, deadline - time.monotonic()))
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=lambda: [i._after_fork() for i in list(_instances)])