deployangel 0.1.0__py3-none-any.whl

deployangel/__init__.py

@@ -0,0 +1,176 @@
+"""DeployAngel agent for Python web apps and job workers.
+
+Framework integrations call start(); the rest of this module is the API they
+record through, plus checkpoint() and notify() for application code. Every
+function here is safe to call anywhere: it never raises and never touches the
+network."""
+
+from __future__ import annotations
+
+import atexit
+import os
+import re
+import sys
+import threading
+from typing import Optional
+
+from deployangel.agent import Agent
+from deployangel.config import Configuration
+from deployangel.version import VERSION
+
+__all__ = ["VERSION", "checkpoint", "configuration", "configure", "notify", "recording", "shutdown", "start"]
+
+# Web servers and job workers report from boot, so idle processes still send
+# heartbeats; other processes (manage.py commands, shells) only report after
+# recording something.
+EAGER_PROGRAMS = re.compile(r"\b(gunicorn|uvicorn|hypercorn|daphne|granian|waitress|uwsgi|celery|rqworker|rq)\b")
+CHECKPOINT_NAME = re.compile(r"\A[a-z0-9][a-z0-9_.:-]{0,99}\Z", re.IGNORECASE)
+
+_configuration: Optional[Configuration] = None
+_agent: Optional[Agent] = None
+_lock = threading.Lock()
+_capabilities = ["http", "exceptions"]
+_metadata_sources: list = []
+_hooks_installed = False
+_warned_checkpoint = False
+
+
+def configuration() -> Configuration:
+    global _configuration
+    if _configuration is None:
+        _configuration = Configuration()
+    return _configuration
+
+
+def configure(**options) -> Configuration:
+    """deployangel.configure(environments=["production", "staging"])"""
+    config = configuration()
+    config.update(**options)
+    return config
+
+
+def agent() -> Optional[Agent]:
+    return _agent
+
+
+def start(environment: Optional[str] = None, root: Optional[str] = None, framework: Optional[str] = None,
+          framework_version: Optional[str] = None, eager: Optional[bool] = None, debug: bool = False) -> Optional[Agent]:
+    """Starts this process's agent, once; later calls return the same agent.
+    Framework integrations call it at boot. debug is the framework's debug
+    flag, which names the environment "development" unless it's configured."""
+    global _agent
+    try:
+        with _lock:
+            if _agent is not None:
+                return _agent
+            config = configuration()
+            environment = config.environment or environment or ("development" if debug else "production")
+            _agent = Agent(config=config, environment=environment, root=config.root or root or os.getcwd(),
+                           framework=framework, framework_version=framework_version,
+                           eager=server_process() if eager is None else eager, capabilities=_capabilities)
+            from deployangel.metadata import Metadata
+
+            _agent.metadata = Metadata(config=config, root=_agent.root, environment=environment, sources=_metadata_sources)
+            _install_process_hooks()
+            return _agent
+    except Exception as error:
+        try:
+            configuration().logger.warning("DeployAngel failed to start: %s: %s", type(error).__name__, error)
+        except Exception:
+            pass
+        return None
+
+
+def recording() -> bool:
+    return _agent is not None and _agent.active
+
+
+def checkpoint(name: str, count: int = 1) -> None:
+    """Counts a business event, such as deployangel.checkpoint("order.fulfilled").
+    DeployAngel learns each checkpoint's normal rate and fails a release after
+    which it drops sharply or stops."""
+    global _warned_checkpoint
+    try:
+        name = str(name)
+        if not CHECKPOINT_NAME.match(name) or not isinstance(count, int) or isinstance(count, bool) or count <= 0:
+            if not _warned_checkpoint:
+                configuration().logger.warning("DeployAngel ignored checkpoint %r: use letters, numbers, and . _ : - (max 100)", name)
+            _warned_checkpoint = True
+            return
+        add_capability("checkpoints")
+        if _agent is not None:
+            _agent.record_checkpoint(name, count)
+    except Exception:
+        pass
+
+
+def notify(exception: BaseException) -> None:
+    """Reports an exception the app caught and handled itself. It's shown for
+    context; only unhandled exceptions count toward verdicts."""
+    if _agent is not None:
+        _agent.record_exception(exception, handled=True)
+
+
+def record_request(route_key: str, status: int, duration_ms: float, unhandled: bool = False, in_totals: bool = True) -> None:
+    if _agent is not None:
+        _agent.record_request(route_key, status, duration_ms, unhandled=unhandled, in_totals=in_totals)
+
+
+def record_job(job_class: str, duration_ms: float, failed: bool = False, discarded: bool = False,
+               queue_latency_ms: Optional[float] = None) -> None:
+    if _agent is not None:
+        _agent.record_job(job_class, duration_ms, failed=failed, discarded=discarded, queue_latency_ms=queue_latency_ms)
+
+
+def record_exception(exception: BaseException, source: Optional[str] = None, handled: bool = False) -> None:
+    if _agent is not None:
+        _agent.record_exception(exception, source=source, handled=handled)
+
+
+def add_capability(name: str) -> None:
+    """Signals this process can observe, announced in every payload so the
+    cloud never claims to verify what the agent cannot see."""
+    if name not in _capabilities:
+        _capabilities.append(name)
+
+
+def add_metadata_source(source) -> None:
+    """An integration that knows part of the app: an object with any of
+    routes(), job_classes(), job_class_files(), and schedules()."""
+    if source not in _metadata_sources:
+        _metadata_sources.append(source)
+
+
+def after_fork() -> None:
+    if _agent is not None:
+        _agent.after_fork()
+
+
+def shutdown() -> None:
+    if _agent is not None:
+        _agent.shutdown()
+
+
+def server_process() -> bool:
+    argv = getattr(sys, "orig_argv", None) or sys.argv
+    return bool(EAGER_PROGRAMS.search(" ".join(os.path.basename(str(arg)) if i == 0 else str(arg)
+                                                for i, arg in enumerate(argv[:4]))))
+
+
+def _install_process_hooks() -> None:
+    global _hooks_installed
+    if _hooks_installed:
+        return
+    _hooks_installed = True
+    atexit.register(shutdown)
+    if hasattr(os, "register_at_fork"):
+        os.register_at_fork(after_in_child=after_fork)
+
+
+def _reset_for_tests() -> None:
+    global _agent, _configuration, _warned_checkpoint
+    _agent = None
+    _configuration = None
+    _warned_checkpoint = False
+    del _capabilities[2:]
+    _metadata_sources.clear()

deployangel/agent.py

@@ -0,0 +1,270 @@
+"""The per-process runtime: records requests and jobs into the aggregator and
+sends completed minutes from a background thread. Every public method fails
+open; nothing here may raise into, or block, the customer's app."""
+
+from __future__ import annotations
+
+import os
+import random
+import threading
+import time
+from typing import Callable, Mapping, Optional
+
+from deployangel.core import fingerprint, protocol, redaction
+from deployangel.core import release as release_module
+from deployangel.core.aggregator import Aggregator
+from deployangel.core.buffer import Buffer
+from deployangel.core.instance import Instance
+from deployangel.core.transport import Transport, encode
+
+TELEMETRY_PATH = "/api/v1/telemetry"
+METADATA_PATH = "/api/v1/application_metadata"
+SHUTDOWN_TIMEOUT = 2.0
+DEFAULT_PAUSE = 60
+# Seconds past each minute a process waits before sending, chosen once per
+# process so the cloud gets a steady stream instead of a burst at the top of
+# the minute. The cloud reads a minute 2 minutes after it ends, so a send that
+# times out and goes again a minute later still lands in time.
+FLUSH_JITTER = (1.0, 50.0)
+# Set on an exception object once it's recorded, whichever instrumentation
+# sees it first.
+SEEN = "__deployangel_recorded__"
+
+
+class Agent:
+    def __init__(self, config, environment: str, root: Optional[str] = None, framework: Optional[str] = None,
+                 framework_version: Optional[str] = None, env: Mapping[str, str] = os.environ, transport=None,
+                 clock: Callable[[], float] = time.time, eager: bool = False, capabilities: Optional[list] = None):
+        self.config = config
+        self.active = config.is_active(environment)
+        self.environment = environment
+        self.root = root
+        self.release = release_module.resolve(config, env=env, root=root) if self.active else release_module.Release(None, None, "unknown")
+        self.runtime = protocol.runtime(framework, framework_version)
+        self.metadata = None
+        self.capabilities = capabilities if capabilities is not None else ["http", "exceptions"]
+        self._transport = transport or Transport(config)
+        self._env = env
+        self._clock = clock
+        self._eager = eager
+        self._warned: set = set()
+        self._reset_process_state()
+        if self.active and self.release.unknown:
+            self._warn_once("unknown_release", "DeployAngel could not determine the release; set DEPLOYANGEL_REVISION "
+                            "or enable Heroku dyno metadata. Telemetry will not be attributed to deployments.")
+        if self.active and eager:
+            self.start_reporter()
+
+    def record_request(self, route_key: str, status: int, duration_ms: float, unhandled: bool = False, in_totals: bool = True) -> None:
+        if not self.active:
+            return
+        try:
+            self._check_fork()
+            self.start_reporter()
+            self._aggregator.record(route_key, status, duration_ms, unhandled=unhandled, in_totals=in_totals)
+        except Exception as error:
+            self._warn_once("record", f"DeployAngel failed to record a request: {type(error).__name__}: {error}")
+
+    def record_job(self, job_class: str, duration_ms: float, failed: bool = False, discarded: bool = False,
+                   queue_latency_ms: Optional[float] = None) -> None:
+        if not self.active:
+            return
+        try:
+            self._check_fork()
+            self.start_reporter()
+            self._aggregator.record_job(job_class, duration_ms, failed=failed, discarded=discarded, queue_latency_ms=queue_latency_ms)
+        except Exception as error:
+            self._warn_once("record_job", f"DeployAngel failed to record a job: {type(error).__name__}: {error}")
+
+    def record_discard(self, job_class: str) -> None:
+        if not self.active:
+            return
+        try:
+            self._check_fork()
+            self._aggregator.record_discard(job_class)
+        except Exception as error:
+            self._warn_once("record_discard", f"DeployAngel failed to record a discarded job: {type(error).__name__}: {error}")
+
+    def record_checkpoint(self, name: str, count: int = 1) -> None:
+        if not self.active:
+            return
+        try:
+            self._check_fork()
+            self.start_reporter()
+            self._aggregator.record_checkpoint(name, count)
+        except Exception as error:
+            self._warn_once("record_checkpoint", f"DeployAngel failed to record a checkpoint: {type(error).__name__}: {error}")
+
+    def record_exception(self, exception: BaseException, source: Optional[str] = None, handled: bool = False) -> None:
+        """Records each exception object once, whichever instrumentation sees it
+        first."""
+        if not self.active or not isinstance(exception, BaseException):
+            return
+        try:
+            if getattr(exception, SEEN, False):
+                return
+            try:
+                setattr(exception, SEEN, True)
+            except Exception:
+                pass
+            messages = self.config.exception_messages
+            details = fingerprint.for_exception(exception, self.root, message=messages,
+                                                redactions=redaction.current() if messages else ())
+            self.record_exception_details(details, fingerprint.backtrace(exception, self.root), source=source, handled=handled)
+        except Exception as error:
+            self._warn_once("record_exception", f"DeployAngel failed to record an exception: {type(error).__name__}: {error}")
+
+    def record_exception_details(self, details: dict, backtrace: Optional[list], source: Optional[str] = None,
+                                 handled: bool = False) -> None:
+        """An exception already fingerprinted, such as one parsed from a
+        formatted traceback."""
+        if not self.active:
+            return
+        try:
+            self._check_fork()
+            self.start_reporter()
+            self._aggregator.record_exception(details, source=source, handled=handled, backtrace=backtrace)
+        except Exception as error:
+            self._warn_once("record_exception", f"DeployAngel failed to record an exception: {type(error).__name__}: {error}")
+
+    def flush(self, include_current: bool = False, deadline: Optional[float] = None) -> int:
+        """Drains completed minutes into the buffer, encoded so unsent minutes
+        stay small while DeployAngel is unreachable, and sends what it can."""
+        if not self.active:
+            return 0
+        try:
+            for period in self._aggregator.drain(include_current=include_current, max_periods=self.config.max_queued_payloads):
+                self._buffer.push(encode(protocol.telemetry(period, self.instance, self.release, self.runtime, self.capabilities)))
+            return self._send_buffered(deadline)
+        except Exception as error:
+            self._warn_once("flush", f"DeployAngel failed to flush telemetry: {type(error).__name__}: {error}")
+            return 0
+
+    def start_reporter(self) -> None:
+        if not self.active or (self._thread is not None and self._thread.is_alive()):
+            return
+        with self._thread_lock:
+            if self._thread is not None and self._thread.is_alive():
+                return
+            self._stopping.clear()
+            thread = threading.Thread(target=self._run_reporter, name="deployangel-reporter", daemon=True)
+            thread.start()
+            self._thread = thread
+
+    def shutdown(self, timeout: float = SHUTDOWN_TIMEOUT) -> None:
+        """Sends the in-progress minute too, bounded by a short timeout, because
+        deployments restart processes. Runs on the calling thread: new threads
+        can't start while the interpreter shuts down."""
+        if not self.active:
+            return
+        try:
+            if os.getpid() != self._pid:
+                return
+            self._stopping.set()
+            self.flush(include_current=True, deadline=time.monotonic() + timeout)
+        except Exception:
+            pass
+
+    def after_fork(self) -> None:
+        """Threads do not survive fork, and the parent's identity and counts
+        must not be reused by the child."""
+        self._reset_process_state()
+        if self.active and self._eager:
+            self.start_reporter()
+
+    def send_metadata(self) -> None:
+        """Sent once per process; retried on the next minute if it fails. File
+        digests are only uploaded when the cloud has not seen the manifest."""
+        if self._metadata_sent or self.metadata is None or self._paused():
+            return
+        try:
+            base = {"protocol_version": protocol.VERSION, "instance": self.instance.to_protocol(),
+                    "release": self.release.to_protocol(), "runtime": self.runtime}
+            base.update(self.metadata.to_protocol())
+            result = self._transport.post(METADATA_PATH, base)
+            if not result.ok:
+                return
+            if isinstance(result.body, dict) and result.body.get("manifest_needed"):
+                if not self._transport.post(METADATA_PATH, dict(base, files=self.metadata.files())).ok:
+                    return
+            self._metadata_sent = True
+        except Exception as error:
+            self._warn_once("metadata", f"DeployAngel failed to send application metadata: {type(error).__name__}: {error}")
+
+    def _check_fork(self) -> None:
+        # A fork the interpreter didn't see, such as uWSGI's, which forks
+        # workers from C.
+        if os.getpid() != self._pid:
+            self.after_fork()
+
+    def _reset_process_state(self) -> None:
+        self._pid = os.getpid()
+        self.instance = Instance(env=self._env, now=self._clock())
+        self._aggregator = Aggregator(max_routes=self.config.max_routes, clock=self._clock)
+        self._buffer = Buffer(self.config.max_queued_payloads)
+        self._paused_until: Optional[float] = None
+        self._metadata_sent = False
+        self._thread: Optional[threading.Thread] = None
+        self._thread_lock = threading.Lock()
+        self._stopping = threading.Event()
+        self._jitter = random.uniform(*FLUSH_JITTER)
+
+    def _run_reporter(self) -> None:
+        try:
+            while not self._stopping.is_set():
+                if self._stopping.wait(self._seconds_until_next_flush()):
+                    break
+                self.send_metadata()
+                self.flush()
+        except Exception as error:
+            self._warn_once("reporter", f"DeployAngel reporter stopped: {type(error).__name__}: {error}")
+
+    def _seconds_until_next_flush(self) -> float:
+        interval = self.config.flush_interval
+        return (interval - (self._clock() % interval)) + self._jitter
+
+    def _send_buffered(self, deadline: Optional[float] = None) -> int:
+        sent = 0
+        while True:
+            payload = self._buffer.shift()
+            if payload is None:
+                break
+            remaining = None
+            if deadline is not None:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    self._buffer.unshift(payload)
+                    break
+            if self._paused():
+                self._buffer.unshift(payload)
+                break
+            result = self._transport.post(TELEMETRY_PATH, payload, timeout=min(remaining, self.config.timeout) if remaining else None)
+            if result.outcome == "ok":
+                sent += 1
+            elif result.outcome == "retry":
+                self._buffer.unshift(payload)
+                break
+            else:
+                self._handle_rejection(result)
+        return sent
+
+    def _handle_rejection(self, result) -> None:
+        if result.status == 429:
+            self._paused_until = self._clock() + (result.retry_after if result.retry_after and result.retry_after > 0 else DEFAULT_PAUSE)
+        elif result.status in (401, 403):
+            self._warn_once("auth", f"DeployAngel rejected the token (HTTP {result.status}); check DEPLOYANGEL_TOKEN "
+                            "and that it has the telemetry scope.")
+        else:
+            self._warn_once(f"rejected_{result.status}", f"DeployAngel rejected a telemetry payload (HTTP {result.status}).")
+
+    def _paused(self) -> bool:
+        return self._paused_until is not None and self._clock() < self._paused_until
+
+    def _warn_once(self, key: str, message: str) -> None:
+        if key in self._warned:
+            return
+        self._warned.add(key)
+        try:
+            self.config.logger.warning(message)
+        except Exception:
+            pass

deployangel/asgi.py

@@ -0,0 +1,216 @@
+"""ASGI integration for Starlette and FastAPI: a middleware that records each
+request against the route the router matched ("GET /items/{item_id}"), and
+the route table for the metadata.
+
+    app = FastAPI()
+    deployangel.fastapi.init(app)
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Optional
+
+import deployangel
+from deployangel import http
+from deployangel.core import redaction
+from deployangel.metadata import module_file
+
+IGNORED_METHODS = ("HEAD", "OPTIONS")
+# The documentation pages FastAPI serves itself.
+FRAMEWORK_MODULES = ("fastapi.", "starlette.")
+
+
+def init(app):
+    """Adds the middleware, lists the app's routes, and starts the agent. Call
+    it where the app is created, before it serves requests."""
+    try:
+        app.add_middleware(DeployAngelMiddleware)
+        deployangel.add_metadata_source(StarletteRoutes(app))
+        framework, version = _framework(app)
+        return deployangel.start(framework=framework, framework_version=version, debug=bool(getattr(app, "debug", False)))
+    except Exception as error:
+        deployangel.configuration().logger.warning("DeployAngel failed to start: %s: %s", type(error).__name__, error)
+        return None
+
+
+class DeployAngelMiddleware:
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope.get("type") != "http" or not deployangel.recording():
+            await self.app(scope, receive, send)
+            return
+        started = time.perf_counter()
+        # The router changes the scope as it goes (a mount moves root_path,
+        # and a mounted app replaces "app"), so note what it was.
+        original = {"type": "http", "path": scope.get("path", ""), "root_path": scope.get("root_path", ""),
+                    "method": scope.get("method", "GET"), "headers": scope.get("headers", [])}
+        outer_app = scope.get("app")
+        redaction.set_request_host(_header(scope, b"host"))
+        status = []
+
+        async def send_with_status(message):
+            if message.get("type") == "http.response.start":
+                status.append(message.get("status", 200))
+            await send(message)
+
+        try:
+            await self.app(scope, receive, send_with_status)
+        except Exception as exception:
+            _record(scope, original, outer_app, 500, started, exception)
+            raise
+        _record(scope, original, outer_app, status[0] if status else 500, started, None)
+
+
+def _record(scope, original, outer_app, status, started, exception) -> None:
+    try:
+        http.record(original["method"], route_pattern(scope, original, outer_app), status, started,
+                    unhandled=exception is not None, exception=exception)
+    except Exception:
+        pass
+
+
+def route_pattern(scope, original, outer_app) -> Optional[str]:
+    """The matched route's path format, with the paths of any mounts it sits
+    under; None when nothing matched or a mounted non-router app (such as
+    StaticFiles) answered."""
+    from starlette.routing import Mount
+
+    route = scope.get("route")
+    if route is None:
+        if scope.get("endpoint") is None:
+            return None
+        route = _match(getattr(outer_app, "routes", []), original)
+        return route
+    if isinstance(route, Mount):
+        return None
+    pattern = _effective_path(scope) or getattr(route, "path_format", None) or getattr(route, "path", None)
+    if not pattern:
+        return None
+    if scope.get("root_path", "") != original["root_path"]:
+        pattern = _mount_prefix(getattr(outer_app, "routes", []), _route_path(original)) + pattern
+    return pattern
+
+
+def _effective_path(scope) -> Optional[str]:
+    """FastAPI 0.14x includes routers lazily: the route in the scope is the
+    router's own, without the prefix include_router() added, and the path the
+    request matched is in FastAPI's part of the scope."""
+    fastapi_scope = scope.get("fastapi")
+    context = fastapi_scope.get("effective_route_context") if isinstance(fastapi_scope, dict) else None
+    if context is None:
+        return None
+    return getattr(getattr(context, "starlette_route", None), "path_format", None) or getattr(context, "path_format", None)
+
+
+def _route_path(scope) -> str:
+    path, root = scope["path"], scope["root_path"]
+    return path[len(root):] if root and path.startswith(root) else path
+
+
+def _mount_prefix(routes, path: str) -> str:
+    from starlette.routing import Mount
+
+    for route in routes:
+        if isinstance(route, Mount):
+            match = route.path_regex.match(path)
+            if match:
+                return route.path + _mount_prefix(getattr(route, "routes", []) or [], "/" + match.groupdict().get("path", ""))
+    return ""
+
+
+def _match(routes, scope, prefix: str = "") -> Optional[str]:
+    """For a router that doesn't name the matched route in the scope: matches
+    the request again against the route table."""
+    from starlette.routing import Match, Mount
+
+    for route in routes:
+        try:
+            if isinstance(route, Mount):
+                match = route.path_regex.match(_route_path(scope))
+                if match and getattr(route, "routes", None):
+                    child = dict(scope, path="/" + match.groupdict().get("path", ""), root_path="")
+                    found = _match(route.routes, child, prefix + route.path)
+                    if found:
+                        return found
+                continue
+            result, _ = route.matches(scope)
+            if result == Match.FULL:
+                return prefix + (getattr(route, "path_format", None) or route.path)
+        except Exception:
+            continue
+    return None
+
+
+def _header(scope, name: bytes) -> Optional[str]:
+    for key, value in scope.get("headers") or []:
+        if key == name:
+            return value.decode("latin-1")
+    return None
+
+
+def _framework(app) -> tuple:
+    try:
+        import fastapi
+
+        if isinstance(app, fastapi.FastAPI):
+            return "fastapi", fastapi.__version__
+    except ImportError:
+        pass
+    import starlette
+
+    return "starlette", getattr(starlette, "__version__", None)
+
+
+class StarletteRoutes:
+    def __init__(self, app):
+        self.app = app
+
+    def routes(self) -> list:
+        agent = deployangel.agent()
+        root = agent.root if agent else None
+        return list(_walk(self.app.routes, "", root))
+
+
+def _walk(routes, prefix: str, root: Optional[str]):
+    from starlette.routing import Mount
+
+    for route in _flatten(routes):
+        if isinstance(route, Mount):
+            if getattr(route, "routes", None):
+                yield from _walk(route.routes, prefix + route.path, root)
+            continue
+        if not getattr(route, "methods", None) or not getattr(route, "endpoint", None):
+            continue
+        endpoint = getattr(route, "endpoint", None)
+        module = getattr(endpoint, "__module__", None) or ""
+        if module.startswith(FRAMEWORK_MODULES):
+            continue
+        path = prefix + (getattr(route, "path_format", None) or route.path)
+        entry = {"controller": module, "action": getattr(endpoint, "__qualname__", None) or getattr(endpoint, "__name__", "")}
+        source = module_file(module, root)
+        if source:
+            entry["files"] = [source]
+        for method in sorted(getattr(route, "methods", None) or ["GET"]):
+            if method not in IGNORED_METHODS:
+                yield dict(entry, key=f"{method} {path}")
+
+
+def _flatten(routes):
+    """The routes as matched, with FastAPI 0.14x's lazily included routers
+    expanded (each with its include_router() prefix)."""
+    try:
+        from fastapi.routing import iter_route_contexts
+    except ImportError:
+        return list(routes)
+    from starlette.routing import Mount
+
+    flat = []
+    for route in routes:
+        if isinstance(route, Mount):
+            flat.append(route)
+        else:
+            flat.extend(iter_route_contexts([route]))
+    return flat