bufferlog 0.1.0__py3-none-any.whl

bufferlog/__init__.py

@@ -0,0 +1,170 @@
+"""
+BufferLog — Python SDK
+
+Buffer logs in memory per-request. Flush only on errors. Save 90%+ on APM costs.
+
+Usage:
+    from bufferlog import BufferLog
+    from bufferlog.adapters import StdOutAdapter
+
+    bl = BufferLog(adapters=[StdOutAdapter(pretty=True)])
+    bl.init_flask(app)
+
+    # Or for FastAPI:
+    from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+    app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from .config import BufferLogConfig, ControlPlaneConfig
+from .ring_buffer import RingBuffer
+from .log_event import LogEvent, LogLevel
+from .buffer_manager import BufferManager, BufferManagerMetrics
+from .flash_controller import FlashController
+from .adapters import StdOutAdapter
+from .integrations import BufferLogHandler
+from .control_plane.policy_fetcher import PolicyFetcher
+from .control_plane.telemetry_reporter import TelemetryReporter
+
+if TYPE_CHECKING:
+    from flask import Flask
+
+__version__ = "0.1.0"
+__all__ = [
+    "BufferLog",
+    "BufferLogConfig",
+    "ControlPlaneConfig",
+    "RingBuffer",
+    "LogEvent",
+    "LogLevel",
+    "BufferManager",
+    "BufferManagerMetrics",
+    "FlashController",
+    "BufferLogHandler",
+    "StdOutAdapter",
+    "PolicyFetcher",
+    "TelemetryReporter",
+]
+
+
+class BufferLog:
+    """High-level SDK entry point.
+
+    Creates and wires all components together. The recommended way to
+    use the BufferLog Python SDK.
+    """
+
+    def __init__(self, config: Optional[BufferLogConfig] = None, **kwargs: Any) -> None:
+        if config is None:
+            config = BufferLogConfig(**kwargs)
+        self.config = config
+
+        self.buffer_manager = BufferManager(config.buffer_capacity)
+
+        adapters = config.adapters if config.adapters else [StdOutAdapter(pretty=True)]
+        self.flash_controller = FlashController(
+            adapters=adapters,
+            fail_open=config.fail_open,
+        )
+
+        self.policy_fetcher: Optional[PolicyFetcher] = None
+        self.telemetry_reporter: Optional[TelemetryReporter] = None
+
+        if config.control_plane:
+            self.policy_fetcher = PolicyFetcher(
+                url=config.control_plane.url,
+                api_key=config.control_plane.api_key,
+                interval_s=config.control_plane.poll_interval_s,
+            )
+            self.telemetry_reporter = TelemetryReporter(
+                url=config.control_plane.url,
+                api_key=config.control_plane.api_key,
+                interval_s=config.control_plane.telemetry_interval_s,
+                metrics_provider=lambda: self.get_metrics_dict(),
+            )
+            self.policy_fetcher.start()
+            self.telemetry_reporter.start()
+
+    # ---- Framework integrations ----
+
+    def init_flask(self, app: "Flask") -> None:
+        """Register BufferLog middleware on a Flask app."""
+        from .middleware.flask_mw import init_flask
+
+        init_flask(app, self.buffer_manager, self.flash_controller, self.config)
+
+    def init_django(self) -> None:
+        """Configure the Django middleware class with this BufferLog instance.
+
+        After calling this, add 'bufferlog.middleware.django_mw.BufferLogDjangoMiddleware'
+        to your MIDDLEWARE list in settings.py.
+        """
+        from .middleware.django_mw import BufferLogDjangoMiddleware
+
+        BufferLogDjangoMiddleware._buffer_manager = self.buffer_manager
+        BufferLogDjangoMiddleware._flash_controller = self.flash_controller
+        BufferLogDjangoMiddleware._config = self.config
+
+    def asgi_kwargs(self) -> Dict[str, Any]:
+        """Return kwargs for adding the ASGI middleware.
+
+        Usage:
+            from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+            app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+        """
+        return {
+            "buffer_manager": self.buffer_manager,
+            "flash_controller": self.flash_controller,
+            "config": self.config,
+        }
+
+    # ---- Logging integration ----
+
+    def logging_handler(self) -> BufferLogHandler:
+        """Create a logging.Handler for Python's built-in logging module."""
+        return BufferLogHandler(
+            flash_controller=self.flash_controller,
+            config=self.config,
+        )
+
+    # ---- Metrics ----
+
+    def get_metrics(self) -> Dict[str, Any]:
+        bm = self.buffer_manager.get_metrics()
+        return {
+            "buffers": {
+                "created": bm.created,
+                "discarded": bm.discarded,
+                "flushed": bm.flushed,
+                "active": bm.active,
+            },
+            "flash": {
+                "flush_count": self.flash_controller.flush_count,
+                "events_flushed": self.flash_controller.events_flushed,
+                "adapter_errors": self.flash_controller.adapter_errors,
+            },
+        }
+
+    def get_metrics_dict(self) -> Dict[str, Any]:
+        """Flat metrics dict for telemetry reporter."""
+        bm = self.buffer_manager.get_metrics()
+        return {
+            "logs_discarded": bm.discarded,
+            "logs_flushed": bm.flushed,
+            "requests_success": bm.discarded,
+            "requests_error": bm.flushed,
+            "buffers_active": bm.active,
+        }
+
+    # ---- Shutdown ----
+
+    def shutdown(self) -> None:
+        """Gracefully stop background tasks."""
+        if self.policy_fetcher:
+            self.policy_fetcher.stop()
+        if self.telemetry_reporter:
+            self.telemetry_reporter.send()  # Final report
+            self.telemetry_reporter.stop()

bufferlog/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""BufferLog adapters."""
+
+from .stdout import StdOutAdapter
+from .datadog import DatadogAdapter
+from .splunk import SplunkAdapter
+
+__all__ = ["StdOutAdapter", "DatadogAdapter", "SplunkAdapter"]

bufferlog/adapters/base.py

@@ -0,0 +1,16 @@
+"""BufferLog — Adapter protocol (abstract base)."""
+
+from __future__ import annotations
+
+from typing import List, Protocol, runtime_checkable
+
+from ..log_event import LogEvent
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """Interface that all downstream adapters must implement."""
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        """Send a batch of log events to the downstream target."""
+        ...

bufferlog/adapters/datadog.py

@@ -0,0 +1,58 @@
+"""BufferLog — Datadog Adapter. Sends flushed logs to Datadog Logs API."""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import List, Optional
+
+from ..log_event import LogEvent
+
+
+class DatadogAdapter:
+    """Ships flushed log events to Datadog via the HTTP Logs API."""
+
+    DEFAULT_URL = "https://http-intake.logs.datadoghq.com/api/v2/logs"
+
+    def __init__(
+        self,
+        api_key: str,
+        url: Optional[str] = None,
+        service: str = "bufferlog",
+        source: str = "python",
+    ) -> None:
+        self._api_key = api_key
+        self._url = url or self.DEFAULT_URL
+        self._service = service
+        self._source = source
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        payload = [
+            {
+                "message": e.message,
+                "ddtags": f"context_id:{context_id}",
+                "ddsource": self._source,
+                "service": self._service,
+                "level": e.to_dict()["level"],
+                "timestamp": int(e.timestamp * 1000),
+                **(e.metadata or {}),
+            }
+            for e in events
+        ]
+
+        data = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            self._url,
+            data=data,
+            headers={
+                "Content-Type": "application/json",
+                "DD-API-KEY": self._api_key,
+            },
+            method="POST",
+        )
+
+        try:
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                resp.read()
+        except Exception:
+            pass  # Fail-open: don't crash the app if Datadog is unreachable

bufferlog/adapters/splunk.py

@@ -0,0 +1,55 @@
+"""BufferLog — Splunk HEC Adapter."""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import List, Optional
+
+from ..log_event import LogEvent
+
+
+class SplunkAdapter:
+    """Ships flushed log events to Splunk via the HTTP Event Collector."""
+
+    def __init__(
+        self,
+        token: str,
+        url: str,
+        source: str = "bufferlog",
+        sourcetype: str = "_json",
+        index: Optional[str] = None,
+    ) -> None:
+        self._token = token
+        self._url = url.rstrip("/") + "/services/collector/event"
+        self._source = source
+        self._sourcetype = sourcetype
+        self._index = index
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        for event in events:
+            payload = {
+                "event": event.to_dict(),
+                "source": self._source,
+                "sourcetype": self._sourcetype,
+                "time": event.timestamp,
+            }
+            if self._index:
+                payload["index"] = self._index
+
+            data = json.dumps(payload).encode("utf-8")
+            req = urllib.request.Request(
+                self._url,
+                data=data,
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Splunk {self._token}",
+                },
+                method="POST",
+            )
+
+            try:
+                with urllib.request.urlopen(req, timeout=10) as resp:
+                    resp.read()
+            except Exception:
+                pass  # Fail-open

bufferlog/adapters/stdout.py

@@ -0,0 +1,25 @@
+"""BufferLog — StdOut Adapter. Prints flushed logs to stdout."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import List
+
+from ..log_event import LogEvent
+
+
+class StdOutAdapter:
+    """Writes flushed log events to stdout as JSON lines."""
+
+    def __init__(self, pretty: bool = False) -> None:
+        self._pretty = pretty
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        for event in events:
+            data = event.to_dict()
+            if self._pretty:
+                sys.stdout.write(json.dumps(data, indent=2, default=str) + "\n")
+            else:
+                sys.stdout.write(json.dumps(data, default=str) + "\n")
+        sys.stdout.flush()

bufferlog/buffer_manager.py

@@ -0,0 +1,85 @@
+"""
+BufferLog — Buffer Manager
+
+Manages per-request ring buffers. Creates, retrieves, discards, and flushes
+buffers identified by context_id. Tracks metrics for the ROI dashboard.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from .ring_buffer import RingBuffer
+from .log_event import LogEvent
+
+
+@dataclass
+class BufferManagerMetrics:
+    created: int = 0
+    discarded: int = 0
+    flushed: int = 0
+    active: int = 0
+
+
+class BufferManager:
+    """Lifecycle manager for per-request ring buffers."""
+
+    def __init__(self, default_capacity: int = 100) -> None:
+        self._default_capacity = default_capacity
+        self._buffers: Dict[str, RingBuffer[LogEvent]] = {}
+        self._metrics = BufferManagerMetrics()
+        self._lock = threading.Lock()
+
+    def create_buffer(
+        self, context_id: str, capacity: Optional[int] = None
+    ) -> RingBuffer[LogEvent]:
+        """Create a new ring buffer for a request context."""
+        buf = RingBuffer[LogEvent](capacity or self._default_capacity)
+        with self._lock:
+            self._buffers[context_id] = buf
+            self._metrics.created += 1
+            self._metrics.active += 1
+        return buf
+
+    def get_buffer(self, context_id: str) -> Optional[RingBuffer[LogEvent]]:
+        """Retrieve the buffer for a given context ID."""
+        with self._lock:
+            return self._buffers.get(context_id)
+
+    def discard_buffer(self, context_id: str) -> bool:
+        """Discard a buffer — the SUCCESS path. Logs are dropped, saving money."""
+        with self._lock:
+            buf = self._buffers.pop(context_id, None)
+            if buf is None:
+                return False
+            buf.clear()
+            self._metrics.discarded += 1
+            self._metrics.active -= 1
+            return True
+
+    def flush_buffer(self, context_id: str) -> List[LogEvent]:
+        """Flush a buffer — the ERROR path. Returns all buffered log events."""
+        with self._lock:
+            buf = self._buffers.pop(context_id, None)
+            if buf is None:
+                return []
+            self._metrics.flushed += 1
+            self._metrics.active -= 1
+        # drain() is thread-safe on its own
+        return buf.drain()
+
+    def get_metrics(self) -> BufferManagerMetrics:
+        with self._lock:
+            return BufferManagerMetrics(
+                created=self._metrics.created,
+                discarded=self._metrics.discarded,
+                flushed=self._metrics.flushed,
+                active=self._metrics.active,
+            )
+
+    @property
+    def active_count(self) -> int:
+        with self._lock:
+            return len(self._buffers)

bufferlog/config.py

@@ -0,0 +1,47 @@
+"""
+BufferLog — Configuration
+
+Defines all tunable parameters for the Python SDK with sensible defaults.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple
+
+
+@dataclass
+class ControlPlaneConfig:
+    """Control plane connection settings."""
+    url: str
+    api_key: str
+    poll_interval_s: float = 60.0
+    telemetry_interval_s: float = 60.0
+
+
+@dataclass
+class BufferLogConfig:
+    """User-facing configuration for the BufferLog SDK."""
+
+    buffer_capacity: int = 100
+    flush_on_levels: List[str] = field(
+        default_factory=lambda: ["error", "critical"]
+    )
+    flush_on_status_codes: List[int] = field(
+        default_factory=lambda: [500, 501, 502, 503, 504]
+    )
+    adapters: List[Any] = field(default_factory=list)
+    enabled: bool = True
+    fail_open: bool = True
+    scrubber: Optional[
+        Callable[[str, Optional[Dict[str, Any]]], Tuple[str, Optional[Dict[str, Any]]]]
+    ] = None
+    control_plane: Optional[ControlPlaneConfig] = None
+
+    @property
+    def flush_levels_set(self) -> Set[str]:
+        return {l.upper() for l in self.flush_on_levels}
+
+    @property
+    def flush_status_set(self) -> Set[int]:
+        return set(self.flush_on_status_codes)

bufferlog/context.py

@@ -0,0 +1,47 @@
+"""
+BufferLog — Request Context
+
+Uses Python's contextvars to track the active request's buffer.
+This is the Python equivalent of Node.js AsyncLocalStorage —
+each request gets its own isolated context without manual passing.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from typing import Optional
+
+from .ring_buffer import RingBuffer
+from .log_event import LogEvent
+
+# Context variable holding the current request's context_id
+_context_id_var: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
+    "bufferlog_context_id", default=None
+)
+
+# Context variable holding the current request's ring buffer
+_buffer_var: contextvars.ContextVar[Optional[RingBuffer[LogEvent]]] = contextvars.ContextVar(
+    "bufferlog_buffer", default=None
+)
+
+
+def get_context_id() -> Optional[str]:
+    """Get the context_id for the current request."""
+    return _context_id_var.get()
+
+
+def get_buffer() -> Optional[RingBuffer[LogEvent]]:
+    """Get the ring buffer for the current request."""
+    return _buffer_var.get()
+
+
+def set_context(context_id: str, buffer: RingBuffer[LogEvent]) -> None:
+    """Set the context for the current request."""
+    _context_id_var.set(context_id)
+    _buffer_var.set(buffer)
+
+
+def clear_context() -> None:
+    """Clear the context for the current request."""
+    _context_id_var.set(None)
+    _buffer_var.set(None)

bufferlog/control_plane/__init__.py

@@ -0,0 +1 @@
+"""BufferLog control plane package."""

bufferlog/control_plane/policy_fetcher.py

@@ -0,0 +1,112 @@
+"""
+BufferLog — Policy Fetcher
+
+Background thread that polls the control plane for dynamic configuration.
+Uses urllib.request for zero external dependencies.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import urllib.request
+from typing import Any, Callable, Dict, Optional
+import platform
+
+
+class PolicyFetcher:
+    """Periodically fetches policy from the BufferLog Control Plane."""
+
+    def __init__(
+        self,
+        url: str,
+        api_key: str,
+        interval_s: float = 60.0,
+        on_update: Optional[Callable[[Dict[str, Any]], None]] = None,
+        on_error: Optional[Callable[[Exception], None]] = None,
+    ) -> None:
+        base = url.rstrip("/")
+        self._url = f"{base}/api/v1/policy"
+        self._api_key = api_key
+        self._interval = interval_s
+        self._on_update = on_update
+        self._on_error = on_error
+
+        self._timer: Optional[threading.Timer] = None
+        self._running = False
+        self._last_policy: Optional[Dict[str, Any]] = None
+        self._fetch_count = 0
+        self._error_count = 0
+        self._lock = threading.Lock()
+
+    def start(self) -> None:
+        if self._running:
+            return
+        self._running = True
+        # Fetch immediately, then schedule recurring
+        self._do_fetch()
+        self._schedule()
+
+    def stop(self) -> None:
+        self._running = False
+        if self._timer:
+            self._timer.cancel()
+            self._timer = None
+
+    def fetch(self) -> Optional[Dict[str, Any]]:
+        """Perform a single synchronous fetch."""
+        return self._do_fetch()
+
+    def _schedule(self) -> None:
+        if not self._running:
+            return
+        self._timer = threading.Timer(self._interval, self._tick)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def _tick(self) -> None:
+        self._do_fetch()
+        self._schedule()
+
+    def _do_fetch(self) -> Optional[Dict[str, Any]]:
+        req = urllib.request.Request(
+            self._url,
+            headers={
+                "Authorization": f"Bearer {self._api_key}",
+                "User-Agent": f"bufferlog-sdk-python/0.1.0 (python/{platform.python_version()})",
+            },
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+            with self._lock:
+                self._fetch_count += 1
+                self._last_policy = data
+            if self._on_update:
+                self._on_update(data)
+            return data
+        except Exception as e:
+            with self._lock:
+                self._error_count += 1
+            if self._on_error:
+                self._on_error(e)
+            return None
+
+    @property
+    def last_policy(self) -> Optional[Dict[str, Any]]:
+        with self._lock:
+            return self._last_policy
+
+    @property
+    def fetch_count(self) -> int:
+        with self._lock:
+            return self._fetch_count
+
+    @property
+    def error_count(self) -> int:
+        with self._lock:
+            return self._error_count
+
+    @property
+    def is_running(self) -> bool:
+        return self._running