bufferlog 0.1.0__tar.gz

bufferlog-0.1.0/PKG-INFO

@@ -0,0 +1,219 @@
+Metadata-Version: 2.4
+Name: bufferlog
+Version: 0.1.0
+Summary: BufferLog — Buffer logs in memory, flush only on errors. Save 90%+ on APM costs.
+Author: BufferLog.io
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/lehan0328/bufferlog
+Project-URL: Repository, https://github.com/lehan0328/bufferlog
+Project-URL: Issues, https://github.com/lehan0328/bufferlog/issues
+Keywords: logging,apm,buffer,datadog,splunk,observability,cost-reduction
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: System :: Logging
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: flask
+Requires-Dist: flask>=2.0; extra == "flask"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.100; extra == "fastapi"
+Requires-Dist: starlette>=0.27; extra == "fastapi"
+Provides-Extra: django
+Requires-Dist: django>=4.0; extra == "django"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: flask>=2.0; extra == "dev"
+Requires-Dist: fastapi>=0.100; extra == "dev"
+Requires-Dist: starlette>=0.27; extra == "dev"
+Requires-Dist: httpx>=0.24; extra == "dev"
+Requires-Dist: uvicorn>=0.20; extra == "dev"
+
+# bufferlog
+
+Buffer logs in memory per-request. Flush only on errors. Save 90%+ on APM costs.
+
+## Table of Contents
+
+- [Install](#install)
+- [Usage](#usage)
+  - [Flask](#flask)
+  - [FastAPI](#fastapi)
+  - [Django](#django)
+- [Configuration](#configuration)
+- [Control Plane](#control-plane)
+- [Metrics](#metrics)
+- [Shutdown](#shutdown)
+- [License](#license)
+
+## Install
+
+```bash
+pip install bufferlog
+```
+
+With framework extras:
+
+```bash
+pip install bufferlog[flask]
+pip install bufferlog[fastapi]
+pip install bufferlog[django]
+```
+
+## Usage
+
+### Flask
+
+```python
+import logging
+from flask import Flask
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+
+app = Flask(__name__)
+bl = BufferLog(adapters=[StdOutAdapter(pretty=True)])
+bl.init_flask(app)
+
+logger = logging.getLogger("myapp")
+logger.addHandler(bl.logging_handler())
+logger.setLevel(logging.DEBUG)
+
+@app.route("/")
+def index():
+    logger.info("Processing request")   # Buffered (discarded on 200)
+    logger.debug("SQL: SELECT * ...")    # Buffered (discarded on 200)
+    return "OK"                          # 200 → logs discarded ($0)
+
+@app.route("/fail")
+def fail():
+    logger.info("Starting transaction")
+    logger.error("Connection lost")      # Triggers flush
+    return "Error", 500                  # 500 → context sent to APM
+```
+
+### FastAPI
+
+```python
+import logging
+from fastapi import FastAPI
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+
+app = FastAPI()
+bl = BufferLog(adapters=[StdOutAdapter()])
+app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+
+logger = logging.getLogger("myapp")
+logger.addHandler(bl.logging_handler())
+logger.setLevel(logging.DEBUG)
+
+@app.get("/")
+async def index():
+    logger.info("Handling request")
+    return {"ok": True}
+```
+
+### Django
+
+```python
+# settings.py
+MIDDLEWARE = [
+    "bufferlog.middleware.django_mw.BufferLogDjangoMiddleware",
+    # ... other middleware
+]
+
+# apps.py or wsgi.py
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+
+bl = BufferLog(adapters=[StdOutAdapter()])
+bl.init_django()
+```
+
+## Configuration
+
+```python
+from bufferlog import BufferLog, BufferLogConfig
+from bufferlog.adapters import DatadogAdapter, SplunkAdapter, StdOutAdapter
+
+bl = BufferLog(config=BufferLogConfig(
+    # Max logs per request buffer. Oldest overwritten when full.
+    buffer_capacity=100,
+
+    # Log levels that trigger immediate flush.
+    flush_on_levels=["error", "critical"],
+
+    # HTTP status codes that trigger flush.
+    flush_on_status_codes=[500, 501, 502, 503, 504],
+
+    # Downstream targets for flushed logs.
+    adapters=[
+        DatadogAdapter(api_key="your-key"),
+        SplunkAdapter(token="your-token", url="https://splunk.example.com"),
+        StdOutAdapter(),
+    ],
+
+    # Enable/disable buffering.
+    enabled=True,
+
+    # If True, logs fall through to stderr on adapter failure.
+    fail_open=True,
+
+    # Scrub PII before logs enter the buffer.
+    scrubber=lambda msg, meta: (msg.replace("password", "***"), meta),
+))
+```
+
+## Control Plane
+
+Connect to the BufferLog Control Plane for remote policy management and usage tracking.
+
+```python
+from bufferlog import BufferLog, ControlPlaneConfig
+
+bl = BufferLog(
+    control_plane=ControlPlaneConfig(
+        url="https://control.bufferlog.io",
+        api_key="bl_sk_your_key",
+        poll_interval_s=60,
+        telemetry_interval_s=60,
+    )
+)
+```
+
+When connected:
+- **Poll for policy updates** — buffer capacity, sampling rate, flush triggers, and bypass rules can all be changed remotely.
+- **Push usage metrics** — counters (never log data) are sent to power the ROI dashboard.
+
+If the control plane is unreachable, the SDK continues with its last known configuration.
+
+## Metrics
+
+```python
+metrics = bl.get_metrics()
+# {
+#   "buffers":  {"created": 1000, "discarded": 995, "flushed": 5, "active": 0},
+#   "flash":    {"flush_count": 5, "events_flushed": 45, "adapter_errors": 0}
+# }
+```
+
+## Shutdown
+
+Stop background threads and send a final telemetry report.
+
+```python
+import atexit
+
+atexit.register(bl.shutdown)
+```
+
+## License
+
+[MIT](LICENSE)

bufferlog-0.1.0/README.md

@@ -0,0 +1,182 @@
+# bufferlog
+
+Buffer logs in memory per-request. Flush only on errors. Save 90%+ on APM costs.
+
+## Table of Contents
+
+- [Install](#install)
+- [Usage](#usage)
+  - [Flask](#flask)
+  - [FastAPI](#fastapi)
+  - [Django](#django)
+- [Configuration](#configuration)
+- [Control Plane](#control-plane)
+- [Metrics](#metrics)
+- [Shutdown](#shutdown)
+- [License](#license)
+
+## Install
+
+```bash
+pip install bufferlog
+```
+
+With framework extras:
+
+```bash
+pip install bufferlog[flask]
+pip install bufferlog[fastapi]
+pip install bufferlog[django]
+```
+
+## Usage
+
+### Flask
+
+```python
+import logging
+from flask import Flask
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+
+app = Flask(__name__)
+bl = BufferLog(adapters=[StdOutAdapter(pretty=True)])
+bl.init_flask(app)
+
+logger = logging.getLogger("myapp")
+logger.addHandler(bl.logging_handler())
+logger.setLevel(logging.DEBUG)
+
+@app.route("/")
+def index():
+    logger.info("Processing request")   # Buffered (discarded on 200)
+    logger.debug("SQL: SELECT * ...")    # Buffered (discarded on 200)
+    return "OK"                          # 200 → logs discarded ($0)
+
+@app.route("/fail")
+def fail():
+    logger.info("Starting transaction")
+    logger.error("Connection lost")      # Triggers flush
+    return "Error", 500                  # 500 → context sent to APM
+```
+
+### FastAPI
+
+```python
+import logging
+from fastapi import FastAPI
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+
+app = FastAPI()
+bl = BufferLog(adapters=[StdOutAdapter()])
+app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+
+logger = logging.getLogger("myapp")
+logger.addHandler(bl.logging_handler())
+logger.setLevel(logging.DEBUG)
+
+@app.get("/")
+async def index():
+    logger.info("Handling request")
+    return {"ok": True}
+```
+
+### Django
+
+```python
+# settings.py
+MIDDLEWARE = [
+    "bufferlog.middleware.django_mw.BufferLogDjangoMiddleware",
+    # ... other middleware
+]
+
+# apps.py or wsgi.py
+from bufferlog import BufferLog
+from bufferlog.adapters import StdOutAdapter
+
+bl = BufferLog(adapters=[StdOutAdapter()])
+bl.init_django()
+```
+
+## Configuration
+
+```python
+from bufferlog import BufferLog, BufferLogConfig
+from bufferlog.adapters import DatadogAdapter, SplunkAdapter, StdOutAdapter
+
+bl = BufferLog(config=BufferLogConfig(
+    # Max logs per request buffer. Oldest overwritten when full.
+    buffer_capacity=100,
+
+    # Log levels that trigger immediate flush.
+    flush_on_levels=["error", "critical"],
+
+    # HTTP status codes that trigger flush.
+    flush_on_status_codes=[500, 501, 502, 503, 504],
+
+    # Downstream targets for flushed logs.
+    adapters=[
+        DatadogAdapter(api_key="your-key"),
+        SplunkAdapter(token="your-token", url="https://splunk.example.com"),
+        StdOutAdapter(),
+    ],
+
+    # Enable/disable buffering.
+    enabled=True,
+
+    # If True, logs fall through to stderr on adapter failure.
+    fail_open=True,
+
+    # Scrub PII before logs enter the buffer.
+    scrubber=lambda msg, meta: (msg.replace("password", "***"), meta),
+))
+```
+
+## Control Plane
+
+Connect to the BufferLog Control Plane for remote policy management and usage tracking.
+
+```python
+from bufferlog import BufferLog, ControlPlaneConfig
+
+bl = BufferLog(
+    control_plane=ControlPlaneConfig(
+        url="https://control.bufferlog.io",
+        api_key="bl_sk_your_key",
+        poll_interval_s=60,
+        telemetry_interval_s=60,
+    )
+)
+```
+
+When connected:
+- **Poll for policy updates** — buffer capacity, sampling rate, flush triggers, and bypass rules can all be changed remotely.
+- **Push usage metrics** — counters (never log data) are sent to power the ROI dashboard.
+
+If the control plane is unreachable, the SDK continues with its last known configuration.
+
+## Metrics
+
+```python
+metrics = bl.get_metrics()
+# {
+#   "buffers":  {"created": 1000, "discarded": 995, "flushed": 5, "active": 0},
+#   "flash":    {"flush_count": 5, "events_flushed": 45, "adapter_errors": 0}
+# }
+```
+
+## Shutdown
+
+Stop background threads and send a final telemetry report.
+
+```python
+import atexit
+
+atexit.register(bl.shutdown)
+```
+
+## License
+
+[MIT](LICENSE)

bufferlog-0.1.0/bufferlog/__init__.py

@@ -0,0 +1,170 @@
+"""
+BufferLog — Python SDK
+
+Buffer logs in memory per-request. Flush only on errors. Save 90%+ on APM costs.
+
+Usage:
+    from bufferlog import BufferLog
+    from bufferlog.adapters import StdOutAdapter
+
+    bl = BufferLog(adapters=[StdOutAdapter(pretty=True)])
+    bl.init_flask(app)
+
+    # Or for FastAPI:
+    from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+    app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from .config import BufferLogConfig, ControlPlaneConfig
+from .ring_buffer import RingBuffer
+from .log_event import LogEvent, LogLevel
+from .buffer_manager import BufferManager, BufferManagerMetrics
+from .flash_controller import FlashController
+from .adapters import StdOutAdapter
+from .integrations import BufferLogHandler
+from .control_plane.policy_fetcher import PolicyFetcher
+from .control_plane.telemetry_reporter import TelemetryReporter
+
+if TYPE_CHECKING:
+    from flask import Flask
+
+__version__ = "0.1.0"
+__all__ = [
+    "BufferLog",
+    "BufferLogConfig",
+    "ControlPlaneConfig",
+    "RingBuffer",
+    "LogEvent",
+    "LogLevel",
+    "BufferManager",
+    "BufferManagerMetrics",
+    "FlashController",
+    "BufferLogHandler",
+    "StdOutAdapter",
+    "PolicyFetcher",
+    "TelemetryReporter",
+]
+
+
+class BufferLog:
+    """High-level SDK entry point.
+
+    Creates and wires all components together. The recommended way to
+    use the BufferLog Python SDK.
+    """
+
+    def __init__(self, config: Optional[BufferLogConfig] = None, **kwargs: Any) -> None:
+        if config is None:
+            config = BufferLogConfig(**kwargs)
+        self.config = config
+
+        self.buffer_manager = BufferManager(config.buffer_capacity)
+
+        adapters = config.adapters if config.adapters else [StdOutAdapter(pretty=True)]
+        self.flash_controller = FlashController(
+            adapters=adapters,
+            fail_open=config.fail_open,
+        )
+
+        self.policy_fetcher: Optional[PolicyFetcher] = None
+        self.telemetry_reporter: Optional[TelemetryReporter] = None
+
+        if config.control_plane:
+            self.policy_fetcher = PolicyFetcher(
+                url=config.control_plane.url,
+                api_key=config.control_plane.api_key,
+                interval_s=config.control_plane.poll_interval_s,
+            )
+            self.telemetry_reporter = TelemetryReporter(
+                url=config.control_plane.url,
+                api_key=config.control_plane.api_key,
+                interval_s=config.control_plane.telemetry_interval_s,
+                metrics_provider=lambda: self.get_metrics_dict(),
+            )
+            self.policy_fetcher.start()
+            self.telemetry_reporter.start()
+
+    # ---- Framework integrations ----
+
+    def init_flask(self, app: "Flask") -> None:
+        """Register BufferLog middleware on a Flask app."""
+        from .middleware.flask_mw import init_flask
+
+        init_flask(app, self.buffer_manager, self.flash_controller, self.config)
+
+    def init_django(self) -> None:
+        """Configure the Django middleware class with this BufferLog instance.
+
+        After calling this, add 'bufferlog.middleware.django_mw.BufferLogDjangoMiddleware'
+        to your MIDDLEWARE list in settings.py.
+        """
+        from .middleware.django_mw import BufferLogDjangoMiddleware
+
+        BufferLogDjangoMiddleware._buffer_manager = self.buffer_manager
+        BufferLogDjangoMiddleware._flash_controller = self.flash_controller
+        BufferLogDjangoMiddleware._config = self.config
+
+    def asgi_kwargs(self) -> Dict[str, Any]:
+        """Return kwargs for adding the ASGI middleware.
+
+        Usage:
+            from bufferlog.middleware.fastapi_mw import BufferLogMiddleware
+            app.add_middleware(BufferLogMiddleware, **bl.asgi_kwargs())
+        """
+        return {
+            "buffer_manager": self.buffer_manager,
+            "flash_controller": self.flash_controller,
+            "config": self.config,
+        }
+
+    # ---- Logging integration ----
+
+    def logging_handler(self) -> BufferLogHandler:
+        """Create a logging.Handler for Python's built-in logging module."""
+        return BufferLogHandler(
+            flash_controller=self.flash_controller,
+            config=self.config,
+        )
+
+    # ---- Metrics ----
+
+    def get_metrics(self) -> Dict[str, Any]:
+        bm = self.buffer_manager.get_metrics()
+        return {
+            "buffers": {
+                "created": bm.created,
+                "discarded": bm.discarded,
+                "flushed": bm.flushed,
+                "active": bm.active,
+            },
+            "flash": {
+                "flush_count": self.flash_controller.flush_count,
+                "events_flushed": self.flash_controller.events_flushed,
+                "adapter_errors": self.flash_controller.adapter_errors,
+            },
+        }
+
+    def get_metrics_dict(self) -> Dict[str, Any]:
+        """Flat metrics dict for telemetry reporter."""
+        bm = self.buffer_manager.get_metrics()
+        return {
+            "logs_discarded": bm.discarded,
+            "logs_flushed": bm.flushed,
+            "requests_success": bm.discarded,
+            "requests_error": bm.flushed,
+            "buffers_active": bm.active,
+        }
+
+    # ---- Shutdown ----
+
+    def shutdown(self) -> None:
+        """Gracefully stop background tasks."""
+        if self.policy_fetcher:
+            self.policy_fetcher.stop()
+        if self.telemetry_reporter:
+            self.telemetry_reporter.send()  # Final report
+            self.telemetry_reporter.stop()

bufferlog-0.1.0/bufferlog/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""BufferLog adapters."""
+
+from .stdout import StdOutAdapter
+from .datadog import DatadogAdapter
+from .splunk import SplunkAdapter
+
+__all__ = ["StdOutAdapter", "DatadogAdapter", "SplunkAdapter"]

bufferlog-0.1.0/bufferlog/adapters/base.py

@@ -0,0 +1,16 @@
+"""BufferLog — Adapter protocol (abstract base)."""
+
+from __future__ import annotations
+
+from typing import List, Protocol, runtime_checkable
+
+from ..log_event import LogEvent
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """Interface that all downstream adapters must implement."""
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        """Send a batch of log events to the downstream target."""
+        ...

bufferlog-0.1.0/bufferlog/adapters/datadog.py

@@ -0,0 +1,58 @@
+"""BufferLog — Datadog Adapter. Sends flushed logs to Datadog Logs API."""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import List, Optional
+
+from ..log_event import LogEvent
+
+
+class DatadogAdapter:
+    """Ships flushed log events to Datadog via the HTTP Logs API."""
+
+    DEFAULT_URL = "https://http-intake.logs.datadoghq.com/api/v2/logs"
+
+    def __init__(
+        self,
+        api_key: str,
+        url: Optional[str] = None,
+        service: str = "bufferlog",
+        source: str = "python",
+    ) -> None:
+        self._api_key = api_key
+        self._url = url or self.DEFAULT_URL
+        self._service = service
+        self._source = source
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        payload = [
+            {
+                "message": e.message,
+                "ddtags": f"context_id:{context_id}",
+                "ddsource": self._source,
+                "service": self._service,
+                "level": e.to_dict()["level"],
+                "timestamp": int(e.timestamp * 1000),
+                **(e.metadata or {}),
+            }
+            for e in events
+        ]
+
+        data = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            self._url,
+            data=data,
+            headers={
+                "Content-Type": "application/json",
+                "DD-API-KEY": self._api_key,
+            },
+            method="POST",
+        )
+
+        try:
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                resp.read()
+        except Exception:
+            pass  # Fail-open: don't crash the app if Datadog is unreachable

bufferlog-0.1.0/bufferlog/adapters/splunk.py

@@ -0,0 +1,55 @@
+"""BufferLog — Splunk HEC Adapter."""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import List, Optional
+
+from ..log_event import LogEvent
+
+
+class SplunkAdapter:
+    """Ships flushed log events to Splunk via the HTTP Event Collector."""
+
+    def __init__(
+        self,
+        token: str,
+        url: str,
+        source: str = "bufferlog",
+        sourcetype: str = "_json",
+        index: Optional[str] = None,
+    ) -> None:
+        self._token = token
+        self._url = url.rstrip("/") + "/services/collector/event"
+        self._source = source
+        self._sourcetype = sourcetype
+        self._index = index
+
+    def send(self, events: List[LogEvent], context_id: str) -> None:
+        for event in events:
+            payload = {
+                "event": event.to_dict(),
+                "source": self._source,
+                "sourcetype": self._sourcetype,
+                "time": event.timestamp,
+            }
+            if self._index:
+                payload["index"] = self._index
+
+            data = json.dumps(payload).encode("utf-8")
+            req = urllib.request.Request(
+                self._url,
+                data=data,
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Splunk {self._token}",
+                },
+                method="POST",
+            )
+
+            try:
+                with urllib.request.urlopen(req, timeout=10) as resp:
+                    resp.read()
+            except Exception:
+                pass  # Fail-open