ledger-sdk 1.0.0__py3-none-any.whl

ledger/__init__.py

@@ -0,0 +1,4 @@
+from ledger._version import __version__
+from ledger.core.client import LedgerClient
+
+__all__ = ["LedgerClient", "__version__"]

ledger/_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

ledger/core/__init__.py

@@ -0,0 +1,3 @@
+from ledger.core.client import LedgerClient
+
+__all__ = ["LedgerClient"]

ledger/core/buffer.py

@@ -0,0 +1,45 @@
+import asyncio
+import sys
+from collections import deque
+from typing import Any
+
+
+class LogBuffer:
+    def __init__(self, max_size: int = 10000):
+        self.max_size = max_size
+        self._queue: deque[dict[str, Any]] = deque(maxlen=max_size)
+        self._lock = asyncio.Lock()
+        self._dropped_count = 0
+
+    def add(self, log_entry: dict[str, Any]) -> None:
+        if len(self._queue) >= self.max_size:
+            self._queue.popleft()
+            self._dropped_count += 1
+            sys.stderr.write(
+                f"[Ledger SDK] WARNING: Buffer full ({self.max_size} logs), "
+                f"dropped oldest log (total dropped: {self._dropped_count})\n"
+            )
+            sys.stderr.flush()
+
+        self._queue.append(log_entry)
+
+    async def get_batch(self, max_batch_size: int) -> list[dict[str, Any]]:
+        async with self._lock:
+            batch_size = min(len(self._queue), max_batch_size)
+            if batch_size == 0:
+                return []
+
+            batch = [self._queue.popleft() for _ in range(batch_size)]
+            return batch
+
+    def size(self) -> int:
+        return len(self._queue)
+
+    def is_empty(self) -> bool:
+        return len(self._queue) == 0
+
+    def clear(self) -> None:
+        self._queue.clear()
+
+    def get_dropped_count(self) -> int:
+        return self._dropped_count

ledger/core/client.py

@@ -0,0 +1,297 @@
+import traceback
+from datetime import datetime, timezone
+from typing import Any
+
+import ledger.core.buffer as buffer_module
+import ledger.core.flusher as flusher_module
+import ledger.core.http_client as http_client_module
+import ledger.core.rate_limiter as rate_limiter_module
+import ledger.core.settings as settings_module
+import ledger.core.validator as validator_module
+
+
+class LedgerClient:
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "http://localhost:8000",
+        flush_interval: float = 5.0,
+        flush_size: int = 100,
+        max_buffer_size: int = 10000,
+        http_timeout: float = 5.0,
+        http_pool_size: int = 10,
+        rate_limit_buffer: float = 0.9,
+    ):
+        self._validate_config(
+            api_key=api_key,
+            base_url=base_url,
+            flush_interval=flush_interval,
+            flush_size=flush_size,
+            max_buffer_size=max_buffer_size,
+            http_timeout=http_timeout,
+            http_pool_size=http_pool_size,
+            rate_limit_buffer=rate_limit_buffer,
+        )
+
+        self.api_key = api_key
+        self.base_url = base_url
+
+        self._http_client = http_client_module.HTTPClient(
+            base_url=base_url,
+            api_key=api_key,
+            timeout=http_timeout,
+            pool_size=http_pool_size,
+        )
+
+        self._settings_manager = settings_module.SettingsManager()
+
+        self._buffer = buffer_module.LogBuffer(max_size=max_buffer_size)
+
+        rate_limits = self._settings_manager.get_rate_limits()
+        self._rate_limiter = rate_limiter_module.RateLimiter(
+            requests_per_minute=rate_limits["requests_per_minute"],
+            requests_per_hour=rate_limits["requests_per_hour"],
+            buffer=rate_limit_buffer,
+        )
+
+        constraints = self._settings_manager.get_constraints()
+        self._validator = validator_module.Validator(constraints)
+
+        self._flusher = flusher_module.BackgroundFlusher(
+            buffer=self._buffer,
+            http_client=self._http_client,
+            rate_limiter=self._rate_limiter,
+            flush_interval=flush_interval,
+            max_batch_size=constraints["max_batch_size"],
+        )
+
+        self._flusher.start()
+
+        self._sdk_start_time = datetime.now(timezone.utc)
+
+    def _validate_config(
+        self,
+        api_key: str,
+        base_url: str,
+        flush_interval: float,
+        flush_size: int,
+        max_buffer_size: int,
+        http_timeout: float,
+        http_pool_size: int,
+        rate_limit_buffer: float,
+    ) -> None:
+        errors = []
+
+        if not api_key or not isinstance(api_key, str):
+            errors.append("api_key must be a non-empty string")
+
+        if not api_key.startswith("ldg_"):
+            errors.append("api_key must start with 'ldg_' prefix")
+
+        if not base_url or not isinstance(base_url, str):
+            errors.append("base_url must be a non-empty string")
+
+        if not base_url.startswith(("http://", "https://")):
+            errors.append("base_url must start with 'http://' or 'https://'")
+
+        if flush_interval <= 0:
+            errors.append(f"flush_interval must be positive, got {flush_interval}")
+
+        if flush_size <= 0:
+            errors.append(f"flush_size must be positive, got {flush_size}")
+
+        if max_buffer_size <= 0:
+            errors.append(f"max_buffer_size must be positive, got {max_buffer_size}")
+
+        if http_timeout <= 0:
+            errors.append(f"http_timeout must be positive, got {http_timeout}")
+
+        if http_pool_size <= 0:
+            errors.append(f"http_pool_size must be positive, got {http_pool_size}")
+
+        if not 0 < rate_limit_buffer <= 1:
+            errors.append(f"rate_limit_buffer must be between 0 and 1, got {rate_limit_buffer}")
+
+        if errors:
+            raise ValueError("Invalid Ledger SDK configuration:\n  - " + "\n  - ".join(errors))
+
+    def log_info(
+        self,
+        message: str,
+        attributes: dict[str, Any] | None = None,
+    ) -> None:
+        self._log(
+            level="info",
+            log_type="console",
+            importance="standard",
+            message=message,
+            attributes=attributes,
+        )
+
+    def log_error(
+        self,
+        message: str,
+        attributes: dict[str, Any] | None = None,
+    ) -> None:
+        self._log(
+            level="error",
+            log_type="console",
+            importance="high",
+            message=message,
+            attributes=attributes,
+        )
+
+    def log_exception(
+        self,
+        exception: Exception,
+        message: str | None = None,
+        attributes: dict[str, Any] | None = None,
+    ) -> None:
+        stack_trace = "".join(
+            traceback.format_exception(
+                type(exception),
+                exception,
+                exception.__traceback__,
+            )
+        )
+
+        self._log(
+            level="error",
+            log_type="exception",
+            importance="high",
+            message=message or str(exception),
+            error_type=exception.__class__.__name__,
+            error_message=str(exception),
+            stack_trace=stack_trace,
+            attributes=attributes,
+        )
+
+    def _log(
+        self,
+        level: str,
+        log_type: str,
+        importance: str,
+        message: str | None = None,
+        error_type: str | None = None,
+        error_message: str | None = None,
+        stack_trace: str | None = None,
+        attributes: dict[str, Any] | None = None,
+    ) -> None:
+        from ledger._version import __version__
+
+        log_entry: dict[str, Any] = {
+            "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+            "level": level,
+            "log_type": log_type,
+            "importance": importance,
+        }
+
+        if message:
+            log_entry["message"] = message
+
+        if error_type:
+            log_entry["error_type"] = error_type
+
+        if error_message:
+            log_entry["error_message"] = error_message
+
+        if stack_trace:
+            log_entry["stack_trace"] = stack_trace
+
+        if attributes:
+            log_entry["attributes"] = attributes
+
+        log_entry["sdk_version"] = __version__
+        log_entry["platform"] = "python"
+
+        validated_log = self._validator.validate_log(log_entry)
+
+        self._buffer.add(validated_log)
+
+    def is_healthy(self) -> bool:
+        flusher_metrics = self._flusher.get_metrics()
+
+        if flusher_metrics["circuit_breaker_open"]:
+            return False
+
+        if flusher_metrics["consecutive_failures"] >= 3:
+            return False
+
+        buffer_utilization = (self._buffer.size() / self._buffer.max_size) * 100
+        if buffer_utilization > 90:
+            return False
+
+        return True
+
+    def get_health_status(self) -> dict[str, Any]:
+        flusher_metrics = self._flusher.get_metrics()
+        buffer_utilization = (self._buffer.size() / self._buffer.max_size) * 100
+
+        status = "healthy"
+        issues = []
+
+        if flusher_metrics["circuit_breaker_open"]:
+            status = "unhealthy"
+            issues.append("Circuit breaker is open (too many failures)")
+
+        if flusher_metrics["consecutive_failures"] >= 3:
+            status = "degraded" if status == "healthy" else status
+            issues.append(f"Consecutive failures: {flusher_metrics['consecutive_failures']}")
+
+        if buffer_utilization > 90:
+            status = "degraded" if status == "healthy" else status
+            issues.append(f"Buffer nearly full: {buffer_utilization:.1f}%")
+
+        if self._buffer.get_dropped_count() > 0:
+            status = "degraded" if status == "healthy" else status
+            issues.append(f"Dropped logs: {self._buffer.get_dropped_count()}")
+
+        return {
+            "status": status,
+            "healthy": status == "healthy",
+            "issues": issues if issues else None,
+            "buffer_utilization_percent": round(buffer_utilization, 2),
+            "circuit_breaker_open": flusher_metrics["circuit_breaker_open"],
+            "consecutive_failures": flusher_metrics["consecutive_failures"],
+        }
+
+    async def shutdown(self, timeout: float = 10.0) -> None:
+        await self._flusher.shutdown(timeout=timeout)
+        await self._http_client.close()
+
+    def get_metrics(self) -> dict[str, Any]:
+        from ledger._version import __version__
+
+        flusher_metrics = self._flusher.get_metrics()
+        uptime = (datetime.now(timezone.utc) - self._sdk_start_time).total_seconds()
+
+        return {
+            "sdk": {
+                "uptime_seconds": round(uptime, 2),
+                "version": __version__,
+            },
+            "buffer": {
+                "current_size": self._buffer.size(),
+                "max_size": self._buffer.max_size,
+                "total_dropped": self._buffer.get_dropped_count(),
+                "utilization_percent": round(
+                    (self._buffer.size() / self._buffer.max_size) * 100, 2
+                ),
+            },
+            "flusher": {
+                "total_flushes": flusher_metrics["total_flushes"],
+                "successful_flushes": flusher_metrics["successful_flushes"],
+                "failed_flushes": flusher_metrics["failed_flushes"],
+                "total_logs_sent": flusher_metrics["total_logs_sent"],
+                "total_logs_failed": flusher_metrics["total_logs_failed"],
+                "consecutive_failures": flusher_metrics["consecutive_failures"],
+                "circuit_breaker_open": flusher_metrics["circuit_breaker_open"],
+                "last_flush_time": flusher_metrics["last_flush_time"],
+                "last_error": flusher_metrics["last_error"],
+            },
+            "rate_limiter": {
+                "current_rate": self._rate_limiter.get_current_rate(),
+                "limit_per_minute": self._rate_limiter.limit_per_minute,
+            },
+            "errors": flusher_metrics["errors_by_type"],
+        }

ledger/core/flusher.py

@@ -0,0 +1,249 @@
+import asyncio
+import sys
+import time
+from typing import Any
+
+import httpx
+
+import ledger.core.buffer as buffer_module
+import ledger.core.http_client as http_client_module
+import ledger.core.rate_limiter as rate_limiter_module
+
+
+class BackgroundFlusher:
+    def __init__(
+        self,
+        buffer: "buffer_module.LogBuffer",
+        http_client: "http_client_module.HTTPClient",
+        rate_limiter: "rate_limiter_module.RateLimiter",
+        flush_interval: float = 5.0,
+        max_batch_size: int = 1000,
+        max_retries: int = 3,
+        retry_backoff_base: float = 2.0,
+    ):
+        self.buffer = buffer
+        self.http_client = http_client
+        self.rate_limiter = rate_limiter
+        self.flush_interval = flush_interval
+        self.max_batch_size = max_batch_size
+        self.max_retries = max_retries
+        self.retry_backoff_base = retry_backoff_base
+
+        self._task: asyncio.Task[Any] | None = None
+        self._shutdown_event = asyncio.Event()
+
+        self._metrics = {
+            "total_flushes": 0,
+            "successful_flushes": 0,
+            "failed_flushes": 0,
+            "total_logs_sent": 0,
+            "total_logs_failed": 0,
+            "consecutive_failures": 0,
+            "last_flush_time": None,
+            "last_error": None,
+            "errors_by_type": {},
+        }
+
+        self._circuit_breaker_open = False
+        self._circuit_breaker_opened_at = None
+        self._circuit_breaker_threshold = 5
+        self._circuit_breaker_timeout = 60.0
+
+    def start(self) -> None:
+        if self._task is None or self._task.done():
+            self._task = asyncio.create_task(self._run())
+
+    async def _run(self) -> None:
+        while not self._shutdown_event.is_set():
+            try:
+                await asyncio.sleep(self.flush_interval)
+
+                if self.buffer.is_empty():
+                    continue
+
+                await self._flush_with_retry()
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self._log_error(f"Unexpected error in flusher: {e}")
+
+    async def _flush_with_retry(self) -> None:
+        if self._circuit_breaker_open:
+            if time.time() - self._circuit_breaker_opened_at < self._circuit_breaker_timeout:
+                return
+
+            self._log_info("Circuit breaker: attempting recovery")
+            self._circuit_breaker_open = False
+
+        batch = await self.buffer.get_batch(self.max_batch_size)
+        if not batch:
+            return
+
+        self._metrics["total_flushes"] += 1
+
+        for attempt in range(self.max_retries):
+            try:
+                await self.rate_limiter.wait_if_needed()
+
+                success = await self._send_batch(batch)
+
+                if success:
+                    self._metrics["successful_flushes"] += 1
+                    self._metrics["total_logs_sent"] += len(batch)
+                    self._metrics["consecutive_failures"] = 0
+                    self._metrics["last_flush_time"] = time.time()
+                    return
+
+                if attempt < self.max_retries - 1:
+                    backoff = self.retry_backoff_base**attempt
+                    await asyncio.sleep(backoff)
+
+            except httpx.TimeoutException as e:
+                self._handle_network_error("Timeout", e, attempt)
+                if attempt < self.max_retries - 1:
+                    await asyncio.sleep(self.retry_backoff_base**attempt * 5.0)
+
+            except httpx.ConnectError as e:
+                self._handle_network_error("Connection refused", e, attempt)
+                if attempt < self.max_retries - 1:
+                    await asyncio.sleep(self.retry_backoff_base**attempt * 5.0)
+
+            except Exception as e:
+                self._handle_network_error("Unexpected error", e, attempt)
+                if attempt < self.max_retries - 1:
+                    await asyncio.sleep(self.retry_backoff_base**attempt)
+
+        self._metrics["failed_flushes"] += 1
+        self._metrics["total_logs_failed"] += len(batch)
+        self._metrics["consecutive_failures"] += 1
+
+        if self._metrics["consecutive_failures"] >= self._circuit_breaker_threshold:
+            self._circuit_breaker_open = True
+            self._circuit_breaker_opened_at = time.time()
+            self._log_error(
+                f"Circuit breaker OPEN: {self._metrics['consecutive_failures']} consecutive failures"
+            )
+
+    async def _send_batch(self, batch: list[dict[str, Any]]) -> bool:
+        try:
+            response = await self.http_client.post(
+                "/api/v1/ingest/batch",
+                json_data={"logs": batch},
+            )
+
+            if response.status_code == 202:
+                data = response.json()
+                accepted = data.get("accepted", 0)
+                rejected = data.get("rejected", 0)
+
+                if rejected > 0:
+                    self._log_warning(f"Batch: {accepted} accepted, {rejected} rejected")
+                    errors = data.get("errors", [])
+                    for error in errors[:5]:
+                        self._log_warning(f"  - {error}")
+
+                return True
+
+            if response.status_code == 429:
+                retry_after = int(response.headers.get("Retry-After", 60))
+                self._log_warning(f"Rate limit exceeded, sleeping {retry_after}s")
+                self._increment_error_count("rate_limit")
+                await asyncio.sleep(retry_after)
+                return False
+
+            if response.status_code == 503:
+                retry_after = int(response.headers.get("Retry-After", 60))
+                self._log_warning(f"Queue full (503), sleeping {retry_after}s")
+                self._increment_error_count("queue_full")
+                await asyncio.sleep(retry_after)
+                return False
+
+            if response.status_code == 401:
+                self._log_error("Invalid API key (401), stopping ingestion")
+                self._increment_error_count("auth_failure")
+                self._shutdown_event.set()
+                return False
+
+            if response.status_code == 400:
+                self._log_error(f"Bad request (400): {response.text}")
+                self._increment_error_count("validation_error")
+                return True
+
+            self._log_error(f"Unexpected response: {response.status_code} - {response.text}")
+            self._increment_error_count("server_error")
+            return False
+
+        except Exception:
+            raise
+
+    def _handle_network_error(self, error_type: str, error: Exception, attempt: int) -> None:
+        self._log_error(f"{error_type} (attempt {attempt + 1}/{self.max_retries}): {error}")
+        self._increment_error_count("network_error")
+        self._metrics["last_error"] = f"{error_type}: {error}"
+
+    def _increment_error_count(self, error_type: str) -> None:
+        if error_type not in self._metrics["errors_by_type"]:
+            self._metrics["errors_by_type"][error_type] = 0
+        self._metrics["errors_by_type"][error_type] += 1
+
+    def _log_info(self, message: str) -> None:
+        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        sys.stderr.write(f"[{timestamp}] [Ledger SDK] [INFO] {message}\n")
+        sys.stderr.flush()
+
+    def _log_warning(self, message: str) -> None:
+        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        sys.stderr.write(f"[{timestamp}] [Ledger SDK] [WARNING] {message}\n")
+        sys.stderr.flush()
+
+    def _log_error(self, message: str) -> None:
+        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        sys.stderr.write(f"[{timestamp}] [Ledger SDK] [ERROR] {message}\n")
+        sys.stderr.flush()
+
+    async def shutdown(self, timeout: float = 10.0) -> None:
+        self._log_info("Shutting down, flushing remaining logs...")
+        self._shutdown_event.set()
+
+        if self._task and not self._task.done():
+            try:
+                await asyncio.wait_for(self._task, timeout=2.0)
+            except asyncio.TimeoutError:
+                self._task.cancel()
+                try:
+                    await self._task
+                except asyncio.CancelledError:
+                    pass
+
+        remaining_attempts = 3
+        while not self.buffer.is_empty() and remaining_attempts > 0:
+            try:
+                await asyncio.wait_for(self._flush_with_retry(), timeout=timeout / 3)
+                remaining_attempts -= 1
+            except asyncio.TimeoutError:
+                self._log_warning("Flush timeout during shutdown")
+                break
+            except Exception as e:
+                self._log_error(f"Shutdown flush error: {e}")
+                break
+
+        if not self.buffer.is_empty():
+            dropped = self.buffer.size()
+            self._log_warning(f"Shutdown: {dropped} logs still in buffer (not sent)")
+
+        self._log_info("Shutdown complete")
+
+    def get_metrics(self) -> dict[str, Any]:
+        return {
+            "total_flushes": self._metrics["total_flushes"],
+            "successful_flushes": self._metrics["successful_flushes"],
+            "failed_flushes": self._metrics["failed_flushes"],
+            "total_logs_sent": self._metrics["total_logs_sent"],
+            "total_logs_failed": self._metrics["total_logs_failed"],
+            "consecutive_failures": self._metrics["consecutive_failures"],
+            "circuit_breaker_open": self._circuit_breaker_open,
+            "last_flush_time": self._metrics["last_flush_time"],
+            "last_error": self._metrics["last_error"],
+            "errors_by_type": self._metrics["errors_by_type"],
+        }

ledger/core/http_client.py

@@ -0,0 +1,67 @@
+from typing import Any
+
+import httpx
+
+
+class HTTPClient:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: float = 5.0,
+        pool_size: int = 10,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+
+        self._client = httpx.AsyncClient(
+            base_url=self.base_url,
+            timeout=httpx.Timeout(timeout),
+            limits=httpx.Limits(
+                max_connections=pool_size,
+                max_keepalive_connections=pool_size,
+                keepalive_expiry=30.0,
+            ),
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+                "User-Agent": "ledger-sdk-python/1.0.0",
+            },
+        )
+
+    async def post(
+        self,
+        path: str,
+        json_data: dict[str, Any],
+        headers: dict[str, str] | None = None,
+    ) -> httpx.Response:
+        merged_headers = self._client.headers.copy()
+        if headers:
+            merged_headers.update(headers)
+
+        response = await self._client.post(
+            path,
+            json=json_data,
+            headers=merged_headers,
+        )
+        return response
+
+    async def get(
+        self,
+        path: str,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> httpx.Response:
+        merged_headers = self._client.headers.copy()
+        if headers:
+            merged_headers.update(headers)
+
+        response = await self._client.get(
+            path,
+            params=params,
+            headers=merged_headers,
+        )
+        return response
+
+    async def close(self) -> None:
+        await self._client.aclose()