llm-cost-guard 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

llm_cost_guard/tracker.py

@@ -23,6 +23,7 @@ from llm_cost_guard.pricing.loader import PricingLoader, get_pricing_loader
 from llm_cost_guard.providers import detect_provider, get_provider
 from llm_cost_guard.rate_limit import RateLimit, RateLimiter
 from llm_cost_guard.span import Span, get_current_span
+from llm_cost_guard.audit import AuditLogger, AuditBackend, LoggingAuditBackend
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +53,8 @@ class CostTracker:
         budget_mode: Literal["local", "distributed"] = "local",
         streaming_budget_mode: Literal["estimate", "actual"] = "actual",
         streaming_max_output_estimate: int = 4096,
+        audit_enabled: bool = True,
+        audit_backend: Optional[AuditBackend] = None,
         **backend_kwargs: Any,
     ):
         """
@@ -72,6 +75,8 @@ class CostTracker:
             budget_mode: Budget enforcement mode (local or distributed)
             streaming_budget_mode: How to handle streaming budgets
             streaming_max_output_estimate: Max output tokens to estimate for streaming
+            audit_enabled: Enable audit logging for compliance
+            audit_backend: Custom audit backend (defaults to logging)
         """
         self._auto_detect_provider = auto_detect_provider
         self._on_tracking_failure = on_tracking_failure
@@ -83,27 +88,59 @@ class CostTracker:
         self._streaming_budget_mode = streaming_budget_mode
         self._streaming_max_output_estimate = streaming_max_output_estimate
 
+        # Graceful degradation metrics
+        self._metrics = {
+            "backend_failures": 0,
+            "fallback_activations": 0,
+            "budget_checks": 0,
+            "budget_exceeded_count": 0,
+            "rate_limit_exceeded_count": 0,
+            "tracking_errors": 0,
+        }
+        self._metrics_lock = threading.Lock()
+
+        # Initialize audit logging
+        self._audit = AuditLogger(
+            backend=audit_backend or LoggingAuditBackend(),
+            enabled=audit_enabled,
+        )
+
         # Initialize backend
         self._backend_url = backend
         self._fallback_backend: Optional[MemoryBackend] = None
+        self._using_fallback = False
         try:
             self._backend: Backend = get_backend(backend, **backend_kwargs)
         except Exception as e:
+            self._increment_metric("backend_failures")
             if on_tracking_failure == "block":
                 raise TrackingUnavailableError(f"Failed to initialize backend: {e}", backend)
             elif on_tracking_failure == "fallback":
                 logger.warning(f"Failed to initialize backend {backend}, using memory fallback: {e}")
                 self._backend = MemoryBackend()
                 self._fallback_backend = self._backend
+                self._using_fallback = True
+                self._increment_metric("fallback_activations")
+                self._audit.log_fallback_activated(backend, "memory", str(e))
             else:
                 logger.warning(f"Failed to initialize backend {backend}: {e}")
                 self._backend = MemoryBackend()
+                self._audit.log_tracking_failure(str(e), backend, "allow")
 
         # Initialize pricing
         self._pricing = PricingLoader(pricing_overrides=pricing_overrides)
 
         # Initialize budget tracking
         self._budget_tracker = BudgetTracker(budgets)
+        
+        # Log budget creation for audit
+        for budget in (budgets or []):
+            self._audit.log_budget_created(
+                budget.name,
+                budget.limit,
+                budget.period,
+                budget.action.value,
+            )
 
         # Initialize rate limiting
         self._rate_limiter = RateLimiter(rate_limits)
@@ -115,6 +152,11 @@ class CostTracker:
         # Last call tracking
         self._last_record: Optional[CostRecord] = None
         self._lock = threading.Lock()
+    
+    def _increment_metric(self, metric: str, amount: int = 1) -> None:
+        """Thread-safe metric increment."""
+        with self._metrics_lock:
+            self._metrics[metric] = self._metrics.get(metric, 0) + amount
 
     def track(
         self,
@@ -344,13 +386,31 @@ class CostTracker:
                 )
 
         # Check budgets
+        self._increment_metric("budget_checks")
         exceeded = self._budget_tracker.check_budget(total_cost, tags)
         for budget, action in exceeded:
-            if action == BudgetAction.BLOCK:
+            current_spending = self._budget_tracker.get_spending(budget.name)
+            
+            # Log to audit
+            if action == BudgetAction.WARN:
+                self._audit.log_budget_warning(
+                    budget.name,
+                    current_spending,
+                    budget.limit,
+                    current_spending / budget.limit,
+                )
+            elif action == BudgetAction.BLOCK:
+                self._increment_metric("budget_exceeded_count")
+                self._audit.log_budget_exceeded(
+                    budget.name,
+                    current_spending,
+                    budget.limit,
+                    "blocked",
+                )
                 raise BudgetExceededError(
                     f"Budget '{budget.name}' would be exceeded",
                     budget=budget,
-                    current=self._budget_tracker.get_spending(budget.name),
+                    current=current_spending,
                     limit=budget.limit,
                 )
 
@@ -358,6 +418,13 @@ class CostTracker:
         rate_exceeded = self._rate_limiter.check(model=model, provider=provider, tags=tags)
         if rate_exceeded:
             limit, current, retry_after = rate_exceeded[0]
+            self._increment_metric("rate_limit_exceeded_count")
+            self._audit.log_rate_limit_exceeded(
+                limit.name,
+                current,
+                limit.limit,
+                retry_after,
+            )
             raise RateLimitExceededError(
                 f"Rate limit '{limit.name}' exceeded",
                 limit_name=limit.name,
@@ -481,14 +548,24 @@ class CostTracker:
 
     def _handle_tracking_error(self, error: Exception) -> None:
         """Handle errors during tracking based on configuration."""
+        self._increment_metric("tracking_errors")
+        self._increment_metric("backend_failures")
+        
         if self._on_tracking_failure == "block":
+            self._audit.log_tracking_failure(str(error), self._backend_url, "blocked")
             raise TrackingUnavailableError(str(error), self._backend_url)
         elif self._on_tracking_failure == "fallback":
             logger.warning(f"Tracking error, using fallback: {error}")
             if self._fallback_backend is None:
                 self._fallback_backend = MemoryBackend()
+                self._using_fallback = True
+                self._increment_metric("fallback_activations")
+                self._audit.log_fallback_activated(
+                    self._backend_url, "memory", str(error)
+                )
         else:
             logger.warning(f"Tracking error (allowing): {error}")
+            self._audit.log_tracking_failure(str(error), self._backend_url, "allowed")
 
     def _check_tag_cardinality(self, tags: Dict[str, str]) -> None:
         """Check and track tag cardinality."""
@@ -648,6 +725,10 @@ class CostTracker:
         except Exception as e:
             errors.append(f"Backend health check failed: {e}")
 
+        # Check if using fallback
+        if self._using_fallback:
+            errors.append("Using fallback backend (primary unavailable)")
+
         # Check pricing freshness
         pricing_fresh = not self._pricing.is_stale
         if self._pricing.is_stale:
@@ -660,7 +741,7 @@ class CostTracker:
                 last_record_time = self._last_record.timestamp
 
         return HealthStatus(
-            healthy=backend_connected and pricing_fresh and len(errors) == 0,
+            healthy=backend_connected and pricing_fresh and len(errors) == 0 and not self._using_fallback,
             backend_connected=backend_connected,
             pricing_fresh=pricing_fresh,
             last_record_time=last_record_time,
@@ -670,6 +751,36 @@ class CostTracker:
             pricing_last_updated=self._pricing.last_updated,
         )
 
+    def get_metrics(self) -> Dict[str, Any]:
+        """
+        Get tracker metrics for observability.
+        
+        Returns metrics for:
+        - backend_failures: Number of backend operation failures
+        - fallback_activations: Number of times fallback was activated
+        - budget_checks: Total budget checks performed
+        - budget_exceeded_count: Number of budget exceeded events
+        - rate_limit_exceeded_count: Number of rate limit exceeded events
+        - tracking_errors: Total tracking errors
+        - using_fallback: Whether currently using fallback backend
+        """
+        with self._metrics_lock:
+            metrics = dict(self._metrics)
+        
+        metrics["using_fallback"] = self._using_fallback
+        metrics["backend_url"] = self._backend_url
+        
+        # Add backend-specific metrics if available
+        if hasattr(self._backend, "get_metrics"):
+            metrics["backend_metrics"] = self._backend.get_metrics()
+        
+        return metrics
+
+    @property
+    def audit(self) -> AuditLogger:
+        """Get the audit logger for querying audit events."""
+        return self._audit
+
     def on_budget_warning(self, callback: Callable[[Budget, float], None]) -> None:
         """Register a callback for budget warnings."""
         self._budget_tracker.on_warning(callback)

{llm_cost_guard-0.1.2.dist-info → llm_cost_guard-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llm-cost-guard
-Version: 0.1.2
+Version: 0.2.0
 Summary: Real-time cost tracking, budget enforcement, and usage analytics for LLM applications
 Project-URL: Homepage, https://github.com/prashantdudami/llm-cost-guard
 Project-URL: Documentation, https://github.com/prashantdudami/llm-cost-guard#readme
@@ -334,6 +334,57 @@ tracker = CostTracker(
 )
 ```
 
+## Audit Logging (v0.2.0+)
+
+Enterprise-ready audit trails for compliance:
+
+```python
+from llm_cost_guard import CostTracker, FileAuditBackend
+
+# Enable audit logging
+tracker = CostTracker(
+    audit_enabled=True,
+    audit_backend=FileAuditBackend("audit.log"),
+)
+
+# Query audit history
+events = tracker.audit.query(
+    event_type=AuditEventType.BUDGET_EXCEEDED,
+    start_date="2024-01-01",
+)
+
+# Get budget-specific history
+history = tracker.audit.get_budget_history("daily")
+```
+
+Audit events include:
+- Budget created/modified/deleted
+- Budget warnings and exceeded events
+- Rate limit exceeded events
+- Tracking failures and fallback activations
+
+## Observability Metrics (v0.2.0+)
+
+Track health and degradation:
+
+```python
+# Get tracker metrics
+metrics = tracker.get_metrics()
+print(metrics)
+# {
+#   "backend_failures": 0,
+#   "fallback_activations": 0,
+#   "budget_exceeded_count": 3,
+#   "tracking_errors": 0,
+#   "using_fallback": False,
+# }
+
+# Health check
+health = tracker.health_check()
+print(health.healthy)  # True/False
+print(health.errors)   # List of issues
+```
+
 ## Custom Pricing
 
 For negotiated enterprise rates:
@@ -349,6 +400,30 @@ tracker = CostTracker(
 )
 ```
 
+## Current Limitations
+
+Being transparent about what's not yet production-ready:
+
+| Feature | Status | Notes |
+|---------|--------|-------|
+| Distributed budgets (Redis) | ✅ v0.2.0 | Atomic operations with Lua scripts |
+| Audit logging | ✅ v0.2.0 | File and logging backends |
+| Graceful degradation metrics | ✅ v0.2.0 | Track failures and fallbacks |
+| PostgreSQL backend | 🚧 Planned | Use SQLite or Redis for now |
+| DynamoDB backend | 🚧 Planned | Use SQLite or Redis for now |
+| Encryption at rest | 🚧 Planned | Use encrypted volumes as workaround |
+| Multi-tenancy optimization | 🚧 Planned | Use tag-scoped budgets for now |
+| Streaming cost estimation | ⚠️ Limited | Actual cost tracked on completion |
+| Fine-tuning cost tracking | ❌ Not supported | |
+
+### Recommended for Production
+
+| Deployment Size | Backend | Notes |
+|-----------------|---------|-------|
+| Single instance | SQLite | Simple, no setup |
+| Multiple instances | Redis | Distributed budget enforcement |
+| High-volume (>1k req/s) | Redis | With sampling (coming soon) |
+
 ## Contributing
 
 Contributions are welcome! Please read our contributing guidelines and submit pull requests.

{llm_cost_guard-0.1.2.dist-info → llm_cost_guard-0.2.0.dist-info}/RECORD

@@ -1,14 +1,16 @@
-llm_cost_guard/__init__.py,sha256=XTXp-hAHbHD36YrJsZzWEQ-POPzcOq1WVGvSoUGrhCY,948
+llm_cost_guard/__init__.py,sha256=o-AIhQ_c-Zr0XqH7KRhG6U4RjpXi8fSZbEzTJ3qKsEE,1245
+llm_cost_guard/audit.py,sha256=WKJ-Q6arfhsP0y_Ibjkrd6-DBde7UWCIkQe3rGpJ5es,14280
 llm_cost_guard/budget.py,sha256=oT8m_7ir5hXfgxjfaaKDIAscRMuuk4PwseVz43IVAoA,10944
 llm_cost_guard/cli.py,sha256=GJZYwFkUHZ5txQhzBQkbnqQiZDr96Hf_zZ_hQNMvr1E,15166
 llm_cost_guard/exceptions.py,sha256=-WayjSAXLAY51FLdsFgGf0W8fMqGswq1zdnfY6ZUAhs,1727
 llm_cost_guard/models.py,sha256=ioH8-ZFt3PIx0AuDHEmPWtpOiW5HWw9tzI05B3bGI4U,3304
 llm_cost_guard/rate_limit.py,sha256=a3tG8RYSMAqirnmzSvNsJYOT7nDFAMoOU56OeUGuTFQ,7702
 llm_cost_guard/span.py,sha256=uXAIBVVLAfe7TDJwNl1TvG4Ay8RNNYSjJD2ZvLN7vYc,4828
-llm_cost_guard/tracker.py,sha256=RPOxXuocwISG-vycsPKZ8qZ2OuMxSN2tyGFG0Iqjffg,25322
-llm_cost_guard/backends/__init__.py,sha256=OSCbQRP_UHCyeOcHdRGEqBgA3o0TQGBqFEX8wE6DgP4,1594
+llm_cost_guard/tracker.py,sha256=39p1LOQ0LoA4N003AhyA9c3gtemyPlIcYK7fuwcvM0g,29748
+llm_cost_guard/backends/__init__.py,sha256=Jf6j8NWC4qapSU7o2Qt49-7tek20aQOmJQpsgFscM4o,1602
 llm_cost_guard/backends/base.py,sha256=8r9mm4mLNJ4jW9zhaDpvLpfFNg-uhdHCqWMp2L7K-SE,3289
 llm_cost_guard/backends/memory.py,sha256=O7FB2m_3qEoFG7cyQGbA1NanO-CalUkJCk8T666AkVE,9176
+llm_cost_guard/backends/redis_backend.py,sha256=QbYzOfD8uL1XifYC5ZK8hONOzwAs02A2FWAMGUKudu8,19361
 llm_cost_guard/backends/sqlite.py,sha256=Qps7opmUm07zDYl1YPC12uLGWhwNQrBZymtaVWOaePU,14411
 llm_cost_guard/clients/__init__.py,sha256=yY-8_u7nW03gZVrnNiAyvwjarCcP4exo2WfnfgwuUNU,237
 llm_cost_guard/clients/anthropic.py,sha256=pG9wqCCKdMGu8igIlFVSb96ndifUm9N8CseaDcXTuaU,6639
@@ -29,8 +31,8 @@ llm_cost_guard/providers/bedrock.py,sha256=WglOxb4a5so3kecsdUMzxIdS5LrvjRnH8rQvD
 llm_cost_guard/providers/openai.py,sha256=dPYwYpwsYLdGHg0O6UYExl9-NFb6JN3m6xgrqN14gGM,4073
 llm_cost_guard/tokenizers/__init__.py,sha256=qCdf9onm6uDeCrVIH6oLTTBte1IsmbCeyef5gm09dWc,178
 llm_cost_guard/tokenizers/base.py,sha256=roDyOMPuuGR2WCdR1Vns45UThrEB8fVYV-4iE6mu89Q,6901
-llm_cost_guard-0.1.2.dist-info/METADATA,sha256=mRBH7gLKgj_hAeVKF53-i-TnKPlyFqs7xf9pFVXKtGM,9970
-llm_cost_guard-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-llm_cost_guard-0.1.2.dist-info/entry_points.txt,sha256=k383VtNlqTw4JKJOPzQaUKXnTxDdpRzafvhKlzrk9jw,59
-llm_cost_guard-0.1.2.dist-info/licenses/LICENSE,sha256=nIvGzmGHr6xbKpf-EbwVZPpQDx65K1A8JJw50dREvps,1072
-llm_cost_guard-0.1.2.dist-info/RECORD,,
+llm_cost_guard-0.2.0.dist-info/METADATA,sha256=7L25b6dFUUDRtQCQAT_5K0_tW9BFvj1hjTse4zSGFKI,12120
+llm_cost_guard-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+llm_cost_guard-0.2.0.dist-info/entry_points.txt,sha256=k383VtNlqTw4JKJOPzQaUKXnTxDdpRzafvhKlzrk9jw,59
+llm_cost_guard-0.2.0.dist-info/licenses/LICENSE,sha256=nIvGzmGHr6xbKpf-EbwVZPpQDx65K1A8JJw50dREvps,1072
+llm_cost_guard-0.2.0.dist-info/RECORD,,