altcodepro-polydb-python 2.5.9__py3-none-any.whl → 2.5.11__py3-none-any.whl

polydb/multitenancy.py

@@ -2,11 +2,12 @@
 """
 Multi-tenancy enforcement and isolation
 """
+
 import re
-from typing import Dict, Any, List, Optional, Callable
 from contextvars import ContextVar
 from dataclasses import dataclass, field
 from enum import Enum
+from typing import Any, Dict, List, Optional
 
 from .errors import ValidationError
 
@@ -42,6 +43,7 @@ def _validate_sql_identifier(value: str, *, kind: str) -> str:
 
 class IsolationLevel(Enum):
     """Tenant isolation levels"""
+
     SHARED_SCHEMA = "shared"  # Shared tables with tenant_id
     SEPARATE_SCHEMA = "schema"  # Separate schema per tenant
     SEPARATE_DATABASE = "database"  # Separate DB per tenant
@@ -50,6 +52,7 @@ class IsolationLevel(Enum):
 @dataclass
 class TenantConfig:
     """Tenant configuration"""
+
     tenant_id: str
     isolation_level: IsolationLevel
     schema_name: Optional[str] = None
@@ -61,18 +64,18 @@ class TenantConfig:
 
 class TenantRegistry:
     """Registry of tenant configurations"""
-    
+
     def __init__(self):
         self._tenants: Dict[str, TenantConfig] = {}
-    
+
     def register(self, config: TenantConfig):
         """Register tenant"""
         self._tenants[config.tenant_id] = config
-    
+
     def get(self, tenant_id: str) -> Optional[TenantConfig]:
         """Get tenant config"""
         return self._tenants.get(tenant_id)
-    
+
     def list_all(self) -> List[TenantConfig]:
         """List all tenants"""
         return list(self._tenants.values())
@@ -80,24 +83,23 @@ class TenantRegistry:
 
 class TenantContext:
     """Tenant context management"""
-    
-    current_tenant: ContextVar[Optional[TenantConfig]] = \
-        ContextVar("current_tenant", default=None)
-    
+
+    current_tenant: ContextVar[Optional[TenantConfig]] = ContextVar("current_tenant", default=None)
+
     @classmethod
     def set_tenant(cls, tenant_id: str, registry: TenantRegistry):
         """Set current tenant"""
         config = registry.get(tenant_id)
         if not config:
             raise ValueError(f"Tenant not found: {tenant_id}")
-        
+
         cls.current_tenant.set(config)
-    
+
     @classmethod
     def get_tenant(cls) -> Optional[TenantConfig]:
         """Get current tenant"""
         return cls.current_tenant.get()
-    
+
     @classmethod
     def clear(cls):
         """Clear tenant context"""
@@ -106,53 +108,45 @@ class TenantContext:
 
 class TenantIsolationEnforcer:
     """Enforces tenant isolation at query level"""
-    
+
     def __init__(self, registry: TenantRegistry):
         self.registry = registry
-    
-    def enforce_read(
-        self,
-        model: str,
-        query: Dict[str, Any]
-    ) -> Dict[str, Any]:
+
+    def enforce_read(self, model: str, query: Dict[str, Any]) -> Dict[str, Any]:
         """Enforce tenant isolation on read"""
         tenant = TenantContext.get_tenant()
-        
+
         if not tenant:
             raise ValueError("No tenant context set")
-        
+
         if tenant.isolation_level == IsolationLevel.SHARED_SCHEMA:
             # Add tenant_id filter
             query = query.copy()
-            query['tenant_id'] = tenant.tenant_id
-        
+            query["tenant_id"] = tenant.tenant_id
+
         return query
-    
-    def enforce_write(
-        self,
-        model: str,
-        data: Dict[str, Any]
-    ) -> Dict[str, Any]:
+
+    def enforce_write(self, model: str, data: Dict[str, Any]) -> Dict[str, Any]:
         """Enforce tenant isolation on write"""
         tenant = TenantContext.get_tenant()
-        
+
         if not tenant:
             raise ValueError("No tenant context set")
-        
+
         if tenant.isolation_level == IsolationLevel.SHARED_SCHEMA:
             # Add tenant_id
             data = data.copy()
-            data['tenant_id'] = tenant.tenant_id
-        
+            data["tenant_id"] = tenant.tenant_id
+
         return data
-    
+
     def get_table_name(self, base_table: str) -> str:
         """Get tenant-specific table name"""
         tenant = TenantContext.get_tenant()
-        
+
         if not tenant:
             raise ValueError("No tenant context set")
-        
+
         if tenant.isolation_level == IsolationLevel.SEPARATE_SCHEMA:
             return f"{tenant.schema_name}.{base_table}"
         elif tenant.isolation_level == IsolationLevel.SEPARATE_DATABASE:
@@ -163,28 +157,29 @@ class TenantIsolationEnforcer:
 
 class TenantQuotaManager:
     """Manages tenant resource quotas"""
-    
+
     def __init__(self, registry: TenantRegistry):
         self.registry = registry
         self._usage: Dict[str, Dict[str, float]] = {}
-    
+
     def check_storage_quota(self, tenant_id: str, size_gb: float) -> bool:
         """Check if operation would exceed storage quota"""
         config = self.registry.get(tenant_id)
         if not config or not config.storage_quota_gb:
             return True
-        
-        current_usage = self._usage.get(tenant_id, {}).get('storage_gb', 0.0)
+
+        current_usage = self._usage.get(tenant_id, {}).get("storage_gb", 0.0)
         return (current_usage + size_gb) <= config.storage_quota_gb
-    
+
     def record_storage_usage(self, tenant_id: str, size_gb: float):
         """Record storage usage"""
         if tenant_id not in self._usage:
             self._usage[tenant_id] = {}
-        
-        self._usage[tenant_id]['storage_gb'] = \
-            self._usage[tenant_id].get('storage_gb', 0.0) + size_gb
-    
+
+        self._usage[tenant_id]["storage_gb"] = (
+            self._usage[tenant_id].get("storage_gb", 0.0) + size_gb
+        )
+
     def get_usage(self, tenant_id: str) -> Dict[str, float]:
         """Get tenant resource usage"""
         return self._usage.get(tenant_id, {})
@@ -192,46 +187,42 @@ class TenantQuotaManager:
 
 class TenantMigrationManager:
     """Manages tenant migrations and onboarding"""
-    
+
     def __init__(self, factory, registry: TenantRegistry):
         self.factory = factory
         self.registry = registry
-    
+
     def provision_tenant(self, config: TenantConfig):
         """Provision new tenant"""
         # Register tenant
         self.registry.register(config)
-        
+
         if config.isolation_level == IsolationLevel.SEPARATE_SCHEMA:
             # Create schema
             schema = _validate_sql_identifier(config.schema_name, kind="schema_name")
             schema_sql = f"CREATE SCHEMA IF NOT EXISTS {schema};"
             self.factory._sql.execute(schema_sql)
-        
+
         elif config.isolation_level == IsolationLevel.SEPARATE_DATABASE:
             # Create database (requires superuser)
-            database = _validate_sql_identifier(
-                config.database_name, kind="database_name"
-            )
+            database = _validate_sql_identifier(config.database_name, kind="database_name")
             db_sql = f"CREATE DATABASE {database};"
             self.factory._sql.execute(db_sql)
-    
+
     def deprovision_tenant(self, tenant_id: str):
         """Deprovision tenant"""
         config = self.registry.get(tenant_id)
         if not config:
             return
-        
+
         if config.isolation_level == IsolationLevel.SEPARATE_SCHEMA:
             # Drop schema
             schema = _validate_sql_identifier(config.schema_name, kind="schema_name")
             schema_sql = f"DROP SCHEMA IF EXISTS {schema} CASCADE;"
             self.factory._sql.execute(schema_sql)
-        
+
         elif config.isolation_level == IsolationLevel.SEPARATE_DATABASE:
             # Drop database
-            database = _validate_sql_identifier(
-                config.database_name, kind="database_name"
-            )
+            database = _validate_sql_identifier(config.database_name, kind="database_name")
             db_sql = f"DROP DATABASE IF EXISTS {database};"
-            self.factory._sql.execute(db_sql)
+            self.factory._sql.execute(db_sql)

polydb/observability/logging.py

@@ -15,7 +15,6 @@ import contextvars
 import json
 import logging
 import os
-import time
 from typing import Any, Optional
 
 # ---------------------------------------------------------------------------

polydb/overflow_gc.py

@@ -0,0 +1,179 @@
+"""
+src/polydb/overflow_gc.py
+
+Garbage collection for orphaned NoSQL KV overflow blobs.
+
+Every `NoSQLKVAdapter` spills an oversized record's JSON to the paired
+object store under a content-addressed key (``overflow/<md5>.json``) and
+keeps only a small reference row (``_overflow``/``_blob_key``/``_size``/
+``_checksum``) in the KV store itself (see
+``NoSQLKVAdapter._check_overflow``). An update to an already-overflowed
+record writes a *new* blob (a new md5 for the new content) and leaves the
+old one behind with nothing referencing it any more -- ``delete()`` only
+ever removes the reference row, never the blob it pointed at. Nothing in
+this codebase has ever swept those orphans, so overflow storage grows
+without bound for the life of a deployment (see CLAUDE.md's "Overflow
+blobs are never garbage-collected").
+
+This module closes that gap with a conservative two-pass mark-and-sweep:
+a blob under the given prefix that isn't referenced by any live row is
+only a *candidate* the first time it's seen -- it's recorded, with the
+wall-clock time it was first seen, in a small JSON state blob written
+back into the same object store (``<prefix>_gc_state.json``). A candidate
+is only actually deleted once ``grace_seconds`` have elapsed since that
+first sighting *and* it is still unreferenced on this run. This gives a
+real grace window without needing last-modified timestamps from the
+object store, which the ``ObjectStorageAdapter`` protocol does not
+expose on any backend.
+
+A blob that becomes referenced again between two sweeps (a genuine race
+with a concurrent write, not merely a slow one -- content-addressed keys
+mean a live row can only ever reference the blob matching its own
+current content) drops out of the candidate list rather than being
+deleted, so a sweep can never delete a blob a live row still points at.
+
+PolyDB doesn't own a model registry (see databaseFactory.py's own "dumb
+storage layer" design stance) -- the caller supplies the list of models
+to scan, e.g.:
+
+    from polydb.overflow_gc import sweep_overflow_blobs
+    from polydb.cloudDatabaseFactory import CloudDatabaseFactory
+
+    factory = CloudDatabaseFactory()
+    report = sweep_overflow_blobs(
+        factory.get_object_storage(),
+        factory.get_nosql_kv(),
+        models=[Order, Widget],
+    )
+
+Run on a schedule (daily is plenty) -- a blob only gets deleted on the
+*second* sweep that finds it orphaned, at least `grace_seconds` after the
+first.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
+
+from .json_safe import json_safe
+
+if TYPE_CHECKING:
+    from .base.NoSQLKVAdapter import NoSQLKVAdapter
+    from .base.ObjectStorageAdapter import ObjectStorageAdapter
+
+
+DEFAULT_PREFIX = "overflow/"
+DEFAULT_GRACE_SECONDS = 24 * 60 * 60  # 24h
+
+
+@dataclass
+class GCReport:
+    """Result of one sweep_overflow_blobs() call."""
+
+    scanned_blobs: int = 0
+    referenced_blobs: int = 0
+    new_candidates: List[str] = field(default_factory=list)
+    still_pending: List[str] = field(default_factory=list)
+    deleted: List[str] = field(default_factory=list)
+    errors: List[Tuple[str, str]] = field(default_factory=list)
+
+
+def _state_key(prefix: str) -> str:
+    return f"{prefix.rstrip('/')}/_gc_state.json"
+
+
+def _load_state(object_storage: "ObjectStorageAdapter", prefix: str) -> dict:
+    try:
+        raw = object_storage.get(_state_key(prefix))
+    except Exception:
+        return {}
+    if not raw:
+        return {}
+    try:
+        return json.loads(raw.decode() if isinstance(raw, bytes) else raw)
+    except Exception:
+        return {}
+
+
+def _save_state(object_storage: "ObjectStorageAdapter", prefix: str, state: dict) -> None:
+    object_storage.put(_state_key(prefix), json.dumps(state, default=json_safe).encode())
+
+
+def _referenced_blob_keys(adapter: "NoSQLKVAdapter", models: Iterable[type]) -> set:
+    """Every _blob_key a live row currently points at, across the given models."""
+    referenced: set = set()
+    for model in models:
+        for row in adapter._query_raw(model, {}, None):
+            blob_key = row.get("_blob_key")
+            if blob_key:
+                referenced.add(blob_key)
+    return referenced
+
+
+def sweep_overflow_blobs(
+    object_storage: "ObjectStorageAdapter",
+    adapter: "NoSQLKVAdapter",
+    models: Iterable[type],
+    *,
+    prefix: str = DEFAULT_PREFIX,
+    grace_seconds: int = DEFAULT_GRACE_SECONDS,
+    dry_run: bool = False,
+    now: Optional[float] = None,
+) -> GCReport:
+    """
+    Run one GC pass over `prefix` in `object_storage`, treating every
+    `_blob_key` referenced by a live row of `models` (read through
+    `adapter`) as live. Anything else is orphaned; an orphan is only
+    deleted once it has been orphaned for at least `grace_seconds` across
+    two separate sweeps. `dry_run=True` reports what *would* be deleted
+    without deleting anything or advancing the grace clock.
+    """
+    now = time.time() if now is None else now
+    report = GCReport()
+
+    state_key_name = _state_key(prefix)
+    all_keys = [k for k in object_storage.list(prefix) if k != state_key_name]
+    report.scanned_blobs = len(all_keys)
+
+    referenced = _referenced_blob_keys(adapter, models)
+    report.referenced_blobs = len(referenced)
+
+    state = _load_state(object_storage, prefix)
+    next_state: dict = {}
+
+    for key in all_keys:
+        if key in referenced:
+            continue  # live -- never a candidate, never written into next_state
+
+        first_seen = state.get(key)
+        if first_seen is None:
+            # First sweep to see this one orphaned: start its clock, don't delete yet.
+            next_state[key] = now
+            report.new_candidates.append(key)
+            continue
+
+        if now - first_seen < grace_seconds:
+            next_state[key] = first_seen
+            report.still_pending.append(key)
+            continue
+
+        # Orphaned on this sweep AND the previous one, past the grace window.
+        if dry_run:
+            report.deleted.append(key)  # what WOULD be deleted
+            next_state[key] = first_seen
+            continue
+
+        try:
+            object_storage.delete(key)
+            report.deleted.append(key)
+        except Exception as e:
+            report.errors.append((key, str(e)))
+            next_state[key] = first_seen  # retry on the next sweep
+
+    if not dry_run:
+        _save_state(object_storage, prefix, next_state)
+
+    return report

polydb/query.py

@@ -1,9 +1,11 @@
 # src/polydb/query.py
 
 from __future__ import annotations
+
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Mapping, Optional, Union
 from enum import Enum
+from typing import Any, Dict, List, Mapping, Optional, Union
+
 from .errors import ValidationError
 from .utils import validate_column_name
 

polydb/retry.py

@@ -3,9 +3,9 @@
 Retry logic with exponential backoff and metrics hooks
 """
 
+import logging
 import os
 import time
-import logging
 from functools import wraps
 from typing import Callable, Tuple, Type
 

polydb/schema.py

@@ -2,12 +2,11 @@
 """
 Schema management and migrations
 """
-from typing import Dict, List, Optional, Any
+
 from dataclasses import dataclass
 from decimal import Decimal
 from enum import Enum
-import json
-from datetime import datetime
+from typing import Any, Dict, List, Optional
 
 from .errors import ValidationError
 from .utils import validate_column_name, validate_table_name
@@ -66,22 +65,22 @@ class Index:
 
 class SchemaBuilder:
     """Build SQL schema definitions"""
-    
+
     def __init__(self):
         self.columns: List[Column] = []
         self.indexes: List[Index] = []
         self.primary_keys: List[str] = []
-    
-    def add_column(self, column: Column) -> 'SchemaBuilder':
+
+    def add_column(self, column: Column) -> "SchemaBuilder":
         self.columns.append(column)
         if column.primary_key:
             self.primary_keys.append(column.name)
         return self
-    
-    def add_index(self, index: Index) -> 'SchemaBuilder':
+
+    def add_index(self, index: Index) -> "SchemaBuilder":
         self.indexes.append(index)
         return self
-    
+
     def to_create_table(self, table_name: str) -> str:
         """Generate CREATE TABLE statement"""
         validate_table_name(table_name)
@@ -89,42 +88,42 @@ class SchemaBuilder:
 
         for col in self.columns:
             parts = [validate_column_name(col.name)]
-            
+
             # Type
             if col.type == ColumnType.VARCHAR and col.max_length:
                 parts.append(f"VARCHAR({col.max_length})")
             else:
                 parts.append(col.type.value)
-            
+
             # Nullable
             if not col.nullable:
                 parts.append("NOT NULL")
-            
+
             # Default
             if col.default is not None:
                 parts.append(_render_default(col.default))
-            
+
             # Unique
             if col.unique:
                 parts.append("UNIQUE")
-            
+
             col_defs.append(" ".join(parts))
-        
+
         # Primary key
         if self.primary_keys:
             pk_cols = [validate_column_name(c) for c in self.primary_keys]
             col_defs.append(f"PRIMARY KEY ({', '.join(pk_cols)})")
-        
+
         sql = f"CREATE TABLE IF NOT EXISTS {table_name} (\n"
         sql += ",\n".join(f"  {col}" for col in col_defs)
         sql += "\n);"
-        
+
         return sql
-    
+
     def to_create_indexes(self, table_name: str) -> List[str]:
         """Generate CREATE INDEX statements"""
         statements = []
-        
+
         validate_table_name(table_name)
 
         for idx in self.indexes:
@@ -135,17 +134,17 @@ class SchemaBuilder:
                 f"ON {table_name}({cols});"
             )
             statements.append(sql)
-        
+
         return statements
 
 
 class MigrationManager:
     """Database migration management"""
-    
+
     def __init__(self, sql_adapter):
         self.sql = sql_adapter
         self._ensure_migrations_table()
-    
+
     def _ensure_migrations_table(self):
         """Create migrations tracking table"""
         schema = """
@@ -159,74 +158,60 @@ class MigrationManager:
         );
         """
         self.sql.execute(schema)
-    
+
     def apply_migration(
-        self,
-        version: str,
-        name: str,
-        up_sql: str,
-        down_sql: Optional[str] = None
+        self, version: str, name: str, up_sql: str, down_sql: Optional[str] = None
     ) -> bool:
         """Apply a migration"""
         import hashlib
-        
+
         # Check if already applied
         existing = self.sql.execute(
-            "SELECT version FROM polydb_migrations WHERE version = %s",
-            [version],
-            fetch_one=True
+            "SELECT version FROM polydb_migrations WHERE version = %s", [version], fetch_one=True
         )
-        
+
         if existing:
             return False
-        
+
         # Calculate checksum
         checksum = hashlib.sha256(up_sql.encode()).hexdigest()
-        
+
         try:
             # Execute migration
             self.sql.execute(up_sql)
-            
+
             # Record migration
-            self.sql.insert('polydb_migrations', {
-                'version': version,
-                'name': name,
-                'rollback_sql': down_sql,
-                'checksum': checksum
-            })
-            
+            self.sql.insert(
+                "polydb_migrations",
+                {"version": version, "name": name, "rollback_sql": down_sql, "checksum": checksum},
+            )
+
             return True
         except Exception as e:
             raise Exception(f"Migration {version} failed: {str(e)}")
-    
+
     def rollback_migration(self, version: str) -> bool:
         """Rollback a migration"""
         migration = self.sql.execute(
             "SELECT rollback_sql FROM polydb_migrations WHERE version = %s",
             [version],
-            fetch_one=True
+            fetch_one=True,
         )
-        
-        if not migration or not migration.get('rollback_sql'):
+
+        if not migration or not migration.get("rollback_sql"):
             raise Exception(f"No rollback available for {version}")
-        
+
         try:
             # Execute rollback
-            self.sql.execute(migration['rollback_sql'])
-            
+            self.sql.execute(migration["rollback_sql"])
+
             # Remove from migrations
-            self.sql.execute(
-                "DELETE FROM polydb_migrations WHERE version = %s",
-                [version]
-            )
-            
+            self.sql.execute("DELETE FROM polydb_migrations WHERE version = %s", [version])
+
             return True
         except Exception as e:
             raise Exception(f"Rollback {version} failed: {str(e)}")
-    
+
     def get_applied_migrations(self) -> List[Dict[str, Any]]:
         """Get list of applied migrations"""
-        return self.sql.execute(
-            "SELECT * FROM polydb_migrations ORDER BY applied_at",
-            fetch=True
-        )
+        return self.sql.execute("SELECT * FROM polydb_migrations ORDER BY applied_at", fetch=True)