altcodepro-polydb-python 2.1.0__py3-none-any.whl

polydb/audit/AuditStorage.py

@@ -0,0 +1,136 @@
+# src/polydb/audit/AuditStorage.py
+from __future__ import annotations
+
+import threading
+from typing import Optional, Dict, Any
+
+from .models import AuditRecord
+from ..factory import CloudDatabaseFactory
+
+
+class AuditStorage:
+    """Audit log with distributed-safe hash chaining"""
+    
+    _lock = threading.Lock()
+    
+    def __init__(self):
+        self.factory = CloudDatabaseFactory()
+        self.sql = self.factory.get_sql()
+        self._ensure_table()
+    
+    def _ensure_table(self):
+        """Create audit table if not exists"""
+        try:
+            schema = """
+            CREATE TABLE IF NOT EXISTS polydb_audit_log (
+                audit_id UUID PRIMARY KEY,
+                timestamp TIMESTAMP NOT NULL,
+                tenant_id VARCHAR(255),
+                actor_id VARCHAR(255),
+                roles TEXT[],
+                action VARCHAR(50) NOT NULL,
+                model VARCHAR(255) NOT NULL,
+                entity_id VARCHAR(255),
+                storage_type VARCHAR(20) NOT NULL,
+                provider VARCHAR(50) NOT NULL,
+                success BOOLEAN NOT NULL,
+                before JSONB,
+                after JSONB,
+                changed_fields TEXT[],
+                trace_id VARCHAR(255),
+                request_id VARCHAR(255),
+                ip_address VARCHAR(45),
+                user_agent TEXT,
+                error TEXT,
+                hash VARCHAR(64) NOT NULL,
+                previous_hash VARCHAR(64),
+                created_at TIMESTAMP DEFAULT NOW()
+            );
+            
+            CREATE INDEX IF NOT EXISTS idx_audit_tenant_timestamp 
+                ON polydb_audit_log(tenant_id, timestamp DESC);
+            CREATE INDEX IF NOT EXISTS idx_audit_model_entity 
+                ON polydb_audit_log(model, entity_id);
+            CREATE INDEX IF NOT EXISTS idx_audit_actor 
+                ON polydb_audit_log(actor_id, timestamp DESC);
+            CREATE INDEX IF NOT EXISTS idx_audit_hash_chain 
+                ON polydb_audit_log(tenant_id, timestamp DESC, previous_hash);
+            """
+            
+            self.sql.execute(schema)
+        except Exception:
+            # Table may already exist
+            pass
+    
+    def get_last_hash(self, tenant_id: Optional[str]) -> Optional[str]:
+        """Get most recent hash with strict ordering (distributed-safe)"""
+        with self._lock:
+            try:
+                from ..query import QueryBuilder, Operator
+                
+                builder = QueryBuilder()
+                
+                if tenant_id is not None:
+                    builder.where('tenant_id', Operator.EQ, tenant_id)
+                
+                builder.order_by('timestamp', descending=True).take(1)
+                
+                results = self.sql.query_linq('polydb_audit_log', builder)
+                
+                if results and len(results) > 0:
+                    return results[0].get('hash')
+                
+                return None
+            except Exception:
+                return None
+    
+    def persist(self, record: AuditRecord) -> None:
+        """Persist with lock to ensure chain integrity"""
+        with self._lock:
+            self.sql.insert('polydb_audit_log', {
+                'audit_id': record.audit_id,
+                'timestamp': record.timestamp,
+                'tenant_id': record.tenant_id,
+                'actor_id': record.actor_id,
+                'roles': record.roles,
+                'action': record.action,
+                'model': record.model,
+                'entity_id': record.entity_id,
+                'storage_type': record.storage_type,
+                'provider': record.provider,
+                'success': record.success,
+                'before': record.before,
+                'after': record.after,
+                'changed_fields': record.changed_fields,
+                'trace_id': record.trace_id,
+                'request_id': record.request_id,
+                'ip_address': record.ip_address,
+                'user_agent': record.user_agent,
+                'error': record.error,
+                'hash': record.hash,
+                'previous_hash': record.previous_hash,
+            })
+    
+    def verify_chain(self, tenant_id: Optional[str] = None) -> bool:
+        """Verify hash chain integrity"""
+        from ..query import QueryBuilder, Operator
+        
+        builder = QueryBuilder()
+        
+        if tenant_id is not None:
+            builder.where('tenant_id', Operator.EQ, tenant_id)
+        
+        builder.order_by('timestamp', descending=False)
+        
+        records = self.sql.query_linq('polydb_audit_log', builder)
+        
+        if not records:
+            return True
+        
+        prev_hash = None
+        for record in records:
+            if record.get('previous_hash') != prev_hash:
+                return False
+            prev_hash = record.get('hash')
+        
+        return True

polydb/audit/__init__.py

@@ -0,0 +1,7 @@
+# src/polydb/audit/__init__.py
+from .models import AuditRecord
+from .context import AuditContext
+from .manager import AuditManager
+from .AuditStorage import AuditStorage
+
+__all__ = ['AuditRecord', 'AuditContext', 'AuditManager', 'AuditStorage']

polydb/audit/context.py

@@ -0,0 +1,53 @@
+# src/polydb/audit/AuditContext.py
+from contextvars import ContextVar
+from typing import Optional, List
+
+class AuditContext:
+    """Context variables for audit trail"""
+    
+    actor_id: ContextVar[Optional[str]] = ContextVar("actor_id", default=None)
+    roles: ContextVar[List[str]] = ContextVar("roles", default=[])
+    tenant_id: ContextVar[Optional[str]] = ContextVar("tenant_id", default=None)
+    trace_id: ContextVar[Optional[str]] = ContextVar("trace_id", default=None)
+    request_id: ContextVar[Optional[str]] = ContextVar("request_id", default=None)
+    ip_address: ContextVar[Optional[str]] = ContextVar("ip_address", default=None)
+    user_agent: ContextVar[Optional[str]] = ContextVar("user_agent", default=None)
+    
+    @classmethod
+    def set(
+        cls,
+        *,
+        actor_id: Optional[str] = None,
+        roles: Optional[List[str]] = None,
+        tenant_id: Optional[str] = None,
+        trace_id: Optional[str] = None,
+        request_id: Optional[str] = None,
+        ip_address: Optional[str] = None,
+        user_agent: Optional[str] = None,
+    ):
+        """Set audit context for current request"""
+        if actor_id is not None:
+            cls.actor_id.set(actor_id)
+        if roles is not None:
+            cls.roles.set(roles)
+        if tenant_id is not None:
+            cls.tenant_id.set(tenant_id)
+        if trace_id is not None:
+            cls.trace_id.set(trace_id)
+        if request_id is not None:
+            cls.request_id.set(request_id)
+        if ip_address is not None:
+            cls.ip_address.set(ip_address)
+        if user_agent is not None:
+            cls.user_agent.set(user_agent)
+    
+    @classmethod
+    def clear(cls):
+        """Clear all context variables"""
+        cls.actor_id.set(None)
+        cls.roles.set([])
+        cls.tenant_id.set(None)
+        cls.trace_id.set(None)
+        cls.request_id.set(None)
+        cls.ip_address.set(None)
+        cls.user_agent.set(None)

polydb/audit/manager.py

@@ -0,0 +1,47 @@
+# src/polydb/audit/manager.py
+from __future__ import annotations
+
+from typing import Optional, Dict, Any, List
+
+from .models import AuditRecord
+from .AuditStorage import AuditStorage
+from .context import AuditContext
+
+
+class AuditManager:
+    def __init__(self):
+        self.storage = AuditStorage()
+
+    def record(
+        self,
+        *,
+        action: str,
+        model: str,
+        entity_id: Optional[str],
+        storage_type: str,
+        provider: str,
+        success: bool,
+        before: Optional[Dict[str, Any]],
+        after: Optional[Dict[str, Any]],
+        error: Optional[str],
+        changed_fields: List[str] | None
+    ) -> None:
+        tenant_id = AuditContext.tenant_id.get()
+        previous_hash = self.storage.get_last_hash(tenant_id)
+
+        record = AuditRecord.create(
+            action=action,
+            model=model,
+            entity_id=entity_id,
+            storage_type=storage_type,
+            provider=provider,
+            success=success,
+            before=before,
+            after=after,
+            changed_fields=changed_fields,
+            error=error,
+            context=AuditContext,
+            previous_hash=previous_hash,
+        )
+
+        self.storage.persist(record)

polydb/audit/models.py

@@ -0,0 +1,86 @@
+# src/polydb/audit/models.py
+
+from dataclasses import dataclass, asdict
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+import uuid
+import hashlib
+import json
+
+
+@dataclass
+class AuditRecord:
+    audit_id: str
+    timestamp: str
+    tenant_id: Optional[str]
+    actor_id: Optional[str]
+    roles: List[str]
+    action: str
+    model: str
+    entity_id: Optional[str]
+    storage_type: str
+    provider: str
+    success: bool
+
+    before: Optional[Dict[str, Any]]
+    after: Optional[Dict[str, Any]]
+    changed_fields: Optional[List[str]]
+
+    trace_id: Optional[str]
+    request_id: Optional[str]
+    ip_address: Optional[str]
+    user_agent: Optional[str]
+
+    error: Optional[str]
+
+    hash: Optional[str] = None
+    previous_hash: Optional[str] = None
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        action: str,
+        model: str,
+        entity_id: Optional[str],
+        storage_type: str,
+        provider: str,
+        success: bool,
+        before: Optional[Dict[str, Any]],
+        after: Optional[Dict[str, Any]],
+        changed_fields: Optional[List[str]],
+        error: Optional[str],
+        context,
+        previous_hash: Optional[str] = None,
+    ):
+        now = datetime.utcnow().isoformat()
+        audit_id = str(uuid.uuid4())
+
+        record = cls(
+            audit_id=audit_id,
+            timestamp=now,
+            tenant_id=context.tenant_id.get(),
+            actor_id=context.actor_id.get(),
+            roles=context.roles.get(),
+            action=action,
+            model=model,
+            entity_id=entity_id,
+            storage_type=storage_type,
+            provider=provider,
+            success=success,
+            before=before,
+            after=after,
+            changed_fields=changed_fields,
+            trace_id=context.trace_id.get(),
+            request_id=context.request_id.get(),
+            ip_address=context.ip_address.get(),
+            user_agent=context.user_agent.get(),
+            error=error,
+            previous_hash=previous_hash,
+        )
+
+        record.hash = hashlib.sha256(
+            json.dumps(asdict(record), sort_keys=True).encode()
+        ).hexdigest()
+
+        return record

polydb/base/NoSQLKVAdapter.py

@@ -0,0 +1,301 @@
+# src/polydb/adapters/NoSQLKVAdapter.py
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+from typing import Any, Dict, List, Optional, Tuple, Union, TYPE_CHECKING
+
+from ..errors import NoSQLError, StorageError
+from ..retry import retry
+from ..query import QueryBuilder, Operator
+from ..types import JsonDict, Lookup
+
+if TYPE_CHECKING:
+    from ..models import PartitionConfig
+
+
+class NoSQLKVAdapter:
+    """Base with auto-overflow and LINQ support"""
+    
+    def __init__(self, partition_config: Optional[PartitionConfig] = None):
+        from ..utils import setup_logger
+        self.logger = setup_logger(self.__class__.__name__)
+        self.partition_config = partition_config
+        self.object_storage = None
+        self._lock = threading.Lock()
+        self.max_size = 1024 * 1024  # 1MB
+    
+    def _get_pk_rk(self, model: type, data: JsonDict) -> Tuple[str, str]:
+        """Extract PK/RK from model metadata"""
+        meta = getattr(model, '__polydb__', {})
+        pk_field = meta.get('pk_field', 'id')
+        rk_field = meta.get('rk_field')
+        
+        if self.partition_config:
+            try:
+                pk = self.partition_config.partition_key_template.format(**data)
+            except KeyError:
+                pk = f"default_{data.get(pk_field, hashlib.md5(json.dumps(data, sort_keys=True).encode()).hexdigest()[:8])}"
+        else:
+            pk = str(data.get(pk_field, 'default'))
+        
+        if rk_field and rk_field in data:
+            rk = str(data[rk_field])
+        elif self.partition_config and self.partition_config.row_key_template:
+            try:
+                rk = self.partition_config.row_key_template.format(**data)
+            except KeyError:
+                rk = hashlib.md5(json.dumps(data, sort_keys=True).encode()).hexdigest()
+        else:
+            rk = data.get('id', hashlib.md5(json.dumps(data, sort_keys=True).encode()).hexdigest())
+        
+        return str(pk), str(rk)
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _check_overflow(self, data: JsonDict) -> Tuple[JsonDict, Optional[str]]:
+        """Check size and store in blob if needed"""
+        data_bytes = json.dumps(data).encode()
+        data_size = len(data_bytes)
+        
+        if data_size > self.max_size:
+            with self._lock:
+                if not self.object_storage:
+                    from ..factory import CloudDatabaseFactory
+                    factory = CloudDatabaseFactory()
+                    self.object_storage = factory.get_object_storage()
+            
+            blob_id = hashlib.md5(data_bytes).hexdigest()
+            blob_key = f"overflow/{blob_id}.json"
+            
+            try:
+                self.object_storage.put(blob_key, data_bytes)
+            except Exception as e:
+                raise StorageError(f"Overflow storage failed: {str(e)}")
+            
+            return {
+                "_overflow": True,
+                "_blob_key": blob_key,
+                "_size": data_size,
+                "_checksum": blob_id
+            }, blob_key
+        
+        return data, None
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
+    def _retrieve_overflow(self, data: JsonDict) -> JsonDict:
+        """Retrieve from blob if overflow"""
+        if not data.get("_overflow"):
+            return data
+        
+        with self._lock:
+            if not self.object_storage:
+                from ..factory import CloudDatabaseFactory
+                factory = CloudDatabaseFactory()
+                self.object_storage = factory.get_object_storage()
+        
+        try:
+            blob_data = self.object_storage.get(data["_blob_key"])
+            retrieved = json.loads(blob_data.decode())
+            
+            # Verify checksum
+            checksum = hashlib.md5(blob_data).hexdigest()
+            if checksum != data.get("_checksum"):
+                raise StorageError("Checksum mismatch on overflow retrieval")
+            
+            return retrieved
+        except Exception as e:
+            raise StorageError(f"Overflow retrieval failed: {str(e)}")
+    
+    def _apply_filters(self, results: List[JsonDict], builder: QueryBuilder) -> List[JsonDict]:
+        """Apply filters in-memory for NoSQL"""
+        if not builder.filters:
+            return results
+        
+        filtered = []
+        for item in results:
+            match = True
+            for f in builder.filters:
+                value = item.get(f.field)
+                
+                if f.operator == Operator.EQ and value != f.value:
+                    match = False
+                elif f.operator == Operator.NE and value == f.value:
+                    match = False
+                elif f.operator == Operator.GT and not (value and value > f.value):
+                    match = False
+                elif f.operator == Operator.GTE and not (value and value >= f.value):
+                    match = False
+                elif f.operator == Operator.LT and not (value and value < f.value):
+                    match = False
+                elif f.operator == Operator.LTE and not (value and value <= f.value):
+                    match = False
+                elif f.operator == Operator.IN and value not in f.value:
+                    match = False
+                elif f.operator == Operator.NOT_IN and value in f.value:
+                    match = False
+                elif f.operator == Operator.CONTAINS and (not value or f.value not in str(value)):
+                    match = False
+                elif f.operator == Operator.STARTS_WITH and (not value or not str(value).startswith(f.value)):
+                    match = False
+                elif f.operator == Operator.ENDS_WITH and (not value or not str(value).endswith(f.value)):
+                    match = False
+                
+                if not match:
+                    break
+            
+            if match:
+                filtered.append(item)
+        
+        return filtered
+    
+    def _apply_ordering(self, results: List[JsonDict], builder: QueryBuilder) -> List[JsonDict]:
+        """Apply ordering"""
+        if not builder.order_by_fields:
+            return results
+        
+        for field, desc in reversed(builder.order_by_fields):
+            results = sorted(
+                results,
+                key=lambda x: x.get(field, ''),
+                reverse=desc
+            )
+        
+        return results
+    
+    def _apply_pagination(self, results: List[JsonDict], builder: QueryBuilder) -> List[JsonDict]:
+        """Apply skip/take"""
+        if builder.skip_count:
+            results = results[builder.skip_count:]
+        
+        if builder.take_count:
+            results = results[:builder.take_count]
+        
+        return results
+    
+    def _apply_projection(self, results: List[JsonDict], builder: QueryBuilder) -> List[JsonDict]:
+        """Apply field selection"""
+        if not builder.select_fields:
+            return results
+        
+        return [
+            {k: v for k, v in item.items() if k in builder.select_fields}
+            for item in results
+        ]
+    
+    # Abstract methods to implement
+    def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
+        raise NotImplementedError
+    
+    def _get_raw(self, model: type, pk: str, rk: str) -> Optional[JsonDict]:
+        raise NotImplementedError
+    
+    def _query_raw(self, model: type, filters: Dict[str, Any], limit: Optional[int]) -> List[JsonDict]:
+        raise NotImplementedError
+    
+    def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
+        raise NotImplementedError
+    
+    # Protocol implementation
+    def put(self, model: type, data: JsonDict) -> JsonDict:
+        pk, rk = self._get_pk_rk(model, data)
+        store_data, _ = self._check_overflow(data)
+        return self._put_raw(model, pk, rk, store_data)
+    
+    def query(
+        self,
+        model: type,
+        query: Optional[Lookup] = None,
+        limit: Optional[int] = None,
+        no_cache: bool = False,
+        cache_ttl: Optional[int] = None
+    ) -> List[JsonDict]:
+        results = self._query_raw(model, query or {}, limit)
+        return [self._retrieve_overflow(r) for r in results]
+    
+    def query_page(
+        self,
+        model: type,
+        query: Lookup,
+        page_size: int,
+        continuation_token: Optional[str] = None
+    ) -> Tuple[List[JsonDict], Optional[str]]:
+        # Basic implementation - override per provider
+        offset = int(continuation_token) if continuation_token else 0
+        results = self.query(model, query, limit=page_size + 1)
+        
+        has_more = len(results) > page_size
+        if has_more:
+            results = results[:page_size]
+        
+        next_token = str(offset + page_size) if has_more else None
+        return results, next_token
+    
+    def patch(
+        self,
+        model: type,
+        entity_id: Union[Any, Lookup],
+        data: JsonDict,
+        *,
+        etag: Optional[str] = None,
+        replace: bool = False
+    ) -> JsonDict:
+        if isinstance(entity_id, dict):
+            pk = entity_id.get('partition_key') or entity_id.get('pk')
+            rk = entity_id.get('row_key') or entity_id.get('rk') or entity_id.get('id')
+        else:
+            pk, rk = self._get_pk_rk(model, {'id': entity_id})
+        
+        if not replace:
+            existing = self._get_raw(model, pk, rk) # type: ignore
+            if existing:
+                existing = self._retrieve_overflow(existing)
+                existing.update(data)
+                data = existing
+        
+        store_data, _ = self._check_overflow(data)
+        return self._put_raw(model, pk, rk, store_data) # type: ignore
+    
+    def upsert(self, model: type, data: JsonDict, *, replace: bool = False) -> JsonDict:
+        return self.put(model, data)
+    
+    def delete(
+        self,
+        model: type,
+        entity_id: Union[Any, Lookup],
+        *,
+        etag: Optional[str] = None
+    ) -> JsonDict:
+        if isinstance(entity_id, dict):
+            pk = entity_id.get('partition_key') or entity_id.get('pk')
+            rk = entity_id.get('row_key') or entity_id.get('rk') or entity_id.get('id')
+        else:
+            pk, rk = self._get_pk_rk(model, {'id': entity_id})
+        
+        return self._delete_raw(model, pk, rk, etag) # type: ignore
+    
+    def query_linq(self, model: type, builder: QueryBuilder) -> Union[List[JsonDict], int]:
+        """LINQ-style query"""
+        results = self._query_raw(model, {}, None)
+        results = [self._retrieve_overflow(r) for r in results]
+        
+        results = self._apply_filters(results, builder)
+        
+        if builder.count_only:
+            return len(results)
+        
+        results = self._apply_ordering(results, builder)
+        results = self._apply_pagination(results, builder)
+        results = self._apply_projection(results, builder)
+        
+        if builder.distinct:
+            seen = set()
+            unique = []
+            for r in results:
+                key = json.dumps(r, sort_keys=True)
+                if key not in seen:
+                    seen.add(key)
+                    unique.append(r)
+            results = unique
+        
+        return results

polydb/base/ObjectStorageAdapter.py

@@ -0,0 +1,42 @@
+from polydb.utils import setup_logger
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+
+class ObjectStorageAdapter(ABC):
+    """Base class for Object Storage with automatic optimization"""
+
+    def __init__(self):
+        self.logger = setup_logger(self.__class__.__name__)
+
+    def put(
+        self, key: str, data: bytes, optimize: bool = True, media_type: Optional[str] = None
+    ) -> str:
+        """Store object with optional optimization"""
+        if optimize and media_type:
+            data = self._optimize_media(data, media_type)
+        return self._put_raw(key, data)
+
+    def _optimize_media(self, data: bytes, media_type: str) -> bytes:
+        """Optimize images and videos - placeholder for implementation"""
+        return data
+
+    @abstractmethod
+    def _put_raw(self, key: str, data: bytes) -> str:
+        """Provider-specific put"""
+        pass
+
+    @abstractmethod
+    def get(self, key: str) -> bytes:
+        """Get object"""
+        pass
+
+    @abstractmethod
+    def delete(self, key: str) -> bool:
+        """Delete object"""
+        pass
+
+    @abstractmethod
+    def list(self, prefix: str = "") -> List[str]:
+        """List objects with prefix"""
+        pass