altcodepro-polydb-python 2.1.0__py3-none-any.whl

polydb/adapters/AzureQueueAdapter.py

@@ -0,0 +1,61 @@
+# src/polydb/adapters/AzureQueueAdapter.py
+from polydb.base.QueueAdapter import QueueAdapter
+from polydb.errors import ConnectionError, QueueError
+from polydb.retry import retry
+import os
+import threading
+from typing import Any, Dict, List
+
+class AzureQueueAdapter(QueueAdapter):
+    """Azure Queue Storage with client reuse"""
+
+    def __init__(self):
+        super().__init__()
+        self.connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") or ""
+        self._client = None
+        self._lock = threading.Lock()
+        self._initialize_client()
+
+    def _initialize_client(self):
+        """Initialize Azure Queue Storage client once"""
+        try:
+            from azure.storage.queue import QueueServiceClient
+
+            with self._lock:
+                if not self._client:
+                    self._client = QueueServiceClient.from_connection_string(self.connection_string)
+                    self.logger.info("Initialized Azure Queue Storage client")
+        except Exception as e:
+            raise ConnectionError(f"Failed to initialize Azure Queue Storage: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
+    def send(self, message: Dict[str, Any], queue_name: str = "default") -> str:
+        """Send message to queue"""
+        try:
+            import json
+
+            if self._client:
+                queue_client = self._client.get_queue_client(queue_name)
+                response = queue_client.send_message(json.dumps(message))
+                return response.id
+            return ""
+        except Exception as e:
+            raise QueueError(f"Azure Queue send failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
+    def receive(self, queue_name: str = "default", max_messages: int = 1) -> List[Dict[str, Any]]:
+        """Receive messages from queue"""
+        try:
+            import json
+
+            if self._client:
+                queue_client = self._client.get_queue_client(queue_name)
+                messages = queue_client.receive_messages(max_messages=max_messages)
+                return [json.loads(msg.content) for msg in messages]
+            return []
+        except Exception as e:
+            raise QueueError(f"Azure Queue receive failed: {str(e)}")
+
+    def delete(self, message_id: str, queue_name: str = "default") -> bool:
+        """Delete message from queue (requires receipt handle)"""
+        return True

polydb/adapters/AzureTableStorageAdapter.py

@@ -0,0 +1,182 @@
+# src/polydb/adapters/AzureTableStorageAdapter.py
+import os
+import threading
+from typing import Any, Dict, List, Optional
+from polydb.base.NoSQLKVAdapter import NoSQLKVAdapter
+from ..errors import NoSQLError, ConnectionError
+from ..retry import retry
+from ..types import JsonDict
+from ..models import PartitionConfig
+
+
+class AzureTableStorageAdapter(NoSQLKVAdapter):
+    """Azure Table Storage with Azure Blob overflow (limit: 1MB per entity)"""
+    
+    AZURE_TABLE_MAX_SIZE = 1024 * 1024  # 1MB
+    
+    def __init__(self, partition_config: Optional[PartitionConfig] = None):
+        super().__init__(partition_config)
+        self.max_size = self.AZURE_TABLE_MAX_SIZE
+        self.connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") or ""
+        self.table_name = os.getenv("AZURE_TABLE_NAME", "defaulttable") or ""
+        self.container_name = os.getenv("AZURE_CONTAINER_NAME", "overflow") or ""
+        self._client = None
+        self._table_client = None
+        self._blob_service = None
+        self._client_lock = threading.Lock()
+        self._initialize_client()
+    
+    def _initialize_client(self):
+        try:
+            from azure.data.tables import TableServiceClient
+            from azure.storage.blob import BlobServiceClient
+            
+            with self._client_lock:
+                if not self._client:
+                    self._client = TableServiceClient.from_connection_string(self.connection_string)
+                    self._table_client = self._client.get_table_client(self.table_name)
+                    self._blob_service = BlobServiceClient.from_connection_string(self.connection_string)
+                    try:
+                        self._blob_service.create_container(self.container_name)
+                    except:
+                        pass  # Already exists
+                    
+                    self.logger.info("Azure Table Storage initialized with Blob overflow")
+        except Exception as e:
+            raise ConnectionError(f"Azure Table init failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
+        try:
+            import json
+            import hashlib
+            
+            data_copy = dict(data)
+            data_copy['PartitionKey'] = pk
+            data_copy['RowKey'] = rk
+            
+            # Check size
+            data_bytes = json.dumps(data_copy).encode()
+            data_size = len(data_bytes)
+            
+            if data_size > self.AZURE_TABLE_MAX_SIZE:
+                # Store in Blob
+                blob_id = hashlib.md5(data_bytes).hexdigest()
+                blob_key = f"overflow/{pk}/{rk}/{blob_id}.json"
+                
+                if self._blob_service:
+                    blob_client = self._blob_service.get_blob_client(self.container_name, blob_key)
+                    blob_client.upload_blob(data_bytes, overwrite=True)
+                    self.logger.info(f"Stored overflow to Blob: {blob_key} ({data_size} bytes)")
+                
+                # Store reference in table
+                reference_data = {
+                    'PartitionKey': pk,
+                    'RowKey': rk,
+                    '_overflow': True,
+                    '_blob_key': blob_key,
+                    '_size': data_size,
+                    '_checksum': blob_id,
+                }
+                
+                if self._table_client:
+                    self._table_client.upsert_entity(reference_data)
+            else:
+                # Store directly in table
+                if self._table_client:
+                    self._table_client.upsert_entity(data_copy)
+            
+            return {'PartitionKey': pk, 'RowKey': rk}
+        except Exception as e:
+            raise NoSQLError(f"Azure Table put failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _get_raw(self, model: type, pk: str, rk: str) -> Optional[JsonDict]:
+        try:
+            import json
+            import hashlib
+            
+            if not self._table_client:
+                return None
+            
+            entity = self._table_client.get_entity(pk, rk)
+            entity_dict = dict(entity)
+            
+            # Check if overflow
+            if entity_dict.get('_overflow'):
+                blob_key = entity_dict.get('_blob_key')
+                checksum = entity_dict.get('_checksum')
+                
+                if blob_key and self._blob_service:
+                    blob_client = self._blob_service.get_blob_client(self.container_name, blob_key)
+                    blob_data = blob_client.download_blob().readall()
+                    
+                    # Verify checksum
+                    actual_checksum = hashlib.md5(blob_data).hexdigest()
+                    if actual_checksum != checksum:
+                        raise NoSQLError(f"Checksum mismatch: expected {checksum}, got {actual_checksum}")
+                    
+                    retrieved = json.loads(blob_data.decode())
+                    self.logger.debug(f"Retrieved overflow from Blob: {blob_key}")
+                    return retrieved
+            
+            return entity_dict
+        except Exception as e:
+            if "ResourceNotFound" in str(e):
+                return None
+            raise NoSQLError(f"Azure Table get failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _query_raw(self, model: type, filters: Dict[str, Any], limit: Optional[int]) -> List[JsonDict]:
+        try:
+            if not self._table_client:
+                return []
+            
+            query_filter = None
+            if filters:
+                filter_parts = []
+                for k, v in filters.items():
+                    if isinstance(v, str):
+                        filter_parts.append(f"{k} eq '{v}'")
+                    elif isinstance(v, bool):
+                        filter_parts.append(f"{k} eq {str(v).lower()}")
+                    else:
+                        filter_parts.append(f"{k} eq {v}")
+                
+                query_filter = " and ".join(filter_parts)
+            
+            entities = self._table_client.query_entities(
+                query_filter=query_filter, # type: ignore
+                results_per_page=limit
+            )
+            
+            return [dict(entity) for entity in entities]
+        except Exception as e:
+            raise NoSQLError(f"Azure Table query failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
+        try:
+            if not self._table_client:
+                return {'deleted': False}
+            
+            # Check if overflow before deleting
+            try:
+                entity = self._table_client.get_entity(pk, rk)
+                entity_dict = dict(entity)
+                
+                if entity_dict.get('_overflow'):
+                    blob_key = entity_dict.get('_blob_key')
+                    if blob_key and self._blob_service:
+                        blob_client = self._blob_service.get_blob_client(self.container_name, blob_key)
+                        blob_client.delete_blob()
+                        self.logger.debug(f"Deleted overflow blob: {blob_key}")
+            except:
+                pass  # Entity might not exist or no overflow
+            
+            # Delete table entity
+            self._table_client.delete_entity(pk, rk, etag=etag)
+            return {'deleted': True, 'PartitionKey': pk, 'RowKey': rk}
+        except Exception as e:
+            raise NoSQLError(f"Azure Table delete failed: {str(e)}")
+

polydb/adapters/DynamoDBAdapter.py

@@ -0,0 +1,216 @@
+# src/polydb/adapters/DynamoDBAdapter.py
+import os
+import threading
+from typing import Any, Dict, List, Optional
+import boto3
+from boto3.dynamodb.conditions import Key, Attr
+
+from polydb.base.NoSQLKVAdapter import NoSQLKVAdapter
+
+from ..errors import NoSQLError, ConnectionError
+from ..retry import retry
+from ..types import JsonDict
+from ..models import PartitionConfig
+
+
+class DynamoDBAdapter(NoSQLKVAdapter):
+    """DynamoDB with S3 overflow (limit: 400KB per item)"""
+    
+    DYNAMODB_MAX_SIZE = 400 * 1024  # 400KB
+    
+    def __init__(self, partition_config: Optional[PartitionConfig] = None):
+        super().__init__(partition_config)
+        self.max_size = self.DYNAMODB_MAX_SIZE
+        self.table_name = os.getenv("DYNAMODB_TABLE_NAME", "default")
+        self.bucket_name = os.getenv("S3_OVERFLOW_BUCKET", "dynamodb-overflow")
+        self._resource = None
+        self._s3_client = None
+        self._client_lock = threading.Lock()
+        self._initialize()
+    
+    def _initialize(self):
+        try:
+            with self._client_lock:
+                if not self._resource:
+                    self._resource = boto3.resource('dynamodb')
+                    self._s3_client = boto3.client('s3')
+                    
+                    # Ensure bucket exists
+                    try:
+                        self._s3_client.create_bucket(Bucket=self.bucket_name)
+                    except:
+                        pass  # Already exists
+                    
+                    self.logger.info("DynamoDB initialized with S3 overflow")
+        except Exception as e:
+            raise ConnectionError(f"DynamoDB init failed: {str(e)}")
+    
+    def _get_table(self, model: type):
+        if not self._resource:
+            self._initialize()
+        
+        meta = getattr(model, '__polydb__', {})
+        table_name = meta.get('table') or self.table_name or model.__name__.lower()
+        return self._resource.Table(table_name) # type: ignore
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
+        try:
+            import json
+            import hashlib
+            
+            data_copy = dict(data)
+            data_copy['PK'] = pk
+            data_copy['SK'] = rk
+            
+            # Check size
+            data_bytes = json.dumps(data_copy).encode()
+            data_size = len(data_bytes)
+            
+            if data_size > self.DYNAMODB_MAX_SIZE:
+                # Store in S3
+                blob_id = hashlib.md5(data_bytes).hexdigest()
+                blob_key = f"overflow/{pk}/{rk}/{blob_id}.json"
+                
+                if self._s3_client:
+                    self._s3_client.put_object(
+                        Bucket=self.bucket_name,
+                        Key=blob_key,
+                        Body=data_bytes
+                    )
+                    self.logger.info(f"Stored overflow to S3: {blob_key} ({data_size} bytes)")
+                
+                # Store reference in DynamoDB
+                reference_data = {
+                    'PK': pk,
+                    'SK': rk,
+                    '_overflow': True,
+                    '_blob_key': blob_key,
+                    '_size': data_size,
+                    '_checksum': blob_id,
+                }
+                
+                table = self._get_table(model)
+                table.put_item(Item=reference_data)
+            else:
+                # Store directly in DynamoDB
+                table = self._get_table(model)
+                table.put_item(Item=data_copy)
+            
+            return {'PK': pk, 'SK': rk}
+        except Exception as e:
+            raise NoSQLError(f"DynamoDB put failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _get_raw(self, model: type, pk: str, rk: str) -> Optional[JsonDict]:
+        try:
+            import json
+            import hashlib
+            
+            table = self._get_table(model)
+            response = table.get_item(Key={'PK': pk, 'SK': rk})
+            
+            if 'Item' not in response:
+                return None
+            
+            item = response['Item']
+            
+            # Check if overflow
+            if item.get('_overflow'):
+                blob_key = item.get('_blob_key')
+                checksum = item.get('_checksum')
+                
+                if blob_key and self._s3_client:
+                    s3_response = self._s3_client.get_object(
+                        Bucket=self.bucket_name,
+                        Key=blob_key
+                    )
+                    blob_data = s3_response['Body'].read()
+                    
+                    # Verify checksum
+                    actual_checksum = hashlib.md5(blob_data).hexdigest()
+                    if actual_checksum != checksum:
+                        raise NoSQLError(f"Checksum mismatch: expected {checksum}, got {actual_checksum}")
+                    
+                    retrieved = json.loads(blob_data.decode())
+                    self.logger.debug(f"Retrieved overflow from S3: {blob_key}")
+                    return retrieved
+            
+            return item
+        except Exception as e:
+            raise NoSQLError(f"DynamoDB get failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _query_raw(self, model: type, filters: Dict[str, Any], limit: Optional[int]) -> List[JsonDict]:
+        try:
+            table = self._get_table(model)
+            
+            # If PK in filters, use query, else scan
+            if 'PK' in filters or 'partition_key' in filters:
+                pk_value = filters.get('PK') or filters.get('partition_key')
+                key_condition = Key('PK').eq(pk_value)
+                
+                if 'SK' in filters:
+                    key_condition = key_condition & Key('SK').eq(filters['SK'])
+                
+                kwargs = {'KeyConditionExpression': key_condition}
+                
+                # Other filters as FilterExpression
+                other_filters = {k: v for k, v in filters.items() if k not in ['PK', 'SK', 'partition_key']}
+                if other_filters:
+                    filter_expr = None
+                    for k, v in other_filters.items():
+                        expr = Attr(k).eq(v)
+                        filter_expr = expr if filter_expr is None else filter_expr & expr
+                    kwargs['FilterExpression'] = filter_expr # type: ignore
+                
+                if limit:
+                    kwargs['Limit'] = limit # type: ignore
+                
+                response = table.query(**kwargs)
+            else:
+                # Scan with filters
+                kwargs = {}
+                if filters:
+                    filter_expr = None
+                    for k, v in filters.items():
+                        expr = Attr(k).eq(v)
+                        filter_expr = expr if filter_expr is None else filter_expr & expr
+                    kwargs['FilterExpression'] = filter_expr
+                
+                if limit:
+                    kwargs['Limit'] = limit
+                
+                response = table.scan(**kwargs)
+            
+            return response.get('Items', [])
+        except Exception as e:
+            raise NoSQLError(f"DynamoDB query failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
+        try:
+            table = self._get_table(model)
+            
+            # Check if overflow before deleting
+            try:
+                response = table.get_item(Key={'PK': pk, 'SK': rk})
+                if 'Item' in response:
+                    item = response['Item']
+                    
+                    if item.get('_overflow'):
+                        blob_key = item.get('_blob_key')
+                        if blob_key and self._s3_client:
+                            self._s3_client.delete_object(
+                                Bucket=self.bucket_name,
+                                Key=blob_key
+                            )
+                            self.logger.debug(f"Deleted overflow S3 object: {blob_key}")
+            except:
+                pass  # Item might not exist or no overflow
+            
+            # Delete DynamoDB item
+            table.delete_item(Key={'PK': pk, 'SK': rk})
+            return {'deleted': True, 'PK': pk, 'SK': rk}
+        except Exception as e:
+            raise NoSQLError(f"DynamoDB delete failed: {str(e)}")

polydb/adapters/EFSAdapter.py

@@ -0,0 +1,50 @@
+# src/polydb/adapters/EFSAdapter.py
+
+from polydb.base.SharedFilesAdapter import SharedFilesAdapter
+from polydb.errors import StorageError
+import os
+from typing import List
+
+class EFSAdapter(SharedFilesAdapter):
+    """AWS EFS (mounted filesystem)"""
+
+    def __init__(self):
+        super().__init__()
+        self.mount_point = os.getenv("EFS_MOUNT_POINT", "/mnt/efs")
+
+    def write(self, path: str, data: bytes) -> bool:
+        """Write file"""
+        try:
+            full_path = os.path.join(self.mount_point, path)
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+            with open(full_path, "wb") as f:
+                f.write(data)
+            return True
+        except Exception as e:
+            raise StorageError(f"EFS write failed: {str(e)}")
+
+    def read(self, path: str) -> bytes:
+        """Read file"""
+        try:
+            full_path = os.path.join(self.mount_point, path)
+            with open(full_path, "rb") as f:
+                return f.read()
+        except Exception as e:
+            raise StorageError(f"EFS read failed: {str(e)}")
+
+    def delete(self, path: str) -> bool:
+        """Delete file"""
+        try:
+            full_path = os.path.join(self.mount_point, path)
+            os.remove(full_path)
+            return True
+        except Exception as e:
+            raise StorageError(f"EFS delete failed: {str(e)}")
+
+    def list(self, directory: str = "/") -> List[str]:
+        """List files in directory"""
+        try:
+            full_path = os.path.join(self.mount_point, directory)
+            return os.listdir(full_path)
+        except Exception as e:
+            raise StorageError(f"EFS list failed: {str(e)}")