altcodepro-polydb-python 2.1.0__py3-none-any.whl

polydb/adapters/FirestoreAdapter.py

@@ -0,0 +1,193 @@
+# src/polydb/adapters/FirestoreAdapter.py
+import os
+import threading
+from typing import Any, Dict, List, Optional
+from google.cloud import firestore
+from google.cloud import storage
+from google.cloud.firestore import Client
+from polydb.base.NoSQLKVAdapter import NoSQLKVAdapter
+
+from ..errors import NoSQLError, ConnectionError
+from ..retry import retry
+from ..types import JsonDict
+from ..models import PartitionConfig
+
+
+class FirestoreAdapter(NoSQLKVAdapter):
+    """Firestore with GCS overflow (limit: 1MB per document)"""
+    
+    FIRESTORE_MAX_SIZE = 1024 * 1024  # 1MB
+    
+    def __init__(self, partition_config: Optional[PartitionConfig] = None):
+        super().__init__(partition_config)
+        self.max_size = self.FIRESTORE_MAX_SIZE
+        self.bucket_name = os.getenv("GCS_OVERFLOW_BUCKET", "firestore-overflow")
+        self._client: Optional[Client] = None
+        self._storage_client = None
+        self._bucket = None
+        self._client_lock = threading.Lock()
+        self._initialize_client()
+    
+    def _initialize_client(self):
+        try:
+            with self._client_lock:
+                if not self._client:
+                    self._client = firestore.Client()
+                    self._storage_client = storage.Client()
+                    self._bucket = self._storage_client.bucket(self.bucket_name)
+                    
+                    # Ensure bucket exists
+                    try:
+                        self._bucket.create()
+                    except:
+                        pass  # Already exists
+                    
+                    self.logger.info("Firestore initialized with GCS overflow")
+        except Exception as e:
+            raise ConnectionError(f"Firestore init failed: {str(e)}")
+    
+    def _get_collection(self, model: type) -> Any:
+        if not self._client:
+            self._initialize_client()
+        
+        meta = getattr(model, '__polydb__', {})
+        collection_name = meta.get('collection') or meta.get('table') or model.__name__.lower()
+        return self._client.collection(collection_name) # type: ignore
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
+        try:
+            import json
+            import hashlib
+            
+            doc_id = f"{pk}_{rk}"
+            data_copy = dict(data)
+            data_copy['_pk'] = pk
+            data_copy['_rk'] = rk
+            
+            # Check size
+            data_bytes = json.dumps(data_copy).encode()
+            data_size = len(data_bytes)
+            
+            if data_size > self.FIRESTORE_MAX_SIZE:
+                # Store in GCS
+                blob_id = hashlib.md5(data_bytes).hexdigest()
+                blob_key = f"overflow/{pk}/{rk}/{blob_id}.json"
+                
+                if self._bucket:
+                    blob = self._bucket.blob(blob_key)
+                    blob.upload_from_string(data_bytes)
+                    self.logger.info(f"Stored overflow to GCS: {blob_key} ({data_size} bytes)")
+                
+                # Store reference in Firestore
+                reference_data = {
+                    '_pk': pk,
+                    '_rk': rk,
+                    '_overflow': True,
+                    '_blob_key': blob_key,
+                    '_size': data_size,
+                    '_checksum': blob_id,
+                }
+                
+                collection = self._get_collection(model)
+                collection.document(doc_id).set(reference_data)
+            else:
+                # Store directly in Firestore
+                collection = self._get_collection(model)
+                collection.document(doc_id).set(data_copy)
+            
+            return {'_pk': pk, '_rk': rk, 'id': doc_id}
+        except Exception as e:
+            raise NoSQLError(f"Firestore put failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _get_raw(self, model: type, pk: str, rk: str) -> Optional[JsonDict]:
+        try:
+            import json
+            import hashlib
+            
+            doc_id = f"{pk}_{rk}"
+            collection = self._get_collection(model)
+            doc = collection.document(doc_id).get()
+            
+            if not doc.exists:
+                return None
+            
+            doc_data = doc.to_dict()
+            
+            # Check if overflow
+            if doc_data.get('_overflow'):
+                blob_key = doc_data.get('_blob_key')
+                checksum = doc_data.get('_checksum')
+                
+                if blob_key and self._bucket:
+                    blob = self._bucket.blob(blob_key)
+                    blob_data = blob.download_as_bytes()
+                    
+                    # Verify checksum
+                    actual_checksum = hashlib.md5(blob_data).hexdigest()
+                    if actual_checksum != checksum:
+                        raise NoSQLError(f"Checksum mismatch: expected {checksum}, got {actual_checksum}")
+                    
+                    retrieved = json.loads(blob_data.decode())
+                    self.logger.debug(f"Retrieved overflow from GCS: {blob_key}")
+                    return retrieved
+            
+            return doc_data
+        except Exception as e:
+            raise NoSQLError(f"Firestore get failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _query_raw(self, model: type, filters: Dict[str, Any], limit: Optional[int]) -> List[JsonDict]:
+        try:
+            collection = self._get_collection(model)
+            query = collection
+            
+            for field, value in filters.items():
+                if field.endswith('__gt'):
+                    query = query.where(field[:-4], '>', value)
+                elif field.endswith('__gte'):
+                    query = query.where(field[:-5], '>=', value)
+                elif field.endswith('__lt'):
+                    query = query.where(field[:-4], '<', value)
+                elif field.endswith('__lte'):
+                    query = query.where(field[:-5], '<=', value)
+                elif field.endswith('__in'):
+                    query = query.where(field[:-4], 'in', value)
+                else:
+                    query = query.where(field, '==', value)
+            
+            if limit:
+                query = query.limit(limit)
+            
+            docs = query.stream()
+            return [doc.to_dict() for doc in docs]
+        except Exception as e:
+            raise NoSQLError(f"Firestore query failed: {str(e)}")
+    
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
+        try:
+            doc_id = f"{pk}_{rk}"
+            collection = self._get_collection(model)
+            
+            # Check if overflow before deleting
+            try:
+                doc = collection.document(doc_id).get()
+                if doc.exists:
+                    doc_data = doc.to_dict()
+                    
+                    if doc_data.get('_overflow'):
+                        blob_key = doc_data.get('_blob_key')
+                        if blob_key and self._bucket:
+                            blob = self._bucket.blob(blob_key)
+                            blob.delete()
+                            self.logger.debug(f"Deleted overflow GCS object: {blob_key}")
+            except:
+                pass  # Doc might not exist or no overflow
+            
+            # Delete Firestore document
+            collection.document(doc_id).delete()
+            return {'deleted': True, 'id': doc_id}
+        except Exception as e:
+            raise NoSQLError(f"Firestore delete failed: {str(e)}")

polydb/adapters/GCPStorageAdapter.py

@@ -0,0 +1,81 @@
+# src/polydb/adapters/GCPStorageAdapter.py
+import os
+import threading
+from typing import List, Optional, cast
+from google.cloud.firestore import DocumentSnapshot
+from ..base.ObjectStorageAdapter import ObjectStorageAdapter
+from ..errors import StorageError, ConnectionError
+from ..retry import retry
+
+
+class GCPStorageAdapter(ObjectStorageAdapter):
+    """GCP Cloud Storage with client reuse"""
+
+    def __init__(self):
+        super().__init__()
+        self.bucket_name = os.getenv("GCS_BUCKET_NAME", "default")
+        self._client = None
+        self._bucket = None
+        self._lock = threading.Lock()
+        self._initialize_client()
+
+    def _initialize_client(self):
+        """Initialize GCS client once"""
+        try:
+            from google.cloud import storage
+
+            with self._lock:
+                if not self._client:
+                    self._client = storage.Client()
+                    self._bucket = self._client.bucket(self.bucket_name)
+                    self.logger.info("Initialized GCS client")
+        except Exception as e:
+            raise ConnectionError(f"Failed to initialize GCS: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
+    def _put_raw(self, key: str, data: bytes) -> str:
+        """Store object"""
+        try:
+            if self._bucket:
+                blob = self._bucket.blob(key)
+                blob.upload_from_string(data)
+                self.logger.debug(f"Uploaded blob: {key}")
+            return key
+        except Exception as e:
+            raise StorageError(f"GCS put failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
+    def get(self, key: str) -> bytes | None:
+        """Get object"""
+        try:
+            if self._bucket:
+                blob = self._bucket.blob(key)
+                return blob.download_as_bytes()
+            return None
+        except Exception as e:
+            raise StorageError(f"GCS get failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
+    def delete(self, key: str) -> bool:
+        """Delete object"""
+        try:
+            if self._bucket:
+                blob = self._bucket.blob(key)
+                blob.delete()
+                return True
+            return False
+        except Exception as e:
+            raise StorageError(f"GCS delete failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
+    def list(self, prefix: str = "") -> List[str]:
+        """List objects with prefix"""
+        try:
+            if self._bucket:
+                blobs = self._bucket.list_blobs(prefix=prefix)
+                return [blob.name for blob in blobs]
+            return []
+        except Exception as e:
+            raise StorageError(f"GCS list failed: {str(e)}")
+
+

polydb/adapters/MongoDBAdapter.py

@@ -0,0 +1,136 @@
+# src/polydb/adapters/mongodb.py
+import os
+import re
+import threading
+from typing import Any, Dict, List, Optional
+from polydb.base.NoSQLKVAdapter import NoSQLKVAdapter
+
+from ..errors import NoSQLError, ConnectionError
+from ..retry import retry
+from ..types import JsonDict
+from ..models import PartitionConfig
+
+
+class MongoDBAdapter(NoSQLKVAdapter):
+    """MongoDB with overflow and LINQ"""
+
+    def __init__(self, partition_config: Optional[PartitionConfig] = None):
+        super().__init__(partition_config)
+        self.mongo_uri = os.getenv("MONGODB_URI", "mongodb://localhost:27017")
+        self.db_name = os.getenv("MONGODB_DATABASE", "default")
+        self._client = None
+        self._client_lock = threading.Lock()
+        self._initialize_client()
+
+    def _initialize_client(self):
+        try:
+            from pymongo import MongoClient
+
+            with self._client_lock:
+                if not self._client:
+                    self._client = MongoClient(
+                        self.mongo_uri,
+                        maxPoolSize=int(os.getenv("MONGODB_MAX_POOL_SIZE", "10")),
+                        minPoolSize=int(os.getenv("MONGODB_MIN_POOL_SIZE", "1")),
+                        serverSelectionTimeoutMS=5000,
+                    )
+                    self._client.server_info()
+                    self.logger.info("MongoDB initialized")
+        except Exception as e:
+            raise ConnectionError(f"MongoDB init failed: {str(e)}")
+
+    def _get_collection(self, model: type):
+        if not self._client:
+            self._initialize_client()
+
+        meta = getattr(model, "__polydb__", {})
+        collection_name = meta.get("collection") or meta.get("table") or model.__name__.lower()
+        return self._client[self.db_name][collection_name]  # type: ignore
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _put_raw(self, model: type, pk: str, rk: str, data: JsonDict) -> JsonDict:
+        try:
+            data_copy = dict(data)
+            data_copy["_pk"] = pk
+            data_copy["_rk"] = rk
+
+            collection = self._get_collection(model)
+            result = collection.update_one(
+                {"_pk": pk, "_rk": rk},
+                {"$set": data_copy},
+                upsert=True,
+            )
+
+            return {"_pk": pk, "_rk": rk}
+        except Exception as e:
+            raise NoSQLError(f"MongoDB put failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _get_raw(self, model: type, pk: str, rk: str) -> Optional[JsonDict]:
+        try:
+            collection = self._get_collection(model)
+            doc = collection.find_one({"_pk": pk, "_rk": rk})
+
+            if doc:
+                doc.pop("_id", None)
+                return doc
+
+            return None
+        except Exception as e:
+            raise NoSQLError(f"MongoDB get failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _query_raw(
+        self, model: type, filters: Dict[str, Any], limit: Optional[int]
+    ) -> List[JsonDict]:
+        try:
+            collection = self._get_collection(model)
+
+            query = {}
+            for k, v in filters.items():
+                if k.endswith("__gt"):
+                    query[k[:-4]] = {"$gt": v}
+                elif k.endswith("__gte"):
+                    query[k[:-5]] = {"$gte": v}
+                elif k.endswith("__lt"):
+                    query[k[:-4]] = {"$lt": v}
+                elif k.endswith("__lte"):
+                    query[k[:-5]] = {"$lte": v}
+                elif k.endswith("__in"):
+                    query[k[:-4]] = {"$in": v}
+                elif k.endswith("__contains"):
+                    safe_pattern = re.escape(str(v))
+                    query[k[:-10]] = {"$regex": safe_pattern, "$options": "i"}  # case-insensitive
+                else:
+                    query[k] = v
+
+            cursor = collection.find(query)
+
+            if limit:
+                cursor = cursor.limit(limit)
+
+            results = []
+            for doc in cursor:
+                doc.pop("_id", None)
+                results.append(doc)
+
+            return results
+        except Exception as e:
+            raise NoSQLError(f"MongoDB query failed: {str(e)}")
+
+    @retry(max_attempts=3, delay=1.0, exceptions=(NoSQLError,))
+    def _delete_raw(self, model: type, pk: str, rk: str, etag: Optional[str]) -> JsonDict:
+        try:
+            collection = self._get_collection(model)
+            result = collection.delete_one({"_pk": pk, "_rk": rk})
+
+            return {"deleted": result.deleted_count > 0, "_pk": pk, "_rk": rk}
+        except Exception as e:
+            raise NoSQLError(f"MongoDB delete failed: {str(e)}")
+
+    def __del__(self):
+        if self._client:
+            try:
+                self._client.close()
+            except:
+                pass