altcodepro-polydb-python 2.2.2__py3-none-any.whl → 2.2.4__py3-none-any.whl

polydb/adapters/S3Adapter.py

@@ -1,61 +1,190 @@
 # src/polydb/adapters/S3Adapter.py
+
 """
-S3 adapter
+S3 adapter (AWS + LocalStack compatible)
 """
 
+from __future__ import annotations
+
+import mimetypes
 import os
 import threading
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
+
+import boto3
+from botocore.exceptions import ClientError
+
 from ..base.ObjectStorageAdapter import ObjectStorageAdapter
 from ..errors import StorageError, ConnectionError
 from ..retry import retry
+
+
 class S3Adapter(ObjectStorageAdapter):
-    """AWS S3 with client reuse"""
+    """AWS S3 adapter with client reuse and automatic bucket creation"""
 
-    def __init__(self):
+    def __init__(self, bucket_name: str = "", region: str = "", endpoint_url: str = ""):
         super().__init__()
-        self.bucket_name = os.getenv("S3_BUCKET_NAME", "default")
-        self._client = None
+
+        self.bucket_name = bucket_name or os.getenv("S3_BUCKET_NAME", "polydb-test")
+
+        self.region = (
+            region or os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
+        )
+
+        self.endpoint_url = endpoint_url or os.getenv("AWS_ENDPOINT_URL")
+
+        self._client: Any = None
         self._lock = threading.Lock()
+
         self._initialize_client()
 
+    # ---------------------------------------------------------
+    # Client initialization
+    # ---------------------------------------------------------
+
     def _initialize_client(self):
         """Initialize S3 client once"""
         try:
-            import boto3
-
             with self._lock:
-                if not self._client:
-                    self._client = boto3.client("s3")
-                    self.logger.info("Initialized S3 client")
+                if self._client:
+                    return
+
+                self._client = boto3.client(
+                    "s3",
+                    region_name=self.region,
+                    endpoint_url=self.endpoint_url,
+                    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID", "test"),
+                    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "test"),
+                )
+
+                self._ensure_bucket_exists()
+
+                self.logger.info(
+                    f"Initialized S3 client (region={self.region}, endpoint={self.endpoint_url or 'aws'})"
+                )
+
         except Exception as e:
-            raise ConnectionError(f"Failed to initialize S3 client: {str(e)}")
+            raise ConnectionError(f"Failed to initialize S3 client: {e}")
+
+    # ---------------------------------------------------------
+    # Bucket management
+    # ---------------------------------------------------------
+
+    def _ensure_bucket_exists(self):
+        """Create bucket if it doesn't exist (safe for AWS + LocalStack)"""
+        if not self._client:
+            return
+
+        try:
+            self._client.head_bucket(Bucket=self.bucket_name)
+            return
+        except ClientError:
+            pass
+
+        try:
+            if self.region == "us-east-1":
+                self._client.create_bucket(Bucket=self.bucket_name)
+            else:
+                self._client.create_bucket(
+                    Bucket=self.bucket_name,
+                    CreateBucketConfiguration={"LocationConstraint": self.region},
+                )
+
+            self.logger.info(f"Created S3 bucket: {self.bucket_name}")
+
+        except ClientError as e:
+            code = e.response.get("Error", {}).get("Code")
+            if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
+                raise StorageError(f"S3 bucket creation failed: {e}")
+
+    # ---------------------------------------------------------
+    # Core operations
+    # ---------------------------------------------------------
 
     @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
-    def _put_raw(self, key: str, data: bytes) -> str:
-        """Store object"""
+    def _put_raw(
+        self,
+        key: str,
+        data: bytes,
+        fileName: str = "",
+        media_type: Optional[str] = None,
+        metadata: Dict[str, Any] | None = None,
+    ) -> str:
+        """Upload object to S3-compatible storage with metadata and media type"""
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                self._client.put_object(Bucket=self.bucket_name, Key=key, Body=data)
-                self.logger.debug(f"Uploaded to S3: {key}")
-            return key
+
+            # --------------------------------------------------
+            # Resolve filename
+            # --------------------------------------------------
+            filename = fileName or os.path.basename(key)
+
+            # --------------------------------------------------
+            # Ensure extension from media_type
+            # --------------------------------------------------
+            if media_type:
+                ext = mimetypes.guess_extension(media_type) or ""
+                if ext and not filename.lower().endswith(ext):
+                    filename += ext
+
+            # --------------------------------------------------
+            # Final key
+            # --------------------------------------------------
+            blob_key = f"{key.rstrip('/')}/{filename}" if fileName else key
+
+            # --------------------------------------------------
+            # Metadata (string only)
+            # --------------------------------------------------
+            safe_metadata = {k: str(v) for k, v in (metadata or {}).items()}
+            safe_metadata["filename"] = filename
+
+            # --------------------------------------------------
+            # Upload
+            # --------------------------------------------------
+            self._client.put_object(  # type: ignore
+                Bucket=self.bucket_name,
+                Key=blob_key,
+                Body=data,
+                ContentType=media_type or "application/octet-stream",
+                Metadata=safe_metadata,
+            )
+
+            self.logger.debug(f"S3 uploaded: {blob_key}, type={media_type}")
+
+            # --------------------------------------------------
+            # Return URL
+            # --------------------------------------------------
+            if self.endpoint_url:
+                # MinIO / Spaces / custom endpoint
+                url = f"{self.endpoint_url.rstrip('/')}/{self.bucket_name}/{blob_key}"
+            else:
+                # AWS S3 default
+                url = f"https://{self.bucket_name}.s3.amazonaws.com/{blob_key}"
+
+            return url
+
         except Exception as e:
-            raise StorageError(f"S3 put failed: {str(e)}")
+            raise StorageError(f"S3-compatible put failed: {str(e)}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
     def get(self, key: str) -> bytes | None:
-        """Get object"""
+        """Download object"""
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                response = self._client.get_object(Bucket=self.bucket_name, Key=key)
-                return response["Body"].read()
+
+            response = self._client.get_object(
+                Bucket=self.bucket_name,
+                Key=key,
+            )
+
+            return response["Body"].read()
+
+        except self._client.exceptions.NoSuchKey:  # type: ignore
             return None
         except Exception as e:
-            raise StorageError(f"S3 get failed: {str(e)}")
+            raise StorageError(f"S3 get failed: {e}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
     def delete(self, key: str) -> bool:
@@ -63,12 +192,16 @@ class S3Adapter(ObjectStorageAdapter):
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                self._client.delete_object(Bucket=self.bucket_name, Key=key)
-                return True
-            return False
+
+            self._client.delete_object(
+                Bucket=self.bucket_name,
+                Key=key,
+            )
+
+            return True
+
         except Exception as e:
-            raise StorageError(f"S3 delete failed: {str(e)}")
+            raise StorageError(f"S3 delete failed: {e}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
     def list(self, prefix: str = "") -> List[str]:
@@ -76,11 +209,15 @@ class S3Adapter(ObjectStorageAdapter):
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                response = self._client.list_objects_v2(Bucket=self.bucket_name, Prefix=prefix)
-                return [obj["Key"] for obj in response.get("Contents", [])]
-            return []
-        except Exception as e:
-            raise StorageError(f"S3 list failed: {str(e)}")
 
+            response = self._client.list_objects_v2(
+                Bucket=self.bucket_name,
+                Prefix=prefix,
+            )
+
+            contents = response.get("Contents", [])
 
+            return [obj["Key"] for obj in contents]
+
+        except Exception as e:
+            raise StorageError(f"S3 list failed: {e}")

polydb/adapters/S3CompatibleAdapter.py

@@ -1,11 +1,13 @@
 # src/polydb/adapters/S3CompatibleAdapter.py
+import mimetypes
 import os
 import threading
-from typing import List
+from typing import Any, Dict, List, Optional
 from ..base.ObjectStorageAdapter import ObjectStorageAdapter
 from ..errors import StorageError, ConnectionError
 from ..retry import retry
 
+
 class S3CompatibleAdapter(ObjectStorageAdapter):
     """S3-compatible storage (MinIO, DigitalOcean Spaces) with client reuse"""
 
@@ -37,15 +39,68 @@ class S3CompatibleAdapter(ObjectStorageAdapter):
             raise ConnectionError(f"Failed to initialize S3-compatible client: {str(e)}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(StorageError,))
-    def _put_raw(self, key: str, data: bytes) -> str:
-        """Store object"""
+    def _put_raw(
+        self,
+        key: str,
+        data: bytes,
+        fileName: str = "",
+        media_type: Optional[str] = None,
+        metadata: Dict[str, Any] | None = None,
+    ) -> str:
+        """Upload object to S3-compatible storage with metadata and media type"""
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                self._client.put_object(Bucket=self.bucket_name, Key=key, Body=data)
-                self.logger.debug(f"Uploaded to S3-compatible: {key}")
-            return key
+
+            # --------------------------------------------------
+            # Resolve filename
+            # --------------------------------------------------
+            filename = fileName or os.path.basename(key)
+
+            # --------------------------------------------------
+            # Ensure extension from media_type
+            # --------------------------------------------------
+            if media_type:
+                ext = mimetypes.guess_extension(media_type) or ""
+                if ext and not filename.lower().endswith(ext):
+                    filename += ext
+
+            # --------------------------------------------------
+            # Final key
+            # --------------------------------------------------
+            blob_key = f"{key.rstrip('/')}/{filename}" if fileName else key
+
+            # --------------------------------------------------
+            # Metadata (string only)
+            # --------------------------------------------------
+            safe_metadata = {k: str(v) for k, v in (metadata or {}).items()}
+            safe_metadata["filename"] = filename
+
+            # --------------------------------------------------
+            # Upload
+            # --------------------------------------------------
+            self._client.put_object(  # type: ignore
+                Bucket=self.bucket_name,
+                Key=blob_key,
+                Body=data,
+                ContentType=media_type or "application/octet-stream",
+                Metadata=safe_metadata,
+            )
+
+            self.logger.debug(f"S3 uploaded: {blob_key}, type={media_type}")
+
+            # --------------------------------------------------
+            # Return URL
+            # --------------------------------------------------
+            if self.endpoint:
+                # MinIO / Spaces / custom endpoint
+                url = f"{self.endpoint.rstrip('/')}/{self.bucket_name}/{blob_key}"
+            else:
+                # AWS S3 default
+                url = f"https://{self.bucket_name}.s3.amazonaws.com/{blob_key}"
+
+            return url
+
         except Exception as e:
             raise StorageError(f"S3-compatible put failed: {str(e)}")
 
@@ -87,4 +142,3 @@ class S3CompatibleAdapter(ObjectStorageAdapter):
             return []
         except Exception as e:
             raise StorageError(f"S3-compatible list failed: {str(e)}")
-

polydb/adapters/SQSAdapter.py

@@ -1,86 +1,163 @@
-from polydb.base.QueueAdapter import QueueAdapter
-from polydb.errors import ConnectionError, QueueError
-from polydb.retry import retry
-
-
-import boto3
-from botocore.client import BaseClient
-
+from __future__ import annotations
 
 import json
 import os
 import threading
 from typing import Any, Dict, List
 
+import boto3
+from botocore.exceptions import ClientError
+
+from ..base.QueueAdapter import QueueAdapter
+from ..errors import ConnectionError, QueueError
+from ..retry import retry
 from ..json_safe import json_safe
 
 
 class SQSAdapter(QueueAdapter):
-    """AWS SQS with client reuse"""
+    """AWS SQS adapter with automatic queue creation (AWS + LocalStack compatible)"""
 
-    def __init__(self):
+    def __init__(self, queue_name: str = "", region: str = "", endpoint_url: str = ""):
         super().__init__()
-        self.queue_url = os.getenv("SQS_QUEUE_URL")
-        self._client: BaseClient | None = None
+
+        self.queue_name = queue_name or os.getenv("SQS_QUEUE_NAME", "polydb-queue")
+
+        self.region = (
+            region or os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
+        )
+
+        self.endpoint_url = endpoint_url or os.getenv("AWS_ENDPOINT_URL")
+
+        self._client: Any = None
+        self._queue_url = None
         self._lock = threading.Lock()
+
         self._initialize_client()
 
+    # ---------------------------------------------------------
+    # Client initialization
+    # ---------------------------------------------------------
+
     def _initialize_client(self):
-        """Initialize SQS client once"""
         try:
-            import boto3
-
             with self._lock:
-                if not self._client:
-                    self._client = boto3.client("sqs")
-                    self.logger.info("Initialized SQS client")
+                if self._client:
+                    return
+
+                self._client = boto3.client(
+                    "sqs",
+                    region_name=self.region,
+                    endpoint_url=self.endpoint_url,
+                    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID", "test"),
+                    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "test"),
+                )
+
+                self._queue_url = self._ensure_queue_exists(self.queue_name)
+
+                self.logger.info(
+                    f"Initialized SQS client (queue={self.queue_name}, endpoint={self.endpoint_url or 'aws'})"
+                )
+
         except Exception as e:
-            raise ConnectionError(f"Failed to initialize SQS client: {str(e)}")
+            raise ConnectionError(f"SQS init failed: {e}")
+
+    # ---------------------------------------------------------
+    # Queue management
+    # ---------------------------------------------------------
+
+    def _ensure_queue_exists(self, queue_name: str) -> str:
+        """Create queue if it does not exist"""
+        if not self._client:
+            raise ConnectionError("SQS client not initialized")
+
+        try:
+            response = self._client.get_queue_url(QueueName=queue_name)
+            return response["QueueUrl"]
+
+        except self._client.exceptions.QueueDoesNotExist:  # type: ignore
+
+            response = self._client.create_queue(QueueName=queue_name)
+            return response["QueueUrl"]
+
+        except ClientError as e:
+            raise QueueError(f"SQS queue creation failed: {e}")
+
+    # ---------------------------------------------------------
+    # Queue operations
+    # ---------------------------------------------------------
 
     @retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
-    def send(self, message: Dict[str, Any], queue_name: str = "default") -> str:
+    def send(self, message: Any, queue_name: str = "default") -> str:
         """Send message to queue"""
         try:
-            import json
-
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                response = self._client.send_message(
-                    QueueUrl=self.queue_url, MessageBody=json.dumps(message,default=json_safe)
-                )
-                return response["MessageId"]
-            return ""
+
+            body = (
+                json.dumps(message, default=json_safe) if not isinstance(message, str) else message
+            )
+
+            resp = self._client.send_message(
+                QueueUrl=self._queue_url,
+                MessageBody=body,
+            )
+
+            return resp["MessageId"]
+
         except Exception as e:
-            raise QueueError(f"SQS send failed: {str(e)}")
+            raise QueueError(f"SQS send failed: {e}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
     def receive(self, queue_name: str = "default", max_messages: int = 1) -> List[Dict[str, Any]]:
         """Receive messages from queue"""
         try:
-            import json
-
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                response = self._client.receive_message(
-                    QueueUrl=self.queue_url, MaxNumberOfMessages=max_messages, WaitTimeSeconds=5
+
+            resp = self._client.receive_message(
+                QueueUrl=self._queue_url,
+                MaxNumberOfMessages=max_messages,
+                WaitTimeSeconds=1,
+            )
+
+            messages = resp.get("Messages", [])
+
+            out: List[Dict[str, Any]] = []
+
+            for m in messages:
+                body = m.get("Body")
+
+                try:
+                    body = json.loads(body)
+                except Exception:
+                    pass
+
+                out.append(
+                    {
+                        "body": body,
+                        "receipt_handle": m["ReceiptHandle"],
+                        "message_id": m["MessageId"],
+                    }
                 )
-                messages = response.get("Messages", [])
-                return [json.loads(msg["Body"]) for msg in messages]
-            return []
+
+            return out
+
         except Exception as e:
-            raise QueueError(f"SQS receive failed: {str(e)}")
+            raise QueueError(f"SQS receive failed: {e}")
 
     @retry(max_attempts=3, delay=1.0, exceptions=(QueueError,))
-    def delete(self, message_id: str, queue_name: str = "default") -> bool:
+    def delete(self, receipt_handle: str, queue_name: str = "default") -> bool:
         """Delete message from queue"""
         try:
             if not self._client:
                 self._initialize_client()
-            if self._client:
-                self._client.delete_message(QueueUrl=self.queue_url, ReceiptHandle=message_id)
-                return True
-            return False
+
+            self._client.delete_message(
+                QueueUrl=self._queue_url,
+                ReceiptHandle=receipt_handle,
+            )
+
+            return True
+
         except Exception as e:
-            raise QueueError(f"SQS delete failed: {str(e)}")
+            raise QueueError(f"SQS delete failed: {e}")