solace-agent-mesh 0.0.1__py3-none-any.whl → 0.1.0__py3-none-any.whl

solace_agent_mesh/services/common/auto_expiry.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+import time 
+import threading
+
+from solace_ai_connector.common.log import log
+
+class AutoExpiry(ABC):
+    _expiry_thread = None
+    expiration_check_interval = None
+    _stop_expiry_thread = None
+
+
+    def _start_auto_expiry_thread(self, expiration_check_interval):
+        """Starts a background thread to handle auto-expiry."""
+        self.expiration_check_interval = expiration_check_interval
+        self._stop_expiry_thread = threading.Event()
+        self._expiry_thread = threading.Thread(
+            target=self._auto_expiry_task, daemon=True
+        )
+        self._expiry_thread.start()
+
+    def _auto_expiry_task(self):
+        """Background task to check and delete expired items."""
+        while not self._stop_expiry_thread.is_set():
+            try:
+                self._delete_expired_items()
+            except Exception as e:
+                log.error(f"Error during auto-expiry process: {e}")
+            time.sleep(self.expiration_check_interval)
+
+    @abstractmethod
+    def _delete_expired_items(self):
+        """Checks all item and deletes those that have exceeded max_time_to_live."""
+        raise NotImplementedError
+        
+    def stop_auto_expiry(self):
+        """Stops the auto-expiry background thread."""
+        if self._stop_expiry_thread:
+            self._stop_expiry_thread.set()
+        if self._expiry_thread and self._expiry_thread.is_alive():
+            self._expiry_thread.join()
+
+    def __del__(self):
+        """Ensure the expiry thread stops when the service is destroyed."""
+        self.stop_auto_expiry()

solace_agent_mesh/services/common/singleton.py

@@ -0,0 +1,18 @@
+from .auto_expiry import AutoExpiry
+
+# Singleton pattern - Ensuring that only one instance of the class is created per given identifier
+class SingletonMeta(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        identifier = kwargs.get("identifier", "default")
+        if cls not in cls._instances:
+            cls._instances[cls] = {}
+        if identifier not in cls._instances[cls]:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls][identifier] = instance
+        return cls._instances[cls][identifier]
+
+
+class AutoExpirySingletonMeta(SingletonMeta, type(AutoExpiry)):
+    pass

solace_agent_mesh/services/file_service/__init__.py

@@ -0,0 +1,14 @@
+from .file_service import FileService, FileServicePermissionError, FS_URL_REGEX
+from .file_utils import Types
+from .file_transformations import LLM_QUERY_OPTIONS, TRANSFORMERS
+from .file_service_constants import FS_PROTOCOL
+
+__all__ = [
+    "FileService",
+    "FS_PROTOCOL",
+    "Types",
+    "FileServicePermissionError",
+    "FS_URL_REGEX",
+    "LLM_QUERY_OPTIONS",
+    "TRANSFORMERS"
+]

solace_agent_mesh/services/file_service/file_manager/bucket_file_manager.py

@@ -0,0 +1,149 @@
+import boto3
+import os
+import json
+from io import BytesIO
+from botocore.exceptions import NoCredentialsError, ClientError
+
+from .file_manager_base import FileManagerBase
+
+
+class BucketFileManager(FileManagerBase):
+    def __init__(self, config, ttl):
+        self.config = config
+        self.ttl = ttl
+        self.bucket_name = config.get("bucket_name")
+        self.boto3_config = config.get("boto3_config", {})
+        self.endpoint_url = config.get("endpoint_url", None)
+        session = boto3.Session(**self.boto3_config)
+        s3_resource = session.resource("s3", endpoint_url=self.endpoint_url)
+        try:
+            if not self.bucket_name:
+                raise Exception("Bucket name not provided.")
+
+            bucket = s3_resource.Bucket(self.bucket_name)
+
+            # Check if the bucket exists by attempting to load it
+            bucket.load()
+            self.bucket = bucket
+
+        except ClientError as e:
+            # Handle the case where the bucket doesn't exist or access is denied
+            error_code = e.response["Error"]["Code"]
+            if error_code == "404":
+                raise Exception((f"Bucket '{self.bucket_name}' does not exist."), e)
+            elif error_code == "403":
+                raise Exception(
+                    (f"Access to bucket '{self.bucket_name}' is denied."), e
+                )
+            else:
+                raise Exception(f"Unexpected error: {e}")
+
+    def _save_metadata(self, file_signature: str, metadata: dict):
+        metadata_key = self._get_metadata_name(file_signature)
+        metadata_content = json.dumps(metadata, indent=4)
+
+        try:
+            self.bucket.put_object(
+                Key=metadata_key,
+                Body=metadata_content,
+                ContentType="application/json",
+            )
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to save metadata to S3: {str(e)}")
+
+    def upload_from_buffer(self, buffer: bytes, file_name: str, **kwargs) -> dict:
+        file_signature = self._generate_file_signature(file_name)
+        metadata = self._create_metadata(file_signature, file_name, buffer, kwargs)
+
+        try:
+            self.bucket.put_object(
+                Key=file_signature,
+                Body=buffer,
+                ContentType=metadata.get("mime_type") or "application/octet-stream",
+            )
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to upload file to S3: {str(e)}")
+
+        self._save_metadata(file_signature, metadata)
+
+        return metadata
+
+    def upload_from_file(self, file_path: str, **kwargs) -> dict:
+        if not os.path.isfile(file_path):
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+        file_name = os.path.basename(file_path)
+        file_signature = self._generate_file_signature(file_name)
+        content_type = kwargs.get("mime_type") or self._get_mime_type(file_name) or "application/octet-stream"
+
+        try:
+            with open(file_path, "rb") as file_data:
+                self.bucket.put_object(
+                    Key=file_signature,
+                    Body=file_data,
+                    ContentType=content_type,
+                )
+                metadata = self._create_metadata(file_signature, file_name, file_data.read(), kwargs)
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to upload file to S3: {str(e)}")
+
+        self._save_metadata(file_signature, metadata)
+
+        return metadata
+
+    def download_to_buffer(self, file_name: str) -> bytes:
+        try:
+            buffer = BytesIO()
+            obj = self.bucket.Object(file_name)
+            obj.download_fileobj(buffer)
+            buffer.seek(0)
+            return buffer.read()
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to download file from S3: {str(e)}")
+
+    def download_to_file(self, file_name: str, destination_path: str):
+        buffer = self.download_to_buffer(file_name)
+
+        try:
+            with open(destination_path, "wb") as destination_file:
+                destination_file.write(buffer)
+        except IOError as e:
+            raise RuntimeError(f"Failed to write file to destination: {str(e)}")
+
+    def get_metadata(self, file_name: str) -> dict:
+        metadata_key = self._get_metadata_name(file_name)
+        meta_buffer = self.download_to_buffer(metadata_key)
+        return json.loads(meta_buffer)
+
+    def delete_by_name(self, file_name: str):
+        metadata_key = self._get_metadata_name(file_name)
+
+        try:
+            # the main file
+            file = self.bucket.Object(file_name)
+            # the metadata file
+            meta = self.bucket.Object(metadata_key)
+
+            # Delete the files
+            file.delete()
+            meta.delete()
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to delete file from S3: {str(e)}")
+        
+    def update_file_expiration(self, file_signature, expiration_timestamp):
+        metadata = self.get_metadata(file_signature)
+        metadata["expiration_timestamp"] = expiration_timestamp
+        self._save_metadata(file_signature, metadata)
+
+    def list_all_metadata(self) -> list:
+        all_metadata = []
+        try:
+            for obj in self.bucket.objects.all():
+                if obj.key.endswith(".metadata"):
+                    meta_buffer = self.download_to_buffer(obj.key)
+                    metadata = json.loads(meta_buffer)
+
+                    all_metadata.append(metadata)
+        except (NoCredentialsError, ClientError) as e:
+            raise RuntimeError(f"Failed to list metadata from S3: {str(e)}")
+        return all_metadata

solace_agent_mesh/services/file_service/file_manager/file_manager_base.py

@@ -0,0 +1,162 @@
+import uuid
+import mimetypes
+import re
+import time
+from abc import ABC, abstractmethod
+
+from ..file_service_constants import FS_PROTOCOL, META_FILE_EXTENSION
+from ..file_utils import get_file_schema_and_shape
+
+
+MAX_NAME_LENGTH = 255 - (
+    len(META_FILE_EXTENSION) + len(FS_PROTOCOL) + 3 + 36 + 1
+)  # 36 is the length of a UUID, 3 is ://
+
+
+class FileManagerBase(ABC):
+    ttl: int
+
+    def _generate_file_signature(self, file_name: str) -> str:
+        """
+        Generate a file signature from the original file name.
+        """
+        file_signature = str(uuid.uuid4())
+        # Remove any leading or trailing whitespace
+        file_name = file_name.strip()
+        # Replace invalid characters and space/ampersand with underscores
+        cleaned_name = re.sub(r'[<>:"/\\|?*& ]', "_", file_name)
+        # replace unicode and binary characters with underscores
+        cleaned_name = re.sub(r"[^\x00-\x7F]+", "_", cleaned_name)
+        cleaned_name = re.sub(r"\\u[0-9a-fA-F]{4}", "_", cleaned_name)
+        # Replace multiple underscores with a single underscore
+        cleaned_name = re.sub(r"_+", "_", cleaned_name)
+        # Truncate the signature if it is too long
+        if len(cleaned_name) > MAX_NAME_LENGTH:
+            start_index = len(cleaned_name) - MAX_NAME_LENGTH
+            # Keep the latter portion of the name
+            cleaned_name = cleaned_name[start_index:]
+        signature = f"{file_signature}_{cleaned_name}"
+        return signature
+
+    def _get_url_from_signature(self, file_signature: str) -> str:
+        """
+        Generate a file URL from a file signature.
+        """
+        return f"{FS_PROTOCOL}://{file_signature}"
+
+    def _get_mime_type(self, file_name: str) -> str:
+        """
+        Get the MIME type of a file.
+        """
+        mime_type, _ = mimetypes.guess_type(file_name)
+        return mime_type
+
+    def _get_metadata_name(self, name: str) -> dict:
+        """
+        Get metadata from a file URL.
+        """
+        metadata_path = f"{name}{META_FILE_EXTENSION}"
+        return metadata_path
+
+    def _create_metadata(self, file_signature: str, file_name: str, file: bytes, metadata: dict):
+        """
+        Extend metadata with schema and shape if not present
+        Add file_size if not present
+        """
+
+        mime_type = self._get_mime_type(file_name)
+        file_url = self._get_url_from_signature(file_signature)
+
+        if not isinstance(metadata, dict):
+            metadata = {}
+        meta_copy = metadata.copy()
+
+        # Can not be provided by user
+        meta_copy["url"] = file_url
+
+        if "mime_type" not in meta_copy:
+            meta_copy["mime_type"] = mime_type
+
+        if "name" not in meta_copy:
+            meta_copy["name"] = file_name
+
+        if (
+            "schema-yaml" not in meta_copy
+            or "schema_yaml" not in meta_copy
+            or "shape" not in meta_copy
+        ):
+            schema, shape = get_file_schema_and_shape(file, meta_copy)
+            if "schema-yaml" not in meta_copy and schema:
+                meta_copy["schema-yaml"] = schema
+
+            if "shape" not in meta_copy and shape:
+                meta_copy["shape"] = shape
+    
+        if "file_size" not in meta_copy:
+            meta_copy["file_size"] = len(file)
+
+        if "upload_timestamp" not in meta_copy:
+            meta_copy["upload_timestamp"] = time.time()
+
+        if "expiration_timestamp" not in meta_copy:
+            meta_copy["expiration_timestamp"] = meta_copy["upload_timestamp"] + self.ttl
+
+        return meta_copy
+    
+    @abstractmethod
+    def update_file_expiration(self, file_signature: str, expiration_timestamp: float):
+        """
+        Update the expiration timestamp for a file.
+        """
+        pass
+
+    @abstractmethod
+    def upload_from_buffer(self, buffer: bytes, file_name: str, **kwargs) -> dict:
+        """
+        Upload a file from a buffer.
+        kwargs are added to metadata
+        """
+        pass
+
+    @abstractmethod
+    def upload_from_file(self, file_path: str, **kwargs) -> dict:
+        """
+        Upload a file from a file path.
+        kwargs are added to metadata
+        """
+        pass
+
+    @abstractmethod
+    def download_to_buffer(self, file_name: str) -> bytes:
+        """
+        Download a file to a buffer.
+        """
+        pass
+
+    @abstractmethod
+    def download_to_file(self, file_name: str, destination_path: str):
+        """
+        Download a file to a destination path.
+        """
+        pass
+
+    @abstractmethod
+    def delete_by_name(self, file_name: str):
+        """
+        Delete a file by name.
+        """
+        pass
+
+    @abstractmethod
+    def get_metadata(self, file_name: str) -> dict:
+        """
+        Get metadata for a file.
+        """
+        pass
+
+    @abstractmethod
+    def list_all_metadata(self) -> list:
+        """
+        List all file metadata in the storage.
+        """
+        pass

solace_agent_mesh/services/file_service/file_manager/memory_file_manager.py

@@ -0,0 +1,64 @@
+import os
+
+from .file_manager_base import FileManagerBase
+
+
+class MemoryFileManager(FileManagerBase):
+    storage = {}
+
+    def __init__(self, config, ttl):
+        self.config = config
+        self.ttl = ttl
+
+    def upload_from_buffer(self, buffer: bytes, file_name: str, **kwargs) -> dict:
+        file_signature = self._generate_file_signature(file_name)
+        metadata = self._create_metadata(
+            file_signature, file_name, buffer, kwargs
+        )
+        metadata_name = self._get_metadata_name(file_signature)
+        self.storage[file_signature] = buffer
+        self.storage[metadata_name] = metadata
+        return metadata
+
+    def upload_from_file(self, file_path: str, **kwargs) -> dict:
+        """
+        Upload a file from a file path.
+        kwargs are added to metadata
+        """
+        with open(file_path, "rb") as file:
+            buffer = file.read()
+        return self.upload_from_buffer(buffer, os.path.basename(file_path), **kwargs)
+
+    def download_to_buffer(self, file_name: str) -> bytes:
+        if not file_name in self.storage:
+            raise FileNotFoundError(f"The file {file_name} does not exist.")
+        return self.storage[file_name]
+
+    def download_to_file(self, file_name: str, destination_path: str):
+        with open(destination_path, "wb") as file:
+            file.write(self.download_to_buffer(file_name))
+
+    def delete_by_name(self, file_name: str):
+        metadata_name = self._get_metadata_name(file_name)
+        if file_name in self.storage:
+            del self.storage[file_name]
+        if metadata_name in self.storage:
+            del self.storage[metadata_name]
+
+    def get_metadata(self, file_name: str) -> dict:
+        metadata_name = self._get_metadata_name(file_name)
+        if metadata_name in self.storage:
+            return self.storage[metadata_name]
+        raise FileNotFoundError(f"The file {file_name} does not exist.")
+    
+    def update_file_expiration(self, file_signature, expiration_timestamp):
+        metadata_name = self._get_metadata_name(file_signature)
+        if metadata_name in self.storage:
+            metadata = self.storage[metadata_name]
+            metadata["expiration_timestamp"] = expiration_timestamp
+            self.storage[metadata_name] = metadata
+        else:
+            raise FileNotFoundError(f"The file {file_signature} does not exist.")
+
+    def list_all_metadata(self) -> list:
+        return [data for key, data in self.storage.items() if key.endswith(".metadata")]

solace_agent_mesh/services/file_service/file_manager/volume_file_manager.py

@@ -0,0 +1,106 @@
+import os
+import json
+
+from .file_manager_base import FileManagerBase
+from ..file_service_constants import FS_PROTOCOL
+
+DEFAULT_DIRECTORY = f"/tmp/{FS_PROTOCOL}"
+
+
+class VolumeFileManager(FileManagerBase):
+
+    def __init__(self, config, ttl):
+        self.config = config
+        self.ttl = ttl
+        self.shared_volume_directory = config.get("directory", DEFAULT_DIRECTORY)
+        if not os.path.exists(self.shared_volume_directory):
+            os.makedirs(self.shared_volume_directory)
+
+    def _save_metadata(self, file_path: str, metadata: dict):
+        metadata_path = self._get_metadata_name(file_path)
+        with open(metadata_path, "w", encoding="utf-8") as metadata_file:
+            json.dump(metadata, metadata_file, indent=4)
+
+    def upload_from_buffer(self, buffer: bytes, file_name: str, **kwargs) -> dict:
+        file_signature = self._generate_file_signature(file_name)
+        file_path = os.path.join(self.shared_volume_directory, file_signature)
+
+        metadata = self._create_metadata(file_signature, file_name, buffer, kwargs)
+
+        with open(file_path, "wb") as file:
+            file.write(buffer)
+
+        self._save_metadata(file_path, metadata)
+
+        return metadata
+
+    def upload_from_file(self, file_path: str, **kwargs) -> dict:
+        if not os.path.isfile(file_path):
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+        file_name = os.path.basename(file_path)
+
+        with open(file_path, "rb") as source_file:
+            buffer = source_file.read()
+
+        return self.upload_from_buffer(buffer, file_name, **kwargs)
+
+    def download_to_buffer(self, file_name: str) -> bytes:
+        file_path = os.path.join(self.shared_volume_directory, file_name)
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(
+                f"The file at {file_name} does not exist in the shared volume."
+            )
+
+        with open(file_path, "rb") as file:
+            buffer = file.read()
+
+        return buffer
+
+    def download_to_file(self, file_name: str, destination_path: str):
+        buffer = self.download_to_buffer(file_name)
+        with open(destination_path, "wb") as destination_file:
+            destination_file.write(buffer)
+
+    def get_metadata(self, file_name: str) -> dict:
+        metadata_name = self._get_metadata_name(file_name)
+        metadata_path = os.path.join(self.shared_volume_directory, metadata_name)
+
+        if not os.path.exists(metadata_path):
+            raise FileNotFoundError(
+                f"The metadata for the file at {file_name} does not exist."
+            )
+
+        with open(metadata_path, "r", encoding="utf-8") as metadata_file:
+            metadata = json.load(metadata_file)
+
+        return metadata
+
+    def delete_by_name(self, file_name: str):
+        file_path = os.path.join(self.shared_volume_directory, file_name)
+        metadata_path = self._get_metadata_name(file_path)
+
+        # Delete the main file
+        if os.path.exists(file_path):
+            os.remove(file_path)
+            os.remove(metadata_path)
+        else:
+            raise FileNotFoundError(
+                f"The file at {file_name} does not exist in the shared volume."
+            )
+        
+    def update_file_expiration(self, file_signature, expiration_timestamp):
+        metadata = self.get_metadata(file_signature)
+        metadata["expiration_timestamp"] = expiration_timestamp
+        file_path = os.path.join(self.shared_volume_directory, file_signature)
+        self._save_metadata(file_path, metadata)
+
+    def list_all_metadata(self) -> list:
+        all_metadata = []
+        for file_name in os.listdir(self.shared_volume_directory):
+            if file_name.endswith(".metadata"):
+                metadata_path = os.path.join(self.shared_volume_directory, file_name)
+                with open(metadata_path, "r", encoding="utf-8") as metadata_file:
+                    metadata = json.load(metadata_file)
+                    all_metadata.append(metadata)
+        return all_metadata