naas-abi-core 1.4.1__py3-none-any.whl

naas_abi_core/services/object_storage/adapters/secondary/ObjectStorageSecondaryAdapterNaas.py

@@ -0,0 +1,131 @@
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from queue import Queue
+from typing import Optional
+
+import pydash
+import requests
+
+# Load S3 secondary adapter as we will use it.
+from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterS3 import (
+    ObjectStorageSecondaryAdapterS3,
+)
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    IObjectStorageAdapter,
+)
+
+NAAS_API_URL = "https://api.naas.ai/"
+CREDENTIALS_EXPIRATION_TIME = timedelta(minutes=20)
+
+
+@dataclass
+class Credentials:
+    bucket_name: str
+    bucket_prefix: str
+    access_key_id: str
+    secret_key: str
+    session_token: str
+    region_name: str
+    created_at: datetime
+
+
+class ObjectStorageSecondaryAdapterNaas(IObjectStorageAdapter):
+    __naas_api_key: str
+    __workspace_id: str
+    __storage_name: str
+    __credentials: Credentials | None
+    __s3_adapter: ObjectStorageSecondaryAdapterS3 | None
+
+    def __init__(
+        self,
+        naas_api_key: str,
+        workspace_id: str,
+        storage_name: str,
+        base_prefix: str = "",
+    ):
+        self.__naas_api_key = naas_api_key
+        self.__workspace_id = workspace_id
+        self.__storage_name = storage_name
+        self.__base_prefix = base_prefix
+        self.__credentials: Credentials | None = None
+        self.__s3_adapter: ObjectStorageSecondaryAdapterS3 | None = None
+
+    def ensure_credentials(self) -> Credentials:
+        if self.__credentials is None:
+            # TODO: Handle raise_for_status.
+            self.__refresh_credentials()
+
+        assert self.__credentials is not None
+
+        # If credentials are older than 10 minutes, refresh them
+        if self.__credentials.created_at < datetime.now() - CREDENTIALS_EXPIRATION_TIME:
+            self.__refresh_credentials()
+
+        return self.__credentials
+
+    def __refresh_credentials(self) -> None:
+        response = requests.post(
+            f"{NAAS_API_URL}/workspace/{self.__workspace_id}/storage/credentials/",
+            headers={"Authorization": f"Bearer {self.__naas_api_key}"},
+            json={
+                "name": self.__storage_name,
+            },
+        )
+
+        response.raise_for_status()
+
+        credentials = response.json()
+
+        self.__credentials = Credentials(
+            bucket_name=pydash.get(credentials, "credentials.s3.endpoint_url").split(
+                "/"
+            )[2],
+            bucket_prefix="/".join(
+                pydash.get(credentials, "credentials.s3.endpoint_url").split("/")[3:]
+            ),
+            access_key_id=pydash.get(credentials, "credentials.s3.access_key_id"),
+            secret_key=pydash.get(credentials, "credentials.s3.secret_key"),
+            session_token=pydash.get(credentials, "credentials.s3.session_token"),
+            region_name=pydash.get(credentials, "credentials.s3.region_name"),
+            created_at=datetime.now(),
+        )
+
+        # Re instantiate the S3 adapter with the new credentials
+        self.__s3_adapter = ObjectStorageSecondaryAdapterS3(
+            bucket_name=self.__credentials.bucket_name,
+            access_key_id=self.__credentials.access_key_id,
+            secret_access_key=self.__credentials.secret_key,
+            base_prefix=os.path.join(
+                self.__credentials.bucket_prefix, self.__base_prefix
+            ),
+            session_token=self.__credentials.session_token,
+        )
+
+    def get_object(self, prefix: str, key: str) -> bytes:
+        self.ensure_credentials()
+
+        assert self.__s3_adapter is not None
+
+        return self.__s3_adapter.get_object(prefix, key)
+
+    def put_object(self, prefix: str, key: str, content: bytes):
+        self.ensure_credentials()
+
+        assert self.__s3_adapter is not None
+
+        return self.__s3_adapter.put_object(prefix, key, content)
+
+    def delete_object(self, prefix: str, key: str):
+        self.ensure_credentials()
+
+        assert self.__s3_adapter is not None
+
+        return self.__s3_adapter.delete_object(prefix, key)
+
+    def list_objects(self, prefix: str, queue: Optional[Queue] = None) -> list[str]:
+        self.ensure_credentials()
+
+        assert self.__s3_adapter is not None
+
+        return self.__s3_adapter.list_objects(prefix, queue)

naas_abi_core/services/object_storage/adapters/secondary/ObjectStorageSecondaryAdapterS3.py

@@ -0,0 +1,171 @@
+from queue import Queue
+from typing import Optional
+
+import boto3
+from botocore.exceptions import ClientError
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    Exceptions,
+    IObjectStorageAdapter,
+)
+
+
+class ObjectStorageSecondaryAdapterS3(IObjectStorageAdapter):
+    """S3 implementation of the Object Storage adapter using boto3."""
+
+    def __init__(
+        self,
+        bucket_name: str,
+        access_key_id: str,
+        secret_access_key: str,
+        base_prefix: str = "",
+        session_token: str | None = None,
+    ):
+        """Initialize S3 adapter with bucket name and credentials.
+
+        Args:
+            bucket_name (str): Name of the S3 bucket to use
+            access_key_id (str): AWS access key ID
+            secret_access_key (str): AWS secret access key
+            base_prefix (str, optional): Base prefix to prepend to all operations. Defaults to ""
+            session_token (str, optional): AWS session token. Defaults to None
+        """
+        self.bucket_name = bucket_name
+        self.base_prefix = base_prefix.rstrip("/")  # Remove trailing slash if present
+        self.s3_client = boto3.client(
+            "s3",
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            aws_session_token=session_token,
+        )
+
+    def __get_full_key(self, prefix: str, key: str | None = None) -> str:
+        """Construct full key path including base prefix.
+
+        Args:
+            prefix (str): Prefix/folder path
+            key (str, optional): Object key name
+
+        Returns:
+            str: Full key path
+        """
+        # if key is None:
+        #     return f"{self.base_prefix}/{prefix}".lstrip('/')
+        return f"{self.base_prefix + '/' if self.base_prefix else ''}{prefix + '/' if prefix else ''}{key if key else ''}".lstrip(
+            "/"
+        )
+
+    def __object_exists(self, prefix: str, key: str | None = None) -> bool:
+        """Check if an object exists in S3.
+
+        Args:
+            prefix (str): Prefix/folder path
+            key (str, optional): Object key name
+
+        Returns:
+            bool: True if exists, raises exception if not
+        """
+        try:
+            if key is None:
+                # Check if prefix exists by listing objects
+                response = self.s3_client.list_objects_v2(
+                    Bucket=self.bucket_name,
+                    Prefix=self.__get_full_key(prefix),
+                    MaxKeys=1,
+                )
+                if "Contents" not in response:
+                    raise Exceptions.ObjectNotFound(f"Prefix {prefix} not found")
+            else:
+                # Check specific object
+                self.s3_client.head_object(
+                    Bucket=self.bucket_name, Key=self.__get_full_key(prefix, key)
+                )
+            return True
+        except ClientError as e:
+            if e.response["Error"]["Code"] in ["404", "NoSuchKey"]:
+                raise Exceptions.ObjectNotFound(f"Object {prefix}/{key} not found")
+            raise e
+
+    def get_object(self, prefix: str, key: str) -> bytes:
+        """Get object from S3 bucket.
+
+        Args:
+            prefix (str): Prefix/folder path
+            key (str): Object key name
+
+        Returns:
+            bytes: Object content
+        """
+        self.__object_exists(prefix, key)
+
+        response = self.s3_client.get_object(
+            Bucket=self.bucket_name, Key=self.__get_full_key(prefix, key)
+        )
+        return response["Body"].read()
+
+    def put_object(self, prefix: str, key: str, content: bytes) -> None:
+        """Put object into S3 bucket.
+
+        Args:
+            prefix (str): Prefix/folder path
+            key (str): Object key name
+            content (bytes): Content to upload
+        """
+        self.s3_client.put_object(
+            Bucket=self.bucket_name, Key=self.__get_full_key(prefix, key), Body=content
+        )
+
+    def delete_object(self, prefix: str, key: str) -> None:
+        """Delete object from S3 bucket.
+
+        Args:
+            prefix (str): Prefix/folder path
+            key (str): Object key name
+        """
+        self.__object_exists(prefix, key)
+
+        self.s3_client.delete_object(
+            Bucket=self.bucket_name, Key=self.__get_full_key(prefix, key)
+        )
+
+    def list_objects(self, prefix: str, queue: Optional[Queue] = None) -> list[str]:
+        """List objects in S3 bucket with given prefix.
+
+        Args:
+            prefix (str): Prefix/folder path to list
+
+        Returns:
+            list[str]: List of object keys at depth 1 only (direct children)
+        """
+        self.__object_exists(prefix)
+
+        objects = []
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+
+        for page in paginator.paginate(
+            Bucket=self.bucket_name,
+            Prefix=self.__get_full_key(prefix),
+            Delimiter="/",  # This makes it list only one level deep
+        ):
+            # Get regular objects at this level
+            if "Contents" in page:
+                for obj in page["Contents"]:
+                    # Remove base prefix from returned keys
+                    key = obj["Key"]
+                    if self.base_prefix:
+                        key = key.replace(f"{self.base_prefix}/", "", 1)
+                    if key != "":
+                        objects.append(key)
+                        if queue:
+                            queue.put(key)
+
+            # Get subfolders at this level
+            if "CommonPrefixes" in page:
+                for prefix_obj in page["CommonPrefixes"]:
+                    prefix_key = prefix_obj["Prefix"]
+                    if self.base_prefix:
+                        prefix_key = prefix_key.replace(f"{self.base_prefix}/", "", 1)
+                    if prefix_key != "":
+                        objects.append(prefix_key)
+                        if queue:
+                            queue.put(prefix_key)
+        return objects

naas_abi_core/services/ontology/OntologyPorts.py

@@ -0,0 +1,36 @@
+from abc import ABC, abstractmethod
+from rdflib import Graph
+from langchain_core.language_models import BaseChatModel
+
+
+class IOntologyNERPort(ABC):
+    __chat_model: BaseChatModel
+
+    @abstractmethod
+    def named_entity_recognition(self, input: str, ontology_str: str) -> Graph:
+        """Apply Named Entity Recognition (NER) on input text to map entities to ontology concepts.
+
+        Args:
+            input (str): The unstructured text to process with NER
+            ontology_str (str): The ontology in Turtle format to map entities against
+
+        Returns:
+            Graph: An RDFLib Graph containing the mapped entities and their relationships from the input text
+        """
+        pass
+
+
+class IOntologyService(ABC):
+    __ner_adaptor: IOntologyNERPort
+
+    @abstractmethod
+    def named_entity_recognition(self, input: str) -> Graph:
+        """Apply Named Entity Recognition (NER) on input text to map entities to ontology concepts.
+
+        Args:
+            input (str): The input text to process with NER
+
+        Returns:
+            Graph: An RDFLib Graph containing the mapped entities and their relationships
+        """
+        pass

naas_abi_core/services/ontology/OntologyService.py

@@ -0,0 +1,17 @@
+from naas_abi_core.services.ontology.OntologyPorts import (
+    IOntologyNERPort,
+    IOntologyService,
+)
+from rdflib import Graph
+
+
+class OntologyService(IOntologyService):
+    __ontology_str: str
+    __ner_adaptor: IOntologyNERPort
+
+    def __init__(self, ner_adaptor: IOntologyNERPort, ontology_str: str):
+        self.__ner_adaptor = ner_adaptor
+        self.__ontology_str = ontology_str
+
+    def named_entity_recognition(self, input: str) -> Graph:
+        return self.__ner_adaptor.named_entity_recognition(input, self.__ontology_str)

naas_abi_core/services/ontology/adaptors/secondary/OntologyService_SecondaryAdaptor_NERPort.py

@@ -0,0 +1,37 @@
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import HumanMessage, SystemMessage
+from naas_abi_core.services.ontology.OntologyPorts import IOntologyNERPort
+from rdflib import Graph
+
+
+class OntologyService_SecondaryAdaptor_NERPort(IOntologyNERPort):
+    __chat_model: BaseChatModel
+
+    def __init__(self, chat_model: BaseChatModel):
+        self.__chat_model = chat_model
+
+    def named_entity_recognition(self, input: str, ontology_str: str) -> Graph:
+        messages = [
+            SystemMessage(
+                content=f"""
+You are an expert in Named Entity Recognition (NER).
+You are given a string of unstructured text and an ontology in Turtle format.
+Your task is to map the entities in the text to the ontology concepts.
+
+Here is the ontology in Turtle format:
+{ontology_str}
+
+
+Only output the Turtle formated file ready to be stored in a file. You must add all prefixes.
+"""
+            ),
+            HumanMessage(content=input),
+        ]
+        response = self.__chat_model.invoke(messages)
+        assert isinstance(response.content, str)
+        sanitized_response = response.content.replace("```turtle", "").replace(
+            "```", ""
+        )
+        g = Graph()
+        g.parse(data=sanitized_response, format="turtle")
+        return g

naas_abi_core/services/secret/Secret.py

@@ -0,0 +1,138 @@
+"""Secret Service Module
+
+This module provides a unified interface for managing secrets across different storage systems.
+The Secret service acts as a facade that coordinates multiple secret adapters, allowing applications
+to retrieve, store, and manage secrets from various sources like environment variables, files,
+cloud services, or other secret management systems.
+
+The module implements the adapter pattern to support pluggable secret storage backends while
+providing a consistent API for secret operations.
+
+Classes:
+    Secret: Main service class implementing ISecretService interface
+
+Example:
+    >>> from naas_abi_core.services.secret.adapters import EnvVarSecretAdapter, FileSecretAdapter
+    >>> adapters = [EnvVarSecretAdapter(), FileSecretAdapter("/path/to/secrets")]
+    >>> secret_service = Secret(adapters)
+    >>> api_key = secret_service.get("API_KEY", "default_key")
+"""
+
+from typing import Any, Dict, List
+
+from naas_abi_core.services.secret.SecretPorts import ISecretAdapter, ISecretService
+
+
+class Secret(ISecretService):
+    """Secret service for managing and retrieving secrets.
+
+    This service provides a unified interface for accessing secrets regardless of their storage location
+    (environment variables, files, cloud services, etc.) through the use of adapters.
+
+    Attributes:
+        __adapter (ISecretAdapter): The adapter implementation used for retrieving secrets
+            from the underlying storage system.
+
+    Example:
+        >>> secret_service = Secret(EnvVarSecretAdapter())
+        >>> api_key = secret_service.get("API_KEY")
+    """
+
+    __adapters: List[ISecretAdapter]
+
+    def __init__(self, adapters: List[ISecretAdapter]):
+        self.__adapters = adapters
+
+    def get(self, key: str, default: Any = None) -> str:
+        """Retrieve a secret value by its key.
+
+        Searches for the secret across all configured adapters in order. Returns the value
+        from the first adapter that contains the key, or the default value if not found.
+
+        Args:
+            key (str): The secret key to retrieve.
+            default (Any, optional): The value to return if the key is not found.
+                Defaults to None.
+
+        Returns:
+            str: The secret value if found, otherwise the default value.
+
+        Example:
+            >>> secret_service = Secret([EnvVarSecretAdapter()])
+            >>> database_url = secret_service.get("DATABASE_URL", "sqlite:///:memory:")
+        """
+        for adapter in self.__adapters:
+            value = adapter.get(key, None)
+
+            if value is not None:
+                return value
+
+        return default
+
+    def set(self, key: str, value: str):
+        """Set a secret value across all configured adapters.
+
+        Stores the secret key-value pair in all adapters. This ensures consistency
+        across different secret storage systems.
+
+        Args:
+            key (str): The secret key to set.
+            value (str): The secret value to store.
+
+        Note:
+            If any adapter fails to set the value, the operation continues with
+            remaining adapters. Consider implementing error handling based on
+            your specific requirements.
+
+        Example:
+            >>> secret_service = Secret([EnvVarSecretAdapter(), FileSecretAdapter()])
+            >>> secret_service.set("API_KEY", "your-secret-api-key")
+        """
+        for adapter in self.__adapters:
+            adapter.set(key, value)
+
+    def remove(self, key: str):
+        """Remove a secret key from all configured adapters.
+
+        Deletes the specified key from all adapters. This ensures the secret
+        is completely removed from all storage systems.
+
+        Args:
+            key (str): The secret key to remove.
+
+        Note:
+            If any adapter fails to remove the key, the operation continues with
+            remaining adapters. The method does not raise exceptions for missing keys.
+
+        Example:
+            >>> secret_service = Secret([EnvVarSecretAdapter(), FileSecretAdapter()])
+            >>> secret_service.remove("OLD_API_KEY")
+        """
+        for adapter in self.__adapters:
+            adapter.remove(key)
+
+    def list(self) -> Dict[str, str | None]:
+        """Retrieve all secrets from all configured adapters.
+
+        Combines secrets from all adapters into a single dictionary. When the same
+        key exists in multiple adapters, the value from the adapter with higher
+        priority (earlier in the list) takes precedence.
+
+        Returns:
+            Dict[str, str]: A dictionary containing all secret key-value pairs.
+
+        Note:
+            The order of adapters matters for conflict resolution. Adapters are
+            processed in reverse order, so earlier adapters override later ones.
+
+        Example:
+            >>> secret_service = Secret([EnvVarSecretAdapter(), FileSecretAdapter()])
+            >>> all_secrets = secret_service.list()
+            >>> print(f"Found {len(all_secrets)} secrets")
+        """
+        secrets = {}
+
+        for adapter in [adapter for adapter in self.__adapters][::-1]:
+            secrets.update(adapter.list())
+
+        return secrets

naas_abi_core/services/secret/SecretPorts.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+
+class SecretAuthenticationError(Exception):
+    pass
+
+
+class ISecretAdapter(ABC):
+    @abstractmethod
+    def get(self, key: str, default: Any = None) -> str | Any | None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def set(self, key: str, value: str):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def remove(self, key: str):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list(self) -> Dict[str, str | None]:
+        raise NotImplementedError()
+
+
+class ISecretService(ABC):
+    __adapter: List[ISecretAdapter]
+
+    @abstractmethod
+    def get(self, key: str, default: Any = None) -> str | Any | None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def set(self, key: str, value: str):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def remove(self, key: str):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def list(self) -> Dict[str, str | None]:
+        raise NotImplementedError()
+        raise NotImplementedError()

naas_abi_core/services/secret/Secret_test.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict
+
+import pytest
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.secret.SecretPorts import ISecretAdapter
+
+
+@pytest.fixture
+def TestSecretAdapter():
+    class TestSecretAdapter(ISecretAdapter):
+        def __init__(self, secrets: Dict[str, str | None]):
+            self.secrets = secrets or {}
+
+        def get(self, key: str, default: Any = None) -> str | Any | None:
+            return self.secrets.get(key, default)
+
+        def set(self, key: str, value: str):
+            self.secrets[key] = value
+
+        def remove(self, key: str):
+            self.secrets.pop(key, None)
+
+        def list(self) -> Dict[str, str | None]:
+            return self.secrets
+
+    return TestSecretAdapter
+
+
+def test_secret(TestSecretAdapter):
+    secret = Secret(
+        [
+            TestSecretAdapter(
+                {
+                    "hello": "world",
+                }
+            ),
+            TestSecretAdapter(
+                {
+                    "hello": "abi",
+                    "second": "second",
+                }
+            ),
+        ]
+    )
+
+    assert secret.get("hello") == "world"
+    assert secret.list() == {
+        "hello": "world",
+        "second": "second",
+    }
+
+    secret.remove("hello")
+    assert secret.get("hello") is None
+    assert secret.list() == {
+        "second": "second",
+    }
+
+    secret.remove("second")
+    assert secret.list() == {}
+
+    secret.set("hello", "world")
+    assert secret.get("hello") == "world"
+    assert secret.list() == {
+        "hello": "world",
+    }