naas-abi-core 1.0.0__py3-none-any.whl

naas_abi_core/engine/IEngine.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Dict, List, Union
+
+from naas_abi_core.services.object_storage.ObjectStorageService import (
+    ObjectStorageService,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.triple_store.TripleStoreService import TripleStoreService
+from naas_abi_core.services.vector_store.VectorStoreService import VectorStoreService
+
+if TYPE_CHECKING:
+    from naas_abi_core.module.Module import BaseModule
+
+
+class IEngine:
+    class Services:
+        __object_storage: ObjectStorageService | None
+        __triple_store: TripleStoreService | None
+        __vector_store: VectorStoreService | None
+        __secret: Secret | None
+
+        def __init__(
+            self,
+            object_storage: ObjectStorageService | None = None,
+            triple_store: TripleStoreService | None = None,
+            vector_store: VectorStoreService | None = None,
+            secret: Secret | None = None,
+        ):
+            self.__object_storage = object_storage
+            self.__triple_store = triple_store
+            self.__vector_store = vector_store
+            self.__secret = secret
+
+        @property
+        def object_storage(self) -> ObjectStorageService:
+            assert self.__object_storage is not None, (
+                "Object storage service is not initialized"
+            )
+            return self.__object_storage
+
+        @property
+        def triple_store(self) -> TripleStoreService:
+            assert self.__triple_store is not None, (
+                "Triple store service is not initialized"
+            )
+            return self.__triple_store
+
+        def triple_store_available(self) -> bool:
+            return self.__triple_store is not None
+
+        @property
+        def vector_store(self) -> VectorStoreService:
+            assert self.__vector_store is not None, (
+                "Vector store service is not initialized"
+            )
+            return self.__vector_store
+
+        @property
+        def secret(self) -> Secret:
+            assert self.__secret is not None, "Secret service is not initialized"
+            return self.__secret
+
+        @property
+        def all(
+            self,
+        ) -> List[
+            Union[
+                ObjectStorageService | None,
+                TripleStoreService | None,
+                VectorStoreService | None,
+                Secret | None,
+            ]
+        ]:
+            return [
+                self.__object_storage,
+                self.__triple_store,
+                self.__vector_store,
+                self.__secret,
+            ]
+
+    __services: Services
+    __modules: Dict[str, BaseModule]
+
+    @property
+    def services(self) -> Services:
+        return self.__services
+
+    @property
+    def modules(self) -> Dict[str, BaseModule]:
+        return self.__modules

naas_abi_core/engine/conftest.py

@@ -0,0 +1,45 @@
+from pytest import fixture
+
+
+@fixture
+def test_configuration():
+    return """
+workspace_id: "1234567890"
+storage_name: "test"
+github_repository: "test"
+github_project_id: 1234567890
+triple_store_path: "test"
+api_title: "test"
+api_description: "test"
+logo_path: "test"
+favicon_path: "test"
+space_name: "test"
+cors_origins:
+  - "http://localhost:9879"
+
+services:
+    object_storage: &object_storage_service
+      object_storage_adapter:
+        adapter: "fs"
+        config:
+          base_path: "test"
+          test: "{{ secret.OXIGRAPH_URL }}"
+
+    triple_store:
+      triple_store_adapter:
+        adapter: "object_storage"
+        config:
+          triples_prefix: "test"
+          object_storage_service: *object_storage_service
+
+        
+    vector_store:
+      vector_store_adapter:
+        adapter: "qdrant"
+        config: {}
+        
+    secret:
+      secret_adapters:
+        - adapter: "dotenv"
+          config: {}
+"""

naas_abi_core/engine/engine_configuration/EngineConfiguration.py

@@ -0,0 +1,160 @@
+import os
+from io import StringIO
+from typing import List
+
+import yaml
+from jinja2 import Template
+from naas_abi_core import logger
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_ObjectStorageService import (
+    ObjectStorageServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_SecretService import (
+    SecretServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_TripleStoreService import (
+    TripleStoreServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_VectorStoreService import (
+    VectorStoreServiceConfiguration,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from pydantic import BaseModel, model_validator
+from typing_extensions import Literal
+
+
+class ServicesConfiguration(BaseModel):
+    object_storage: ObjectStorageServiceConfiguration
+    triple_store: TripleStoreServiceConfiguration
+    vector_store: VectorStoreServiceConfiguration
+    secret: SecretServiceConfiguration
+
+
+class ApiConfiguration(BaseModel):
+    title: str = "ABI API"
+    description: str = "API for ABI, your Artifical Business Intelligence"
+    logo_path: str = "assets/logo.png"
+    favicon_path: str = "assets/favicon.ico"
+    cors_origins: List[str] = ["http://localhost:9879"]
+
+
+class FirstPassConfiguration(BaseModel):
+    """This is a first pass configuration that is used to load the secret service.
+
+    It is used to load the secret service before the other services are loaded.
+    This is because the secret service needs to be loaded before the other services
+    are loaded to be able to resolve the secrets.
+    This is a first pass configuration that is used to load the secret service.
+    """
+
+    class FirstPassServicesConfiguration(BaseModel):
+        secret: SecretServiceConfiguration
+
+    services: FirstPassServicesConfiguration
+
+
+class ModuleConfig(BaseModel):
+    path: str | None = None
+    module: str | None = None
+    enabled: bool
+    config: dict = {}
+
+    @model_validator(mode="after")
+    def validate_path_or_module(self):
+        if self.path is None and self.module is None:
+            raise ValueError("Either path or module must be provided")
+
+        if self.path is None:
+            assert self.module is not None, (
+                "module must be provided if path is not provided"
+            )
+        if self.module is None:
+            assert self.path is not None, (
+                "path must be provided if module is not provided"
+            )
+        return self
+
+
+class GlobalConfig(BaseModel):
+    ai_mode: Literal["cloud", "local", "airgap"]
+
+
+class EngineConfiguration(BaseModel):
+    api: ApiConfiguration
+
+    services: ServicesConfiguration
+
+    global_config: GlobalConfig
+
+    modules: List[ModuleConfig]
+
+    @classmethod
+    def from_yaml(cls, yaml_path: str) -> "EngineConfiguration":
+        with open(yaml_path, "r") as file:
+            return cls.from_yaml_content(file.read())
+
+    @classmethod
+    def from_yaml_content(cls, yaml_content: str) -> "EngineConfiguration":
+        # First we do a pass with the minimal configuration to load the secret service.
+        class SecretServiceWrapper:
+            secret_service: Secret | None = None
+
+            def __init__(self, secret_service: Secret | None = None):
+                self.secret_service = secret_service
+
+            def __getattr__(self, name):
+                if self.secret_service is None:
+                    return 0
+                secret = self.secret_service.get(name)
+                if secret is None:
+                    raise ValueError(f"Secret {name} not found")
+                return secret
+
+            def get(self, key, default=None):
+                return 0
+
+        first_pass_data = yaml.safe_load(
+            StringIO(Template(yaml_content).render(secret=SecretServiceWrapper()))
+        )
+
+        first_pass_configuration = FirstPassConfiguration(**first_pass_data)
+        secret_service = first_pass_configuration.services.secret.load()
+
+        # Here we can now template the yaml by using `yaml_content` and the secret service.
+        # Using Jinja2 template engine.
+
+        logger.debug("Yaml content: {yaml_content}")
+
+        template = Template(yaml_content)
+        templated_yaml = template.render(secret=SecretServiceWrapper(secret_service))
+
+        logger.debug(f"Templated yaml: {templated_yaml}")
+
+        data = yaml.safe_load(StringIO(templated_yaml))
+
+        logger.debug(f"Data: {data}")
+
+        return cls(**data)
+
+    @classmethod
+    def load_configuration(
+        cls, configuration_yaml: str | None = None
+    ) -> "EngineConfiguration":
+        # This is useful for testing.
+        if configuration_yaml is not None:
+            return cls.from_yaml_content(configuration_yaml)
+
+        logger.debug(f"Loading configuration from {os.getenv('ENV')}")
+
+        if os.path.exists(f"config.{os.getenv('ENV')}.yaml"):
+            return cls.from_yaml(f"config.{os.getenv('ENV')}.yaml")
+        elif os.path.exists("config.yaml"):
+            return cls.from_yaml("config.yaml")
+        else:
+            raise FileNotFoundError(
+                "Configuration file not found. Please create a config.yaml file or config.{env}.yaml file."
+            )
+
+
+if __name__ == "__main__":
+    config = EngineConfiguration.load_configuration()
+    print(config)

naas_abi_core/engine/engine_configuration/EngineConfiguration_GenericLoader.py

@@ -0,0 +1,49 @@
+import importlib
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+
+class GenericLoader(BaseModel):
+    """
+    Generic loader for dynamically importing and instantiating Python callables.
+
+    This class enables dynamic loading of Python classes or functions by specifying
+    the module path and callable name. The callable is invoked with the provided
+    configuration dictionary unpacked as keyword arguments.
+
+    Usage:
+        1. Use importlib.import_module(python_module) to import the target module
+        2. Use getattr(module, module_callable) to retrieve the callable
+        3. Invoke the callable with the custom_config as **custom_config
+
+    Example:
+        loader = GenericLoader(
+            python_module="my.package.module",
+            module_callable="MyClass",
+            custom_config={"param1": "value1", "param2": 42}
+        )
+        # This would effectively do:
+        # import importlib
+        # module = importlib.import_module("my.package.module")
+        # callable_obj = getattr(module, "MyClass")
+        # instance = callable_obj(param1="value1", param2=42)
+
+    Attributes:
+        python_module: The fully qualified Python module path (e.g., "package.subpackage.module")
+        module_callable: The name of the callable (class or function) to retrieve from the module
+        custom_config: Dictionary of configuration parameters to pass as keyword arguments to the callable
+    """
+
+    python_module: str | None = None
+    module_callable: str | None = None
+    custom_config: Dict[str, Any] | None = None
+
+    def load(self) -> Any:
+        assert self.python_module is not None, "python_module is required"
+        assert self.module_callable is not None, "module_callable is required"
+        assert self.custom_config is not None, "custom_config is required"
+
+        module = importlib.import_module(self.python_module)
+        callable_obj = getattr(module, self.module_callable)
+        return callable_obj(**self.custom_config)

naas_abi_core/engine/engine_configuration/EngineConfiguration_ObjectStorageService.py

@@ -0,0 +1,131 @@
+from typing import Dict, Literal, Tuple, Union
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    IObjectStorageAdapter,
+)
+from naas_abi_core.services.object_storage.ObjectStorageService import (
+    ObjectStorageService,
+)
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+
+class ObjectStorageAdapterFSConfiguration(BaseModel):
+    base_path: str
+
+
+class ObjectStorageAdapterS3Configuration(BaseModel):
+    bucket_name: str
+    access_key_id: str
+    secret_access_key: str
+    base_prefix: str
+    session_token: str | None = None
+
+
+class ObjectStorageAdapterNaasConfiguration(BaseModel):
+    naas_api_key: str
+    workspace_id: str
+    storage_name: str
+    base_prefix: str = ""
+
+
+class ObjectStorageAdapterConfiguration(GenericLoader):
+    adapter: Literal["fs", "s3", "naas", "custom"]
+    config: (
+        Union[
+            ObjectStorageAdapterFSConfiguration,
+            ObjectStorageAdapterS3Configuration,
+            ObjectStorageAdapterNaasConfiguration,
+        ]
+        | None
+    ) = None
+
+    __MAPPING: Dict[Literal["fs", "s3", "naas"], Tuple[str, str]] = {
+        "fs": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS",
+            "ObjectStorageSecondaryAdapterFS",
+        ),
+        "s3": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterS3",
+            "ObjectStorageSecondaryAdapterS3",
+        ),
+        "naas": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterNaas",
+            "ObjectStorageSecondaryAdapterNaas",
+        ),
+    }
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> Self:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+        if self.adapter == "fs":
+            pydantic_model_validator(
+                ObjectStorageAdapterFSConfiguration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 'fs' adapter",
+            )
+        if self.adapter == "s3":
+            pydantic_model_validator(
+                ObjectStorageAdapterS3Configuration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 's3' adapter",
+            )
+        if self.adapter == "naas":
+            pydantic_model_validator(
+                ObjectStorageAdapterNaasConfiguration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 'naas' adapter",
+            )
+
+        return self
+
+    def load(self) -> IObjectStorageAdapter:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+            if self.adapter == "fs":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS import (
+                    ObjectStorageSecondaryAdapterFS,
+                )
+
+                return ObjectStorageSecondaryAdapterFS(**self.config.model_dump())
+            elif self.adapter == "s3":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterS3 import (
+                    ObjectStorageSecondaryAdapterS3,
+                )
+
+                return ObjectStorageSecondaryAdapterS3(**self.config.model_dump())
+            elif self.adapter == "naas":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterNaas import (
+                    ObjectStorageSecondaryAdapterNaas,
+                )
+
+                return ObjectStorageSecondaryAdapterNaas(**self.config.model_dump())
+            else:
+                raise ValueError(f"Unknown adapter: {self.adapter}")
+            # return GenericLoader(
+            #     python_module=self.__MAPPING[self.adapter][0],
+            #     module_callable=self.__MAPPING[self.adapter][1],
+            #     custom_config=self.config.model_dump(),
+            # ).load()
+        else:
+            return super().load()
+
+
+class ObjectStorageServiceConfiguration(BaseModel):
+    object_storage_adapter: ObjectStorageAdapterConfiguration
+
+    def load(self) -> ObjectStorageService:
+        return ObjectStorageService(adapter=self.object_storage_adapter.load())

naas_abi_core/engine/engine_configuration/EngineConfiguration_ObjectStorageService_test.py

@@ -0,0 +1,26 @@
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_ObjectStorageService import (
+    ObjectStorageAdapterConfiguration,
+    ObjectStorageAdapterFSConfiguration,
+    ObjectStorageServiceConfiguration,
+)
+from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS import (
+    ObjectStorageSecondaryAdapterFS,
+)
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    IObjectStorageAdapter,
+)
+
+
+def test_object_storage_service_configuration():
+    configuration = ObjectStorageServiceConfiguration(
+        object_storage_adapter=ObjectStorageAdapterConfiguration(
+            adapter="fs", config=ObjectStorageAdapterFSConfiguration(base_path="test")
+        )
+    )
+    assert configuration.object_storage_adapter is not None
+
+    object_storage_adapter = configuration.object_storage_adapter.load()
+
+    assert object_storage_adapter is not None
+    assert isinstance(object_storage_adapter, IObjectStorageAdapter)
+    assert isinstance(object_storage_adapter, ObjectStorageSecondaryAdapterFS)

naas_abi_core/engine/engine_configuration/EngineConfiguration_SecretService.py

@@ -0,0 +1,116 @@
+from typing import TYPE_CHECKING, List, Literal, Union
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.secret.SecretPorts import ISecretAdapter
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+# Only import for type checking, not at runtime
+if TYPE_CHECKING:
+    from naas_abi_core.services.secret.adaptors.secondary.Base64Secret import (
+        Base64Secret,
+    )
+
+
+class Base64SecretConfiguration(BaseModel):
+    secret_adapter: "SecretAdapterConfiguration"
+    base64_secret_key: str
+
+    def load(self) -> "Base64Secret":
+        from naas_abi_core.services.secret.adaptors.secondary.Base64Secret import (
+            Base64Secret,
+        )
+
+        return Base64Secret(self.secret_adapter.load(), self.base64_secret_key)
+
+
+class NaasSecretConfiguration(BaseModel):
+    naas_api_key: str
+    naas_api_url: str
+
+
+class DotenvSecretConfiguration(BaseModel):
+    pass
+
+
+class SecretAdapterConfiguration(GenericLoader):
+    adapter: Literal["dotenv", "naas", "base64", "custom"]
+    config: (
+        Union[
+            DotenvSecretConfiguration,
+            NaasSecretConfiguration,
+            Base64SecretConfiguration,
+        ]
+        | None
+    ) = None
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> Self:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+        if self.adapter == "base64":
+            pydantic_model_validator(
+                Base64SecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'base64' adapter",
+            )
+        if self.adapter == "dotenv":
+            pydantic_model_validator(
+                DotenvSecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'dotenv' adapter",
+            )
+        if self.adapter == "naas":
+            pydantic_model_validator(
+                NaasSecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'naas' adapter",
+            )
+        return self
+
+    def load(self) -> ISecretAdapter:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+            # Lazy import: only import the adapter that's actually configured
+            if self.adapter == "base64":
+                assert isinstance(self.config, Base64SecretConfiguration), (
+                    "config must be a Base64SecretConfiguration if adapter is base64"
+                )
+                assert hasattr(self.config, "load"), (
+                    "config must have a load method if adapter is base64"
+                )
+                return self.config.load()
+            elif self.adapter == "dotenv":
+                from naas_abi_core.services.secret.adaptors.secondary.dotenv_secret_secondaryadaptor import (
+                    DotenvSecretSecondaryAdaptor,
+                )
+
+                return DotenvSecretSecondaryAdaptor(**self.config.model_dump())
+            elif self.adapter == "naas":
+                from naas_abi_core.services.secret.adaptors.secondary.NaasSecret import (
+                    NaasSecret,
+                )
+
+                return NaasSecret(**self.config.model_dump())
+            else:
+                raise ValueError(f"Unknown adapter: {self.adapter}")
+        else:
+            return super().load()
+
+
+class SecretServiceConfiguration(BaseModel):
+    secret_adapters: List[SecretAdapterConfiguration]
+
+    def load(self) -> Secret:
+        return Secret(adapters=[adapter.load() for adapter in self.secret_adapters])