naas-abi-core 1.4.1__py3-none-any.whl

naas_abi_core/engine/engine_configuration/EngineConfiguration.py

@@ -0,0 +1,216 @@
+import os
+import sys
+from io import StringIO
+from typing import List
+
+import yaml
+from jinja2 import Template
+from naas_abi_core import logger
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_Deploy import (
+    DeployConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_ObjectStorageService import (
+    ObjectStorageServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_SecretService import (
+    SecretServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_TripleStoreService import (
+    TripleStoreServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_VectorStoreService import (
+    VectorStoreServiceConfiguration,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from pydantic import BaseModel, model_validator
+from rich.prompt import Prompt
+from typing_extensions import Literal, Self
+
+
+class ServicesConfiguration(BaseModel):
+    object_storage: ObjectStorageServiceConfiguration
+    triple_store: TripleStoreServiceConfiguration
+    vector_store: VectorStoreServiceConfiguration
+    secret: SecretServiceConfiguration
+
+
+class ApiConfiguration(BaseModel):
+    title: str = "ABI API"
+    description: str = "API for ABI, your Artifical Business Intelligence"
+    logo_path: str = "assets/logo.png"
+    favicon_path: str = "assets/favicon.ico"
+    cors_origins: List[str] = ["http://localhost:9879"]
+
+
+class FirstPassConfiguration(BaseModel):
+    """This is a first pass configuration that is used to load the secret service.
+
+    It is used to load the secret service before the other services are loaded.
+    This is because the secret service needs to be loaded before the other services
+    are loaded to be able to resolve the secrets.
+    This is a first pass configuration that is used to load the secret service.
+    """
+
+    class FirstPassServicesConfiguration(BaseModel):
+        secret: SecretServiceConfiguration
+
+    services: FirstPassServicesConfiguration
+
+
+class ModuleConfig(BaseModel):
+    path: str | None = None
+    module: str | None = None
+    enabled: bool
+    config: dict = {}
+
+    @model_validator(mode="after")
+    def validate_path_or_module(self):
+        if self.path is None and self.module is None:
+            raise ValueError("Either path or module must be provided")
+
+        if self.path is None:
+            assert self.module is not None, (
+                "module must be provided if path is not provided"
+            )
+        if self.module is None:
+            assert self.path is not None, (
+                "path must be provided if module is not provided"
+            )
+        return self
+
+
+class GlobalConfig(BaseModel):
+    ai_mode: Literal["cloud", "local", "airgap"]
+
+
+class EngineConfiguration(BaseModel):
+    api: ApiConfiguration
+
+    deploy: DeployConfiguration | None = None
+
+    services: ServicesConfiguration
+
+    global_config: GlobalConfig
+
+    modules: List[ModuleConfig]
+
+    def ensure_default_modules(self) -> None:
+        if not any(
+            m.path == "naas_abi_core.modules.templatablesparqlquery"
+            or m.module == "naas_abi_core.modules.templatablesparqlquery"
+            for m in self.modules
+        ):
+            self.modules.append(
+                ModuleConfig(
+                    module="naas_abi_core.modules.templatablesparqlquery",
+                    enabled=True,
+                    config={},
+                )
+            )
+
+    @model_validator(mode="after")
+    def validate_modules(self) -> Self:
+        self.ensure_default_modules()
+        return self
+
+    @classmethod
+    def from_yaml(cls, yaml_path: str) -> "EngineConfiguration":
+        with open(yaml_path, "r") as file:
+            return cls.from_yaml_content(file.read())
+
+    @classmethod
+    def from_yaml_content(cls, yaml_content: str) -> "EngineConfiguration":
+        # First we do a pass with the minimal configuration to load the secret service.
+        class SecretServiceWrapper:
+            secret_service: Secret | None = None
+
+            def __init__(self, secret_service: Secret | None = None):
+                self.secret_service = secret_service
+
+            def __getattr__(self, name):
+                if self.secret_service is None:
+                    # This rule is used only when doing the first pass configuration to load the secret service.
+
+                    # First priority is to check the environment variables.
+                    if name in os.environ:
+                        return os.environ.get(name)
+                    else:
+                        # If the environment variable is not found, we check the .env file. ONLY FOR THE FIRST PASS CONFIGURATION.
+                        from dotenv import dotenv_values
+
+                        secrets = dotenv_values()
+                        if name in secrets:
+                            return secrets.get(name)
+                        else:
+                            return f"Secret '{name}' not found while loading the secret service. Please provide it via the environment variables or .env file."
+                elif name in os.environ:
+                    return os.environ.get(name)
+                secret = self.secret_service.get(name)
+                if secret is None:
+                    if not sys.stdin.isatty():
+                        raise ValueError(
+                            f"Secret '{name}' not found and no TTY available to prompt. Please provide it via the configured secret service or environment."
+                        )
+                    value = Prompt.ask(
+                        f"[bold yellow]Secret '{name}' not found.[/bold yellow] Please enter the value for [cyan]{name}[/cyan]",
+                        password=False,
+                    )
+                    self.secret_service.set(name, value)
+                    return value
+                return secret
+
+        first_pass_data = yaml.safe_load(
+            StringIO(Template(yaml_content).render(secret=SecretServiceWrapper()))
+        )
+
+        first_pass_configuration = FirstPassConfiguration(**first_pass_data)
+        secret_service = first_pass_configuration.services.secret.load()
+
+        # Here we can now template the yaml by using `yaml_content` and the secret service.
+        # Using Jinja2 template engine.
+
+        logger.debug(f"Yaml content: {yaml_content}")
+
+        template = Template(yaml_content)
+        templated_yaml = template.render(secret=SecretServiceWrapper(secret_service))
+
+        data = yaml.safe_load(StringIO(templated_yaml))
+
+        logger.debug(f"Data: {data}")
+
+        return cls(**data)
+
+    @classmethod
+    def load_configuration(
+        cls, configuration_yaml: str | None = None
+    ) -> "EngineConfiguration":
+        # This is useful for testing.
+        if configuration_yaml is not None:
+            return cls.from_yaml_content(configuration_yaml)
+
+        # Get ENV value
+        from dotenv import dotenv_values
+
+        env = os.getenv("ENV")
+        if not env:
+            env = dotenv_values().get("ENV")
+
+        # First we check the environment variable.
+        if os.path.exists(f"config.{env}.yaml"):
+            config_file = f"config.{env}.yaml"
+        # If the config.{env}.yaml file is not found, we check the config.yaml file.
+        elif os.path.exists("config.yaml"):
+            config_file = "config.yaml"
+        else:
+            raise FileNotFoundError(
+                "Configuration file not found. Please create a config.yaml file or config.{env}.yaml file."
+            )
+
+        logger.debug(f"Loading configuration from {config_file}")
+
+        return cls.from_yaml(config_file)
+
+
+if __name__ == "__main__":
+    config = EngineConfiguration.load_configuration()
+    print(config)

naas_abi_core/engine/engine_configuration/EngineConfiguration_Deploy.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+class DeployConfiguration(BaseModel):
+    workspace_id: str
+    space_name: str
+    naas_api_key: str
+    env: dict[str, str] = {}

naas_abi_core/engine/engine_configuration/EngineConfiguration_GenericLoader.py

@@ -0,0 +1,49 @@
+import importlib
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+
+class GenericLoader(BaseModel):
+    """
+    Generic loader for dynamically importing and instantiating Python callables.
+
+    This class enables dynamic loading of Python classes or functions by specifying
+    the module path and callable name. The callable is invoked with the provided
+    configuration dictionary unpacked as keyword arguments.
+
+    Usage:
+        1. Use importlib.import_module(python_module) to import the target module
+        2. Use getattr(module, module_callable) to retrieve the callable
+        3. Invoke the callable with the custom_config as **custom_config
+
+    Example:
+        loader = GenericLoader(
+            python_module="my.package.module",
+            module_callable="MyClass",
+            custom_config={"param1": "value1", "param2": 42}
+        )
+        # This would effectively do:
+        # import importlib
+        # module = importlib.import_module("my.package.module")
+        # callable_obj = getattr(module, "MyClass")
+        # instance = callable_obj(param1="value1", param2=42)
+
+    Attributes:
+        python_module: The fully qualified Python module path (e.g., "package.subpackage.module")
+        module_callable: The name of the callable (class or function) to retrieve from the module
+        custom_config: Dictionary of configuration parameters to pass as keyword arguments to the callable
+    """
+
+    python_module: str | None = None
+    module_callable: str | None = None
+    custom_config: Dict[str, Any] | None = None
+
+    def load(self) -> Any:
+        assert self.python_module is not None, "python_module is required"
+        assert self.module_callable is not None, "module_callable is required"
+        assert self.custom_config is not None, "custom_config is required"
+
+        module = importlib.import_module(self.python_module)
+        callable_obj = getattr(module, self.module_callable)
+        return callable_obj(**self.custom_config)

naas_abi_core/engine/engine_configuration/EngineConfiguration_ObjectStorageService.py

@@ -0,0 +1,159 @@
+from typing import Dict, Literal, Tuple, Union
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    IObjectStorageAdapter,
+)
+from naas_abi_core.services.object_storage.ObjectStorageService import (
+    ObjectStorageService,
+)
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+
+class ObjectStorageAdapterFSConfiguration(BaseModel):
+    """Object storage adapter filesystem configuration.
+
+    object_storage_adapter:
+      adapter: "fs"
+      config:
+        base_path: "storage/datastore"
+    """
+    base_path: str
+
+
+class ObjectStorageAdapterS3Configuration(BaseModel):
+    """Object storage adapter S3 configuration.
+
+    object_storage_adapter:
+      adapter: "s3"
+      config:
+        bucket_name: "my-bucket"
+        base_prefix: "my-prefix"
+        access_key_id: "{{ secret.AWS_ACCESS_KEY_ID }}"
+        secret_access_key: "{{ secret.AWS_SECRET_ACCESS_KEY }}"
+        session_token: "{{ secret.AWS_SESSION_TOKEN }}"
+    """
+    bucket_name: str
+    base_prefix: str
+    access_key_id: str
+    secret_access_key: str
+    session_token: str | None = None
+
+
+class ObjectStorageAdapterNaasConfiguration(BaseModel):
+    """Object storage adapter Naas configuration.
+
+    object_storage_adapter:
+      adapter: "naas"
+      config:
+        naas_api_key: "{{ secret.NAAS_API_KEY }}"
+        workspace_id: "{{ secret.WORKSPACE_ID }}"
+        storage_name: "{{ secret.STORAGE_NAME }}"
+        base_prefix: "my-prefix"
+    """
+    naas_api_key: str
+    workspace_id: str
+    storage_name: str
+    base_prefix: str = ""
+
+
+class ObjectStorageAdapterConfiguration(GenericLoader):
+    adapter: Literal["fs", "s3", "naas", "custom"]
+    config: (
+        Union[
+            ObjectStorageAdapterFSConfiguration,
+            ObjectStorageAdapterS3Configuration,
+            ObjectStorageAdapterNaasConfiguration,
+        ]
+        | None
+    ) = None
+
+    __MAPPING: Dict[Literal["fs", "s3", "naas"], Tuple[str, str]] = {
+        "fs": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS",
+            "ObjectStorageSecondaryAdapterFS",
+        ),
+        "s3": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterS3",
+            "ObjectStorageSecondaryAdapterS3",
+        ),
+        "naas": (
+            "abi.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterNaas",
+            "ObjectStorageSecondaryAdapterNaas",
+        ),
+    }
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> Self:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+        if self.adapter == "fs":
+            pydantic_model_validator(
+                ObjectStorageAdapterFSConfiguration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 'fs' adapter",
+            )
+        if self.adapter == "s3":
+            pydantic_model_validator(
+                ObjectStorageAdapterS3Configuration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 's3' adapter",
+            )
+        if self.adapter == "naas":
+            pydantic_model_validator(
+                ObjectStorageAdapterNaasConfiguration,
+                self.config,
+                "Invalid configuration for services.object_storage.object_storage_adapter 'naas' adapter",
+            )
+
+        return self
+
+    def load(self) -> IObjectStorageAdapter:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+            if self.adapter == "fs":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS import (
+                    ObjectStorageSecondaryAdapterFS,
+                )
+
+                return ObjectStorageSecondaryAdapterFS(**self.config.model_dump())
+            elif self.adapter == "s3":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterS3 import (
+                    ObjectStorageSecondaryAdapterS3,
+                )
+
+                return ObjectStorageSecondaryAdapterS3(**self.config.model_dump())
+            elif self.adapter == "naas":
+                from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterNaas import (
+                    ObjectStorageSecondaryAdapterNaas,
+                )
+
+                return ObjectStorageSecondaryAdapterNaas(**self.config.model_dump())
+            else:
+                raise ValueError(f"Unknown adapter: {self.adapter}")
+            # return GenericLoader(
+            #     python_module=self.__MAPPING[self.adapter][0],
+            #     module_callable=self.__MAPPING[self.adapter][1],
+            #     custom_config=self.config.model_dump(),
+            # ).load()
+        else:
+            return super().load()
+
+
+class ObjectStorageServiceConfiguration(BaseModel):
+    object_storage_adapter: ObjectStorageAdapterConfiguration
+
+    def load(self) -> ObjectStorageService:
+        return ObjectStorageService(adapter=self.object_storage_adapter.load())

naas_abi_core/engine/engine_configuration/EngineConfiguration_ObjectStorageService_test.py

@@ -0,0 +1,26 @@
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_ObjectStorageService import (
+    ObjectStorageAdapterConfiguration,
+    ObjectStorageAdapterFSConfiguration,
+    ObjectStorageServiceConfiguration,
+)
+from naas_abi_core.services.object_storage.adapters.secondary.ObjectStorageSecondaryAdapterFS import (
+    ObjectStorageSecondaryAdapterFS,
+)
+from naas_abi_core.services.object_storage.ObjectStoragePort import (
+    IObjectStorageAdapter,
+)
+
+
+def test_object_storage_service_configuration():
+    configuration = ObjectStorageServiceConfiguration(
+        object_storage_adapter=ObjectStorageAdapterConfiguration(
+            adapter="fs", config=ObjectStorageAdapterFSConfiguration(base_path="test")
+        )
+    )
+    assert configuration.object_storage_adapter is not None
+
+    object_storage_adapter = configuration.object_storage_adapter.load()
+
+    assert object_storage_adapter is not None
+    assert isinstance(object_storage_adapter, IObjectStorageAdapter)
+    assert isinstance(object_storage_adapter, ObjectStorageSecondaryAdapterFS)

naas_abi_core/engine/engine_configuration/EngineConfiguration_SecretService.py

@@ -0,0 +1,138 @@
+from typing import TYPE_CHECKING, List, Literal, Union
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.secret.SecretPorts import ISecretAdapter
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+# Only import for type checking, not at runtime
+if TYPE_CHECKING:
+    from naas_abi_core.services.secret.adaptors.secondary.Base64Secret import (
+        Base64Secret,
+    )
+
+
+class DotenvSecretConfiguration(BaseModel):
+    """Dotenv secret configuration.
+
+    secret_adapters:
+      - adapter: "dotenv"
+        config: {}
+    """
+    pass
+
+
+class NaasSecretConfiguration(BaseModel):
+    """Naas secret configuration.
+
+    secret_adapters:
+      - adapter: "naas"
+        config:
+          naas_api_key: "{{ secret.NAAS_API_KEY }}"
+          naas_api_url: "https://api.naas.ai"
+    """
+    naas_api_key: str
+    naas_api_url: str
+
+
+class Base64SecretConfiguration(BaseModel):
+    """Base64 secret configuration.
+
+    secret_adapters:
+      - adapter: "base64"
+        config:
+          secret_adapter: *secret_adapter
+          base64_secret_key: "{{ secret.BASE64_SECRET_KEY }}"
+    """
+    secret_adapter: "SecretAdapterConfiguration"
+    base64_secret_key: str
+
+    def load(self) -> "Base64Secret":
+        from naas_abi_core.services.secret.adaptors.secondary.Base64Secret import (
+            Base64Secret,
+        )
+
+        return Base64Secret(self.secret_adapter.load(), self.base64_secret_key)
+
+
+class SecretAdapterConfiguration(GenericLoader):
+    adapter: Literal["dotenv", "naas", "base64", "custom"]
+    config: (
+        Union[
+            DotenvSecretConfiguration,
+            NaasSecretConfiguration,
+            Base64SecretConfiguration,
+        ]
+        | None
+    ) = None
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> Self:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+        if self.adapter == "base64":
+            pydantic_model_validator(
+                Base64SecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'base64' adapter",
+            )
+        if self.adapter == "dotenv":
+            pydantic_model_validator(
+                DotenvSecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'dotenv' adapter",
+            )
+        if self.adapter == "naas":
+            pydantic_model_validator(
+                NaasSecretConfiguration,
+                self.config,
+                "Invalid configuration for services.secret.secret_adapters 'naas' adapter",
+            )
+        return self
+
+    def load(self) -> ISecretAdapter:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+            # Lazy import: only import the adapter that's actually configured
+            if self.adapter == "base64":
+                assert isinstance(self.config, Base64SecretConfiguration), (
+                    "config must be a Base64SecretConfiguration if adapter is base64"
+                )
+                assert hasattr(self.config, "load"), (
+                    "config must have a load method if adapter is base64"
+                )
+                return self.config.load()
+            elif self.adapter == "dotenv":
+                from naas_abi_core.services.secret.adaptors.secondary.dotenv_secret_secondaryadaptor import (
+                    DotenvSecretSecondaryAdaptor,
+                )
+
+                return DotenvSecretSecondaryAdaptor(**self.config.model_dump())
+            elif self.adapter == "naas":
+                from naas_abi_core.services.secret.adaptors.secondary.NaasSecret import (
+                    NaasSecret,
+                )
+
+                return NaasSecret(**self.config.model_dump())
+            else:
+                raise ValueError(f"Unknown adapter: {self.adapter}")
+        else:
+            return super().load()
+
+
+class SecretServiceConfiguration(BaseModel):
+    secret_adapters: List[SecretAdapterConfiguration]
+
+    def load(self) -> Secret:
+        return Secret(adapters=[adapter.load() for adapter in self.secret_adapters])

naas_abi_core/engine/engine_configuration/EngineConfiguration_SecretService_test.py

@@ -0,0 +1,74 @@
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_SecretService import (
+    DotenvSecretConfiguration,
+    NaasSecretConfiguration,
+    SecretAdapterConfiguration,
+    SecretServiceConfiguration,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.secret.SecretPorts import ISecretAdapter
+
+
+def test_secret_service_configuration():
+    from naas_abi_core.services.secret.adaptors.secondary.dotenv_secret_secondaryadaptor import (
+        DotenvSecretSecondaryAdaptor,
+    )
+
+    configuration = SecretServiceConfiguration(
+        secret_adapters=[
+            SecretAdapterConfiguration(
+                adapter="dotenv", config=DotenvSecretConfiguration()
+            )
+        ]
+    )
+    assert configuration.secret_adapters is not None
+    assert len(configuration.secret_adapters) == 1
+
+    secret_adapter = configuration.secret_adapters[0].load()
+
+    assert secret_adapter is not None
+    assert isinstance(secret_adapter, ISecretAdapter)
+    assert isinstance(secret_adapter, DotenvSecretSecondaryAdaptor)
+
+    secret_service = configuration.load()
+
+    assert secret_service is not None
+    assert isinstance(secret_service, Secret)
+
+
+def test_secret_service_configuration_naas():
+    import os
+
+    from dotenv import load_dotenv
+    from naas_abi_core.services.secret.adaptors.secondary.NaasSecret import (
+        NaasSecret,
+    )
+
+    load_dotenv()
+
+    naas_api_key = os.getenv("NAAS_API_KEY")
+    if naas_api_key is None:
+        raise ValueError("NAAS_API_KEY is not set")
+
+    naas_api_url = os.getenv("NAAS_API_URL")
+    if naas_api_url is None:
+        raise ValueError("NAAS_API_URL is not set")
+
+    configuration = SecretServiceConfiguration(
+        secret_adapters=[
+            SecretAdapterConfiguration(
+                adapter="naas",
+                config=NaasSecretConfiguration(
+                    naas_api_key=naas_api_key,
+                    naas_api_url=naas_api_url,
+                ),
+            )
+        ]
+    )
+    assert configuration.secret_adapters is not None
+    assert len(configuration.secret_adapters) == 1
+
+    secret_adapter = configuration.secret_adapters[0].load()
+
+    assert secret_adapter is not None
+    assert isinstance(secret_adapter, ISecretAdapter)
+    assert isinstance(secret_adapter, NaasSecret)