naas-abi-core 1.0.0__py3-none-any.whl

naas_abi_core/engine/engine_configuration/EngineConfiguration_TripleStoreService.py

@@ -0,0 +1,171 @@
+from typing import TYPE_CHECKING, Literal, Union
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_ObjectStorageService import (
+    ObjectStorageServiceConfiguration,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.triple_store.TripleStorePorts import ITripleStorePort
+from naas_abi_core.services.triple_store.TripleStoreService import TripleStoreService
+from pydantic import BaseModel, model_validator
+from typing_extensions import Self
+
+# Only import for type checking, not at runtime
+if TYPE_CHECKING:
+    pass
+
+
+class OxigraphAdapterConfiguration(BaseModel):
+    oxigraph_url: str = "http://localhost:7878"
+    timeout: int = 60
+
+
+class AWSNeptuneAdapterConfiguration(BaseModel):
+    aws_region_name: str
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    db_instance_identifier: str
+
+
+class AWSNeptuneSSHTunnelAdapterConfiguration(AWSNeptuneAdapterConfiguration):
+    bastion_host: str
+    bastion_port: int
+    bastion_user: str
+    bastion_private_key: str
+
+
+class TripleStoreAdapterFilesystemConfiguration(BaseModel):
+    store_path: str
+    triples_path: str = "triples"
+
+
+class TripleStoreAdapterObjectStorageConfiguration(BaseModel):
+    object_storage_service: ObjectStorageServiceConfiguration
+    triples_prefix: str = "triples"
+
+
+class TripleStoreAdapterConfiguration(GenericLoader):
+    adapter: Literal[
+        "oxigraph",
+        "aws_neptune_sshtunnel",
+        "aws_neptune",
+        "fs",
+        "object_storage",
+        "custom",
+    ]
+    config: (
+        Union[
+            OxigraphAdapterConfiguration,
+            AWSNeptuneAdapterConfiguration,
+            AWSNeptuneSSHTunnelAdapterConfiguration,
+            TripleStoreAdapterFilesystemConfiguration,
+            TripleStoreAdapterObjectStorageConfiguration,
+            dict,
+        ]
+        | None
+    ) = None
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> Self:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+        if self.adapter == "fs":
+            pydantic_model_validator(
+                TripleStoreAdapterFilesystemConfiguration,
+                self.config,
+                "Invalid configuration for services.triple_store.triple_store_adapter 'fs' adapter",
+            )
+        if self.adapter == "object_storage":
+            pydantic_model_validator(
+                TripleStoreAdapterObjectStorageConfiguration,
+                self.config,
+                "Invalid configuration for services.triple_store.triple_store_adapter 'object_storage' adapter",
+            )
+        if self.adapter == "oxigraph":
+            pydantic_model_validator(
+                OxigraphAdapterConfiguration,
+                self.config,
+                "Invalid configuration for services.triple_store.triple_store_adapter 'oxigraph' adapter",
+            )
+        if self.adapter == "aws_neptune":
+            pydantic_model_validator(
+                AWSNeptuneAdapterConfiguration,
+                self.config,
+                "Invalid configuration for services.triple_store.triple_store_adapter 'aws_neptune' adapter",
+            )
+        if self.adapter == "aws_neptune_sshtunnel":
+            pydantic_model_validator(
+                AWSNeptuneSSHTunnelAdapterConfiguration,
+                self.config,
+                "Invalid configuration for services.triple_store.triple_store_adapter 'aws_neptune_sshtunnel' adapter",
+            )
+
+        return self
+
+    def load(self) -> ITripleStorePort:
+        if self.adapter != "custom":
+            assert self.config is not None, "config is required"
+
+            arguments = (
+                self.config.model_dump()
+                if not isinstance(self.config, dict)
+                else self.config
+            )
+
+            # Lazy import: only import the adapter that's actually configured
+            if self.adapter == "oxigraph":
+                from naas_abi_core.services.triple_store.adaptors.secondary.Oxigraph import (
+                    Oxigraph,
+                )
+
+                OxigraphAdapterConfiguration.model_validate(arguments)
+
+                return Oxigraph(**arguments)
+            elif self.adapter == "aws_neptune":
+                from naas_abi_core.services.triple_store.adaptors.secondary.AWSNeptune import (
+                    AWSNeptune,
+                )
+
+                AWSNeptuneAdapterConfiguration.model_validate(arguments)
+
+                return AWSNeptune(**arguments)
+            elif self.adapter == "aws_neptune_sshtunnel":
+                from naas_abi_core.services.triple_store.adaptors.secondary.AWSNeptune import (
+                    AWSNeptuneSSHTunnel,
+                )
+
+                AWSNeptuneSSHTunnelAdapterConfiguration.model_validate(arguments)
+
+                return AWSNeptuneSSHTunnel(**arguments)
+            elif self.adapter == "fs":
+                from naas_abi_core.services.triple_store.adaptors.secondary.TripleStoreService__SecondaryAdaptor__Filesystem import (
+                    TripleStoreService__SecondaryAdaptor__Filesystem,
+                )
+
+                TripleStoreAdapterFilesystemConfiguration.model_validate(arguments)
+
+                return TripleStoreService__SecondaryAdaptor__Filesystem(**arguments)
+            elif self.adapter == "object_storage":
+                from naas_abi_core.services.triple_store.adaptors.secondary.TripleStoreService__SecondaryAdaptor__ObjectStorage import (
+                    TripleStoreService__SecondaryAdaptor__ObjectStorage,
+                )
+
+                return TripleStoreService__SecondaryAdaptor__ObjectStorage(**arguments)
+            else:
+                raise ValueError(f"Adapter {self.adapter} not supported")
+        else:
+            return super().load()
+
+
+class TripleStoreServiceConfiguration(BaseModel):
+    triple_store_adapter: TripleStoreAdapterConfiguration
+
+    def load(self) -> TripleStoreService:
+        return TripleStoreService(triple_store_adapter=self.triple_store_adapter.load())

naas_abi_core/engine/engine_configuration/EngineConfiguration_VectorStoreService.py

@@ -0,0 +1,65 @@
+from typing import Literal
+
+from naas_abi_core.engine.engine_configuration.EngineConfiguration_GenericLoader import (
+    GenericLoader,
+)
+from naas_abi_core.engine.engine_configuration.utils.PydanticModelValidator import (
+    pydantic_model_validator,
+)
+from naas_abi_core.services.vector_store.IVectorStorePort import IVectorStorePort
+from naas_abi_core.services.vector_store.VectorStoreService import VectorStoreService
+from pydantic import BaseModel, model_validator
+
+
+class VectorStoreAdapterQdrantConfiguration(BaseModel):
+    host: str = "localhost"
+    port: int = 6333
+    api_key: str | None = None
+    https: bool = False
+    timeout: int = 30
+
+
+class VectorStoreAdapterConfiguration(GenericLoader):
+    adapter: Literal["qdrant", "custom"]
+    config: dict | None = None
+
+    @model_validator(mode="after")
+    def validate_adapter(self) -> "VectorStoreAdapterConfiguration":
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+        if self.adapter == "qdrant":
+            pydantic_model_validator(
+                VectorStoreAdapterQdrantConfiguration,
+                self.config,
+                "Invalid configuration for services.vector_store.vector_store_adapter 'qdrant' adapter",
+            )
+
+        return self
+
+    def load(self) -> IVectorStorePort:
+        if self.adapter != "custom":
+            assert self.config is not None, (
+                "config is required if adapter is not custom"
+            )
+
+            # Lazy import: only import when actually loading
+            if self.adapter == "qdrant":
+                from naas_abi_core.services.vector_store.adapters.QdrantAdapter import (
+                    QdrantAdapter,
+                )
+
+                return QdrantAdapter(**self.config)
+            else:
+                raise ValueError(f"Unknown adapter: {self.adapter}")
+        else:
+            return super().load()
+
+
+class VectorStoreServiceConfiguration(BaseModel):
+    vector_store_adapter: VectorStoreAdapterConfiguration
+
+    def load(self) -> VectorStoreService:
+        return VectorStoreService(adapter=self.vector_store_adapter.load())

naas_abi_core/engine/engine_configuration/EngineConfiguration_test.py

@@ -0,0 +1,9 @@
+from naas_abi_core.engine.engine_configuration.EngineConfiguration import (
+    EngineConfiguration,
+)
+
+
+def test_configuration_load(test_configuration: str):
+    config = EngineConfiguration.load_configuration(test_configuration)
+    assert config is not None
+    assert False is True

naas_abi_core/engine/engine_configuration/utils/PydanticModelValidator.py

@@ -0,0 +1,15 @@
+from typing import Any
+
+from naas_abi_core.utils.Logger import logger
+from pydantic import BaseModel, ValidationError
+
+
+def pydantic_model_validator(model: Any, payload: Any, message: str):
+    if not isinstance(model, type) or not issubclass(model, BaseModel):
+        raise TypeError(
+            "The model argument must be a class that inherits from pydantic.BaseModel"
+        )
+    try:
+        model.model_validate(payload)
+    except ValidationError as e:
+        logger.error(f"{message}: {e}")

naas_abi_core/engine/engine_loaders/EngineModuleLoader.py

@@ -0,0 +1,302 @@
+import importlib
+from typing import Dict, List
+
+import pydantic_core
+from naas_abi_core import logger
+from naas_abi_core.engine.engine_configuration.EngineConfiguration import (
+    EngineConfiguration,
+)
+from naas_abi_core.engine.EngineProxy import EngineProxy
+from naas_abi_core.engine.IEngine import IEngine
+from naas_abi_core.module.Module import (
+    BaseModule,
+    ModuleConfiguration,
+    ModuleDependencies,
+)
+
+
+class EngineModuleLoader:
+    __configuration: EngineConfiguration
+
+    __module_load_order: List[str] = []
+
+    __modules: Dict[str, BaseModule] = {}
+
+    __module_dependencies: Dict[str, ModuleDependencies] | None = None
+
+    def __init__(self, configuration: EngineConfiguration):
+        self.__configuration = configuration
+
+    @property
+    def modules(self) -> Dict[str, BaseModule]:
+        return self.__modules
+
+    @property
+    def module_load_order(self) -> List[str]:
+        return self.__module_load_order
+
+    @property
+    def ordered_modules(self) -> List[BaseModule]:
+        return [self.__modules[module_name] for module_name in self.__module_load_order]
+
+    def __topological_sort(self, dependencies: Dict[str, List[str]]) -> List[str]:
+        """
+        Perform topological sort on modules based on dependencies.
+        Returns a list of module names in load order.
+        Raises ValueError if circular dependency is detected.
+        """
+        # Build adjacency list and in-degree count
+        in_degree = {node: 0 for node in dependencies}
+        adj_list: Dict[str, List[str]] = {node: [] for node in dependencies}
+
+        for node, deps in dependencies.items():
+            for dep in deps:
+                soft_dependency = dep.endswith("#soft")
+                if dep.endswith("#soft"):
+                    dep = dep.replace("#soft", "")
+
+                if (
+                    dep not in dependencies and not soft_dependency
+                ):  # Soft modules are not required to be in the dependencies.
+                    logger.error(
+                        f"Module '{node}' depends on '{dep}' which is not enabled. "
+                        "Aborting."
+                    )
+                    raise ValueError(
+                        f"Module '{node}' depends on '{dep}' which is not enabled. "
+                        "Aborting."
+                    )
+                    continue
+
+                if dep in adj_list:
+                    adj_list[dep].append(node)
+                    in_degree[node] += 1
+
+        # Kahn's algorithm
+        queue = [node for node in in_degree if in_degree[node] == 0]
+        sorted_list = []
+
+        while queue:
+            # Sort queue for deterministic order when multiple nodes have in-degree 0
+            queue.sort()
+            node = queue.pop(0)
+            sorted_list.append(node)
+
+            for neighbor in adj_list[node]:
+                in_degree[neighbor] -= 1
+                if in_degree[neighbor] == 0:
+                    queue.append(neighbor)
+
+        # Check for circular dependencies
+        if len(sorted_list) != len(dependencies):
+            unprocessed = [node for node in dependencies if node not in sorted_list]
+            raise ValueError(
+                f"Circular dependency detected among modules: {unprocessed}. "
+                "Cannot determine load order."
+            )
+
+        logger.debug(f"Module load order: {sorted_list}")
+        return sorted_list
+
+    def module_dependencies_recursive(
+        self, module_name: str, module_dependencies: Dict[str, ModuleDependencies]
+    ) -> List[str]:
+        required_modules: List[str] = [module_name]
+        # This is not protected against circular dependencies.
+        # This is why it must be ran after the topological sort.
+        for dependency in module_dependencies[module_name].modules:
+            required_modules.append(dependency)
+            required_modules.extend(
+                self.module_dependencies_recursive(dependency, module_dependencies)
+            )
+        return list(set(required_modules))
+
+    def modules_dependencies_recursive(
+        self,
+        module_names: List[str],
+        module_dependencies: Dict[str, ModuleDependencies],
+    ) -> List[str]:
+        dependencies: List[str] = []
+        for module_name in module_names:
+            dependencies.extend(
+                self.module_dependencies_recursive(module_name, module_dependencies)
+            )
+        return list(set(dependencies))
+
+    def get_module_dependencies(
+        self, module_name: str, scanned_modules: List[str] = []
+    ) -> Dict[str, ModuleDependencies]:
+        dependencies: Dict[str, ModuleDependencies] = {}
+        logger.debug(f"Getting module dependencies for {module_name}")
+        is_soft_module = module_name.endswith("#soft")
+        module_name = module_name.replace("#soft", "")
+        if module_name in scanned_modules:
+            raise ValueError(
+                f"Circular dependency detected: {scanned_modules + [module_name]}"
+            )
+
+        module_config = next(
+            (m for m in self.__configuration.modules if m.module == module_name),
+            None,
+        )
+
+        if module_config is None and not is_soft_module:
+            raise ValueError(f"Module {module_name} not found in configuration")
+        elif module_config is None and is_soft_module:
+            # Soft modules are not required to be in the configuration.
+            # We return an empty dictionary.
+            return {}
+
+        assert module_config is not None, (
+            f"Module {module_name} configuration is not set in the configuration file"
+        )
+
+        if module_config.module is None:
+            raise ValueError(
+                f"Module {module_name} configuration module is not set in the configuration file"
+            )
+
+        module = importlib.import_module(module_config.module)
+        if module is None:
+            raise ValueError(
+                f"Error while trying to import module {module_config.module}"
+            )
+
+        assert hasattr(module, "ABIModule"), (
+            f"Module {module_config.module} does not have a ABIModule class"
+        )
+        assert hasattr(module.ABIModule, "get_dependencies"), (
+            f"Module {module_config.module} does not have a get_dependencies method"
+        )
+        dependencies[module_name] = module.ABIModule.get_dependencies()
+        assert isinstance(dependencies[module_name], ModuleDependencies), (
+            f"Module {module_config.module} get_dependencies method must return a ModuleDependencies object"
+        )
+
+        # We recursively get the dependencies of the module.
+        for module_dependency in dependencies[module_name].modules:
+            submodule_dependencies = self.get_module_dependencies(
+                module_dependency, scanned_modules + [module_name]
+            )
+            dependencies.update(submodule_dependencies)
+
+            # dependencies.modules.extend(submodule_dependencies.modules)
+            # dependencies.services.extend(submodule_dependencies.services)
+
+        # # Make sure we have unique modules and services.
+        # dependencies.modules = list(set(dependencies.modules))
+        # dependencies.services = list(set(dependencies.services))
+
+        logger.debug(f"Module dependencies for {module_name}: {dependencies}")
+
+        return dependencies
+
+    def get_modules_dependencies(
+        self, module_names: List[str] = []
+    ) -> Dict[str, ModuleDependencies]:
+        module_dependencies: Dict[str, ModuleDependencies] = {}
+        for module_config in self.__configuration.modules:
+            # We check if the module is required by the configuration.
+            if (
+                len(module_names) > 0
+                and module_config.module in module_names
+                and not module_config.enabled
+            ):
+                raise ValueError(
+                    f"Module {module_config.module} is not enabled but is required by the configuration"
+                )
+
+            if len(module_names) > 0 and module_config.module not in module_names:
+                continue
+
+            assert module_config.module is not None, (
+                f"Module {module_config.module} configuration module is not set in the configuration file"
+            )
+            if module_config.enabled:
+                module_dependencies.update(
+                    self.get_module_dependencies(module_config.module)
+                )
+        self.__module_dependencies = module_dependencies
+        return self.__module_dependencies
+
+    def load_modules(
+        self,
+        engine: IEngine,
+        module_names: List[str] = [],
+    ) -> Dict[str, BaseModule]:
+        self.__modules: Dict[str, BaseModule] = {}
+
+        if self.__module_dependencies is None:
+            # Call this to hydrate the __module_dependencies attribute.
+            self.__module_dependencies = self.get_modules_dependencies(module_names)
+
+        assert self.__module_dependencies is not None, (
+            "Module dependencies are not set after getting modules dependencies"
+        )
+
+        logger.debug(f"Module dependencies: {self.__module_dependencies}")
+
+        module_load_order = self.__topological_sort(
+            {
+                module_name: dependencies.modules
+                for module_name, dependencies in self.__module_dependencies.items()
+            }
+        )
+
+        self.__module_load_order = module_load_order
+
+        # We load the modules in the order of the topological sort
+        for module_name in self.__module_load_order:
+            module_config = next(
+                (m for m in self.__configuration.modules if m.module == module_name),
+                None,
+            )
+            if module_config is None:
+                raise ValueError(f"Module {module_name} not found in configuration")
+
+            if module_config.enabled:
+                if module_config.module:
+                    module = importlib.import_module(module_config.module)
+                    if not hasattr(module, "ABIModule"):
+                        raise ValueError(
+                            f"Module {module_config.module} does not have a Module class"
+                        )
+
+                    if not hasattr(module.ABIModule, "Configuration"):
+                        raise ValueError(
+                            f"Module {module_config.module} does not have a Configuration class"
+                        )
+
+                    assert type(module.ABIModule.Configuration) is type(
+                        ModuleConfiguration
+                    ), (
+                        f"Module {module_config.module} Configuration must be a subclass of ModuleConfiguration"
+                    )
+
+                    try:
+                        cfg = module.ABIModule.Configuration(
+                            global_config=self.__configuration.global_config,
+                            **module_config.config,
+                        )
+                    except pydantic_core._pydantic_core.ValidationError as e:
+                        raise ValueError(
+                            f"Error loading configuration for module {module_config.module}: {e}"
+                        ) from e
+
+                    # This effectively creates a new instance of the module.
+                    # It will call the constructor of the ABIModule class inside the module.
+                    module = module.ABIModule(
+                        EngineProxy(
+                            engine, module_name, self.__module_dependencies[module_name]
+                        ),
+                        cfg,
+                    )
+                    assert isinstance(module, BaseModule), (
+                        f"Module {module_config.module} is not a subclass of BaseModule"
+                    )
+                    self.__modules[module_name] = module
+                    module.on_load()
+                else:
+                    raise ValueError("module must be provided for a module")
+
+        return self.__modules

naas_abi_core/engine/engine_loaders/EngineOntologyLoader.py

@@ -0,0 +1,16 @@
+from typing import List
+
+from naas_abi_core import logger
+from naas_abi_core.module.Module import BaseModule
+from naas_abi_core.services.triple_store.TripleStoreService import TripleStoreService
+
+
+class EngineOntologyLoader:
+    @classmethod
+    def load_ontologies(
+        cls, triple_store: TripleStoreService, modules: List[BaseModule]
+    ) -> None:
+        logger.debug("Loading ontologies")
+        for module in modules:
+            for ontology in module.ontologies:
+                triple_store.load_schema(ontology)

naas_abi_core/engine/engine_loaders/EngineServiceLoader.py

@@ -0,0 +1,47 @@
+from typing import Dict, List
+
+from naas_abi_core import logger
+from naas_abi_core.engine.engine_configuration.EngineConfiguration import (
+    EngineConfiguration,
+)
+from naas_abi_core.engine.IEngine import IEngine
+from naas_abi_core.module.Module import ModuleDependencies
+from naas_abi_core.services.object_storage.ObjectStorageService import (
+    ObjectStorageService,
+)
+from naas_abi_core.services.secret.Secret import Secret
+from naas_abi_core.services.triple_store.TripleStoreService import TripleStoreService
+from naas_abi_core.services.vector_store.VectorStoreService import VectorStoreService
+
+
+class EngineServiceLoader:
+    __configuration: EngineConfiguration
+
+    def __init__(self, configuration: EngineConfiguration):
+        self.__configuration = configuration
+
+    def load_services(
+        self, module_dependencies: Dict[str, ModuleDependencies]
+    ) -> IEngine.Services:
+        services_to_load: List[type] = []
+
+        for _, module_dependency in module_dependencies.items():
+            services_to_load.extend(module_dependency.services)
+
+        services_to_load = list(set(services_to_load))
+        logger.debug(f"Services to load: {services_to_load}")
+
+        return IEngine.Services(
+            self.__configuration.services.object_storage.load()
+            if ObjectStorageService in services_to_load
+            else None,
+            self.__configuration.services.triple_store.load()
+            if TripleStoreService in services_to_load
+            else None,
+            self.__configuration.services.vector_store.load()
+            if VectorStoreService in services_to_load
+            else None,
+            self.__configuration.services.secret.load()
+            if Secret in services_to_load
+            else None,
+        )

naas_abi_core/integration/__init__.py

@@ -0,0 +1,7 @@
+from naas_abi_core.integration.integration import Integration as Integration
+from naas_abi_core.integration.integration import (
+    IntegrationConfiguration as IntegrationConfiguration,
+)
+from naas_abi_core.integration.integration import (
+    IntegrationConnectionError as IntegrationConnectionError,
+)

naas_abi_core/integration/integration.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+
+
+class IntegrationConnectionError(Exception):
+    pass
+
+
+@dataclass
+class IntegrationConfiguration:
+    pass
+
+
+class Integration:
+    """An Integration represents a way to interact with a third-party tool.
+
+    The Integration class serves as a base class for implementing connections to external services,
+    APIs, or tools. It provides a standardized interface for configuring and establishing these
+    connections.
+
+    Attributes:
+        __configuration (IntegrationConfiguration): Configuration instance containing
+            necessary credentials and settings for connecting to the third-party tool.
+    """
+
+    __configuration: IntegrationConfiguration
+
+    def __init__(self, configuration: IntegrationConfiguration):
+        self.__configuration = configuration