naas-abi 1.0.0__py3-none-any.whl

naas_abi/pipelines/InsertDataSPARQLPipeline_test.py

@@ -0,0 +1,96 @@
+import pytest
+from naas_abi import services
+from naas_abi.pipelines.InsertDataSPARQLPipeline import (
+    InsertDataSPARQLPipeline,
+    InsertDataSPARQLPipelineConfiguration,
+    InsertDataSPARQLPipelineParameters,
+)
+
+
+@pytest.fixture
+def pipeline() -> InsertDataSPARQLPipeline:
+    pipeline = InsertDataSPARQLPipeline(
+        configuration=InsertDataSPARQLPipelineConfiguration(
+            triple_store=services.triple_store_service
+        )
+    )
+    return pipeline
+
+
+def test_insert_data_sparql_pipeline(pipeline: InsertDataSPARQLPipeline):
+    from naas_abi.utils.SPARQL import results_to_list
+    from rdflib import Literal, Namespace, URIRef
+
+    sparql_statement = """
+    PREFIX abi: <http://ontology.naas.ai/abi/>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX cco: <https://www.commoncoreontologies.org/>
+    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+
+    INSERT DATA {
+        abi:john a cco:ont00001262, owl:NamedIndividual ;
+                    abi:name "John Doe" ;
+                    abi:age 30 ;
+                    abi:email "john.doe@example.com" .
+        
+        abi:jane a cco:ont00001262, owl:NamedIndividual ;
+                    abi:name "Jane Smith" ;
+                    abi:age 28 ;
+                    abi:email "jane.smith@example.com" .
+    }
+    """
+    graph = pipeline.run(
+        InsertDataSPARQLPipelineParameters(sparql_statement=sparql_statement)
+    )
+    ABI = Namespace("http://ontology.naas.ai/abi/")
+
+    assert graph is not None, graph.serialize(format="turtle")
+    assert (
+        len(list(graph.triples((URIRef(ABI.john), ABI.name, Literal("John Doe"))))) == 1
+    ), graph.serialize(format="turtle")
+    assert (
+        len(list(graph.triples((URIRef(ABI.jane), ABI.name, Literal("Jane Smith")))))
+        == 1
+    ), graph.serialize(format="turtle")
+
+    # Remove graph
+    services.triple_store_service.remove(graph)
+
+    # Check triples are removed from the triple store
+    sparql_query = """
+    PREFIX abi: <http://ontology.naas.ai/abi/>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX cco: <https://www.commoncoreontologies.org/>
+    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+
+    SELECT ?s ?p ?o
+    WHERE {
+        ?s ?p ?o .
+        FILTER(?s = abi:john || ?s = abi:jane)
+    }
+    """
+    results = services.triple_store_service.query(sparql_query)
+    results_list = results_to_list(results)
+    assert results_list is None, results_list
+
+
+def test_extract_sparql_from_text(pipeline: InsertDataSPARQLPipeline):
+    text = """
+    ```sparql
+    INSERT DATA {
+        <http://ontology.naas.ai/abi/john> <http://www.w3.org/2000/01/rdf-schema#label> "John Doe" .
+    }
+    ```
+    """
+    sparql_statement = pipeline.get_sparql_from_text(
+        InsertDataSPARQLPipelineParameters(sparql_statement=text)
+    )
+    assert sparql_statement is not None, sparql_statement
+
+
+def test_extract_sparql_from_text_no_sparql(pipeline: InsertDataSPARQLPipeline):
+    text = "This is a text without a SPARQL INSERT DATA statement"
+    sparql_statement = pipeline.get_sparql_from_text(
+        InsertDataSPARQLPipelineParameters(sparql_statement=text)
+    )
+    assert sparql_statement == "No SPARQL INSERT DATA statement found", sparql_statement

naas_abi/pipelines/MergeIndividualsPipeline.py

@@ -0,0 +1,245 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Annotated
+
+from fastapi import APIRouter
+from langchain_core.tools import BaseTool, StructuredTool
+from naas_abi import ABIModule
+from naas_abi_core import logger
+from naas_abi_core.pipeline import Pipeline, PipelineConfiguration, PipelineParameters
+from naas_abi_core.services.triple_store.TripleStorePorts import ITripleStoreService
+from naas_abi_core.utils.Graph import URI_REGEX
+from naas_abi_core.utils.SPARQL import SPARQLUtils
+from naas_abi_core.utils.StorageUtils import StorageUtils
+from pydantic import Field
+from rdflib import RDFS, SKOS, Graph, Literal, Namespace, URIRef
+
+# Define namespaces
+BFO = Namespace("http://purl.obolibrary.org/obo/")
+CCO = Namespace("https://www.commoncoreontologies.org/")
+ABI = Namespace("http://ontology.naas.ai/abi/")
+
+
+@dataclass
+class MergeIndividualsPipelineConfiguration(PipelineConfiguration):
+    """Configuration for MergeIndividualsPipeline.
+
+    Attributes:
+        triple_store (ITripleStoreService): The triple store service to use
+    """
+
+    triple_store: ITripleStoreService
+    datastore_path: str = "datastore/ontology/merged_individual"
+
+
+class MergeIndividualsPipelineParameters(PipelineParameters):
+    uri_to_keep: Annotated[
+        str,
+        Field(
+            description="The URI that will remain and receive the merged triples",
+            pattern=URI_REGEX,
+        ),
+    ]
+    uri_to_merge: Annotated[
+        str,
+        Field(
+            description="The URI that will be merged into uri_to_keep and then removed",
+            pattern=URI_REGEX,
+        ),
+    ]
+
+
+class MergeIndividualsPipeline(Pipeline):
+    """Pipeline for merging two individuals in the ontology."""
+
+    __configuration: MergeIndividualsPipelineConfiguration
+    __sparql_utils: SPARQLUtils
+    __storage_utils: StorageUtils
+
+    def __init__(self, configuration: MergeIndividualsPipelineConfiguration):
+        super().__init__(configuration)
+        self.__configuration = configuration
+        self.__sparql_utils: SPARQLUtils = SPARQLUtils(
+            ABIModule.get_instance().engine.services.triple_store
+        )
+        self.__storage_utils: StorageUtils = StorageUtils(
+            ABIModule.get_instance().engine.services.object_storage
+        )
+
+    def get_all_triples_for_uri(self, uri: str):
+        """
+        Retrieve all triples where the given URI appears as either a subject or object.
+
+        Args:
+            uri (str): The URI to search for
+
+        Returns:
+            rdflib.query.Result: Query results containing all triples where the URI appears
+        """
+        sparql_query = f"""
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX owl: <http://www.w3.org/2002/07/owl#>
+        
+        SELECT ?s ?p ?o
+        WHERE {{
+            {{
+                # Find triples where the URI is the subject
+                <{uri}> ?p ?o .
+                BIND(<{uri}> AS ?s)
+            }}
+            UNION
+            {{
+                # Find triples where the URI is the object
+                ?s ?p <{uri}> .
+                BIND(<{uri}> AS ?o)
+            }}
+        }}
+        """
+
+        return self.__configuration.triple_store.query(sparql_query)
+
+    def run(self, parameters: PipelineParameters) -> Graph:
+        if not isinstance(parameters, MergeIndividualsPipelineParameters):
+            raise ValueError(
+                "Parameters must be of type MergeIndividualsPipelineParameters"
+            )
+
+        output_dir = self.__configuration.datastore_path
+
+        # Get all triples for both URIs
+        keep_results = self.get_all_triples_for_uri(parameters.uri_to_keep)
+        keep_graph = Graph()
+        for row in keep_results:
+            s, p, o = row
+            keep_graph.add((s, p, o))
+        logger.info(
+            f"Found {len(keep_results)} triples for URI to keep: {parameters.uri_to_keep}"
+        )
+
+        merge_results = self.get_all_triples_for_uri(parameters.uri_to_merge)
+        merge_graph = Graph()
+        for row in merge_results:
+            s, p, o = row
+            merge_graph.add((s, p, o))
+        logger.info(
+            f"Found {len(merge_results)} triples for URI to merge: {parameters.uri_to_merge}"
+        )
+
+        graph_insert = Graph()
+        graph_insert.bind("bfo", BFO)
+        graph_insert.bind("cco", CCO)
+        graph_insert.bind("abi", ABI)
+        graph_remove = Graph()
+        uri_to_keep_ref = URIRef(parameters.uri_to_keep)
+        uri_to_keep_label = keep_graph.value(uri_to_keep_ref, RDFS.label)
+        uri_to_merge_ref = URIRef(parameters.uri_to_merge)
+        uri_to_merge_label = merge_graph.value(uri_to_merge_ref, RDFS.label)
+
+        # Process triples from uri_to_merge
+        logger.info(
+            f"Merging '{uri_to_merge_label}' ({parameters.uri_to_merge}) into '{uri_to_keep_label}' ({parameters.uri_to_keep})"
+        )
+        for row in merge_results:
+            s, p, o = row
+            if s == uri_to_merge_ref and p not in [RDFS.label, ABI.universal_name]:
+                check_properties = keep_graph.triples((uri_to_keep_ref, p, o))
+                if len(list(check_properties)) == 0:
+                    if isinstance(o, URIRef):
+                        graph_insert.add((uri_to_keep_ref, URIRef(p), URIRef(o)))
+                    elif isinstance(o, Literal):
+                        # Preserve datatype and language tag if present
+                        datatype = o.datatype if hasattr(o, "datatype") else None
+                        lang = o.language if hasattr(o, "language") else None
+                        graph_insert.add(
+                            (
+                                uri_to_keep_ref,
+                                URIRef(p),
+                                Literal(str(o), datatype=datatype, lang=lang),
+                            )
+                        )
+
+            elif s == uri_to_merge_ref and p in [RDFS.label, ABI.universal_name]:
+                datatype = o.datatype if hasattr(o, "datatype") else None
+                lang = o.language if hasattr(o, "language") else None
+                graph_insert.add(
+                    (
+                        uri_to_keep_ref,
+                        SKOS.altLabel,
+                        Literal(str(o), datatype=datatype, lang=lang),
+                    )
+                )
+
+            elif o == uri_to_merge_ref:
+                check_properties = keep_graph.triples((s, p, uri_to_keep_ref))
+                if len(list(check_properties)) == 0:
+                    graph_insert.add((s, p, uri_to_keep_ref))
+
+            # Always add original triple for removal
+            graph_remove.add((s, p, o))
+
+        if len(graph_insert) > 0:
+            logger.info(f"✅ Inserting {len(graph_insert)} triples")
+            logger.info(graph_insert.serialize(format="turtle"))
+            self.__storage_utils.save_triples(
+                graph_insert,
+                output_dir,
+                f"{uri_to_keep_label}_{parameters.uri_to_keep.split('/')[-1]}_merged.ttl",
+            )
+            self.__configuration.triple_store.insert(graph_insert)
+        if len(graph_remove) > 0:
+            logger.info(f"✅ Removing {len(graph_remove)} triples")
+            logger.info(graph_remove.serialize(format="turtle"))
+            self.__storage_utils.save_triples(
+                graph_remove,
+                output_dir,
+                f"{uri_to_merge_label}_{parameters.uri_to_merge.split('/')[-1]}_removed.ttl",
+            )
+            self.__configuration.triple_store.remove(graph_remove)
+
+        return self.__sparql_utils.get_subject_graph(parameters.uri_to_keep)
+
+    def as_tools(self) -> list[BaseTool]:
+        return [
+            StructuredTool(
+                name="merge_individuals",
+                description="Merge two individuals in the triplestore by transferring all triples from one to another",
+                func=lambda **kwargs: self.run(
+                    MergeIndividualsPipelineParameters(**kwargs)
+                ),
+                args_schema=MergeIndividualsPipelineParameters,
+            )
+        ]
+
+    def as_api(
+        self,
+        router: APIRouter,
+        route_name: str = "",
+        name: str = "",
+        description: str = "",
+        description_stream: str = "",
+        tags: list[str | Enum] | None = None,
+    ) -> None:
+        if tags is None:
+            tags = []
+        return None
+
+
+if __name__ == "__main__":
+    from naas_abi import services
+
+    uri_to_keep = "http://ontology.naas.ai/abi/69a231b9-e87a-4503-8f80-a530ed8eaa4b"  # URI that will remain
+    uri_to_merge = "http://ontology.naas.ai/abi/4f92bbdd-e710-4e43-9480-9b6cd6d9af80"  # URI that will be merged and removed
+
+    configuration = MergeIndividualsPipelineConfiguration(
+        triple_store=services.triple_store_service
+    )
+
+    pipeline = MergeIndividualsPipeline(configuration)
+    graph = pipeline.run(
+        MergeIndividualsPipelineParameters(
+            uri_to_keep=uri_to_keep,
+            uri_to_merge=uri_to_merge,
+        )
+    )
+    logger.info(graph.serialize(format="turtle"))

naas_abi/pipelines/MergeIndividualsPipeline_test.py

@@ -0,0 +1,98 @@
+import pytest
+from naas_abi import services
+from naas_abi.pipelines.MergeIndividualsPipeline import (
+    MergeIndividualsPipeline,
+    MergeIndividualsPipelineConfiguration,
+    MergeIndividualsPipelineParameters,
+)
+
+
+@pytest.fixture
+def pipeline() -> MergeIndividualsPipeline:
+    return MergeIndividualsPipeline(
+        MergeIndividualsPipelineConfiguration(
+            triple_store=services.triple_store_service
+        )
+    )
+
+
+def test_merge_individuals_pipeline(pipeline: MergeIndividualsPipeline):
+    import time
+    from uuid import uuid4
+
+    from naas_abi.utils.SPARQL import get_subject_graph
+    from naas_abi_core import logger, services
+    from rdflib import OWL, RDF, RDFS, SKOS, Graph, Literal, Namespace, URIRef
+
+    ABI = Namespace("http://ontology.naas.ai/abi/")
+
+    graph = Graph()
+    uri_to_keep = ABI[str(uuid4())]
+    graph.add(
+        (
+            uri_to_keep,
+            RDF.type,
+            URIRef("https://www.commoncoreontologies.org/ont00000443"),
+        )
+    )
+    graph.add((uri_to_keep, RDF.type, OWL.NamedIndividual))
+    graph.add((uri_to_keep, RDFS.label, Literal("Naas.ai")))
+    graph.add(
+        (
+            uri_to_keep,
+            ABI.logo,
+            Literal(
+                "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ9gXMaBLQZ39W6Pk53PRuzFjUvv_6lLRWPoQ&s"
+            ),
+        )
+    )
+
+    uri_to_merge = ABI[str(uuid4())]
+    graph.add(
+        (
+            uri_to_merge,
+            RDF.type,
+            URIRef("https://www.commoncoreontologies.org/ont00000443"),
+        )
+    )
+    graph.add((uri_to_merge, RDF.type, OWL.NamedIndividual))
+    graph.add((uri_to_merge, RDFS.label, Literal("Naas.ai 2")))
+    graph.add(
+        (
+            uri_to_merge,
+            ABI.logo,
+            Literal(
+                "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ9gXMaBLQZ39W6Pk53PRuzFjUvv_6lLRWPoQ&s"
+            ),
+        )
+    )
+
+    logger.info("Inserting triples to triplestore")
+    services.triple_store_service.insert(graph)
+    time.sleep(3)
+
+    # Run pipeline to merge individuals
+    graph_merged = pipeline.run(
+        MergeIndividualsPipelineParameters(
+            uri_to_keep=uri_to_keep,
+            uri_to_merge=uri_to_merge,
+        )
+    )
+    assert graph_merged is not None, graph_merged.serialize(format="turtle")
+    assert str(graph_merged.value(uri_to_keep, RDFS.label)) == "Naas.ai", (
+        graph_merged.serialize(format="turtle")
+    )
+    assert str(graph_merged.value(uri_to_keep, SKOS.altLabel)) == "Naas.ai 2", (
+        graph_merged.serialize(format="turtle")
+    )
+    assert len(list(graph_merged.triples((uri_to_keep, ABI.logo, None)))) == 1, (
+        graph_merged.serialize(format="turtle")
+    )
+
+    # Check if uri_to_merge is removed in triplestore
+    graph = get_subject_graph(str(uri_to_merge), 1)
+    assert len(graph) == 0, graph.serialize(format="turtle")
+
+    # Remove graphs
+    services.triple_store_service.remove(graph_merged)
+    time.sleep(3)

naas_abi/pipelines/RemoveIndividualPipeline.py

@@ -0,0 +1,166 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Annotated, List
+
+from fastapi import APIRouter
+from langchain_core.tools import BaseTool, StructuredTool
+from naas_abi import ABIModule, logger
+from naas_abi_core.pipeline import Pipeline, PipelineConfiguration, PipelineParameters
+from naas_abi_core.services.triple_store.TripleStorePorts import ITripleStoreService
+from naas_abi_core.utils.StorageUtils import StorageUtils
+from pydantic import Field
+from rdflib import Graph, Namespace
+
+# Define namespaces
+BFO = Namespace("http://purl.obolibrary.org/obo/")
+CCO = Namespace("https://www.commoncoreontologies.org/")
+ABI = Namespace("http://ontology.naas.ai/abi/")
+
+
+@dataclass
+class RemoveIndividualPipelineConfiguration(PipelineConfiguration):
+    """Configuration for RemoveIndividualPipeline.
+
+    Attributes:
+        triple_store (ITripleStoreService): The triple store service to use
+    """
+
+    triple_store: ITripleStoreService
+    datastore_path: str = "datastore/ontology/removed_individual"
+
+
+class RemoveIndividualPipelineParameters(PipelineParameters):
+    uris_to_remove: Annotated[
+        List[str],
+        Field(description="List of URIs to remove from the ontology", min_items=1),
+    ]
+
+
+class RemoveIndividualPipeline(Pipeline):
+    """Pipeline for removing individuals from the ontology."""
+
+    __configuration: RemoveIndividualPipelineConfiguration
+    __storage_utils: StorageUtils
+
+    def __init__(self, configuration: RemoveIndividualPipelineConfiguration):
+        super().__init__(configuration)
+        self.__configuration = configuration
+        self.__storage_utils: StorageUtils = StorageUtils(
+            ABIModule.get_instance().engine.services.object_storage
+        )
+
+    def get_all_triples_for_uri(self, uri: str):
+        """
+        Retrieve all triples where the given URI appears as either a subject or object.
+
+        Args:
+            uri (str): The URI to search for
+
+        Returns:
+            rdflib.query.Result: Query results containing all triples where the URI appears
+        """
+        sparql_query = f"""
+        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX owl: <http://www.w3.org/2002/07/owl#>
+        
+        SELECT ?s ?p ?o
+        WHERE {{
+            {{
+                # Find triples where the URI is the subject
+                <{uri}> ?p ?o .
+                BIND(<{uri}> AS ?s)
+            }}
+            UNION
+            {{
+                # Find triples where the URI is the object
+                ?s ?p <{uri}> .
+                BIND(<{uri}> AS ?o)
+            }}
+        }}
+        """
+
+        return self.__configuration.triple_store.query(sparql_query)
+
+    def run(self, parameters: PipelineParameters) -> Graph:
+        if not isinstance(parameters, RemoveIndividualPipelineParameters):
+            raise ValueError(
+                "Parameters must be of type RemoveIndividualPipelineParameters"
+            )
+
+        output_dir = self.__configuration.datastore_path
+        removed_graph = Graph()
+        removed_graph.bind("bfo", BFO)
+        removed_graph.bind("cco", CCO)
+        removed_graph.bind("abi", ABI)
+
+        for uri in parameters.uris_to_remove:
+            logger.info(f"Getting triples for URI: {uri}")
+            results = self.get_all_triples_for_uri(uri)
+            graph_remove = Graph()
+
+            for row in results:
+                s, p, o = row
+                graph_remove.add((s, p, o))
+
+            if len(graph_remove) > 0:
+                logger.info(f"✅ Removing {len(graph_remove)} triples for URI: {uri}")
+                logger.info(graph_remove.serialize(format="turtle"))
+                self.__storage_utils.save_triples(
+                    graph_remove, output_dir, f"{uri.split('/')[-1]}.ttl"
+                )
+                self.__configuration.triple_store.remove(graph_remove)
+
+                # Add to the combined removed graph for return
+                for triple in graph_remove:
+                    removed_graph.add(triple)
+            else:
+                logger.info(f"No triples found for {uri}")
+
+        return removed_graph
+
+    def as_tools(self) -> list[BaseTool]:
+        return [
+            StructuredTool(
+                name="remove_individuals",
+                description="Remove individuals from the triplestore by deleting all their associated triples",
+                func=lambda **kwargs: self.run(
+                    RemoveIndividualPipelineParameters(**kwargs)
+                ),
+                args_schema=RemoveIndividualPipelineParameters,
+            )
+        ]
+
+    def as_api(
+        self,
+        router: APIRouter,
+        route_name: str = "",
+        name: str = "",
+        description: str = "",
+        description_stream: str = "",
+        tags: list[str | Enum] | None = None,
+    ) -> None:
+        if tags is None:
+            tags = []
+        return None
+
+
+if __name__ == "__main__":
+    from naas_abi import services
+
+    uris_to_remove = [
+        "http://ontology.naas.ai/abi/example-uri-1",
+        "http://ontology.naas.ai/abi/example-uri-2",
+    ]
+
+    configuration = RemoveIndividualPipelineConfiguration(
+        triple_store=services.triple_store_service
+    )
+
+    pipeline = RemoveIndividualPipeline(configuration)
+    graph = pipeline.run(
+        RemoveIndividualPipelineParameters(
+            uris_to_remove=uris_to_remove,
+        )
+    )
+    logger.info(graph.serialize(format="turtle"))

naas_abi/pipelines/RemoveIndividualPipeline_test.py

@@ -0,0 +1,58 @@
+import pytest
+from naas_abi import services
+from naas_abi.pipelines.RemoveIndividualPipeline import (
+    RemoveIndividualPipeline,
+    RemoveIndividualPipelineConfiguration,
+    RemoveIndividualPipelineParameters,
+)
+
+
+@pytest.fixture
+def pipeline() -> RemoveIndividualPipeline:
+    return RemoveIndividualPipeline(
+        RemoveIndividualPipelineConfiguration(
+            triple_store=services.triple_store_service
+        )
+    )
+
+
+def test_remove_individual_pipeline(pipeline: RemoveIndividualPipeline):
+    import time
+    from uuid import uuid4
+
+    from naas_abi import services
+    from naas_abi.utils.SPARQL import get_subject_graph
+    from rdflib import OWL, RDF, RDFS, Graph, Literal, Namespace, URIRef
+
+    ABI = Namespace("http://ontology.naas.ai/abi/")
+
+    graph = Graph()
+    uri = ABI[str(uuid4())]
+    graph.add(
+        (uri, RDF.type, URIRef("https://www.commoncoreontologies.org/ont00000443"))
+    )
+    graph.add((uri, RDF.type, OWL.NamedIndividual))
+    graph.add((uri, RDFS.label, Literal("Naas.ai")))
+    graph.add(
+        (
+            uri,
+            ABI.logo,
+            Literal(
+                "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ9gXMaBLQZ39W6Pk53PRuzFjUvv_6lLRWPoQ&s"
+            ),
+        )
+    )
+    services.triple_store_service.insert(graph)
+    time.sleep(3)
+
+    # Run pipeline to remove triples
+    graph = pipeline.run(
+        RemoveIndividualPipelineParameters(
+            uris_to_remove=[str(uri)],
+        )
+    )
+    assert graph is not None, graph.serialize(format="turtle")
+
+    # Check if uri is removed in triplestore
+    graph = get_subject_graph(str(uri), 1)
+    assert len(graph) == 0, graph.serialize(format="turtle")