naas-abi-core 1.4.1__py3-none-any.whl

naas_abi_core/services/vector_store/adapters/QdrantAdapter.py

@@ -0,0 +1,251 @@
+import logging
+from typing import Any, Dict, List, Optional, Union, cast
+from uuid import UUID
+
+import numpy as np
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance,
+    FieldCondition,
+    Filter,
+    MatchValue,
+    PointIdsList,
+    PointStruct,
+    PointVectors,
+    UpdateStatus,
+    VectorParams,
+)
+
+from ..IVectorStorePort import IVectorStorePort, SearchResult, VectorDocument
+
+logger = logging.getLogger(__name__)
+
+
+class QdrantAdapter(IVectorStorePort):
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 6333,
+        api_key: Optional[str] = None,
+        https: bool = False,
+        timeout: int = 30,
+    ):
+        self.host = host
+        self.port = port
+        self.api_key = api_key
+        self.https = https
+        self.timeout = timeout
+        self.client: Optional[QdrantClient] = None
+
+    def initialize(self) -> None:
+        if self.client is None:
+            self.client = QdrantClient(
+                host=self.host,
+                port=self.port,
+                api_key=self.api_key,
+                https=self.https,
+                timeout=self.timeout,
+            )
+            logger.info(f"Connected to Qdrant at {self.host}:{self.port}")
+
+    def _get_distance_metric(self, metric: str) -> Distance:
+        metric_map = {
+            "cosine": Distance.COSINE,
+            "euclidean": Distance.EUCLID,
+            "dot": Distance.DOT,
+        }
+        return metric_map.get(metric.lower(), Distance.COSINE)
+
+    def create_collection(
+        self,
+        collection_name: str,
+        dimension: int,
+        distance_metric: str = "cosine",
+        **kwargs,
+    ) -> None:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        self.client.create_collection(
+            collection_name=collection_name,
+            vectors_config=VectorParams(
+                size=dimension, distance=self._get_distance_metric(distance_metric)
+            ),
+            **kwargs,
+        )
+        logger.info(f"Created collection: {collection_name}")
+
+    def delete_collection(self, collection_name: str) -> None:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        self.client.delete_collection(collection_name=collection_name)
+        logger.info(f"Deleted collection: {collection_name}")
+
+    def list_collections(self) -> List[str]:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        collections = self.client.get_collections()
+        return [c.name for c in collections.collections]
+
+    def store_vectors(
+        self, collection_name: str, documents: List[VectorDocument]
+    ) -> None:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        points = []
+        for doc in documents:
+            payload = {}
+            if doc.metadata:
+                payload.update(doc.metadata)
+            if doc.payload:
+                payload["payload"] = doc.payload
+
+            point = PointStruct(
+                id=doc.id,
+                vector=doc.vector.tolist(),
+                payload=payload if payload else None,
+            )
+            points.append(point)
+
+        operation_info = self.client.upsert(
+            collection_name=collection_name, points=points
+        )
+
+        if operation_info.status != UpdateStatus.COMPLETED:
+            raise RuntimeError(f"Failed to store vectors: {operation_info}")
+
+        logger.debug(f"Stored {len(documents)} vectors in {collection_name}")
+
+    def search(
+        self,
+        collection_name: str,
+        query_vector: np.ndarray,
+        k: int = 10,
+        filter: Optional[Dict[str, Any]] = None,
+        include_vectors: bool = False,
+        include_metadata: bool = True,
+    ) -> List[SearchResult]:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        search_filter = None
+        if filter:
+            conditions = []
+            for key, value in filter.items():
+                conditions.append(
+                    FieldCondition(key=key, match=MatchValue(value=value))
+                )
+            if conditions:
+                from typing import cast
+
+                search_filter = Filter(must=cast(Any, conditions))
+
+        search_result = self.client.query_points(
+            collection_name=collection_name,
+            query=query_vector.tolist(),
+            limit=k,
+            query_filter=search_filter,
+            with_vectors=include_vectors,
+            with_payload=include_metadata,
+        )
+
+        results = []
+        for hit in search_result.points:
+            result = SearchResult(
+                id=str(hit.id),
+                score=hit.score,
+                vector=np.array(hit.vector) if hit.vector else None,
+                metadata=dict(hit.payload)
+                if hit.payload and include_metadata
+                else None,
+                payload=hit.payload.get("payload")
+                if hit.payload and "payload" in hit.payload
+                else None,
+            )
+            results.append(result)
+
+        return results
+
+    def get_vector(
+        self, collection_name: str, vector_id: str, include_vector: bool = True
+    ) -> Optional[VectorDocument]:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        points = self.client.retrieve(
+            collection_name=collection_name,
+            ids=[vector_id],
+            with_vectors=include_vector,
+            with_payload=True,
+        )
+
+        if not points:
+            return None
+
+        point = points[0]
+        payload = dict(point.payload) if point.payload else {}
+
+        vector_array = np.array(point.vector) if point.vector else np.array([])
+        return VectorDocument(
+            id=str(point.id),
+            vector=vector_array,
+            metadata={k: v for k, v in payload.items() if k != "payload"},
+            payload=payload.get("payload"),
+        )
+
+    def update_vector(
+        self,
+        collection_name: str,
+        vector_id: str,
+        vector: Optional[np.ndarray] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        payload: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        if vector is not None:
+            self.client.update_vectors(
+                collection_name=collection_name,
+                points=[PointVectors(id=vector_id, vector=vector.tolist())],
+            )
+
+        if metadata is not None or payload is not None:
+            new_payload = {}
+            if metadata:
+                new_payload.update(metadata)
+            if payload:
+                new_payload["payload"] = payload
+
+            self.client.set_payload(
+                collection_name=collection_name, payload=new_payload, points=[vector_id]
+            )
+
+    def delete_vectors(self, collection_name: str, vector_ids: List) -> None:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        point_ids = cast(List[Union[int, str, UUID]], vector_ids)
+
+        # Cast to satisfy mypy: list[str] is compatible with list[int | str | UUID | PointId]
+        self.client.delete(
+            collection_name=collection_name,
+            points_selector=PointIdsList(points=point_ids),
+        )
+        logger.debug(f"Deleted {len(vector_ids)} vectors from {collection_name}")
+
+    def count_vectors(self, collection_name: str) -> int:
+        if not self.client:
+            raise RuntimeError("Adapter not initialized")
+
+        collection_info = self.client.get_collection(collection_name=collection_name)
+        return collection_info.indexed_vectors_count or 0
+
+    def close(self) -> None:
+        if self.client:
+            self.client.close()
+            self.client = None
+            logger.info("Closed connection to Qdrant")

naas_abi_core/services/vector_store/adapters/QdrantAdapter_test.py

@@ -0,0 +1,57 @@
+import pytest
+import os
+from ..IVectorStorePort_test import GenericVectorStoreAdapterTest
+from .QdrantAdapter import QdrantAdapter
+
+
+@pytest.mark.skipif(
+    os.getenv("QDRANT_HOST") is None,
+    reason="Qdrant integration tests require QDRANT_HOST to be set"
+)
+class TestQdrantAdapter(GenericVectorStoreAdapterTest):
+    @pytest.fixture
+    def adapter(self):
+        adapter = QdrantAdapter(
+            host=os.getenv("QDRANT_HOST", "localhost"),
+            port=int(os.getenv("QDRANT_PORT", "6333")),
+            api_key=os.getenv("QDRANT_API_KEY"),
+            https=os.getenv("QDRANT_HTTPS", "false").lower() == "true"
+        )
+        
+        adapter.initialize()
+        
+        try:
+            adapter.delete_collection("test_collection")
+        except Exception:
+            pass
+        
+        yield adapter
+        
+        try:
+            adapter.delete_collection("test_collection")
+        except Exception:
+            pass
+        
+        adapter.close()
+
+    def test_distance_metric_mapping(self):
+        adapter = QdrantAdapter()
+        
+        from qdrant_client.models import Distance
+        
+        assert adapter._get_distance_metric("cosine") == Distance.COSINE
+        assert adapter._get_distance_metric("euclidean") == Distance.EUCLID
+        assert adapter._get_distance_metric("dot") == Distance.DOT
+        assert adapter._get_distance_metric("unknown") == Distance.COSINE
+
+    def test_adapter_not_initialized_error(self):
+        adapter = QdrantAdapter()
+        
+        with pytest.raises(RuntimeError, match="Adapter not initialized"):
+            adapter.create_collection("test", 128)
+        
+        with pytest.raises(RuntimeError, match="Adapter not initialized"):
+            adapter.list_collections()
+        
+        with pytest.raises(RuntimeError, match="Adapter not initialized"):
+            adapter.store_vectors("test", [])

naas_abi_core/tests/test_services_imports.py

@@ -0,0 +1,69 @@
+# This is used to test all imports and ensure all libraries are installed and imported correctly.
+
+
+def test_imports():
+    from naas_abi_core.services.agent import Agent, IntentAgent
+    from naas_abi_core.services.cache import CacheFactory, CachePort, CacheService
+    from naas_abi_core.services.cache.adapters.secondary import CacheFSAdapter
+    from naas_abi_core.services.object_storage import ObjectStorageService
+    from naas_abi_core.services.object_storage.adapters.secondary import (
+        ObjectStorageSecondaryAdapterFS,
+        ObjectStorageSecondaryAdapterNaas,
+        ObjectStorageSecondaryAdapterS3,
+    )
+    from naas_abi_core.services.secret import Secret, SecretPorts
+    from naas_abi_core.services.secret.adaptors.secondary import (
+        Base64Secret,
+        NaasSecret,
+        dotenv_secret_secondaryadaptor,
+    )
+    from naas_abi_core.services.triple_store import (
+        TripleStoreFactory,
+        TripleStorePorts,
+        TripleStoreService,
+    )
+    from naas_abi_core.services.triple_store.adaptors.secondary import (
+        AWSNeptune,
+        Oxigraph,
+    )
+    from naas_abi_core.services.vector_store import (
+        IVectorStorePort,
+        VectorStoreFactory,
+        VectorStoreService,
+    )
+    from naas_abi_core.services.vector_store.adapters.QdrantAdapter import QdrantAdapter
+
+    Agent
+    IntentAgent
+    CacheService
+    CacheFactory
+    CachePort
+    CacheFSAdapter
+    ObjectStorageService
+    ObjectStorageSecondaryAdapterFS
+    ObjectStorageSecondaryAdapterNaas
+    ObjectStorageSecondaryAdapterS3
+    ObjectStorageSecondaryAdapterFS
+    ObjectStorageSecondaryAdapterNaas
+    ObjectStorageSecondaryAdapterS3
+    TripleStoreService
+    TripleStoreFactory
+    TripleStorePorts
+    AWSNeptune
+    Oxigraph
+    ObjectStorageSecondaryAdapterNaas
+    ObjectStorageSecondaryAdapterS3
+    TripleStoreService
+    TripleStoreFactory
+    Secret
+    SecretPorts
+    Base64Secret
+    dotenv_secret_secondaryadaptor
+    NaasSecret
+    VectorStoreFactory
+    VectorStoreService
+    IVectorStorePort
+    QdrantAdapter
+    VectorStoreService
+    IVectorStorePort
+    QdrantAdapter

naas_abi_core/utils/Expose.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter
+
+if TYPE_CHECKING:    
+    from langchain_core.tools import BaseTool
+
+
+class Expose(ABC):
+    @abstractmethod
+    def as_tools(self) -> list[BaseTool]:
+        """Returns a list of Tools that can be used by an Agent.
+
+        This method should be implemented by concrete classes to expose their functionality
+        as LangChain StructuredTools that can be used by an Agent.
+
+        Returns:
+            list[StructuredTool]: A list of StructuredTools that expose the class's functionality
+
+        Raises:
+            NotImplementedError: If the concrete class does not implement this method
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def as_api(
+        self,
+        router: APIRouter,
+        route_name: str = "",
+        name: str = "",
+        description: str = "",
+        description_stream: str = "",
+        tags: list[str | Enum] | None = [],
+    ) -> None:
+        """Registers API routes for the class's functionality on the provided FastAPI router.
+
+        This method should be implemented by concrete classes to expose their functionality
+        via HTTP endpoints using FastAPI. The method should register routes on the provided
+        router that allow accessing the class's functionality through HTTP requests.
+
+        Args:
+            router (APIRouter): The FastAPI router on which to register the routes
+
+        Returns:
+            None
+
+        Raises:
+            NotImplementedError: If the concrete class does not implement this method
+        """
+        raise NotImplementedError()
+        raise NotImplementedError()

naas_abi_core/utils/Graph.py

@@ -0,0 +1,182 @@
+from datetime import date, datetime
+from urllib.parse import quote
+
+import pytz
+from rdflib import Graph as rdfgraph
+from rdflib import Literal, Namespace, URIRef
+from rdflib.namespace import OWL, RDF, RDFS, SKOS
+
+from naas_abi_core import logger
+
+BFO = Namespace("http://purl.obolibrary.org/obo/")
+ABI = Namespace("http://ontology.naas.ai/abi/")
+TEST = Namespace("http://ontology.naas.ai/test/")
+TIME = Namespace("http://www.w3.org/2006/time#")
+XSD = Namespace("http://www.w3.org/2001/XMLSchema#")
+CCO = Namespace("https://www.commoncoreontologies.org/")
+DCTERMS = Namespace("http://purl.org/dc/terms/")
+URI_REGEX = r"http:\/\/ontology\.naas\.ai\/.+\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+
+
+class ABIGraph(rdfgraph):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.bind("bfo", BFO)
+        self.bind("skos", SKOS)
+        self.bind("abi", ABI)
+        self.bind("cco", CCO)
+        self.bind("xsd", XSD)
+        self.bind("time", TIME)
+
+    def add_data_properties(self, uri: URIRef, lang="en", **data_properties):
+        for x in data_properties:
+            value = data_properties.get(x)
+            if value is not None:
+                if type(value) is str:
+                    self.add((uri, ABI[x], Literal(value.strip(), lang=lang)))
+                elif type(value) in [int, float]:
+                    self.add((uri, ABI[x], Literal(value, datatype=XSD.integer)))
+                elif isinstance(value, datetime):
+                    if value.tzinfo is None:
+                        value = value.replace(
+                            tzinfo=pytz.utc
+                        )  # Apply UTC timezone if none
+                    self.add(
+                        (
+                            uri,
+                            ABI[x],
+                            Literal(
+                                value.strftime("%Y-%m-%dT%H:%M:%S%z"),
+                                datatype=XSD.dateTime,
+                            ),
+                        )
+                    )
+                elif isinstance(value, date):
+                    self.add(
+                        (
+                            uri,
+                            ABI[x],
+                            Literal(value.strftime("%Y-%m-%d"), datatype=XSD.date),
+                        )
+                    )
+        self.add(
+            (
+                uri,
+                DCTERMS.modified,
+                Literal(
+                    str(datetime.now().strftime("%Y-%m-%dT%H:%M:%S%z")),
+                    datatype=XSD.dateTime,
+                ),
+            )
+        )
+
+    # Add OWL NamedIndividual to Graph
+    def add_individual(
+        self,
+        uri: URIRef,
+        label,
+        is_a,
+        lang="en",
+        skip_if_exists=True,
+        **data_properties,
+    ) -> URIRef:
+        if (uri, RDF.type, is_a) in self and skip_if_exists:
+            logger.debug(f"🟡 '{label}' ({str(uri)}) already exists in ontology.")
+        else:
+            # Add NamedIndividual to ontology
+            self.add((uri, RDF.type, OWL.NamedIndividual))
+            self.add((uri, RDF.type, URIRef(is_a)))
+            self.add((uri, RDFS.label, Literal(str(label), lang=lang)))
+            self.add(
+                (
+                    uri,
+                    DCTERMS.created,
+                    Literal(
+                        str(datetime.now().strftime("%Y-%m-%dT%H:%M:%S%z")),
+                        datatype=XSD.dateTime,
+                    ),
+                )
+            )
+            logger.debug(f"🟢 '{label}' ({str(uri)}) successfully added to ontology.")
+
+        # Add data properties to NamedIndividual
+        self.add_data_properties(uri, lang, **data_properties)
+        return uri
+
+    def add_individual_to_prefix(
+        self,
+        prefix: Namespace,
+        uid: str,
+        label: str,
+        is_a: URIRef,
+        lang="en",
+        skip_if_exists=True,
+        **data_properties,
+    ) -> URIRef:
+        uid = str(uid).split(":")[-1]
+        type_name = str(is_a).split("/")[-1]
+        uri = URIRef(quote(f"{str(prefix)}{type_name}#{uid}", safe=":/#"))
+        return self.add_individual(
+            uri, label, is_a, lang, skip_if_exists, **data_properties
+        )
+
+    def add_process(
+        self,
+        prefix: Namespace,
+        uid: str,
+        label: str,
+        is_a: URIRef,
+        lang="en",
+        participants=[],
+        participants_oprop=BFO.BFO_0000057,
+        participants_oprop_inverse=BFO.BFO_0000056,
+        realizes=[],
+        realizes_oprop=BFO.BFO_0000055,
+        realizes_oprop_inverse=BFO.BFO_0000054,
+        occurs_in=[],
+        occurs_in_oprop=BFO.BFO_0000066,
+        occurs_in_oprop_inverse=BFO.BFO_0000183,
+        concretizes=[],
+        concretizes_oprop=BFO.BFO_0000058,
+        concretizes_oprop_inverse=BFO.BFO_0000059,
+        temporal_region=None,
+        temporal_region_oprop=BFO.BFO_0000199,
+        spatiotemporal_region=None,
+        spatiotemporal_region_oprop=BFO.BFO_0000200,
+        skip_if_exists=True,
+        **data_properties,
+    ):
+        # Init
+        uri = self.add_individual_to_prefix(
+            prefix, uid, label, is_a, lang, skip_if_exists, **data_properties
+        )
+
+        # Add participants
+        for participant in participants:
+            self.add((uri, participants_oprop, participant))
+            self.add((participant, participants_oprop_inverse, uri))
+
+        # Add realizable entities
+        for realize in realizes:
+            self.add((uri, realizes_oprop, realize))
+            self.add((realize, realizes_oprop_inverse, uri))
+
+        # Add occurs_in
+        for oi in occurs_in:
+            self.add((uri, occurs_in_oprop, oi))
+            self.add((oi, occurs_in_oprop_inverse, uri))
+
+        # Add concretizes
+        for concretize in concretizes:
+            self.add((uri, concretizes_oprop, concretize))
+            self.add((concretize, concretizes_oprop_inverse, uri))
+
+        # Add occupies_temporal_region
+        if temporal_region:
+            self.add((uri, temporal_region_oprop, temporal_region))
+
+        # Add occupies_spatiotemporal_region
+        if spatiotemporal_region:
+            self.add((uri, spatiotemporal_region_oprop, spatiotemporal_region))
+        return uri

naas_abi_core/utils/JSON.py

@@ -0,0 +1,49 @@
+import json
+import re
+
+from naas_abi_core import logger
+
+
+def extract_json_from_completion(completion_text: str) -> list | dict:
+    """Extract JSON object from completion text that contains markdown formatting.
+
+    Args:
+        completion_text (str): Raw completion text containing JSON in markdown format
+
+    Returns:
+        dict: Parsed JSON object
+    """
+    # Find JSON content between ```json and ``` markers
+    json_start = completion_text.find("```json\n")
+    json_end = completion_text.rfind("```")
+
+    if json_start == -1 or json_end == -1:
+        # If no markdown markers found, try parsing the whole text
+        json_str = completion_text
+    else:
+        # Extract content between markers and clean it
+        json_start += len("```json\n")
+        json_str = completion_text[json_start:json_end].strip()
+
+    # Try multiple cleanup approaches
+    attempts = [
+        lambda x: x,  # Original string
+        lambda x: x.replace("```", "").strip(),  # Remove markdown artifacts
+        lambda x: x.replace("\n", "").replace(" ", ""),  # Remove all whitespace
+        lambda x: x.replace("'", '"'),  # Replace single quotes with double quotes
+        lambda x: re.sub(
+            r"([{,])\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:", r'\1"\2":', x
+        ),  # Quote unquoted keys
+    ]
+
+    for attempt in attempts:
+        try:
+            cleaned_str = attempt(json_str)
+            return json.loads(cleaned_str)
+        except json.JSONDecodeError as e:
+            logger.debug(f"JSON parse attempt failed: {str(e)}")
+            continue
+
+    # If all attempts fail, return empty dict
+    logger.error("Failed to parse JSON after all cleanup attempts")
+    return {}