guardianhub 0.1.88__py3-none-any.whl

guardianhub/clients/vector_client.py

@@ -0,0 +1,344 @@
+"""
+Vector Service Client for interacting with vector database.
+
+This module provides functionality for:
+- Managing vector collections
+- Storing and retrieving document embeddings (via the service)
+- Handling semantic and agentic tools
+- Chunking large documents
+- Sanitizing metadata
+"""
+
+import json
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+from guardianhub import get_logger
+from guardianhub.config.settings import settings
+
+logger = get_logger(__name__)
+
+def _chunk_text(
+        text: str,
+        max_tokens: int = 500,
+        approx_chars_per_token: int = 4
+) -> List[str]:
+    """Split text into chunks of approximately max_tokens length."""
+    if not text:
+        return []
+
+    max_chars = max_tokens * approx_chars_per_token
+    if len(text) <= max_chars:
+        return [text]
+
+    chunks = []
+    start = 0
+
+    while start < len(text):
+        end = min(len(text), start + max_chars)
+        slice_ = text[start:end]
+
+        # Look for the last newline or space to cut cleanly
+        if end < len(text):
+            last_nl = slice_.rfind("\n")
+            last_space = slice_.rfind(" ")
+            cut = max(last_nl, last_space)
+            if cut > 0:
+                end = start + cut
+
+        chunks.append(text[start:end].strip())
+        start = end
+
+    return [c for c in chunks if c]
+
+def _sanitize_metadata_value(value: Any) -> Any:
+    """Ensure metadata value is vector-DB safe."""
+    if value is None or value == []:
+        return None
+    if isinstance(value, (str, int, float, bool)):
+        return value
+    try:
+        # Complex types like dicts or lists should be JSON serialized
+        return json.dumps(value)
+    except Exception:
+        return str(value)
+
+def sanitize_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    """Clean metadata dictionary by dropping None values and normalizing types."""
+    return {
+        k: _sanitize_metadata_value(v)
+        for k, v in metadata.items()
+        if _sanitize_metadata_value(v) is not None
+    }
+
+class VectorClient:
+    """Client for interacting with the vector database service."""
+
+    def __init__(
+            self,
+            base_url: Optional[str] = None,
+            collection: str = settings.vector.default_collection,
+            http_timeout: float = 30.0,
+            **collection_kwargs
+    ):
+        """Initialize the VectorClient.
+
+        Args:
+            base_url: Base URL for the vector service. If not provided, will try to get from settings.
+            collection_docs: Name of the document collection to use.
+            http_timeout: Timeout for HTTP requests in seconds.
+            **collection_kwargs: Additional collection configuration.
+        """
+
+        self.base_url = base_url or settings.endpoints.get('VECTOR_SERVICE_URL')
+        # self.base_url = base_url.rstrip("/")
+        self.collection = collection
+        self.collection_config = collection_kwargs
+        self._client = httpx.AsyncClient(
+            base_url=self.base_url,
+            timeout=http_timeout
+        )
+        self.initialized = False
+
+    async def initialize(self):
+        """Initialize the client and check connection asynchronously"""
+        try:
+            await self._check_connection()
+            self.initialized = True
+            return self.initialized
+        except Exception as e:
+            logger.error(f"Vector client initialization failed: {str(e)}")
+            self.initialized = False
+            return False
+
+    async def ensure_collection_exists(self, collection_name: str = None, **collection_kwargs) -> bool:
+        """Ensure the specified collection exists, create it if it doesn't.
+
+        Args:
+            collection_name: Name of the collection to check/create
+            **collection_kwargs: Additional arguments for collection creation
+
+        Returns:
+            bool: True if collection exists or was created, False otherwise
+        """
+        try:
+            # Try to create the collection - will succeed if it doesn't exist
+            await self.create_collection(collection_name, **collection_kwargs)
+            logger.info(f"Created collection: {collection_name}")
+            return True
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 409:  # Collection already exists
+                logger.debug(f"Collection {collection_name} already exists")
+                return True
+            logger.error(f"Error ensuring collection {collection_name} exists: {str(e)}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error ensuring collection {collection_name} exists: {str(e)}")
+            return False
+
+    async def _check_connection(self):
+        """Check if the vector service is available."""
+        try:
+            response = await self._client.get("/health")
+            response.raise_for_status()
+            self.initialized = True
+        except Exception as e:
+            logger.error(f"Vector client health check failed: {str(e)}")
+            self.initialized = False
+            raise
+
+    async def close(self):
+        """Close the HTTP client."""
+        await self._client.aclose()
+
+    async def upsert_documents(
+            self,
+            ids: List[str],
+            documents: List[str],
+            metadatas: Optional[List[Dict[str, Any]]],
+            collection: Optional[str] ,
+            embeddings: Optional[List[List[float]]] = None,
+    ) -> Any:
+        """Upsert documents into the vector database in a single batch call."""
+        if not ids or not documents or len(ids) != len(documents):
+            raise ValueError("ids and documents must be non-empty and same length")
+
+        if metadatas and len(metadatas) != len(ids):
+            raise ValueError("metadatas must be same length as ids if provided")
+
+        # Ensure metadata is safe for the vector store
+        sanitized_metas = [
+            sanitize_metadata(m) if m else {}
+            for m in (metadatas or [{}] * len(ids))
+        ]
+
+        payload = {
+            # Note: We do not pass embeddings here; the service will generate them.
+            "documents": documents,
+            "ids": ids,
+            "metadatas": sanitized_metas,
+            "embeddings": embeddings
+        }
+
+        response = await self._client.post(
+            f"/collections/{collection}/add",
+            json=payload
+        )
+        response.raise_for_status()
+        return response.json()
+
+    async def upsert_document_from_text(
+            self,
+            document_content: str,
+            doc_id: str,
+            metadata: Dict[str, Any],
+            collection: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Chunks a single document and upserts all chunks into the vector database.
+
+        The ingestion activity (store_in_vector_db_activity) should call this method.
+        It returns the ID of the first chunk, which acts as the 'vector_id' for the document.
+        """
+        collection_name = collection
+
+        if not document_content:
+            logger.warning(f"Attempted to upsert empty content for document ID: {doc_id}")
+            return {"status": "skipped", "message": "Empty document content"}
+
+        # 1. Chunk the document
+        chunks = _chunk_text(document_content)
+
+        if not chunks:
+             logger.warning(f"Chunking failed for document ID: {doc_id}")
+             return {"status": "skipped", "message": "Chunking failed to produce content"}
+
+        # 2. Prepare IDs and Metadata for upsert
+        chunk_ids = [f"{doc_id}-{i}" for i in range(len(chunks))]
+
+        # All chunks share the same base metadata, plus chunk-specific indices
+        base_metadata = sanitize_metadata(metadata.copy())
+
+        chunk_metadatas = []
+        for i in range(len(chunks)):
+            meta = base_metadata.copy()
+            # Store the primary document ID in the metadata for easy filtering/retrieval
+            meta["original_doc_id"] = doc_id
+            meta["chunk_index"] = i
+            meta["chunk_total"] = len(chunks)
+            chunk_metadatas.append(meta)
+
+        # 3. Upsert the documents (chunks)
+        response = await self.upsert_documents(
+            ids=chunk_ids,
+            documents=chunks,
+            metadatas=chunk_metadatas,
+            collection=collection_name,
+        )
+
+        # Return the ID of the first chunk
+        return {"id": chunk_ids[0], "status": "success", "chunk_count": len(chunks), "service_response": response}
+
+
+    async def delete_document(self, doc_id: str, collection: str) -> None:
+        """Delete a document (and all its chunks) from the vector database.
+        
+        Args:
+            doc_id: The ID of the document to delete
+            collection: The name of the collection containing the document
+            
+        Raises:
+            httpx.HTTPStatusError: If the deletion request fails
+        """
+        # For a complete document deletion, we must delete based on the original_doc_id in metadata
+        response = await self._client.request(
+            method="DELETE",
+            url=f"/collections/{collection}/delete",
+            json={
+                "ids": [doc_id],
+            }
+        )
+        response.raise_for_status()
+        logger.info(f"Deleted document chunks for original ID {doc_id} from {collection}")
+        return response.json()
+
+    # Removed the previous embed_text method as embedding is handled by the service.
+
+    async def query(
+            self,
+            query: str,
+            collection: str,
+            n_results: int = 5,
+            where: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """Query the vector database using a text query."""
+        if not self.initialized:
+            await self.initialize()
+
+        payload = {
+            "query_texts": [query],
+            "n_results": n_results
+        }
+        # Where clause must be sanitized if needed, but we rely on the service to interpret Chroma syntax
+        if where:
+            payload["where"] = where
+
+        response = await self._client.post(
+            f"/collections/{collection}/query",
+            json=payload
+        )
+        response.raise_for_status()
+        return response.json()
+
+    async def create_collection(self, name: str, **kwargs) -> Dict[str, Any]:
+        """Create a new collection."""
+        response = await self._client.post(
+            f"/collections",
+            json={
+                "name": name,
+                **kwargs
+            }
+        )
+        response.raise_for_status()
+        return response.json()
+
+    async def delete_collection(self, name: str) -> None:
+        """Delete a collection."""
+        # response = await self._client.delete(f"/collections/{name}/delete")
+        # response.raise_for_status()
+        logger.info(f"Deleted collection {name}")
+
+    async def get_raw_embedding(self, document_text: str) -> Optional[List[float]]:
+        """
+        Retrieves the raw embedding vector for a single document text
+        via the Vector DB Service's dedicated generation endpoint.
+        """
+        endpoint = "/embed/text"
+
+        # NOTE: The Vector Service endpoint expects a list of texts
+        payload = {"texts": [document_text]}
+
+        try:
+            response = await self._client.post(endpoint, json=payload)
+            response.raise_for_status()
+
+            data = response.json()
+            embeddings = data.get("embeddings")
+
+            if embeddings and len(embeddings) > 0:
+                # We requested one text, so we return the first (and only) embedding
+                return embeddings[0]
+
+            logger.error("Embedding service returned success but embeddings list was empty.")
+            return None
+
+        except httpx.HTTPStatusError as e:
+            logger.error(
+                "HTTP error generating raw embedding: Status %d. Detail: %s",
+                e.response.status_code, e.response.text
+            )
+            return None
+        except httpx.RequestError as e:
+            logger.error("Network error connecting to Vector DB for embedding generation: %s", str(e))
+            return None

guardianhub/config/config_development.json

@@ -0,0 +1,84 @@
+{
+  "service": {
+    "name": "sutram-orchestration-service",
+    "id": "sutram-orchestration-service-01",
+    "host": "0.0.0.0",
+    "port": 7001
+  },
+  "logging": {
+    "level": "INFO",
+    "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  },
+  "vector": {
+    "default_collection": "test_collection"
+  },
+  "endpoints": {
+    "POSTGRES_URL": "postgresql+asyncpg://user:pass@postgres-host:5432/db_name",
+    "CONSUL_HTTP_ADDR": "http://guardian:8500",
+    "TOOL_REGISTRY_URL": "http://tools-registry.guardianhub.com",
+    "TOOL_EXECUTOR_URL": "http://agentic-orchestrator.guardianhub.com",
+    "GRAPH_DB_URL": "http://guardianhub:8009/",
+    "VECTOR_SERVICE_URL": "http://vectordb.guardianhub.com",
+    "LLM_URL": "http://aura-llm.guardianhub.com",
+    "LANGFUSE_HOST": "http://langfuse.guardianhub.com",
+    "OTEL_EXPORTER_OTLP_ENDPOINT": "http://guardianhub:4318",
+    "LANGFUSE_OTLP_TRACES_ENDPOINT": "http://langfuse.guardianhub.com:4318",
+    "KG_INGESTION_URL": "http://graph-db-service:8009/ingest-yaml-schema",
+    "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+    "OTEL_EXPORTER_OTLP_TRACES_PATH": "http/protobuf",
+    "LF_EXCLUDED_URL": "http://guardianhub:3000.*,/v1/metrics",
+    "LANGFUSE_PROJECT_ID": "cmgy74xzc0006xo07iwmvamue"
+  },
+  "llm": {
+    "model_configs": {
+      "judge": {
+        "model_name": "gemini-2.0-flash",
+        "base_url": "https://generativelanguage.googleapis.com/v1beta/models",
+        "api_key": "AIzaSyC7R2Ag1e-6VPf_c15boWPFr99FyvpHcT8",
+        "headers": {
+        },
+        "temperature": 0.1,
+        "max_tokens": 2048,
+        "model_kwargs": {},
+        "provider": "GEMINI",
+        "streaming": false
+      },
+      "tuned": {
+        "model_name": "phi3:3.8b-mini-4k-instruct-q4_K_M",
+        "provider": "OPENAI",
+        "base_url": "http://guardianhub:11434",
+        "temperature": 0.1,
+        "max_tokens": 2048,
+        "streaming": false,
+        "model_kwargs": {}
+      },
+      "coding": {
+        "model_name": "deepseek-coder:1.3b",
+        "temperature": 0.1,
+        "max_tokens": 2048,
+        "base_url": "http://guardianhub:11434",
+        "api_key": "your-api-key-here",
+        "streaming": false,
+        "model_kwargs": {}
+      },
+      "default": {
+        "model_name": "phi3:mini",
+        "provider": "OLLAMA",
+        "base_url": "http://guardianhub:11434",
+        "temperature": 0.1,
+        "max_tokens": 2048,
+        "streaming": false,
+        "model_kwargs": {}
+      },
+      "large": {
+        "model_name": "mistral-7b-instruct",
+        "temperature": 0.1,
+        "max_tokens": 2048,
+        "base_url": "http://guardianhub:11434",
+        "api_key": "your-api-key-here",
+        "streaming": false,
+        "model_kwargs": {}
+      }
+    }
+  }
+}

guardianhub/config/config_prod.json

@@ -0,0 +1,39 @@
+{
+  "service": {
+    "name": "guardianhub-sdk",
+    "id": "guardianhub-sdk-01",
+    "host": "0.0.0.0",
+    "port": 8001
+  },
+  "logging": {
+    "level": "INFO",
+    "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  },
+  "excluded_urls": "http://langfuse-web.langfuse.svc.cluster.local:3000.*,/v1/metrics",
+  "endpoints": {
+    "POSTGRES_URL": "postgresql+asyncpg://user:pass@postgres-host:5432/db_name",
+    "CONSUL_HTTP_ADDR": "http://consul-server.consul.svc.cluster.local:8500",
+    "TOOL_REGISTRY_URL": "http://tool-registry-service.default.svc.cluster.local:8002",
+    "TOOL_EXECUTOR_URL": "http://agentic-orchestrator-service.default.svc.cluster.local:8003",
+    "LLM_URL": "http://aura-llm-service.default.svc.cluster.local:8000/v1",
+    "LANGFUSE_HOST": "http://langfuse-web.langfuse.svc.cluster.local:3000",
+    "LANGFUSE_OTLP_TRACES_ENDPOINT": "http://langfuse-web.langfuse.svc.cluster.local:3000",
+    "OTEL_EXPORTER_OTLP_ENDPOINT": "http://otel-collector-service.monitoring.svc.cluster.local:4318",
+    "KG_INGESTION_URL": "http://graph-db-service:8009/ingest-yaml-schema",
+    "ENVIRONMENT": "kubernetes-dev",
+    "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+    "OTEL_EXPORTER_OTLP_TRACES_PATH": "http/protobuf",
+    "LF_EXCLUDED_URL": "http://langfuse-web.langfuse.svc.cluster.local:3000.*,/v1/metrics",
+    "GRAPH_DB_URL": "http://graph-db-service.default.svc.cluster.local:8009",
+    "VECTOR_SERVICE_URL": "http://vector-db-service.default.svc.cluster.local:8005"
+  },
+  "llm": {
+    "model_name": "phi-4",
+    "temperature": 0.1,
+    "max_tokens": 4096,
+    "base_url": "http://aura-llm-service.default.svc.cluster.local:8000/v1",
+    "api_key": "your-api-key-here",
+    "streaming": false,
+    "model_kwargs": {}
+  }
+}

guardianhub/config/settings.py

@@ -0,0 +1,221 @@
+"""Core settings for the GuardianHub Foundation SDK."""
+from __future__ import annotations
+
+import os
+import json
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+# Correct imports for Pydantic v2 and Pydantic-Settings v2
+from pydantic import BaseModel, Field, ConfigDict, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+def _load_config_defaults(env: str = None) -> dict:
+    """Load configuration defaults from JSON files based on environment."""
+    if env is None:
+        env = os.environ.get("ENVIRONMENT", "development").lower()
+
+    # Mapping common aliases
+    if env == "dev":
+        env = "development"
+
+    config_dir = Path(__file__).parent
+    possible_files = [f"config_{env}.json", "config_dev.json", "config_development.json"]
+
+    for filename in possible_files:
+        config_path = config_dir / filename
+        if config_path.exists():
+            try:
+                with open(config_path) as f:
+                    return json.load(f)
+            except Exception:
+                continue
+    return {}
+
+class ServiceInfo(BaseModel):
+    """Metadata about the microservice using the SDK."""
+    name: str = "guardianhub-service"
+    version: str = "0.0.1"
+    id: str = "guardian-01"
+    host: str = "0.0.0.0"
+    port: int = 8001
+
+
+class LoggingSettings(BaseModel):
+    """Standardized logging configuration."""
+    level: str = "INFO"
+    format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+
+from typing import Dict, Any, Optional
+from pydantic import BaseModel, Field, ConfigDict
+
+class ModelConfig(BaseModel):
+    """Standardized schema for any LLM model configuration."""
+    model_name: str
+    provider: str = "OPENAI"  # GEMINI, OLLAMA, OPENAI, etc.
+    base_url: str
+    api_key: str = "your-api-key-here"
+    temperature: float = 0.1
+    max_tokens: int = 2048
+    streaming: bool = False
+    headers: Dict[str, str] = {}
+    model_kwargs: Dict[str, Any] = {}
+
+class LLMSettings(BaseModel):
+    """Registry of models defined in the config JSON."""
+    # This matches your "model_configs": { "judge": {...}, "tuned": {...} }
+    model_configs: Dict[str, ModelConfig] = Field(default_factory=dict)
+
+    def get_config(self, key: str = "default") -> ModelConfig:
+        """Helper to safely retrieve a model config by its key."""
+        return self.model_configs.get(key, self.model_configs.get("default"))
+
+class ServiceEndpoints(BaseModel):
+    """
+    Endpoints for essential shared services.
+    Supports dictionary-style access for backward compatibility.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    # Infrastructure & Databases
+    POSTGRES_URL: str = "postgresql://user:password@localhost:5432/guardianhub"
+    CONSUL_HTTP_ADDR: str = "http://localhost:8500"
+
+    # Shared GuardianHub Services
+    TOOL_REGISTRY_URL: str = "http://localhost:8000"
+    TOOL_EXECUTOR_URL: str = "http://localhost:8003"
+    LLM_URL: str = "http://localhost:8001"
+    LANGFUSE_HOST: str = "http://localhost:3000"
+    KG_INGESTION_URL: str = "http://localhost:8002"
+    GRAPH_DB_URL: str = "http://localhost:8009"
+    VECTOR_SERVICE_URL: str = "http://localhost:8005"
+
+    # Observability
+    OTEL_EXPORTER_OTLP_ENDPOINT: str = "http://localhost:4318"
+    LANGFUSE_OTLP_TRACES_ENDPOINT: str = "http://localhost:4318"
+    ENVIRONMENT: str = "development"
+
+    def __init__(self, **data):
+        # Handle case-insensitive environment variables
+        processed_data = {}
+        for k, v in data.items():
+            # Convert to uppercase for case-insensitive matching
+            if k.upper() in self.model_fields:
+                processed_data[k.upper()] = v
+            else:
+                processed_data[k] = v
+
+        # Initialize the model with known fields
+        super().__init__(**{k: v for k, v in processed_data.items()
+                          if k in self.model_fields})
+
+        # Store extra fields in __pydantic_extra__
+        extra_data = {k: v for k, v in processed_data.items()
+                     if k not in self.model_fields}
+        if extra_data:
+            if not hasattr(self, '__pydantic_extra__'):
+                object.__setattr__(self, '__pydantic_extra__', {})
+            self.__pydantic_extra__.update(extra_data)
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Dictionary-style get method for backward compatibility."""
+        # First try to get from model fields (case-insensitive)
+        key_upper = key.upper()
+        if key_upper in self.model_fields:
+            return getattr(self, key_upper, default)
+
+        # Then try to get from extra fields (case-sensitive)
+        if hasattr(self, '__pydantic_extra__'):
+            # Try exact match first
+            if key in self.__pydantic_extra__:
+                return self.__pydantic_extra__[key]
+            # Try case-insensitive match
+            for k, v in self.__pydantic_extra__.items():
+                if k.upper() == key_upper:
+                    return v
+
+        # Finally, try direct attribute access
+        return getattr(self, key, default)
+
+    def __getitem__(self, key: str) -> Any:
+        """Enable dictionary-style access."""
+        value = self.get(key)
+        if value is None and not hasattr(self, key) and not (hasattr(self, '__pydantic_extra__') and key in self.__pydantic_extra__):
+            raise KeyError(f"'{key}' not found in ServiceEndpoints")
+        return value
+
+class VectorConfig(BaseModel):
+    """Vector database configuration."""
+    default_collection: str = Field("", description="Default collection name")
+
+
+class CoreSettings(BaseSettings):
+    """
+    The Core Settings class. Automatically merges bundled JSON
+    profiles with environment variables.
+    """
+    model_config = SettingsConfigDict(
+        env_prefix="GH_",
+        env_nested_delimiter="__",
+        extra="allow"
+    )
+
+    service: ServiceInfo = Field(default_factory=ServiceInfo)
+    logging: LoggingSettings = Field(default_factory=LoggingSettings)
+    llm: LLMSettings = Field(default_factory=LLMSettings)
+    endpoints: ServiceEndpoints = Field(default_factory=ServiceEndpoints)
+    vector: VectorConfig = Field(default_factory=VectorConfig)
+    excluded_urls: Optional[str] = None
+    max_retries: int = 3
+    retry_delay: int = 2
+    http_timeout: int = 30
+
+    def __init__(self, **data):
+        # Handle environment variable overrides for ENVIRONMENT
+        env = os.environ.get("ENVIRONMENT")
+        if env:
+            if "endpoints" not in data:
+                data["endpoints"] = {}
+            data["endpoints"]["ENVIRONMENT"] = env
+
+        # Let Pydantic handle the main initialization
+        super().__init__(**data)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _bootstrap_config(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            # Load the appropriate config file based on environment
+            env = data.get("ENVIRONMENT") or os.environ.get("ENVIRONMENT", "development")
+            config_data = _load_config_defaults(env)
+
+            if isinstance(config_data, dict):
+                # Create a copy of the input data to avoid modifying it
+                result = config_data.copy()
+
+                # Update with environment variables, but preserve the endpoints
+                if "endpoints" in data and isinstance(data["endpoints"], dict):
+                    # Create a new endpoints dict with the config defaults
+                    endpoints = result.get("endpoints", {}).copy()
+                    # Update with any environment overrides
+                    endpoints.update(data["endpoints"])
+                    result["endpoints"] = endpoints
+
+                # Update with any other non-endpoint data
+                for key, value in data.items():
+                    if key != "endpoints":
+                        result[key] = value
+
+                return result
+        return data
+
+@lru_cache()
+def get_cached_settings() -> CoreSettings:
+    return CoreSettings()
+
+
+# Exported singleton
+settings: CoreSettings = get_cached_settings()