rakam-systems-vectorstore 0.1.1rc7__py3-none-any.whl

rakam_systems_vectorstore/config.py

@@ -0,0 +1,266 @@
+"""
+Configuration management for Vector Store components.
+
+This module provides a unified configuration system that supports:
+- YAML/JSON configuration files
+- Environment variable overrides
+- Programmatic configuration
+- Validation and defaults
+"""
+
+from __future__ import annotations
+
+import os
+import yaml
+import json
+from typing import Any, Dict, Optional, Union
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+
+
+@dataclass
+class EmbeddingConfig:
+    """Configuration for embedding models."""
+    
+    model_type: str = "sentence_transformer"  # sentence_transformer, openai, cohere, etc.
+    model_name: str = "Snowflake/snowflake-arctic-embed-m"
+    api_key: Optional[str] = None
+    batch_size: int = 128
+    normalize: bool = True
+    dimensions: Optional[int] = None  # Auto-detected if None
+    
+    def __post_init__(self):
+        # Load API key from environment if not provided
+        if self.model_type == "openai" and not self.api_key:
+            self.api_key = os.getenv("OPENAI_API_KEY")
+        elif self.model_type == "cohere" and not self.api_key:
+            self.api_key = os.getenv("COHERE_API_KEY")
+
+
+@dataclass
+class DatabaseConfig:
+    """Configuration for database connection."""
+    
+    host: str = "localhost"
+    port: int = 5432
+    database: str = "vectorstore_db"
+    user: str = "postgres"
+    password: str = "postgres"
+    pool_size: int = 10
+    max_overflow: int = 20
+    
+    def __post_init__(self):
+        # Load from environment variables if available
+        self.host = os.getenv("POSTGRES_HOST", self.host)
+        self.port = int(os.getenv("POSTGRES_PORT", str(self.port)))
+        self.database = os.getenv("POSTGRES_DB", self.database)
+        self.user = os.getenv("POSTGRES_USER", self.user)
+        self.password = os.getenv("POSTGRES_PASSWORD", self.password)
+    
+    def to_connection_string(self) -> str:
+        """Generate PostgreSQL connection string."""
+        return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
+
+
+@dataclass
+class SearchConfig:
+    """Configuration for search operations."""
+    
+    similarity_metric: str = "cosine"  # cosine, l2, dot_product
+    default_top_k: int = 5
+    enable_hybrid_search: bool = True
+    hybrid_alpha: float = 0.7  # Weight for vector similarity (1-alpha for keyword)
+    rerank: bool = True
+    search_buffer_factor: int = 2  # Retrieve more results for reranking
+    
+    # Keyword search configuration
+    keyword_ranking_algorithm: str = "bm25"  # bm25 or ts_rank
+    bm25_k1: float = 1.5  # BM25 term frequency saturation parameter
+    bm25_b: float = 0.75  # BM25 length normalization parameter
+    
+    def validate(self):
+        """Validate search configuration."""
+        if self.similarity_metric not in ["cosine", "l2", "dot_product", "dot"]:
+            raise ValueError(f"Invalid similarity metric: {self.similarity_metric}")
+        if not 0 <= self.hybrid_alpha <= 1:
+            raise ValueError(f"hybrid_alpha must be between 0 and 1, got {self.hybrid_alpha}")
+        if self.default_top_k < 1:
+            raise ValueError(f"default_top_k must be >= 1, got {self.default_top_k}")
+        if self.keyword_ranking_algorithm not in ["bm25", "ts_rank"]:
+            raise ValueError(f"Invalid keyword ranking algorithm: {self.keyword_ranking_algorithm}")
+        if self.bm25_k1 < 0:
+            raise ValueError(f"bm25_k1 must be >= 0, got {self.bm25_k1}")
+        if not 0 <= self.bm25_b <= 1:
+            raise ValueError(f"bm25_b must be between 0 and 1, got {self.bm25_b}")
+
+
+@dataclass
+class IndexConfig:
+    """Configuration for indexing operations."""
+    
+    chunk_size: int = 512
+    chunk_overlap: int = 50
+    enable_parallel_processing: bool = False
+    parallel_workers: int = 4
+    batch_insert_size: int = 10000
+
+
+@dataclass
+class VectorStoreConfig:
+    """Master configuration for Vector Store component."""
+    
+    name: str = "pg_vector_store"
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    search: SearchConfig = field(default_factory=SearchConfig)
+    index: IndexConfig = field(default_factory=IndexConfig)
+    
+    # Component-specific settings
+    enable_caching: bool = True
+    cache_size: int = 1000
+    enable_logging: bool = True
+    log_level: str = "INFO"
+    
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any]) -> "VectorStoreConfig":
+        """Create configuration from dictionary."""
+        # Extract nested configs
+        embedding_config = EmbeddingConfig(**config_dict.get("embedding", {}))
+        database_config = DatabaseConfig(**config_dict.get("database", {}))
+        search_config = SearchConfig(**config_dict.get("search", {}))
+        index_config = IndexConfig(**config_dict.get("index", {}))
+        
+        # Create main config
+        main_config = {
+            "name": config_dict.get("name", "pg_vector_store"),
+            "embedding": embedding_config,
+            "database": database_config,
+            "search": search_config,
+            "index": index_config,
+            "enable_caching": config_dict.get("enable_caching", True),
+            "cache_size": config_dict.get("cache_size", 1000),
+            "enable_logging": config_dict.get("enable_logging", True),
+            "log_level": config_dict.get("log_level", "INFO"),
+        }
+        
+        return cls(**main_config)
+    
+    @classmethod
+    def from_yaml(cls, yaml_path: Union[str, Path]) -> "VectorStoreConfig":
+        """Load configuration from YAML file."""
+        path = Path(yaml_path)
+        if not path.exists():
+            raise FileNotFoundError(f"Configuration file not found: {yaml_path}")
+        
+        with open(path, 'r') as f:
+            config_dict = yaml.safe_load(f)
+        
+        return cls.from_dict(config_dict)
+    
+    @classmethod
+    def from_json(cls, json_path: Union[str, Path]) -> "VectorStoreConfig":
+        """Load configuration from JSON file."""
+        path = Path(json_path)
+        if not path.exists():
+            raise FileNotFoundError(f"Configuration file not found: {json_path}")
+        
+        with open(path, 'r') as f:
+            config_dict = json.load(f)
+        
+        return cls.from_dict(config_dict)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert configuration to dictionary."""
+        return {
+            "name": self.name,
+            "embedding": asdict(self.embedding),
+            "database": asdict(self.database),
+            "search": asdict(self.search),
+            "index": asdict(self.index),
+            "enable_caching": self.enable_caching,
+            "cache_size": self.cache_size,
+            "enable_logging": self.enable_logging,
+            "log_level": self.log_level,
+        }
+    
+    def save_yaml(self, output_path: Union[str, Path]) -> None:
+        """Save configuration to YAML file."""
+        with open(output_path, 'w') as f:
+            yaml.dump(self.to_dict(), f, default_flow_style=False, sort_keys=False)
+    
+    def save_json(self, output_path: Union[str, Path]) -> None:
+        """Save configuration to JSON file."""
+        with open(output_path, 'w') as f:
+            json.dump(self.to_dict(), f, indent=2)
+    
+    def validate(self) -> None:
+        """Validate all configuration settings."""
+        self.search.validate()
+        
+        # Validate embedding config
+        if self.embedding.batch_size < 1:
+            raise ValueError(f"embedding.batch_size must be >= 1, got {self.embedding.batch_size}")
+        
+        # Validate index config
+        if self.index.chunk_size < 1:
+            raise ValueError(f"index.chunk_size must be >= 1, got {self.index.chunk_size}")
+        if self.index.chunk_overlap < 0:
+            raise ValueError(f"index.chunk_overlap must be >= 0, got {self.index.chunk_overlap}")
+        if self.index.chunk_overlap >= self.index.chunk_size:
+            raise ValueError("index.chunk_overlap must be less than index.chunk_size")
+
+
+def load_config(
+    config_source: Optional[Union[str, Path, Dict[str, Any]]] = None,
+    config_type: str = "auto"
+) -> VectorStoreConfig:
+    """
+    Load configuration from various sources.
+    
+    Args:
+        config_source: Path to config file, dict, or None for defaults
+        config_type: Type of config file ('yaml', 'json', 'auto')
+    
+    Returns:
+        VectorStoreConfig instance
+    """
+    if config_source is None:
+        # Return default configuration
+        return VectorStoreConfig()
+    
+    if isinstance(config_source, dict):
+        # Load from dictionary
+        return VectorStoreConfig.from_dict(config_source)
+    
+    # Load from file
+    path = Path(config_source)
+    if not path.exists():
+        raise FileNotFoundError(f"Configuration file not found: {config_source}")
+    
+    if config_type == "auto":
+        # Auto-detect based on file extension
+        suffix = path.suffix.lower()
+        if suffix in ['.yaml', '.yml']:
+            config_type = 'yaml'
+        elif suffix == '.json':
+            config_type = 'json'
+        else:
+            raise ValueError(f"Cannot auto-detect config type for file: {config_source}")
+    
+    if config_type == 'yaml':
+        return VectorStoreConfig.from_yaml(path)
+    elif config_type == 'json':
+        return VectorStoreConfig.from_json(path)
+    else:
+        raise ValueError(f"Unsupported config type: {config_type}")
+
+
+__all__ = [
+    "EmbeddingConfig",
+    "DatabaseConfig", 
+    "SearchConfig",
+    "IndexConfig",
+    "VectorStoreConfig",
+    "load_config",
+]
+

rakam_systems_vectorstore/core.py

@@ -0,0 +1,8 @@
+"""
+Re-export core data structures from rakam-systems-core for backward compatibility.
+These classes are now maintained in rakam_systems_core.ai_core.vs_core
+"""
+
+from rakam_systems_core.ai_core.vs_core import Node, NodeMetadata, VSFile
+
+__all__ = ["Node", "NodeMetadata", "VSFile"]

rakam_systems_vectorstore/pyproject.toml

@@ -0,0 +1,113 @@
+[project]
+name = "rakam-systems-vectorstore"
+version = "0.1.0"
+description = "Modular vector store and RAG components for semantic search and retrieval"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "Apache-2.0"}
+authors = [
+    {name = "Mohamed Hilel", email = "mohammedjassemhlel@gmail.com"},
+    {name = "Peng Zheng", email = "pengzheng990630@outlook.com"}
+]
+keywords = ["vector-store", "embeddings", "rag", "semantic-search", "pgvector", "faiss"]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+# Core dependencies required for base functionality
+dependencies = [
+
+
+    "rakam-system-core",
+    "pyyaml>=6.0",
+    "numpy>=1.24.0",
+    "tqdm>=4.66.0",
+]
+
+[project.optional-dependencies]
+# PostgreSQL backend with Django ORM
+postgres = [
+    "psycopg2-binary>=2.9.9",
+    "django>=4.0.0",
+]
+
+# FAISS backend for in-memory vector search
+faiss = [
+    "faiss-cpu>=1.12.0",
+]
+
+# Local embedding models using SentenceTransformers
+local-embeddings = [
+    "sentence-transformers>=5.1.0",
+    "torch>=2.0.0",
+]
+
+# OpenAI embeddings
+openai = [
+    "openai>=1.0.0",
+]
+
+# Cohere embeddings
+cohere = [
+    "cohere>=4.0.0",
+]
+
+# Document loaders for various file types
+loaders = [
+    "python-magic>=0.4.27",
+    "beautifulsoup4>=4.12.0",
+    "python-docx>=1.2.0",
+    "pymupdf>=1.24.0",
+    "pymupdf4llm>=0.0.17",
+    "docling==2.62.0",
+    "chonkie==1.4.2",
+    "odfpy==1.4.1",
+]
+
+# Complete installation with all backends and features
+all = [
+    "rakam-systems-vectorstore[postgres]",
+    "rakam-systems-vectorstore[faiss]",
+    "rakam-systems-vectorstore[local-embeddings]",
+    "rakam-systems-vectorstore[openai]",
+    "rakam-systems-vectorstore[cohere]",
+    "rakam-systems-vectorstore[loaders]",
+]
+
+# Development dependencies
+dev = [
+    "pytest>=7.0.0",
+    "pytest-django>=4.5.0",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+]
+
+# ref: https://github.com/astral-sh/uv/issues/6371
+[tool.uv.sources]
+rakam-system-core = { workspace = true }
+
+[project.urls]
+Homepage = "https://github.com/Rakam-AI/rakam_systems"
+Documentation = "https://github.com/Rakam-AI/rakam_systems"
+Repository = "https://github.com/Rakam-AI/rakam_systems"
+Issues = "https://github.com/Rakam-AI/rakam_systems/issues"
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[tool.setuptools.packages.find]
+where = [".."]
+include = ["rakam_systems_vectorstore*", "rakam_systems_core*"]
+
+[tool.setuptools.package-data]
+"rakam_systems_vectorstore" = ["**/*.yaml", "**/*.yml", "**/*.json", "**/*.md"]

rakam_systems_vectorstore/server/README.md

@@ -0,0 +1,290 @@
+# Vector Store MCP Server
+
+This module provides an MCP (Model Context Protocol) server for vector store operations, enabling AI agents to interact with vector databases through a standardized interface.
+
+## Overview
+
+The Vector Store MCP Server creates a message-based interface that wraps vector store operations into reusable tool components. This allows AI agents to:
+
+- Search for documents using semantic similarity
+- Add documents to the vector store
+- Query collection information and metadata
+- Perform concurrent operations efficiently
+
+## Components
+
+### 1. VectorSearchTool
+
+Performs semantic search on vector collections.
+
+**Parameters:**
+
+- `query` (str): Search query text
+- `collection_name` (str, optional): Target collection (default: "documents")
+- `top_k` (int, optional): Number of results to return (default: 5)
+
+**Returns:**
+
+```python
+{
+    'query': str,
+    'collection': str,
+    'results_count': int,
+    'results': List[{
+        'content': str,
+        'node_id': str,
+        'source_file': str,
+        'position': int,
+        'metadata': dict
+    }]
+}
+```
+
+### 2. VectorStorageTool
+
+Adds documents to the vector store.
+
+**Parameters:**
+
+- `documents` (List[str]): List of document texts to add
+- `collection_name` (str, optional): Target collection (default: "documents")
+- `doc_metadata` (dict, optional): Metadata to attach to documents
+
+**Returns:**
+
+```python
+{
+    'success': bool,
+    'collection': str,
+    'documents_added': int,
+    'node_ids': List[str]
+}
+```
+
+### 3. VectorInfoTool
+
+Retrieves information about collections.
+
+**Parameters:**
+
+- `collection_name` (str, optional): Specific collection name (lists all if not provided)
+
+**Returns:**
+
+```python
+# For specific collection:
+{
+    'collection_name': str,
+    'node_count': int,
+    'embedding_dim': int
+}
+
+# For all collections:
+{
+    'total_collections': int,
+    'collections': List[str]
+}
+```
+
+## Usage
+
+### Basic Setup
+
+```python
+from rakam_systems_vectorstore.server import run_vector_mcp
+from rakam_systems_vectorstore.components.vectorstore.configurable_pg_vector_store import ConfigurablePgVectorStore
+from rakam_systems_vectorstore.config import VectorStoreConfig, EmbeddingConfig
+
+# Initialize vector store
+config = VectorStoreConfig(
+    name="my_store",
+    embedding=EmbeddingConfig(
+        model_type="sentence_transformer",
+        model_name="Snowflake/snowflake-arctic-embed-m"
+    )
+)
+
+vector_store = ConfigurablePgVectorStore(name="my_store", config=config)
+vector_store.setup()
+
+# Create MCP server
+mcp_server = run_vector_mcp(vector_store)
+
+# List available tools
+print(mcp_server.list_components())
+# Output: ['vector_info', 'vector_search', 'vector_storage']
+```
+
+### Using with Async Messages
+
+```python
+# Search for documents
+result = await mcp_server.asend_message(
+    sender="agent",
+    receiver="vector_search",
+    message={
+        'arguments': {
+            'query': 'machine learning',
+            'collection_name': 'docs',
+            'top_k': 5
+        }
+    }
+)
+
+# Add documents
+result = await mcp_server.asend_message(
+    sender="agent",
+    receiver="vector_storage",
+    message={
+        'arguments': {
+            'documents': ['Doc 1 text', 'Doc 2 text'],
+            'collection_name': 'docs'
+        }
+    }
+)
+
+# Get collection info
+result = await mcp_server.asend_message(
+    sender="agent",
+    receiver="vector_info",
+    message={
+        'arguments': {
+            'collection_name': 'docs'
+        }
+    }
+)
+```
+
+### Integration with Tool Registry
+
+```python
+from rakam_systems_core.ai_core.interfaces import ToolRegistry, ToolInvoker
+
+# Create registry and invoker
+registry = ToolRegistry()
+invoker = ToolInvoker(registry)
+
+# Register MCP server
+invoker.register_mcp_server("vector_store_mcp", mcp_server)
+
+# Register tools
+registry.register_mcp_tool(
+    name="search_docs",
+    mcp_server="vector_store_mcp",
+    mcp_tool_name="vector_search",
+    description="Search documents using semantic similarity",
+    category="vector_store"
+)
+
+# Use through invoker
+results = await invoker.ainvoke(
+    "search_docs",
+    query="What is AI?",
+    top_k=3
+)
+```
+
+### Concurrent Operations
+
+```python
+import asyncio
+
+# Execute multiple operations concurrently
+results = await asyncio.gather(
+    mcp_server.asend_message(
+        sender="agent",
+        receiver="vector_search",
+        message={'arguments': {'query': 'machine learning', 'top_k': 3}}
+    ),
+    mcp_server.asend_message(
+        sender="agent",
+        receiver="vector_search",
+        message={'arguments': {'query': 'deep learning', 'top_k': 3}}
+    ),
+    mcp_server.asend_message(
+        sender="agent",
+        receiver="vector_info",
+        message={'arguments': {}}
+    )
+)
+```
+
+## Function Reference
+
+### run_vector_mcp()
+
+Creates and configures an MCP server with vector store tools.
+
+```python
+def run_vector_mcp(
+    vector_store,
+    name: str = "vector_store_mcp",
+    enable_logging: bool = False
+) -> MCPServer
+```
+
+**Parameters:**
+
+- `vector_store`: ConfigurablePgVectorStore instance (must be set up)
+- `name` (str, optional): Name for the MCP server (default: "vector_store_mcp")
+- `enable_logging` (bool, optional): Enable detailed MCP logging (default: False)
+
+**Returns:**
+
+- `MCPServer`: Configured MCP server instance with registered tools
+
+## Examples
+
+See the comprehensive examples in:
+
+- `/examples/mcp_vector_search_example.py` - Full MCP vector search demonstration
+- `/docs/MCP_VECTOR_STORE_GUIDE.md` - Complete guide to MCP with vector stores
+
+## Architecture
+
+The MCP server follows a modular architecture:
+
+```
+┌─────────────────────────────────────────┐
+│          AI Agent / Client              │
+└──────────────┬──────────────────────────┘
+               │
+               │ send_message / asend_message
+               ▼
+┌─────────────────────────────────────────┐
+│         MCPServer                       │
+│  (Message Router & Component Registry)  │
+└──────────────┬──────────────────────────┘
+               │
+     ┌─────────┼─────────┐
+     ▼         ▼         ▼
+┌─────────┐ ┌──────────┐ ┌──────────┐
+│ Vector  │ │ Vector   │ │ Vector   │
+│ Search  │ │ Storage  │ │ Info     │
+│ Tool    │ │ Tool     │ │ Tool     │
+└────┬────┘ └────┬─────┘ └────┬─────┘
+     │           │             │
+     └───────────┴─────────────┘
+                 │
+                 ▼
+     ┌────────────────────────┐
+     │  ConfigurablePgVector  │
+     │       Store            │
+     └────────────────────────┘
+```
+
+## Benefits
+
+1. **Decoupled Architecture**: Tools don't need direct dependencies on the vector store
+2. **Standardized Interface**: All tools work the same way through MCP
+3. **Easy Extension**: Add new tools without modifying existing code
+4. **Tool Discovery**: Agents can query available tools dynamically
+5. **Async Support**: Efficient concurrent operations
+6. **Integration Ready**: Works seamlessly with BaseAgent and other AI components
+
+## Related Documentation
+
+- [MCP Server API](/app/rakam_systems/rakam_systems/ai_core/mcp/README.md)
+- [MCP Vector Store Guide](/docs/MCP_VECTOR_STORE_GUIDE.md)
+- [MCP Quickstart](/docs/MCP_QUICKSTART.md)
+- [Vector Search Architecture](/docs/VECTOR_SEARCH_ARCHITECTURE.md)

rakam_systems_vectorstore/server/__init__.py

@@ -0,0 +1,20 @@
+"""
+Vector Store MCP Server Module
+
+This module provides MCP (Model Context Protocol) server functionality for vector store operations.
+"""
+
+from .mcp_server_vector import (
+    run_vector_mcp,
+    VectorSearchTool,
+    VectorStorageTool,
+    VectorInfoTool,
+)
+
+__all__ = [
+    "run_vector_mcp",
+    "VectorSearchTool",
+    "VectorStorageTool",
+    "VectorInfoTool",
+]
+