iris-vector-rag 0.2.1__py3-none-any.whl

iris_rag/__init__.py

@@ -0,0 +1,284 @@
+# iris_rag package
+# This file makes the iris_rag directory a Python package.
+
+from typing import Any, Callable, Dict, List, Optional
+
+from common.utils import get_llm_func  # Import get_llm_func
+
+from .config.manager import ConfigurationManager
+from .core.base import RAGPipeline
+from .core.connection import ConnectionManager
+from .pipelines.basic import BasicRAGPipeline
+from .pipelines.crag import CRAGPipeline
+
+# Import validation components
+from .validation.factory import ValidatedPipelineFactory, create_validated_pipeline
+from .validation.orchestrator import SetupOrchestrator
+from .validation.requirements import get_pipeline_requirements
+from .validation.validator import PreConditionValidator
+
+# Package version
+__version__ = "0.2.1"
+__author__ = "InterSystems IRIS RAG Templates Project"
+__description__ = "A comprehensive, production-ready framework for implementing Retrieval Augmented Generation (RAG) pipelines using InterSystems IRIS as the vector database backend."
+
+
+def create_pipeline(
+    pipeline_type: str,
+    config_path: Optional[str] = None,
+    llm_func: Optional[Callable[[str], str]] = None,
+    embedding_func: Optional[Callable[[List[str]], List[List[float]]]] = None,
+    external_connection=None,
+    validate_requirements: bool = True,
+    auto_setup: bool = False,
+    **kwargs,
+) -> RAGPipeline:
+    """
+    Factory function to create RAG pipeline instances with validation.
+
+    Args:
+        pipeline_type: The type of pipeline to create. Available types:
+            - "basic": Basic RAG with vector similarity search
+            - "basic_rerank": Basic RAG with reranking
+            - "crag": Corrective RAG with retrieval evaluation
+            - "graphrag": Graph-based RAG with entity relationships
+            - "pylate_colbert": ColBERT-based dense retrieval
+            - "multi_query_rrf": Multi-query retrieval with RRF fusion
+        config_path: Optional path to configuration file.
+        llm_func: Optional LLM function for answer generation.
+        embedding_func: Optional embedding function for vector generation.
+        external_connection: Optional existing database connection to use.
+        validate_requirements: Whether to validate pipeline requirements before creation.
+        auto_setup: Whether to automatically set up missing requirements.
+        **kwargs: Additional configuration parameters.
+
+    Returns:
+        An instance of a RAGPipeline.
+
+    Raises:
+        ValueError: If the pipeline_type is unknown.
+        PipelineValidationError: If validation fails and auto_setup is False.
+    """
+    # Initialize configuration manager
+    config_manager = ConfigurationManager(config_path)
+
+    # Initialize connection manager
+    if external_connection:
+        # Create a wrapper connection manager that uses the external connection
+        connection_manager = ExternalConnectionWrapper(
+            external_connection, config_manager
+        )
+    else:
+        connection_manager = ConnectionManager(config_manager)
+
+    # Use validated factory if validation is requested
+    if validate_requirements:
+        factory = ValidatedPipelineFactory(connection_manager, config_manager)
+        return factory.create_pipeline(
+            pipeline_type=pipeline_type,
+            llm_func=llm_func,
+            auto_setup=auto_setup,
+            validate_requirements=True,
+            **kwargs,
+        )
+
+    # Legacy creation without validation
+    # Pass embedding_func through kwargs for pipelines that need it
+    if embedding_func:
+        kwargs["embedding_func"] = embedding_func
+
+    # If no llm_func is provided, get one using the utility function
+    effective_llm_func = llm_func
+    if effective_llm_func is None:
+        # Use 'stub' provider for testing if not specified otherwise in config or kwargs
+        llm_provider = config_manager.get("llm.provider", "stub")
+        llm_model_name = config_manager.get("llm.model_name", "stub-model")
+        effective_llm_func = get_llm_func(
+            provider=llm_provider, model_name=llm_model_name, **kwargs
+        )
+
+    return _create_pipeline_legacy(
+        pipeline_type, connection_manager, config_manager, effective_llm_func, **kwargs
+    )
+
+
+def _create_pipeline_legacy(
+    pipeline_type: str,
+    connection_manager: ConnectionManager,
+    config_manager: ConfigurationManager,
+    llm_func: Optional[Callable[[str], str]],
+    **kwargs,
+) -> RAGPipeline:
+    """Legacy pipeline creation without validation."""
+    if pipeline_type == "basic":
+        return BasicRAGPipeline(
+            connection_manager=connection_manager,
+            config_manager=config_manager,
+            llm_func=llm_func,
+        )
+    elif pipeline_type == "basic_rerank":
+        from .pipelines.basic_rerank import BasicRAGRerankingPipeline
+
+        return BasicRAGRerankingPipeline(
+            connection_manager=connection_manager,
+            config_manager=config_manager,
+            llm_func=llm_func,
+        )
+    elif pipeline_type == "crag":
+        # Extract embedding_func from kwargs if provided
+        embedding_func = kwargs.get("embedding_func")
+        return CRAGPipeline(
+            connection_manager=connection_manager,
+            config_manager=config_manager,
+            embedding_func=embedding_func,
+            llm_func=llm_func,
+        )
+    elif pipeline_type == "graphrag":
+        from .pipelines.hybrid_graphrag import HybridGraphRAGPipeline
+
+        return HybridGraphRAGPipeline(
+            connection_manager=connection_manager,
+            config_manager=config_manager,
+            llm_func=llm_func,
+        )
+    elif pipeline_type == "pylate_colbert":
+        from .pipelines.colbert_pylate.pylate_pipeline import PyLateColBERTPipeline
+
+        return PyLateColBERTPipeline(
+            connection_manager=connection_manager,
+            config_manager=config_manager,
+            llm_func=llm_func,
+        )
+    elif pipeline_type == "multi_query_rrf":
+        from .pipelines.multi_query_rrf import MultiQueryRRFPipeline
+
+        return MultiQueryRRFPipeline(
+            num_queries=kwargs.get("num_queries", 4),
+            retrieved_k=kwargs.get("retrieved_k", 20),
+            rrf_k=kwargs.get("rrf_k", 60),
+            use_llm_expansion=kwargs.get("use_llm_expansion", False),
+            llm_model=kwargs.get("llm_model", "gpt-4o-mini"),
+        )
+    else:
+        available_types = [
+            "basic",
+            "basic_rerank",
+            "crag",
+            "graphrag",
+            "pylate_colbert",
+            "multi_query_rrf",
+        ]
+        raise ValueError(
+            f"Unknown pipeline type: {pipeline_type}. Available: {available_types}"
+        )
+
+
+def validate_pipeline(
+    pipeline_type: str, config_path: Optional[str] = None, external_connection=None
+) -> Dict[str, Any]:
+    """
+    Validate pipeline requirements without creating an instance.
+
+    Args:
+        pipeline_type: Type of pipeline to validate
+        config_path: Optional path to configuration file
+        external_connection: Optional existing database connection
+
+    Returns:
+        Validation results dictionary
+    """
+    config_manager = ConfigurationManager(config_path)
+
+    if external_connection:
+        connection_manager = ExternalConnectionWrapper(
+            external_connection, config_manager
+        )
+    else:
+        connection_manager = ConnectionManager(config_manager)
+
+    factory = ValidatedPipelineFactory(connection_manager, config_manager)
+    return factory.validate_pipeline_type(pipeline_type)
+
+
+def setup_pipeline(
+    pipeline_type: str, config_path: Optional[str] = None, external_connection=None
+) -> Dict[str, Any]:
+    """
+    Set up all requirements for a pipeline type.
+
+    Args:
+        pipeline_type: Type of pipeline to set up
+        config_path: Optional path to configuration file
+        external_connection: Optional existing database connection
+
+    Returns:
+        Setup results dictionary
+    """
+    config_manager = ConfigurationManager(config_path)
+
+    if external_connection:
+        connection_manager = ExternalConnectionWrapper(
+            external_connection, config_manager
+        )
+    else:
+        connection_manager = ConnectionManager(config_manager)
+
+    factory = ValidatedPipelineFactory(connection_manager, config_manager)
+    return factory.setup_pipeline_requirements(pipeline_type)
+
+
+def get_pipeline_status(
+    pipeline_type: str, config_path: Optional[str] = None, external_connection=None
+) -> Dict[str, Any]:
+    """
+    Get detailed status information for a pipeline type.
+
+    Args:
+        pipeline_type: Type of pipeline to check
+        config_path: Optional path to configuration file
+        external_connection: Optional existing database connection
+
+    Returns:
+        Detailed status information
+    """
+    config_manager = ConfigurationManager(config_path)
+
+    if external_connection:
+        connection_manager = ExternalConnectionWrapper(
+            external_connection, config_manager
+        )
+    else:
+        connection_manager = ConnectionManager(config_manager)
+
+    factory = ValidatedPipelineFactory(connection_manager, config_manager)
+    return factory.get_pipeline_status(pipeline_type)
+
+
+class ExternalConnectionWrapper:
+    """Wrapper to make external connections work with iris_rag ConnectionManager interface."""
+
+    def __init__(self, external_connection, config_manager):
+        self.external_connection = external_connection
+        self.config_manager = config_manager
+
+    def get_connection(self, backend_name: str = "iris"):
+        """Return the external connection for any backend."""
+        return self.external_connection
+
+
+__all__ = [
+    "create_pipeline",
+    "validate_pipeline",
+    "setup_pipeline",
+    "get_pipeline_status",
+    "create_validated_pipeline",
+    "RAGPipeline",
+    "ConnectionManager",
+    "ConfigurationManager",
+    "BasicRAGPipeline",
+    "CRAGPipeline",
+    "ValidatedPipelineFactory",
+    "PreConditionValidator",
+    "SetupOrchestrator",
+    "get_pipeline_requirements",
+]

iris_rag/config/default_config.yaml

@@ -0,0 +1,295 @@
+# Default configuration for iris_rag package
+database:
+  iris:
+    driver: "intersystems_iris.dbapi._DBAPI"
+    host: "localhost"
+    port: "1974"
+    namespace: "USER"
+    username: "_SYSTEM"
+    password: "SYS"
+
+storage:
+  iris:
+    table_name: "RAG.SourceDocuments"
+    vector_dimension: 384
+
+# Vector index configuration for optimal HNSW performance
+vector_index:
+  type: "HNSW"                    # Index type: HNSW for high-performance vector search
+  M: 16                          # Number of bi-directional links for each node (higher = better recall, more memory)
+  efConstruction: 200            # Size of dynamic candidate list (higher = better quality, slower build)
+  Distance: "COSINE"             # Distance metric: COSINE, EUCLIDEAN, or DOT
+
+pipelines:
+  basic:
+    chunk_size: 1000
+    chunk_overlap: 200
+    default_top_k: 5
+    embedding_batch_size: 32
+  vector_query_mode: "safe_single_param"
+  vector_hnsw:
+    ensure_on_start: true
+    try_acorn: true
+
+# Pipeline validation configuration (Feature 047)
+validation:
+  enabled: true                          # Enable pipeline contract validation
+  strict_mode: false                     # If true, treat warnings as errors
+  validate_on_register: true             # Validate pipelines when registering handlers
+  validate_on_query: false               # Validate responses after each query (performance impact)
+  log_violations: true                   # Log contract violations
+  raise_on_error: true                   # Raise exceptions for validation errors
+
+  # Contract requirements (should match PipelineValidator)
+  contract:
+    required_methods: ["query", "load_documents"]
+    required_response_fields:
+      - "answer"
+      - "retrieved_documents"
+      - "contexts"
+      - "sources"
+      - "execution_time"
+      - "metadata"
+    required_metadata_fields:
+      - "num_retrieved"
+      - "pipeline_type"
+      - "generated_answer"
+      - "processing_time"
+      - "retrieval_method"
+      - "context_count"
+
+embeddings:
+  default_provider: "sentence_transformers"
+  sentence_transformers:
+    model_name: "all-MiniLM-L6-v2"
+    device: "cpu"
+
+# Entity extraction configuration
+entity_extraction:
+  enabled: true
+  method: "llm_basic"                    # ontology_hybrid, llm_basic, pattern_only, hybrid
+  confidence_threshold: 0.7              # Minimum confidence score for entities
+  entity_types:                         # Enabled entity types (matches pattern definitions)
+    - "DRUG"
+    - "DISEASE"
+    - "GENE"
+    - "ANATOMY"
+    - "PROCEDURE"
+    - "SYMPTOM"
+    - "TREATMENT"
+    - "PERSON"
+    - "ORGANIZATION"
+  llm:
+    model: "openai/gpt-oss-120b"         # LLM model: GPT-OSS 120B (OpenAI-compatible)
+    api_base: "http://apps-llm-4.iscinternal.com:8000/v1"  # GPT-OSS endpoint
+    api_key: "dummy"                     # API key (not required for internal endpoints)
+    api_type: "openai"                   # OpenAI-compatible API
+    temperature: 0.1                     # Low temperature for consistent extraction
+    max_tokens: 2000                     # Max tokens for LLM response
+    supports_response_format: false      # GPT-OSS doesn't support JSON mode
+    use_json_mode: false                 # Disable JSON mode
+  batch_processing:
+    enabled: false                       # DISABLED: GPT-OSS doesn't support JSON mode
+  storage:
+    entities_table: "RAG.Entities"      # Database table for entities
+    relationships_table: "RAG.EntityRelationships"  # Table for relationships
+    embeddings_table: "RAG.EntityEmbeddings"        # Table for entity embeddings
+
+# General-purpose ontology configuration for ANY domain
+ontology:
+  enabled: false                         # Enable/disable ontology support
+  type: "general"                        # Single general-purpose type (replaces domain-specific plugins)
+  auto_detect_domain: true               # Auto-detect domain from ontology metadata
+  
+  # Ontology data sources - works with ANY valid ontology
+  sources:
+    - type: "owl"                        # Ontology format: owl, rdf, skos, xml, ttl, n3
+      path: "path/to/any/ontology.owl"   # Path to ontology file (ANY domain)
+      # Optional: override auto-detection
+      # domain: "custom_domain_name"
+    
+    # Examples for different domains:
+    # - type: "owl"
+    #   path: "ontologies/medical/umls.owl"        # Medical domain
+    # - type: "skos" 
+    #   path: "ontologies/legal/legal_terms.skos"  # Legal domain
+    # - type: "rdf"
+    #   path: "ontologies/financial/fibo.rdf"      # Financial domain
+    # - type: "owl"
+    #   path: "ontologies/scientific/research.owl" # Scientific domain
+  
+  # Custom domain definitions (optional)
+  custom_domains:
+    enabled: false                       # Enable custom domain definitions
+    definition_path: null                # Path to JSON file with domain configuration
+    # Example JSON structure:
+    # {
+    #   "domain_name": "biomedical",
+    #   "description": "Biomedical research domain",
+    #   "entity_types": {
+    #     "PROTEIN": ["protein", "enzyme", "antibody"],
+    #     "GENE": ["gene", "allele", "locus"]
+    #   },
+    #   "extraction_patterns": {
+    #     "PROTEIN": ["\\b[A-Z][a-z]+\\d+\\b"]
+    #   },
+    #   "synonyms": {
+    #     "protein": ["enzyme", "polypeptide"]
+    #   }
+    # }
+  
+  # Reasoning engine configuration
+  reasoning:
+    enable_inference: true               # Enable ontology-based reasoning
+    max_inference_depth: 3               # Maximum depth for concept traversal
+    confidence_threshold: 0.6            # Minimum confidence for inferred relationships
+    reasoning_strategies:                # Active reasoning strategies
+      - "subsumption"                    # Hierarchical concept reasoning
+      - "property_based"                 # Property-based inference
+      - "rule_based"                     # Custom rule application
+  
+  # Entity mapping configuration
+  entity_mapping:
+    auto_map: true                       # Automatically map concepts to entity types
+    auto_domain_detection: true          # Auto-detect relevant domains from text
+    expansion_strategy: "synonyms"       # Query expansion: synonyms, hierarchical, semantic
+    # Note: custom_mappings are now auto-generated from loaded ontologies
+    
+  # Performance and optimization settings
+  performance:
+    max_concepts_per_domain: 10000       # Limit concepts loaded per domain
+    cache_ontology_queries: true         # Cache frequently used ontology queries
+    lazy_load_concepts: true             # Load concepts on-demand
+    concept_similarity_threshold: 0.8    # Minimum similarity for concept matching
+
+# GraphRAG pipeline configuration with general ontology support
+graphrag:
+  enabled: true                          # Enable GraphRAG pipeline
+  default_top_k: 10                      # Default number of results to return
+  max_depth: 2                           # Maximum graph traversal depth
+  max_entities: 50                       # Maximum entities per traversal
+  enable_vector_fallback: false          # Fallback to vector search if graph fails
+
+# IRIS Graph Core Hybrid Search Configuration
+iris_graph_core:
+  enabled: true                          # Enable iris_graph_core hybrid search capabilities
+
+  # RRF (Reciprocal Rank Fusion) Configuration
+  rrf:
+    c_parameter: 60                      # RRF constant (typically 60)
+    default_vector_k: 30                 # Default number of vector candidates
+    default_text_k: 30                   # Default number of text candidates
+    default_weights: [0.5, 0.3, 0.2]    # Default fusion weights [vector, text, graph]
+
+  # HNSW Vector Search Optimization
+  hnsw:
+    enabled: true                        # Enable HNSW-optimized vector search
+    fallback_to_csv: true                # Fall back to CSV parsing if HNSW unavailable
+    M: 16                                # HNSW M parameter (bi-directional links)
+    efConstruction: 200                  # HNSW efConstruction parameter
+    distance: "COSINE"                   # Distance metric: COSINE, EUCLIDEAN, DOT
+    try_acorn: false                     # Try ACORN=1 optimization (requires licensed IRIS)
+    community_edition_compatible: true   # Ensure compatibility with IRIS Community Edition
+    # For licensed IRIS users: Set try_acorn: true and use intersystemsdc/iris-ml:latest
+    # For community users: Keep try_acorn: false and use intersystemsdc/iris-community:latest
+
+  # Text Search Configuration
+  text_search:
+    enable_ifind: true                   # Use IRIS iFind for enhanced text search
+    fallback_to_like: true               # Fall back to LIKE patterns if iFind unavailable
+    min_confidence_threshold: 0          # Minimum confidence for text results (0-1000)
+    use_stemming: true                   # Enable stemming in iFind
+    use_stopwords: true                  # Enable stopword filtering in iFind
+
+  # Graph Traversal Configuration
+  graph_traversal:
+    max_expansion_depth: 2               # Maximum depth for neighborhood expansion
+    default_confidence_threshold: 500   # Default confidence threshold (0-1000)
+    max_entities_per_expansion: 50       # Maximum entities per expansion step
+    use_json_table_filtering: true       # Use JSON_TABLE for confidence filtering
+
+  # Hybrid Search Method Configuration
+  hybrid_search:
+    default_method: "rrf"                # Default method: hybrid, rrf, vector, text, kg
+    adaptive_routing: true               # Enable adaptive query routing
+    enable_multi_modal: true             # Enable multi-modal search fusion
+
+  # Schema Management for Graph Core Tables
+  schema:
+    auto_create_tables: true             # Automatically create iris_graph_core tables
+    auto_migrate_schema: true            # Automatically migrate schema changes
+    required_tables:                     # Tables required for iris_graph_core
+      - "rdf_labels"
+      - "rdf_props"
+      - "rdf_edges"
+      - "kg_NodeEmbeddings_optimized"
+
+  # Performance and Optimization
+  performance:
+    cache_vector_queries: true           # Cache frequently used vector queries
+    cache_text_queries: true             # Cache frequently used text queries
+    max_cache_size: 1000                 # Maximum number of cached queries
+    query_timeout_ms: 30000              # Query timeout in milliseconds
+  
+  # General ontology integration for GraphRAG
+  ontology_integration:
+    query_expansion: true                # Expand queries using ontology concepts
+    entity_enrichment: true              # Enrich extracted entities with ontology data
+    relationship_inference: true         # Infer relationships using ontology reasoning
+    semantic_similarity: true            # Use semantic similarity for entity matching
+    
+  # Visualization settings
+  visualization:
+    enabled: true                        # Enable graph visualization
+    default_type: "plotly"               # Default visualization: plotly, d3, traversal
+    highlight_ontology_concepts: true    # Highlight ontology-derived concepts
+    show_inference_paths: true           # Show reasoning inference paths
+
+# Example configurations for different domains:
+
+# Medical/Healthcare Domain Example:
+# ontology:
+#   enabled: true
+#   sources:
+#     - type: "owl"
+#       path: "ontologies/umls/umls_core.owl"
+#     - type: "skos"
+#       path: "ontologies/snomed/snomed_ct.skos"
+
+# Legal Domain Example:
+# ontology:
+#   enabled: true
+#   sources:
+#     - type: "owl"
+#       path: "ontologies/legal/lkif.owl"
+#     - type: "rdf"
+#       path: "ontologies/legal/legislation.rdf"
+
+# Financial Domain Example:
+# ontology:
+#   enabled: true
+#   sources:
+#     - type: "owl"
+#       path: "ontologies/fibo/fibo_core.owl"
+#     - type: "skos"
+#       path: "ontologies/finance/financial_terms.skos"
+
+# Scientific Research Domain Example:
+# ontology:
+#   enabled: true
+#   sources:
+#     - type: "owl"
+#       path: "ontologies/science/research_methods.owl"
+#     - type: "rdf"
+#       path: "ontologies/science/publications.rdf"
+
+# Multi-Domain Example:
+# ontology:
+#   enabled: true
+#   sources:
+#     - type: "owl"
+#       path: "ontologies/medical.owl"
+#     - type: "owl"
+#       path: "ontologies/legal.owl"
+#     - type: "skos"
+#       path: "ontologies/general_terms.skos"