iris-vector-rag 0.5.3__tar.gz → 0.5.8__tar.gz

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.5.3
+Version: 0.5.8
 Summary: Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration
 Author-email: InterSystems <support@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>
@@ -51,9 +51,18 @@ Requires-Dist: torch>=2.0.0
 Provides-Extra: embedding
 Requires-Dist: torch>=2.0.0; extra == "embedding"
 Requires-Dist: sentence-transformers>=2.2.0; extra == "embedding"
-Requires-Dist: iris-vector-graph>=1.1.1; extra == "embedding"
+Requires-Dist: iris-vector-graph>=1.1.7; extra == "embedding"
 Provides-Extra: hybrid-graphrag
-Requires-Dist: iris-vector-graph>=1.1.1; extra == "hybrid-graphrag"
+Requires-Dist: iris-vector-graph>=1.1.7; extra == "hybrid-graphrag"
+Provides-Extra: monitoring
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "monitoring"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "monitoring"
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == "monitoring"
+Provides-Extra: enterprise
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "enterprise"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "enterprise"
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == "enterprise"
+Requires-Dist: tqdm>=4.64.0; extra == "enterprise"
 Provides-Extra: dev
 Requires-Dist: uv>=0.1.0; extra == "dev"
 Requires-Dist: pytest>=7.0.0; extra == "dev"
@@ -100,6 +109,10 @@ Requires-Dist: python-multipart>=0.0.6; extra == "all"
 Requires-Dist: redis>=5.0.0; extra == "all"
 Requires-Dist: bcrypt>=4.0.0; extra == "all"
 Requires-Dist: python-jose[cryptography]>=3.3.0; extra == "all"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "all"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "all"
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == "all"
+Requires-Dist: tqdm>=4.64.0; extra == "all"
 Dynamic: license-file
 
 # IRIS Vector RAG Templates
@@ -108,6 +121,8 @@ Dynamic: license-file
 
 Build intelligent applications that combine large language models with your enterprise data using battle-tested RAG patterns and native vector search capabilities.
 
+**Author: Thomas Dyar** (thomas.dyar@intersystems.com)
+
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![InterSystems IRIS](https://img.shields.io/badge/IRIS-2024.1+-purple.svg)](https://www.intersystems.com/products/intersystems-iris/)

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/README.md

@@ -4,6 +4,8 @@
 
 Build intelligent applications that combine large language models with your enterprise data using battle-tested RAG patterns and native vector search capabilities.
 
+**Author: Thomas Dyar** (thomas.dyar@intersystems.com)
+
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![InterSystems IRIS](https://img.shields.io/badge/IRIS-2024.1+-purple.svg)](https://www.intersystems.com/products/intersystems-iris/)

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/__init__.py

@@ -18,7 +18,7 @@ from .validation.requirements import get_pipeline_requirements
 from .validation.validator import PreConditionValidator
 
 # Package version
-__version__ = "0.5.3"
+__version__ = "0.5.7"
 __author__ = "InterSystems IRIS RAG Templates Project"
 __description__ = "A comprehensive, production-ready framework for implementing Retrieval Augmented Generation (RAG) pipelines using InterSystems IRIS as the vector database backend."
 

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/common/iris_dbapi_connector.py

@@ -165,11 +165,11 @@ def get_iris_dbapi_connection():
     Returns:
         A direct IRIS connection object or None if connection fails.
     """
-    # Use direct iris import instead of DBAPI
+    # Use intersystems_iris DBAPI module
     try:
-        import iris
+        import intersystems_iris.dbapi._DBAPI as iris
     except ImportError as e:
-        logger.error(f"Cannot import iris module: {e}")
+        logger.error(f"Cannot import intersystems_iris.dbapi module: {e}")
         return None
 
     # Get connection parameters from environment with auto-detection fallback

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/config/default_config.yaml

@@ -12,6 +12,16 @@ storage:
   iris:
     table_name: "RAG.SourceDocuments"
     vector_dimension: 384
+    # Custom metadata filter keys for multi-tenant isolation (User Story 1 - P1)
+    # These fields can be used in metadata_filter parameter for queries
+    # Default filter keys (always enabled): document_id, source, title, author,
+    #   created_at, updated_at, content_type, file_type, page_number, section,
+    #   chunk_id, chunk_index, language, collection_id, metadata_hash,
+    #   parent_document_id, version
+    custom_filter_keys:
+      - "tenant_id"              # Tenant identifier for multi-tenant deployments
+      - "security_level"         # Security clearance level (e.g., public, confidential, secret)
+      - "department"             # Department or organizational unit
 
 # Vector index configuration for optimal HNSW performance
 vector_index:
@@ -353,4 +363,105 @@ iris_graph_core:
 #     - type: "owl"
 #       path: "ontologies/legal.owl"
 #     - type: "skos"
-#       path: "ontologies/general_terms.skos"
+#       path: "ontologies/general_terms.skos"
+
+# =============================================================================
+# Enterprise Enhancements (v0.6.0)
+# =============================================================================
+
+# Role-Based Access Control (User Story 3 - P1)
+security:
+  rbac:
+    enabled: false               # Enable RBAC policy enforcement
+    policy_class: null           # Full import path to RBAC policy class (e.g., "myapp.security.MyRBACPolicy")
+    # Example policy classes:
+    # - "iris_vector_rag.security.examples.LDAPRBACPolicy"
+    # - "iris_vector_rag.security.examples.ClearanceLevelPolicy"
+    # - "mycompany.security.CustomRBACPolicy"
+
+# OpenTelemetry Monitoring (User Story 4 - P2)
+telemetry:
+  enabled: false                 # Enable OpenTelemetry instrumentation (zero overhead when disabled)
+  service_name: "iris-vector-rag"
+  endpoint: "http://localhost:4318"  # OTLP HTTP endpoint
+  sampling_ratio: 1.0            # Sampling rate (1.0 = 100%, 0.1 = 10%)
+  export_protocol: "http"        # Export protocol: http or grpc
+  export_timeout_ms: 10000       # Export timeout in milliseconds
+
+  # GenAI semantic conventions (OpenTelemetry)
+  genai_conventions:
+    enabled: true                # Track LLM-specific attributes (gen.ai.*)
+    track_token_usage: true      # Track token counts and costs
+    track_prompts: false         # Include prompts in spans (may contain sensitive data)
+    track_completions: false     # Include completions in spans (may contain sensitive data)
+
+  # Cost tracking
+  cost_tracking:
+    enabled: true                # Calculate LLM API costs
+    pricing_table:               # Model pricing (per 1M tokens)
+      openai:
+        gpt-4o-mini:
+          input: 0.15            # $0.15 per 1M input tokens
+          output: 0.60           # $0.60 per 1M output tokens
+        gpt-4o:
+          input: 2.50            # $2.50 per 1M input tokens
+          output: 10.00          # $10.00 per 1M output tokens
+        gpt-4-turbo:
+          input: 10.00
+          output: 30.00
+      anthropic:
+        claude-3-5-sonnet-20241022:
+          input: 3.00            # $3.00 per 1M input tokens
+          output: 15.00          # $15.00 per 1M output tokens
+        claude-3-opus-20240229:
+          input: 15.00
+          output: 75.00
+
+# Bulk Document Loading (User Story 5 - P2)
+batch_operations:
+  enabled: true                  # Enable batch operations (10x+ speedup)
+  default_batch_size: 1000       # Default batch size for bulk operations
+  max_batch_size: 10000          # Maximum allowed batch size
+  show_progress: true            # Show progress bars for batch operations
+  error_handling: "continue"     # Error handling strategy: continue, stop, rollback
+  max_errors_in_response: 100    # Maximum number of errors to return in response
+
+  # Performance targets
+  performance:
+    target_throughput_docs_per_sec: 1000  # Target: >=1000 docs/sec for 10K documents
+    target_bulk_load_time_ms: 10000       # Target: <10s for 10K documents
+
+# Collection Management (User Story 2 - P1)
+collections:
+  enabled: true                  # Enable collection lifecycle management
+  default_collection_id: "default"
+  max_collection_id_length: 128
+  allowed_collection_id_pattern: "^[a-zA-Z0-9_-]+$"  # Alphanumeric + hyphens/underscores
+
+  # Statistics and metadata
+  track_statistics: true         # Track document counts, sizes, timestamps
+  compute_size_in_bytes: true    # Calculate total size of documents in collections
+
+# Metadata Schema Discovery (User Story 6 - P3)
+schema_discovery:
+  enabled: true                  # Enable metadata schema discovery
+  default_sample_size: 100       # Default number of documents to sample
+  max_sample_size: 200           # Maximum allowed sample size
+  min_sample_size: 10            # Minimum allowed sample size
+
+  # Type inference
+  type_inference:
+    enabled: true                # Enable automatic type inference
+    detect_datetime: true        # Detect ISO 8601 datetime strings
+    detect_numeric: true         # Detect integers and floats
+    detect_boolean: true         # Detect boolean values
+    detect_arrays: true          # Detect array types
+
+  # Statistics
+  compute_statistics:
+    enabled: true                # Compute min/max/avg for numeric fields
+    max_example_values: 5        # Maximum number of example values per field
+
+  # Performance target
+  performance:
+    target_discovery_time_ms: 5000  # Target: <5s for sample_size=200

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/dspy_modules/batch_entity_extraction.py

@@ -14,6 +14,16 @@ from typing import List, Dict, Any, Optional
 logger = logging.getLogger(__name__)
 
 
+# Domain-specific entity type presets
+DOMAIN_PRESETS = {
+    "it_support": ["PRODUCT", "USER", "MODULE", "ERROR", "ACTION", "ORGANIZATION", "VERSION"],
+    "biomedical": ["GENE", "PROTEIN", "DISEASE", "CHEMICAL", "DRUG", "CELL_TYPE", "ORGANISM"],
+    "legal": ["PARTY", "JUDGE", "COURT", "LAW", "DATE", "MONETARY_AMOUNT", "JURISDICTION"],
+    "general": ["PERSON", "ORGANIZATION", "LOCATION", "DATE", "EVENT", "PRODUCT"],
+    "wikipedia": ["PERSON", "ORGANIZATION", "LOCATION", "TITLE", "ROLE", "POSITION", "EVENT"],
+}
+
+
 class BatchEntityExtractionSignature(dspy.Signature):
     """Extract entities from MULTIPLE tickets in one LLM call."""
 
@@ -21,7 +31,7 @@ class BatchEntityExtractionSignature(dspy.Signature):
         desc="JSON array of tickets. Each has: ticket_id, text. Extract entities for ALL tickets."
     )
     entity_types = dspy.InputField(
-        desc="PRODUCT, USER, MODULE, ERROR, ACTION, ORGANIZATION, VERSION"
+        desc="Comma-separated list of entity types to extract"
     )
 
     batch_results = dspy.OutputField(
@@ -138,16 +148,29 @@ class BatchEntityExtractionModule(dspy.Module):
 
         return None
 
-    def forward(self, tickets: List[Dict[str, str]]) -> List[Dict[str, Any]]:
+    def forward(
+        self,
+        tickets: List[Dict[str, str]],
+        entity_types: Optional[List[str]] = None
+    ) -> List[Dict[str, Any]]:
         """
         Extract entities from a batch of tickets with JSON retry logic (T025).
 
         Args:
             tickets: List of dicts with 'id' and 'text' keys
+            entity_types: List of entity types to extract (e.g., ["PERSON", "ORG", "LOCATION"])
+                         If None, defaults to IT support types for backward compatibility
 
         Returns:
             List of extraction results (one per ticket)
         """
+        # Default to IT support types for backward compatibility
+        if entity_types is None:
+            entity_types = ["PRODUCT", "USER", "MODULE", "ERROR", "ACTION", "ORGANIZATION", "VERSION"]
+
+        # Convert list to comma-separated string for DSPy
+        entity_types_str = ", ".join(entity_types)
+
         try:
             # Prepare batch input
             batch_input = json.dumps([
@@ -158,7 +181,7 @@ class BatchEntityExtractionModule(dspy.Module):
             # Single LLM call for entire batch
             prediction = self.extract(
                 tickets_batch=batch_input,
-                entity_types="PRODUCT, USER, MODULE, ERROR, ACTION, ORGANIZATION, VERSION"
+                entity_types=entity_types_str
             )
 
             # Parse batch results with retry logic (T025)

iris_vector_rag-0.5.8/iris_vector_rag/exceptions.py

@@ -0,0 +1,188 @@
+"""
+Exception classes for iris-vector-rag enterprise features.
+
+This module defines custom exceptions for security, configuration, and
+operational errors in the RAG framework.
+
+Classes:
+    VectorStoreConfigurationError: Raised when vector store configuration is invalid
+    PermissionDeniedError: Raised when user lacks required permissions for an operation
+
+Example:
+    >>> from iris_vector_rag.exceptions import VectorStoreConfigurationError
+    >>>
+    >>> # Configuration validation
+    >>> if invalid_filter_key:
+    ...     raise VectorStoreConfigurationError(
+    ...         "Invalid metadata filter key",
+    ...         rejected_keys=["invalid_key"],
+    ...         allowed_keys=["tenant_id", "department"]
+    ...     )
+    >>>
+    >>> # RBAC enforcement
+    >>> from iris_vector_rag.exceptions import PermissionDeniedError
+    >>>
+    >>> if not has_permission(user, "read", collection):
+    ...     raise PermissionDeniedError(
+    ...         user=user,
+    ...         resource=collection_id,
+    ...         operation="read"
+    ...     )
+"""
+
+from typing import Any, Dict, List, Optional
+
+
+class VectorStoreConfigurationError(Exception):
+    """
+    Exception raised when vector store configuration is invalid.
+
+    This exception is raised when attempting to use metadata filter keys
+    that are not in the allowed list, or when other configuration errors occur.
+
+    Attributes:
+        message: Human-readable error message
+        rejected_keys: List of rejected metadata filter keys (if applicable)
+        allowed_keys: List of allowed metadata filter keys (if applicable)
+        details: Additional error context
+
+    Example:
+        >>> raise VectorStoreConfigurationError(
+        ...     "Invalid metadata filter keys",
+        ...     rejected_keys=["invalid_field"],
+        ...     allowed_keys=["tenant_id", "department", "security_level"]
+        ... )
+    """
+
+    def __init__(
+        self,
+        message: str,
+        rejected_keys: Optional[List[str]] = None,
+        allowed_keys: Optional[List[str]] = None,
+        **details: Any
+    ):
+        """
+        Initialize VectorStoreConfigurationError.
+
+        Args:
+            message: Error message describing the configuration issue
+            rejected_keys: Optional list of rejected metadata filter keys
+            allowed_keys: Optional list of allowed metadata filter keys
+            **details: Additional error context (e.g., field_name, rejected_value)
+        """
+        super().__init__(message)
+        self.message = message
+        self.rejected_keys = rejected_keys or []
+        self.allowed_keys = allowed_keys or []
+        self.details = details
+
+    def __str__(self) -> str:
+        """Format error message with context."""
+        parts = [self.message]
+
+        if self.rejected_keys:
+            parts.append(f"Rejected keys: {', '.join(self.rejected_keys)}")
+
+        if self.allowed_keys:
+            parts.append(f"Allowed keys: {', '.join(self.allowed_keys)}")
+
+        if self.details:
+            detail_str = ", ".join(f"{k}={v}" for k, v in self.details.items())
+            parts.append(f"Details: {detail_str}")
+
+        return " | ".join(parts)
+
+
+class PermissionDeniedError(Exception):
+    """
+    Exception raised when user lacks required permissions for an operation.
+
+    This exception is raised by RBAC policy enforcement when a user attempts
+    to access a resource or perform an operation they are not authorized for.
+
+    Attributes:
+        user: User identifier (username, email, or user ID)
+        resource: Resource identifier (collection ID, document ID, etc.)
+        operation: Operation name (read, write, delete, etc.)
+        reason: Optional additional context explaining why permission was denied
+        details: Additional error context
+
+    Example:
+        >>> raise PermissionDeniedError(
+        ...     user="john.doe@example.com",
+        ...     resource="medical_records_collection",
+        ...     operation="read",
+        ...     reason="User does not have 'medical' clearance level"
+        ... )
+    """
+
+    def __init__(
+        self,
+        user: str,
+        resource: str,
+        operation: str,
+        reason: Optional[str] = None,
+        **details: Any
+    ):
+        """
+        Initialize PermissionDeniedError.
+
+        Args:
+            user: User identifier (username, email, or user ID)
+            resource: Resource identifier being accessed
+            operation: Operation being attempted (read, write, delete)
+            reason: Optional explanation for permission denial
+            **details: Additional error context (e.g., required_role, user_role)
+        """
+        message = (
+            f"Permission denied: User '{user}' cannot perform '{operation}' "
+            f"on resource '{resource}'"
+        )
+        if reason:
+            message += f" - {reason}"
+
+        super().__init__(message)
+        self.user = user
+        self.resource = resource
+        self.operation = operation
+        self.reason = reason
+        self.details = details
+
+    def __str__(self) -> str:
+        """Format error message with full context."""
+        parts = [
+            f"User: {self.user}",
+            f"Resource: {self.resource}",
+            f"Operation: {self.operation}",
+        ]
+
+        if self.reason:
+            parts.append(f"Reason: {self.reason}")
+
+        if self.details:
+            detail_str = ", ".join(f"{k}={v}" for k, v in self.details.items())
+            parts.append(f"Details: {detail_str}")
+
+        return " | ".join(parts)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert exception to dictionary for JSON serialization.
+
+        Returns:
+            Dictionary with user, resource, operation, reason, and details
+
+        Example:
+            >>> error = PermissionDeniedError(
+            ...     user="john", resource="docs", operation="read"
+            ... )
+            >>> error.to_dict()
+            {'user': 'john', 'resource': 'docs', 'operation': 'read', ...}
+        """
+        return {
+            "user": self.user,
+            "resource": self.resource,
+            "operation": self.operation,
+            "reason": self.reason,
+            **self.details,
+        }

iris_vector_rag-0.5.8/iris_vector_rag/monitoring/__init__.py

@@ -0,0 +1,58 @@
+"""
+Monitoring and observability module for iris-vector-rag.
+
+This module provides OpenTelemetry integration for production observability,
+including query latency tracking, token usage monitoring, and cost tracking.
+
+Classes:
+    TelemetryManager: Central telemetry manager with lazy initialization
+
+Functions:
+    configure_telemetry: Configure global telemetry settings
+    calculate_llm_cost: Calculate LLM API costs based on token usage
+
+Features:
+    - Zero overhead when disabled (lazy initialization)
+    - <5% overhead when enabled
+    - OpenTelemetry GenAI semantic conventions
+    - OTLP export support (HTTP/gRPC)
+    - Automatic span context propagation
+    - Cost tracking for OpenAI, Anthropic models
+
+Example:
+    >>> from iris_vector_rag.monitoring import configure_telemetry
+    >>>
+    >>> # Enable telemetry with OTLP export
+    >>> configure_telemetry(
+    ...     enabled=True,
+    ...     service_name="my-rag-service",
+    ...     endpoint="http://localhost:4318",
+    ...     sampling_ratio=1.0
+    ... )
+    >>>
+    >>> # Telemetry automatically instruments RAG operations
+    >>> pipeline = create_pipeline("basic")
+    >>> result = pipeline.query("What is diabetes?")  # Auto-traced
+    >>>
+    >>> # Manual instrumentation
+    >>> from iris_vector_rag.monitoring import telemetry
+    >>>
+    >>> with telemetry.trace_operation("custom.operation") as span:
+    ...     span.set_attribute("custom.param", "value")
+    ...     # Your code here
+    ...     pass
+"""
+
+from iris_vector_rag.monitoring.telemetry import (
+    TelemetryManager,
+    configure_telemetry,
+    telemetry,
+)
+from iris_vector_rag.monitoring.cost_tracking import calculate_llm_cost
+
+__all__ = [
+    "TelemetryManager",
+    "configure_telemetry",
+    "telemetry",
+    "calculate_llm_cost",
+]

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/pipelines/graphrag.py

@@ -181,8 +181,8 @@ class GraphRAGPipeline(RAGPipeline):
                                 for entity2 in entities[i+1:]:
                                     # Simple relationship: co-occurrence in same document
                                     rel = Relationship(
-                                        source_entity_id=entity1.text,  # Fixed: was source_entity
-                                        target_entity_id=entity2.text,  # Fixed: was target_entity
+                                        source_entity_id=entity1.id,  # Use entity UUID, not text
+                                        target_entity_id=entity2.id,  # Use entity UUID, not text
                                         relationship_type="co_occurs_with",
                                         confidence=0.8,
                                         source_document_id=doc.id

iris_vector_rag-0.5.8/iris_vector_rag/security/__init__.py

@@ -0,0 +1,34 @@
+"""
+Security module for iris-vector-rag.
+
+This module provides Role-Based Access Control (RBAC) integration for
+multi-tenant deployments and enterprise security requirements.
+
+Classes:
+    RBACPolicy: Abstract base class for implementing custom RBAC policies
+    PermissionDeniedError: Exception raised when user lacks required permissions
+
+Example:
+    >>> from iris_vector_rag.security import RBACPolicy
+    >>>
+    >>> class MyRBACPolicy(RBACPolicy):
+    ...     def check_collection_access(self, user, collection_id, operation):
+    ...         # Custom authorization logic
+    ...         return user.role in ['admin', 'reader']
+    ...
+    ...     def filter_documents(self, user, documents):
+    ...         # Filter documents based on user permissions
+    ...         return [doc for doc in documents if self._can_access(user, doc)]
+    >>>
+    >>> # Use with IRISVectorStore
+    >>> policy = MyRBACPolicy()
+    >>> store = IRISVectorStore(rbac_policy=policy)
+"""
+
+from iris_vector_rag.security.rbac import RBACPolicy
+from iris_vector_rag.exceptions import PermissionDeniedError
+
+__all__ = [
+    "RBACPolicy",
+    "PermissionDeniedError",
+]

{iris_vector_rag-0.5.3 → iris_vector_rag-0.5.8}/iris_vector_rag/services/batch_entity_processor.py

@@ -109,6 +109,13 @@ class BatchEntityProcessor:
             conn = self.connection_manager.get_connection()
             cursor = conn.cursor()
 
+            # Log entity IDs being stored for debugging
+            entity_ids_to_store = [str(e.id) for e in entities]
+            logger.debug(
+                f"Preparing to store {len(entities)} entities. "
+                f"First 5 IDs: {entity_ids_to_store[:5]}"
+            )
+
             # Prepare batch data for executemany()
             batch_data = []
 
@@ -308,10 +315,21 @@ class BatchEntityProcessor:
                             or str(r.target_entity_id) in missing_ids
                         ]
                     )
+
+                    # Enhanced logging to diagnose missing entities
+                    missing_ids_list = sorted(list(missing_ids))[:10]  # Show first 10
                     logger.error(
                         f"Foreign key validation failed: {len(missing_ids)} missing entity IDs, "
                         f"{orphaned_count} orphaned relationships"
                     )
+                    logger.error(
+                        f"Sample missing entity IDs (first 10): {missing_ids_list}"
+                    )
+                    logger.info(
+                        f"Total entity IDs referenced: {len(entity_ids)}, "
+                        f"Found in database: {len(existing_ids)}, "
+                        f"Missing: {len(missing_ids)}"
+                    )
 
                     # Filter out relationships with missing entity IDs
                     relationships = [