iris-vector-rag 0.3.0__tar.gz → 0.3.2__tar.gz

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.3.0
+Version: 0.3.2
 Summary: Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration
 Author-email: InterSystems <support@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>
@@ -305,6 +305,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/README.md

@@ -201,6 +201,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/iris_rag/dspy_modules/entity_extraction_module.py

@@ -379,6 +379,17 @@ def configure_dspy(llm_config: dict):
         max_tokens = llm_config.get("max_tokens", 2000)
         temperature = llm_config.get("temperature", 0.1)
 
+        # Log configuration details before attempting to configure
+        logger.info("=" * 60)
+        logger.info("🔧 Configuring DSPy for Entity Extraction")
+        logger.info("=" * 60)
+        logger.info(f"  API Type:    {api_type}")
+        logger.info(f"  Model:       {model}")
+        logger.info(f"  API Base:    {api_base}")
+        logger.info(f"  Max Tokens:  {max_tokens}")
+        logger.info(f"  Temperature: {temperature}")
+        logger.info("=" * 60)
+
         # FIX for Bug #6: Use custom DirectOpenAILM for GPT-OSS NIM
         # This bypasses LiteLLM entirely to preserve full model name "openai/gpt-oss-120b"
         if model == "openai/gpt-oss-120b" or (api_type == "openai" and "/" in model):

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/iris_rag/pipelines/graphrag.py

@@ -131,11 +131,26 @@ class GraphRAGPipeline(RAGPipeline):
 
         # Process documents in batches of 5 for 3x speedup
         batch_size = 5
+        total_batches = (len(documents) + batch_size - 1) // batch_size  # Ceiling division
+
+        logger.info("=" * 70)
+        logger.info(f"🚀 Starting Entity Extraction")
+        logger.info("=" * 70)
+        logger.info(f"  Total documents: {len(documents)}")
+        logger.info(f"  Batch size:      {batch_size}")
+        logger.info(f"  Total batches:   {total_batches}")
+        logger.info("=" * 70)
+
+        import time
+        extraction_start_time = time.time()
+
         for i in range(0, len(documents), batch_size):
             batch_docs = documents[i:i + batch_size]
+            batch_num = i//batch_size + 1
 
             try:
-                logger.info(f"🚀 Processing batch {i//batch_size + 1} ({len(batch_docs)} documents) for entity extraction")
+                batch_start_time = time.time()
+                logger.info(f"📦 Processing batch {batch_num}/{total_batches} ({len(batch_docs)} documents)...")
 
                 # Batch extract entities (single LLM call for all documents in batch!)
                 batch_results = self.entity_extraction_service.extract_batch_with_dspy(
@@ -209,6 +224,28 @@ class GraphRAGPipeline(RAGPipeline):
                         logger.warning(f"Failed to store entities for document {doc.id}: {e}")
                         failed_documents.append(doc.id)
 
+                # Log batch completion with timing and statistics
+                batch_elapsed = time.time() - batch_start_time
+                batch_entity_count = sum(len(batch_results.get(doc.id, [])) for doc in batch_docs)
+                logger.info(
+                    f"✅ Batch {batch_num}/{total_batches} complete: "
+                    f"{batch_entity_count} entities extracted in {batch_elapsed:.1f}s "
+                    f"(avg: {batch_entity_count/len(batch_docs):.1f} per doc)"
+                )
+
+                # Show overall progress every 10 batches
+                if batch_num % 10 == 0 or batch_num == total_batches:
+                    elapsed_so_far = time.time() - extraction_start_time
+                    docs_processed = min(batch_num * batch_size, len(documents))
+                    avg_time_per_doc = elapsed_so_far / docs_processed if docs_processed > 0 else 0
+                    remaining_docs = len(documents) - docs_processed
+                    eta_seconds = remaining_docs * avg_time_per_doc
+                    logger.info(
+                        f"📊 Progress: {docs_processed}/{len(documents)} documents processed "
+                        f"({total_entities} entities, {total_relationships} relationships) | "
+                        f"ETA: {eta_seconds/60:.1f} min"
+                    )
+
             except Exception as e:
                 # Batch extraction failed - fall back to individual processing for this batch
                 logger.warning(f"Batch entity extraction failed: {e}, falling back to individual processing")
@@ -238,6 +275,20 @@ class GraphRAGPipeline(RAGPipeline):
                         logger.warning(f"Entity extraction error for document {doc.id}: {e}")
                         failed_documents.append(doc.id)
 
+        # Log final extraction summary
+        extraction_elapsed = time.time() - extraction_start_time
+        logger.info("=" * 70)
+        logger.info("✅ Entity Extraction Complete")
+        logger.info("=" * 70)
+        logger.info(f"  Documents processed:     {len(documents)}")
+        logger.info(f"  Total entities:          {total_entities}")
+        logger.info(f"  Total relationships:     {total_relationships}")
+        logger.info(f"  Failed documents:        {len(failed_documents)}")
+        logger.info(f"  Extraction time:         {extraction_elapsed:.1f}s ({extraction_elapsed/60:.2f} min)")
+        logger.info(f"  Throughput:              {len(documents)/extraction_elapsed:.2f} docs/sec")
+        logger.info(f"  Avg entities per doc:    {total_entities/len(documents):.1f}")
+        logger.info("=" * 70)
+
         # Only fail if we got zero entities across ALL documents
         # (suggests a systematic extraction failure rather than content issues or storage failure)
         if total_entities == 0:

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/iris_rag/services/entity_extraction.py

@@ -534,6 +534,7 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
         Bug fix: https://github.com/tdyar/hipporag2-pipeline/BUG_REPORT_RAG_TEMPLATES.md
         Prevents silent fallback to Ollama when OpenAI is configured.
         """
+        # FIX: Use root-level LLM config
         llm_config = self.config_manager.get("llm", {})
 
         if not llm_config:
@@ -581,6 +582,7 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
         Returns:
             Model name string, defaults to "qwen2.5:7b" if not configured
         """
+        # FIX: Use root-level LLM config
         llm_config = self.config_manager.get("llm", {})
 
         # Try both naming conventions (model takes precedence)
@@ -729,7 +731,8 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
         entities = []
         try:
             # Check if DSPy extraction is configured
-            use_dspy = self.config.get("llm", {}).get("use_dspy", False)
+            # FIX: Use root-level LLM config
+            use_dspy = self.config_manager.get("llm", {}).get("use_dspy", False)
 
             if use_dspy:
                 # Use DSPy-powered entity extraction
@@ -766,7 +769,8 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
 
             # Configure DSPy if not already configured
             if not hasattr(self, '_dspy_module'):
-                llm_config = self.config.get("llm", {})
+                # FIX: Use root-level LLM config, fall back to entity_extraction.llm
+                llm_config = self.config_manager.get("llm", {}) or self.config.get("llm", {})
 
                 # Only configure DSPy if not already configured globally
                 # This prevents threading issues when workers have already configured DSPy
@@ -886,7 +890,8 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
                 # Import configuration helper
                 from ..dspy_modules.entity_extraction_module import configure_dspy
 
-                llm_config = self.config.get("llm", {})
+                # FIX: Use root-level LLM config
+                llm_config = self.config_manager.get("llm", {})
 
                 # Configure DSPy if needed
                 import dspy as dspy_module
@@ -1068,9 +1073,9 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
             import requests
             import json
 
-            # Get LLM config from entity_extraction config
-            llm_config = self.config.get("llm", {})
-            model = llm_config.get("model", "qwen2.5:7b")  # Changed from qwen3:14b - faster and works better
+            # FIX: Use root-level LLM config
+            llm_config = self.config_manager.get("llm", {})
+            model = llm_config.get("model") or llm_config.get("model_name") or "qwen2.5:7b"
             temperature = llm_config.get("temperature", 0.1)
             max_tokens = llm_config.get("max_tokens", 2000)
 

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/iris_vector_rag.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.3.0
+Version: 0.3.2
 Summary: Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration
 Author-email: InterSystems <support@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>
@@ -305,6 +305,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.0 → iris_vector_rag-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "iris-vector-rag"
-version = "0.3.0"
+version = "0.3.2"
 description = "Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration"
 readme = "README.md"
 license = {text = "MIT"}