iris-vector-rag 0.3.1__tar.gz → 0.3.3__tar.gz

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.3.1
+Version: 0.3.3
 Summary: Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration
 Author-email: InterSystems <support@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>
@@ -305,6 +305,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/README.md

@@ -201,6 +201,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/iris_rag/dspy_modules/entity_extraction_module.py

@@ -379,6 +379,17 @@ def configure_dspy(llm_config: dict):
         max_tokens = llm_config.get("max_tokens", 2000)
         temperature = llm_config.get("temperature", 0.1)
 
+        # Log configuration details before attempting to configure
+        logger.info("=" * 60)
+        logger.info("🔧 Configuring DSPy for Entity Extraction")
+        logger.info("=" * 60)
+        logger.info(f"  API Type:    {api_type}")
+        logger.info(f"  Model:       {model}")
+        logger.info(f"  API Base:    {api_base}")
+        logger.info(f"  Max Tokens:  {max_tokens}")
+        logger.info(f"  Temperature: {temperature}")
+        logger.info("=" * 60)
+
         # FIX for Bug #6: Use custom DirectOpenAILM for GPT-OSS NIM
         # This bypasses LiteLLM entirely to preserve full model name "openai/gpt-oss-120b"
         if model == "openai/gpt-oss-120b" or (api_type == "openai" and "/" in model):

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/iris_rag/pipelines/graphrag.py

@@ -131,11 +131,26 @@ class GraphRAGPipeline(RAGPipeline):
 
         # Process documents in batches of 5 for 3x speedup
         batch_size = 5
+        total_batches = (len(documents) + batch_size - 1) // batch_size  # Ceiling division
+
+        logger.info("=" * 70)
+        logger.info(f"🚀 Starting Entity Extraction")
+        logger.info("=" * 70)
+        logger.info(f"  Total documents: {len(documents)}")
+        logger.info(f"  Batch size:      {batch_size}")
+        logger.info(f"  Total batches:   {total_batches}")
+        logger.info("=" * 70)
+
+        import time
+        extraction_start_time = time.time()
+
         for i in range(0, len(documents), batch_size):
             batch_docs = documents[i:i + batch_size]
+            batch_num = i//batch_size + 1
 
             try:
-                logger.info(f"🚀 Processing batch {i//batch_size + 1} ({len(batch_docs)} documents) for entity extraction")
+                batch_start_time = time.time()
+                logger.info(f"📦 Processing batch {batch_num}/{total_batches} ({len(batch_docs)} documents)...")
 
                 # Batch extract entities (single LLM call for all documents in batch!)
                 batch_results = self.entity_extraction_service.extract_batch_with_dspy(
@@ -209,6 +224,28 @@ class GraphRAGPipeline(RAGPipeline):
                         logger.warning(f"Failed to store entities for document {doc.id}: {e}")
                         failed_documents.append(doc.id)
 
+                # Log batch completion with timing and statistics
+                batch_elapsed = time.time() - batch_start_time
+                batch_entity_count = sum(len(batch_results.get(doc.id, [])) for doc in batch_docs)
+                logger.info(
+                    f"✅ Batch {batch_num}/{total_batches} complete: "
+                    f"{batch_entity_count} entities extracted in {batch_elapsed:.1f}s "
+                    f"(avg: {batch_entity_count/len(batch_docs):.1f} per doc)"
+                )
+
+                # Show overall progress every 10 batches
+                if batch_num % 10 == 0 or batch_num == total_batches:
+                    elapsed_so_far = time.time() - extraction_start_time
+                    docs_processed = min(batch_num * batch_size, len(documents))
+                    avg_time_per_doc = elapsed_so_far / docs_processed if docs_processed > 0 else 0
+                    remaining_docs = len(documents) - docs_processed
+                    eta_seconds = remaining_docs * avg_time_per_doc
+                    logger.info(
+                        f"📊 Progress: {docs_processed}/{len(documents)} documents processed "
+                        f"({total_entities} entities, {total_relationships} relationships) | "
+                        f"ETA: {eta_seconds/60:.1f} min"
+                    )
+
             except Exception as e:
                 # Batch extraction failed - fall back to individual processing for this batch
                 logger.warning(f"Batch entity extraction failed: {e}, falling back to individual processing")
@@ -238,6 +275,20 @@ class GraphRAGPipeline(RAGPipeline):
                         logger.warning(f"Entity extraction error for document {doc.id}: {e}")
                         failed_documents.append(doc.id)
 
+        # Log final extraction summary
+        extraction_elapsed = time.time() - extraction_start_time
+        logger.info("=" * 70)
+        logger.info("✅ Entity Extraction Complete")
+        logger.info("=" * 70)
+        logger.info(f"  Documents processed:     {len(documents)}")
+        logger.info(f"  Total entities:          {total_entities}")
+        logger.info(f"  Total relationships:     {total_relationships}")
+        logger.info(f"  Failed documents:        {len(failed_documents)}")
+        logger.info(f"  Extraction time:         {extraction_elapsed:.1f}s ({extraction_elapsed/60:.2f} min)")
+        logger.info(f"  Throughput:              {len(documents)/extraction_elapsed:.2f} docs/sec")
+        logger.info(f"  Avg entities per doc:    {total_entities/len(documents):.1f}")
+        logger.info("=" * 70)
+
         # Only fail if we got zero entities across ALL documents
         # (suggests a systematic extraction failure rather than content issues or storage failure)
         if total_entities == 0:

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/iris_rag/services/entity_extraction.py

@@ -527,6 +527,9 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
         # Validate LLM configuration (Bug fix: silent fallback to Ollama)
         self._validate_llm_config()
 
+        # Log LLM configuration on startup for visibility
+        self._log_llm_configuration()
+
     def _validate_llm_config(self):
         """
         Validate LLM configuration and warn about silent fallbacks.
@@ -601,6 +604,34 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
 
         return model
 
+    def _log_llm_configuration(self):
+        """
+        Log LLM configuration at service startup for visibility.
+
+        Helps users diagnose configuration issues and understand what model
+        is being used for entity extraction.
+        """
+        llm_config = self.config_manager.get("llm", {})
+
+        if not llm_config:
+            logger.warning("⚠️  No LLM configuration found - entity extraction may fail")
+            return
+
+        model = llm_config.get("model") or llm_config.get("model_name") or "qwen2.5:7b"
+        provider = llm_config.get("provider", "unknown")
+        api_type = llm_config.get("api_type", "unknown")
+        api_base = llm_config.get("api_base", "http://localhost:11434")
+
+        logger.info("=" * 70)
+        logger.info("🤖 Entity Extraction Service - LLM Configuration")
+        logger.info("=" * 70)
+        logger.info(f"  Provider:    {provider}")
+        logger.info(f"  API Type:    {api_type}")
+        logger.info(f"  Model:       {model}")
+        logger.info(f"  API Base:    {api_base}")
+        logger.info(f"  Method:      {self.method}")
+        logger.info("=" * 70)
+
     def extract_entities(self, document: Document) -> List[Entity]:
         """Extract entities with automatic ontology enhancement if enabled."""
         if hasattr(self, "ontology_enabled") and self.ontology_enabled:
@@ -863,20 +894,40 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
         Returns:
             Dict mapping document IDs to their extracted entities
         """
+        import time
+
+        # Start timing
+        batch_start_time = time.time()
+
+        # Log batch start
+        logger.info(f"📦 Processing batch of {len(documents)} documents...")
+
         # Check if batch processing is enabled
         batch_config = self.config.get("batch_processing", {})
         batch_enabled = batch_config.get("enabled", True)
 
         if not batch_enabled:
-            logger.info("Batch processing disabled - falling back to individual extraction")
+            logger.info("⚠️  Batch processing disabled - falling back to individual extraction")
+            logger.info(f"Processing {len(documents)} documents individually...")
+
             # Process documents individually
             result_map = {}
-            for doc in documents:
+            for i, doc in enumerate(documents, 1):
+                logger.debug(f"  Processing document {i}/{len(documents)}...")
                 result = self.process_document(doc)
                 if result.get("stored", False):
                     # Extract entities from the result
                     entities = result.get("entities", [])
                     result_map[doc.id] = entities
+                    logger.debug(f"    Extracted {len(entities)} entities")
+
+            batch_elapsed = time.time() - batch_start_time
+            total_entities = sum(len(ents) for ents in result_map.values())
+            logger.info(
+                f"✅ Individual processing complete: {len(documents)} documents → {total_entities} entities "
+                f"in {batch_elapsed:.1f}s"
+            )
+
             return result_map
 
         try:
@@ -946,9 +997,12 @@ class EntityExtractionService(OntologyAwareEntityExtractor):
                 logger.debug(f"  Ticket {ticket_id}: {len(entities)} entities extracted")
 
             total_entities = sum(len(ents) for ents in result_map.values())
+            batch_elapsed = time.time() - batch_start_time
+
             logger.info(
-                f"✅ Batch complete: {len(tickets)} tickets → {total_entities} entities "
-                f"(avg: {total_entities/len(tickets):.1f} per ticket)"
+                f"✅ Batch complete: {len(tickets)} documents → {total_entities} entities "
+                f"in {batch_elapsed:.1f}s (avg: {total_entities/len(tickets):.1f} entities/doc, "
+                f"{batch_elapsed/len(tickets):.1f}s/doc)"
             )
 
             return result_map

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/iris_vector_rag.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.3.1
+Version: 0.3.3
 Summary: Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration
 Author-email: InterSystems <support@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>
@@ -305,6 +305,31 @@ print(f"Relationships: {result['metadata']['relationships']}")
 print(f"Graph depth: {result['metadata']['graph_depth']}")
 ```
 
+**⚠️ IMPORTANT:** To enable entity extraction during document ingestion, you MUST set `entity_extraction_enabled: True` in your pipeline configuration:
+
+```python
+from iris_rag.config import ConfigurationManager
+
+config = ConfigurationManager({
+    "pipelines": {
+        "graphrag": {
+            "entity_extraction_enabled": True,  # REQUIRED for entity extraction!
+        }
+    },
+    "llm": {
+        "provider": "openai",
+        "api_type": "openai",
+        "model": "gpt-4o-mini",
+        "api_key": "your-api-key",
+        "temperature": 0.0
+    }
+})
+
+pipeline = create_pipeline('graphrag', config_manager=config)
+```
+
+**Without this setting**, entity extraction will not run during `load_documents()` and no knowledge graph will be built, making graph-based retrieval unavailable.
+
 ### MultiQueryRRF: Multi-Perspective Retrieval
 
 Expands queries into multiple perspectives and fuses results:

{iris_vector_rag-0.3.1 → iris_vector_rag-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "iris-vector-rag"
-version = "0.3.1"
+version = "0.3.3"
 description = "Production-ready, extensible RAG framework with native IRIS vector search - unified API for basic, CRAG, GraphRAG, and ColBERT pipelines with RAGAS and DSPy integration"
 readme = "README.md"
 license = {text = "MIT"}