iris-vector-rag 0.6.0__tar.gz → 0.7.0__tar.gz

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.6.0
+Version: 0.7.0
 Summary: Enterprise RAG pipelines with native IRIS vector search. 6 production implementations with RAGAS evaluation, LangChain, AWS/Azure configs. No external VectorDB required.
 Author-email: Thomas Dyar <thomas.dyar@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/iris_vector_rag/__init__.py

@@ -19,7 +19,7 @@ from .validation.requirements import get_pipeline_requirements
 from .validation.validator import PreConditionValidator
 
 # Package version
-__version__ = "0.5.19"
+__version__ = "0.7.0"
 __author__ = "InterSystems IRIS RAG Templates Project"
 __description__ = "A comprehensive, production-ready framework for implementing Retrieval Augmented Generation (RAG) pipelines using InterSystems IRIS as the vector database backend."
 

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/iris_vector_rag/pipelines/hybrid_graphrag.py

@@ -10,7 +10,7 @@ robust error handling, and modular architecture.
 
 import logging
 import time
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, TypedDict
 
 from ..config.manager import ConfigurationManager
 from ..core.connection import ConnectionManager
@@ -22,6 +22,20 @@ from .hybrid_graphrag_retrieval import HybridRetrievalMethods
 logger = logging.getLogger(__name__)
 
 
+class AttachResult(TypedDict):
+    table: str
+    label: str
+    id_col: str
+    embedding_col: str
+    dimension: int
+    row_count: int
+    has_hnsw_index: Optional[bool]
+
+
+class DimensionMismatchError(ValueError):
+    pass
+
+
 class HybridGraphRAGPipeline(GraphRAGPipeline):
     """
     Enhanced GraphRAG pipeline with hybrid search capabilities.
@@ -48,7 +62,13 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
         embedding_config: Optional[str] = None,
         executor=None,
     ):
-        super().__init__(connection_manager, config_manager, llm_func, vector_store, executor=executor)
+        super().__init__(
+            connection_manager,
+            config_manager,
+            llm_func,
+            vector_store,
+            executor=executor,
+        )
 
         # Store schema manager for iris_vector_graph table management
         self.schema_manager = schema_manager
@@ -73,6 +93,8 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
         self.text_engine = None
         self.vector_optimizer = None
 
+        self._attached_corpora: Dict[str, AttachResult] = {}
+
         # Initialize graph core integration
         self._initialize_graph_core()
 
@@ -147,6 +169,116 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
             logger.error(f"Failed to initialize iris_vector_graph components: {e}")
             raise
 
+    def _detect_hnsw_index(
+        self, source_table: str, embedding_col: str
+    ) -> Optional[bool]:
+        try:
+            schema, tbl = (
+                source_table.split(".", 1)
+                if "." in source_table
+                else ("", source_table)
+            )
+            conn = self.iris_engine.conn if self.iris_engine else None
+            if conn is None:
+                return None
+            cur = conn.cursor()
+            try:
+                cur.execute(
+                    "SELECT COUNT(*) FROM INFORMATION_SCHEMA.INDEXES "
+                    "WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?",
+                    [schema or "USER", tbl, embedding_col],
+                )
+                row = cur.fetchone()
+                return row is not None and int(row[0]) > 0
+            finally:
+                cur.close()
+        except Exception:
+            return None
+
+    def attach_existing_corpus(
+        self,
+        source_table: str,
+        id_col: str,
+        text_col: str,
+        embedding_col: str,
+        graph_label: str,
+    ) -> AttachResult:
+        if self.iris_engine is None:
+            raise RuntimeError(
+                "iris_vector_graph engine not initialized. "
+                "Call _initialize_graph_core() first."
+            )
+
+        validation = self.iris_engine.validate_vector_table(source_table, embedding_col)
+        dimension = validation.get("dimension")
+        row_count = validation.get("row_count", 0)
+
+        if dimension is None and row_count > 0:
+            cur = self.iris_engine.conn.cursor()
+            try:
+                cur.execute(
+                    f"SELECT TOP 1 {embedding_col} "
+                    f"FROM {source_table} WHERE {embedding_col} IS NOT NULL"
+                )
+                row = cur.fetchone()
+                if row and row[0]:
+                    raw = row[0]
+                    if isinstance(raw, str):
+                        dimension = raw.count(",") + 1
+                    elif isinstance(raw, (list, tuple)):
+                        dimension = len(raw)
+                    elif isinstance(raw, bytes):
+                        dimension = len(raw) // 8
+            except Exception:
+                pass
+            finally:
+                cur.close()
+
+        if dimension is None or dimension == 0:
+            logger.warning(
+                f"No non-NULL embeddings found in {source_table}.{embedding_col} — "
+                f"vector search will return no results until embeddings are populated."
+            )
+
+        self.iris_engine.map_sql_table(
+            source_table,
+            id_col,
+            graph_label,
+            property_columns=[text_col],
+        )
+
+        has_hnsw = self._detect_hnsw_index(source_table, embedding_col)
+        if has_hnsw is False:
+            logger.warning(
+                f"No HNSW index found on {source_table}.{embedding_col} — "
+                f"vector search will use brute force. "
+                f"Consider running BUILD INDEX for better performance."
+            )
+
+        result: AttachResult = {
+            "table": source_table,
+            "label": graph_label,
+            "id_col": id_col,
+            "embedding_col": embedding_col,
+            "dimension": dimension if dimension else 0,
+            "row_count": row_count,
+            "has_hnsw_index": has_hnsw,
+        }
+        self._attached_corpora[graph_label] = result
+        return result
+
+    def _validate_query_dimension(self, graph_label: str, query_vec) -> None:
+        if graph_label not in self._attached_corpora:
+            return
+        expected_dim = self._attached_corpora[graph_label].get("dimension", 0)
+        if expected_dim and expected_dim > 0:
+            actual_dim = len(query_vec) if hasattr(query_vec, "__len__") else 0
+            if actual_dim > 0 and actual_dim != expected_dim:
+                raise DimensionMismatchError(
+                    f"Query vector dimension ({actual_dim}) does not match "
+                    f"table embedding dimension ({expected_dim})"
+                )
+
     def query(
         self,
         query_text: Optional[str] = None,
@@ -282,7 +414,11 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
         normalized: List[Document] = []
         for doc in documents:
             if isinstance(doc, Document):
-                score = doc.metadata.get("similarity_score") if hasattr(doc, "metadata") else None
+                score = (
+                    doc.metadata.get("similarity_score")
+                    if hasattr(doc, "metadata")
+                    else None
+                )
                 score_value = float(score) if score is not None else 1.0
                 normalized.append(
                     Document(
@@ -354,7 +490,9 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
             )
             # If text search returns 0 results, fall back to vector search
             if not documents:
-                logger.warning("Text search returned 0 results. Falling back to vector search.")
+                logger.warning(
+                    "Text search returned 0 results. Falling back to vector search."
+                )
                 fallback_docs = self._fallback_to_vector_search(query_text, top_k)
                 return fallback_docs, "vector_fallback"
             return documents, method
@@ -374,7 +512,9 @@ class HybridGraphRAGPipeline(GraphRAGPipeline):
             )
             # If HNSW returns 0 results, fall back to IRISVectorStore
             if not documents:
-                logger.warning("HNSW vector search returned 0 results. Falling back to IRISVectorStore.")
+                logger.warning(
+                    "HNSW vector search returned 0 results. Falling back to IRISVectorStore."
+                )
                 fallback_docs = self._fallback_to_vector_search(query_text, top_k)
                 return fallback_docs, "vector_fallback"
             return documents, method

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/iris_vector_rag.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iris-vector-rag
-Version: 0.6.0
+Version: 0.7.0
 Summary: Enterprise RAG pipelines with native IRIS vector search. 6 production implementations with RAGAS evaluation, LangChain, AWS/Azure configs. No external VectorDB required.
 Author-email: Thomas Dyar <thomas.dyar@intersystems.com>
 Maintainer-email: Thomas Dyar <thomas.dyar@intersystems.com>

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/iris_vector_rag.egg-info/SOURCES.txt

@@ -182,6 +182,7 @@ iris_vector_rag/validation/factory.py
 iris_vector_rag/validation/orchestrator.py
 iris_vector_rag/validation/requirements.py
 iris_vector_rag/validation/validator.py
+tests/test_attach_corpus.py
 tests/test_comprehensive_pipeline_validation_e2e.py
 tests/test_comprehensive_pipeline_validation_e2e_fixed.py
 tests/test_enterprise_storage.py

{iris_vector_rag-0.6.0 → iris_vector_rag-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "iris-vector-rag"
-version = "0.6.0"
+version = "0.7.0"
 description = "Enterprise RAG pipelines with native IRIS vector search. 6 production implementations with RAGAS evaluation, LangChain, AWS/Azure configs. No external VectorDB required."
 readme = "README.md"
 license = {text = "MIT"}

iris_vector_rag-0.7.0/tests/test_attach_corpus.py

@@ -0,0 +1,362 @@
+import logging
+import os
+
+import numpy as np
+import pytest
+
+IRIS_HOST = os.environ.get("IRIS_HOSTNAME", "localhost")
+IRIS_PORT = int(os.environ.get("IRIS_PORT", "1972"))
+TOKEN_DIM = 384
+
+logger = logging.getLogger(__name__)
+
+
+def _get_conn():
+    import iris.dbapi as dbapi
+
+    return dbapi.connect(
+        hostname=IRIS_HOST,
+        port=IRIS_PORT,
+        namespace="USER",
+        username="_SYSTEM",
+        password="SYS",
+    )
+
+
+def _get_engine(conn):
+    import intersystems_iris
+    from iris_vector_graph import IRISGraphEngine
+
+    native = intersystems_iris.createConnection(
+        hostname=IRIS_HOST,
+        port=IRIS_PORT,
+        namespace="USER",
+        username="_SYSTEM",
+        password="SYS",
+    )
+    return IRISGraphEngine(native)
+
+
+def _create_test_table(conn, table_name, n_rows, dim=TOKEN_DIM):
+    cur = conn.cursor()
+    try:
+        cur.execute(f"DROP TABLE IF EXISTS {table_name}")
+        conn.commit()
+        cur.execute(
+            f"CREATE TABLE {table_name} ("
+            f"doc_id VARCHAR(50) PRIMARY KEY, "
+            f"text_content VARCHAR(2000), "
+            f"embedding VECTOR(DOUBLE, {dim}))"
+        )
+        conn.commit()
+        rng = np.random.default_rng(42)
+        for i in range(n_rows):
+            vec = rng.random(dim).astype(np.float32)
+            vec /= np.linalg.norm(vec)
+            vec_str = ",".join(str(float(v)) for v in vec)
+            cur.execute(
+                f"INSERT INTO {table_name} (doc_id, text_content, embedding) "
+                f"VALUES (?, ?, TO_VECTOR(?, DOUBLE, {dim}))",
+                [f"doc_{i:04d}", f"Test document {i} about topic {i % 10}.", vec_str],
+            )
+        conn.commit()
+    finally:
+        cur.close()
+    return n_rows
+
+
+def _drop_test_table(conn, table_name):
+    cur = conn.cursor()
+    try:
+        cur.execute(f"DROP TABLE IF EXISTS {table_name}")
+        conn.commit()
+    except Exception:
+        pass
+    finally:
+        cur.close()
+
+
+def _make_pipeline(conn, engine):
+    from unittest.mock import MagicMock
+
+    from iris_vector_rag.pipelines.hybrid_graphrag import HybridGraphRAGPipeline
+
+    pipeline = HybridGraphRAGPipeline.__new__(HybridGraphRAGPipeline)
+    pipeline.iris_engine = engine
+    pipeline._attached_corpora = {}
+    pipeline.connection_manager = MagicMock()
+    pipeline.config_manager = MagicMock()
+    return pipeline
+
+
+@pytest.fixture(scope="module")
+def conn():
+    c = _get_conn()
+    yield c
+    c.close()
+
+
+@pytest.fixture(scope="module")
+def engine(conn):
+    import intersystems_iris
+
+    native = intersystems_iris.createConnection(
+        hostname=IRIS_HOST,
+        port=IRIS_PORT,
+        namespace="USER",
+        username="_SYSTEM",
+        password="SYS",
+    )
+    from iris_vector_graph import IRISGraphEngine
+
+    e = IRISGraphEngine(native, embedding_dimension=TOKEN_DIM)
+    e.initialize_schema(auto_deploy_objectscript=False)
+    return e
+
+
+@pytest.fixture(scope="module")
+def pipeline(conn, engine):
+    return _make_pipeline(conn, engine)
+
+
+@pytest.fixture(scope="module")
+def test_table(conn):
+    table = "Test069.CorpusA"
+    _create_test_table(conn, table, 100)
+    yield table
+    _drop_test_table(conn, table)
+
+
+# --- US1: Attach RAG corpus ---
+
+
+@pytest.mark.integration
+class TestAttachCorpusBasic:
+    def test_attach_returns_result(self, pipeline, test_table):
+        result = pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="TestDoc",
+        )
+        assert result["table"] == test_table
+        assert result["label"] == "TestDoc"
+        assert result["dimension"] == TOKEN_DIM
+        assert result["row_count"] == 100
+        assert isinstance(result["has_hnsw_index"], (bool, type(None)))
+
+    def test_graph_query_returns_rows(self, pipeline, engine, test_table):
+        pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="TestDoc2",
+        )
+        results = engine.execute_cypher("MATCH (d:TestDoc2) RETURN d.doc_id LIMIT 5")
+        assert len(results) > 0
+
+    def test_vector_search_returns_results(self, pipeline, engine, test_table):
+        pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="TestDocVS",
+        )
+        rng = np.random.default_rng(99)
+        q = rng.random(TOKEN_DIM).astype(np.float32)
+        q /= np.linalg.norm(q)
+        results = engine.vector_search(
+            test_table, "embedding", q.tolist(), top_k=5, id_col="doc_id"
+        )
+        assert len(results) > 0
+
+
+@pytest.mark.integration
+class TestAttachCorpusIdempotent:
+    def test_double_attach_no_error(self, pipeline, test_table):
+        r1 = pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="IdempotentLabel",
+        )
+        r2 = pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="IdempotentLabel",
+        )
+        assert r1["dimension"] == r2["dimension"]
+        assert r1["label"] == r2["label"]
+
+
+@pytest.mark.integration
+class TestNewRowsVisible:
+    def test_new_row_in_graph_after_insert(self, pipeline, conn, engine, test_table):
+        pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="NewRowLabel",
+        )
+        cur = conn.cursor()
+        rng = np.random.default_rng(777)
+        vec = rng.random(TOKEN_DIM).astype(np.float32)
+        vec /= np.linalg.norm(vec)
+        vec_str = ",".join(str(float(v)) for v in vec)
+        try:
+            cur.execute(
+                f"INSERT INTO {test_table} (doc_id, text_content, embedding) "
+                f"VALUES (?, ?, TO_VECTOR(?, FLOAT, {TOKEN_DIM}))",
+                ["doc_new_0001", "Newly inserted document.", vec_str],
+            )
+            conn.commit()
+        finally:
+            cur.close()
+        results = engine.execute_cypher(
+            "MATCH (d:NewRowLabel) WHERE d.doc_id = 'doc_new_0001' RETURN d.doc_id"
+        )
+        assert len(results) > 0
+
+
+# --- US2: Custom IRIS table ---
+
+
+@pytest.mark.integration
+class TestAttachCustomTable:
+    def test_custom_schema_table(self, conn, engine):
+        p = _make_pipeline(conn, engine)
+        table = "Test069.ClinicalNotes"
+        _create_test_table(conn, table, 50, dim=384)
+        try:
+            result = p.attach_existing_corpus(
+                source_table=table,
+                id_col="doc_id",
+                text_col="text_content",
+                embedding_col="embedding",
+                graph_label="ClinNote",
+            )
+            assert result["label"] == "ClinNote"
+            assert result["row_count"] == 50
+            results = engine.execute_cypher(
+                "MATCH (n:ClinNote) RETURN n.doc_id LIMIT 3"
+            )
+            assert len(results) > 0
+        finally:
+            _drop_test_table(conn, table)
+
+
+# --- US3: Dimension mismatch ---
+
+
+@pytest.mark.integration
+class TestDimensionMismatch:
+    def test_mismatch_raises_error(self, pipeline, test_table):
+        from iris_vector_rag.pipelines.hybrid_graphrag import DimensionMismatchError
+
+        pipeline.attach_existing_corpus(
+            source_table=test_table,
+            id_col="doc_id",
+            text_col="text_content",
+            embedding_col="embedding",
+            graph_label="DimTest",
+        )
+        wrong_dim_vec = np.random.default_rng(0).random(768).tolist()
+        with pytest.raises(DimensionMismatchError):
+            pipeline._validate_query_dimension("DimTest", wrong_dim_vec)
+
+    def test_all_null_embeddings_warns(self, conn, engine, caplog):
+        p = _make_pipeline(conn, engine)
+        table = "Test069.NullEmb"
+        cur = conn.cursor()
+        try:
+            cur.execute(f"DROP TABLE IF EXISTS {table}")
+            conn.commit()
+            cur.execute(
+                f"CREATE TABLE {table} ("
+                f"doc_id VARCHAR(50) PRIMARY KEY, "
+                f"text_content VARCHAR(200), "
+                f"embedding VECTOR(DOUBLE, 384))"
+            )
+            conn.commit()
+            cur.execute(
+                f"INSERT INTO {table} (doc_id, text_content) VALUES ('n1', 'No embedding')"
+            )
+            conn.commit()
+        finally:
+            cur.close()
+        try:
+            with caplog.at_level(logging.WARNING):
+                result = p.attach_existing_corpus(
+                    source_table=table,
+                    id_col="doc_id",
+                    text_col="text_content",
+                    embedding_col="embedding",
+                    graph_label="NullLabel",
+                )
+            assert result["dimension"] is None or result["dimension"] == 0
+            assert any(
+                "non-NULL" in r.message or "NULL" in r.message for r in caplog.records
+            )
+        finally:
+            _drop_test_table(conn, table)
+
+
+# --- Polish: Edge cases ---
+
+
+@pytest.mark.integration
+class TestEdgeCases:
+    def test_nonexistent_table_raises(self, pipeline):
+        with pytest.raises(ValueError, match="not found"):
+            pipeline.attach_existing_corpus(
+                source_table="NoSuch.Table",
+                id_col="id",
+                text_col="txt",
+                embedding_col="emb",
+                graph_label="Ghost",
+            )
+
+    def test_nonexistent_column_raises(self, pipeline, test_table):
+        with pytest.raises(ValueError, match="not found"):
+            pipeline.attach_existing_corpus(
+                source_table=test_table,
+                id_col="doc_id",
+                text_col="text_content",
+                embedding_col="no_such_column",
+                graph_label="BadCol",
+            )
+
+    def test_upsert_repoint_label(self, conn, engine):
+        p = _make_pipeline(conn, engine)
+        table_a = "Test069.TableA"
+        table_b = "Test069.TableB"
+        _create_test_table(conn, table_a, 10)
+        _create_test_table(conn, table_b, 20)
+        try:
+            r1 = p.attach_existing_corpus(
+                source_table=table_a,
+                id_col="doc_id",
+                text_col="text_content",
+                embedding_col="embedding",
+                graph_label="Repoint",
+            )
+            assert r1["row_count"] == 10
+            r2 = p.attach_existing_corpus(
+                source_table=table_b,
+                id_col="doc_id",
+                text_col="text_content",
+                embedding_col="embedding",
+                graph_label="Repoint",
+            )
+            assert r2["row_count"] == 20
+            assert r2["table"] == table_b
+        finally:
+            _drop_test_table(conn, table_a)
+            _drop_test_table(conn, table_b)