visual-rag-toolkit 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

demo/ui/upload.py

@@ -9,6 +9,7 @@ import inspect
 from datetime import datetime
 from pathlib import Path
 
+import numpy as np
 import streamlit as st
 
 from demo.config import AVAILABLE_MODELS
@@ -17,6 +18,10 @@ from demo.qdrant_utils import (
     get_collection_stats,
     sample_points_cached,
 )
+from visual_rag.embedding.visual_embedder import VisualEmbedder
+from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
+from visual_rag.indexing.pipeline import ProcessingPipeline
 
 
 VECTOR_TYPES = ["initial", "mean_pooling", "experimental_pooling", "global_pooling"]
@@ -251,10 +256,6 @@ def process_pdfs(uploaded_files, config):
             model_short = model_name.split("/")[-1]
             model_status.info(f"Loading `{model_short}`...")
             
-            import numpy as np
-            from visual_rag import VisualEmbedder
-            from visual_rag.indexing import QdrantIndexer, CloudinaryUploader, ProcessingPipeline
-            
             output_dtype = np.float16 if vector_dtype == "float16" else np.float32
             embedder_key = f"{model_name}::{vector_dtype}"
             embedder = None

visual_rag/__init__.py

@@ -31,7 +31,47 @@ Quick Start:
 Each component works independently - use only what you need.
 """
 
-__version__ = "0.1.0"
+import logging
+
+__version__ = "0.1.3"
+
+
+def setup_logging(level: str = "INFO", format: str = None) -> None:
+    """
+    Configure logging for visual_rag package.
+    
+    Args:
+        level: Log level ("DEBUG", "INFO", "WARNING", "ERROR")
+        format: Custom format string. Default shows time, level, and message.
+    
+    Example:
+        >>> import visual_rag
+        >>> visual_rag.setup_logging("INFO")
+        >>> # Now you'll see processing logs
+    """
+    if format is None:
+        format = "[%(asctime)s] %(levelname)s - %(message)s"
+    
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format=format,
+        datefmt="%H:%M:%S",
+    )
+    
+    # Also set the visual_rag logger specifically
+    logger = logging.getLogger("visual_rag")
+    logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+
+
+# Enable INFO logging by default for visual_rag package and all submodules
+# This ensures logs like "Processing PDF...", "Embedding pages..." are visible
+_logger = logging.getLogger("visual_rag")
+if not _logger.handlers:
+    _handler = logging.StreamHandler()
+    _handler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s", datefmt="%H:%M:%S"))
+    _logger.addHandler(_handler)
+_logger.setLevel(logging.INFO)
+_logger.propagate = False  # Don't duplicate to root logger
 
 # Import main classes at package level for convenience
 # These are optional - if dependencies aren't installed, we catch the error
@@ -95,4 +135,6 @@ __all__ = [
     "load_config",
     "get",
     "get_section",
+    # Logging
+    "setup_logging",
 ]

visual_rag/config.py

@@ -21,16 +21,13 @@ _raw_config_cache_path: Optional[str] = None
 
 
 def _env_qdrant_url() -> Optional[str]:
-    return os.getenv("SIGIR_QDRANT_URL") or os.getenv("DEST_QDRANT_URL") or os.getenv("QDRANT_URL")
+    """Get Qdrant URL from environment. Prefers QDRANT_URL."""
+    return os.getenv("QDRANT_URL") or os.getenv("SIGIR_QDRANT_URL")  # legacy fallback
 
 
 def _env_qdrant_api_key() -> Optional[str]:
-    return (
-        os.getenv("SIGIR_QDRANT_KEY")
-        or os.getenv("SIGIR_QDRANT_API_KEY")
-        or os.getenv("DEST_QDRANT_API_KEY")
-        or os.getenv("QDRANT_API_KEY")
-    )
+    """Get Qdrant API key from environment. Prefers QDRANT_API_KEY."""
+    return os.getenv("QDRANT_API_KEY") or os.getenv("SIGIR_QDRANT_KEY")  # legacy fallback
 
 
 def load_config(

visual_rag/indexing/__init__.py

@@ -8,10 +8,27 @@ Components:
 - ProcessingPipeline: End-to-end PDF → Qdrant pipeline
 """
 
-from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
-from visual_rag.indexing.pdf_processor import PDFProcessor
-from visual_rag.indexing.pipeline import ProcessingPipeline
-from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+# Lazy imports to avoid failures when optional dependencies aren't installed
+
+try:
+    from visual_rag.indexing.pdf_processor import PDFProcessor
+except ImportError:
+    PDFProcessor = None
+
+try:
+    from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+except ImportError:
+    QdrantIndexer = None
+
+try:
+    from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
+except ImportError:
+    CloudinaryUploader = None
+
+try:
+    from visual_rag.indexing.pipeline import ProcessingPipeline
+except ImportError:
+    ProcessingPipeline = None
 
 __all__ = [
     "PDFProcessor",

visual_rag/indexing/qdrant_indexer.py

@@ -19,6 +19,23 @@ from urllib.parse import urlparse
 
 import numpy as np
 
+try:
+    from qdrant_client import QdrantClient
+    from qdrant_client.http import models as qdrant_models
+    from qdrant_client.http.models import Distance, VectorParams
+    from qdrant_client.models import FieldCondition, Filter, MatchValue
+
+    QDRANT_AVAILABLE = True
+except ImportError:
+    QDRANT_AVAILABLE = False
+    QdrantClient = None
+    qdrant_models = None
+    Distance = None
+    VectorParams = None
+    FieldCondition = None
+    Filter = None
+    MatchValue = None
+
 logger = logging.getLogger(__name__)
 
 
@@ -58,9 +75,7 @@ class QdrantIndexer:
         prefer_grpc: bool = False,
         vector_datatype: str = "float32",
     ):
-        try:
-            from qdrant_client import QdrantClient
-        except ImportError:
+        if not QDRANT_AVAILABLE:
             raise ImportError(
                 "Qdrant client not installed. "
                 "Install with: pip install visual-rag-toolkit[qdrant]"
@@ -139,9 +154,6 @@ class QdrantIndexer:
         Returns:
             True if created, False if already existed
         """
-        from qdrant_client.http import models
-        from qdrant_client.http.models import Distance, VectorParams
-
         if self.collection_exists():
             if force_recreate:
                 logger.info(f"🗑️ Deleting existing collection: {self.collection_name}")
@@ -153,15 +165,15 @@ class QdrantIndexer:
         logger.info(f"📦 Creating collection: {self.collection_name}")
 
         # Multi-vector config for ColBERT-style MaxSim
-        multivector_config = models.MultiVectorConfig(
-            comparator=models.MultiVectorComparator.MAX_SIM
+        multivector_config = qdrant_models.MultiVectorConfig(
+            comparator=qdrant_models.MultiVectorComparator.MAX_SIM
         )
 
         # Vector configs - simplified for compatibility
         datatype = (
-            models.Datatype.FLOAT16
+            qdrant_models.Datatype.FLOAT16
             if self.vector_datatype == "float16"
-            else models.Datatype.FLOAT32
+            else qdrant_models.Datatype.FLOAT32
         )
         vectors_config = {
             "initial": VectorParams(
@@ -198,6 +210,18 @@ class QdrantIndexer:
             vectors_config=vectors_config,
         )
 
+        # Create required payload index for skip_existing functionality
+        # This index is needed for filtering by filename when checking existing docs
+        try:
+            self.client.create_payload_index(
+                collection_name=self.collection_name,
+                field_name="filename",
+                field_schema=qdrant_models.PayloadSchemaType.KEYWORD,
+            )
+            logger.info("   📇 Created payload index: filename")
+        except Exception as e:
+            logger.warning(f"   ⚠️ Could not create filename index: {e}")
+
         logger.info(f"✅ Collection created: {self.collection_name}")
         return True
 
@@ -212,14 +236,12 @@ class QdrantIndexer:
             fields: List of {field, type} dicts
                    type can be: integer, keyword, bool, float, text
         """
-        from qdrant_client.http import models
-
         type_mapping = {
-            "integer": models.PayloadSchemaType.INTEGER,
-            "keyword": models.PayloadSchemaType.KEYWORD,
-            "bool": models.PayloadSchemaType.BOOL,
-            "float": models.PayloadSchemaType.FLOAT,
-            "text": models.PayloadSchemaType.TEXT,
+            "integer": qdrant_models.PayloadSchemaType.INTEGER,
+            "keyword": qdrant_models.PayloadSchemaType.KEYWORD,
+            "bool": qdrant_models.PayloadSchemaType.BOOL,
+            "float": qdrant_models.PayloadSchemaType.FLOAT,
+            "text": qdrant_models.PayloadSchemaType.TEXT,
         }
 
         if not fields:
@@ -230,7 +252,7 @@ class QdrantIndexer:
         for field_config in fields:
             field_name = field_config["field"]
             field_type_str = field_config.get("type", "keyword")
-            field_type = type_mapping.get(field_type_str, models.PayloadSchemaType.KEYWORD)
+            field_type = type_mapping.get(field_type_str, qdrant_models.PayloadSchemaType.KEYWORD)
 
             try:
                 self.client.create_payload_index(
@@ -271,8 +293,6 @@ class QdrantIndexer:
         Returns:
             Number of successfully uploaded points
         """
-        from qdrant_client.http import models
-
         if not points:
             return 0
 
@@ -315,8 +335,8 @@ class QdrantIndexer:
                 return val.tolist()
             return val
 
-        def _build_qdrant_points(batch_points: List[Dict[str, Any]]) -> List[models.PointStruct]:
-            qdrant_points: List[models.PointStruct] = []
+        def _build_qdrant_points(batch_points: List[Dict[str, Any]]) -> List[qdrant_models.PointStruct]:
+            qdrant_points: List[qdrant_models.PointStruct] = []
             for p in batch_points:
                 global_pooled = p.get("global_pooled_embedding")
                 if global_pooled is None:
@@ -336,7 +356,7 @@ class QdrantIndexer:
                 global_pooling = global_pooled.astype(self._np_vector_dtype, copy=False)
 
                 qdrant_points.append(
-                    models.PointStruct(
+                    qdrant_models.PointStruct(
                         id=p["id"],
                         vector={
                             "initial": _to_list(initial),
@@ -361,6 +381,8 @@ class QdrantIndexer:
                     wait=wait,
                 )
 
+                logger.info(f"   ✅ Uploaded {len(points)} points to Qdrant")
+                
                 if delay_between_batches > 0:
                     if _is_cancelled():
                         return 0
@@ -413,32 +435,60 @@ class QdrantIndexer:
             return False
 
     def get_existing_ids(self, filename: str) -> Set[str]:
-        """Get all point IDs for a specific file."""
-        from qdrant_client.models import FieldCondition, Filter, MatchValue
+        """Get all point IDs for a specific file.
 
+        Requires a payload index on 'filename' field. If the index doesn't exist,
+        this method will attempt to create it automatically.
+        """
         existing_ids = set()
         offset = None
 
-        while True:
-            results = self.client.scroll(
-                collection_name=self.collection_name,
-                scroll_filter=Filter(
-                    must=[FieldCondition(key="filename", match=MatchValue(value=filename))]
-                ),
-                limit=100,
-                offset=offset,
-                with_payload=["page_number"],
-                with_vectors=False,
-            )
+        try:
+            while True:
+                results = self.client.scroll(
+                    collection_name=self.collection_name,
+                    scroll_filter=Filter(
+                        must=[FieldCondition(key="filename", match=MatchValue(value=filename))]
+                    ),
+                    limit=100,
+                    offset=offset,
+                    with_payload=["page_number"],
+                    with_vectors=False,
+                )
+
+                points, next_offset = results
 
-            points, next_offset = results
+                for point in points:
+                    existing_ids.add(str(point.id))
 
-            for point in points:
-                existing_ids.add(str(point.id))
+                if next_offset is None or len(points) == 0:
+                    break
+                offset = next_offset
 
-            if next_offset is None or len(points) == 0:
-                break
-            offset = next_offset
+        except Exception as e:
+            error_msg = str(e).lower()
+            if "index required" in error_msg or "index" in error_msg and "filename" in error_msg:
+                # Missing payload index - try to create it
+                logger.warning(
+                    "⚠️ Missing 'filename' payload index. Creating it now... "
+                    "(skip_existing requires this index for filtering)"
+                )
+                try:
+                    self.client.create_payload_index(
+                        collection_name=self.collection_name,
+                        field_name="filename",
+                        field_schema=qdrant_models.PayloadSchemaType.KEYWORD,
+                    )
+                    logger.info("   ✅ Created 'filename' index. Retrying query...")
+                    # Retry the query
+                    return self.get_existing_ids(filename)
+                except Exception as idx_err:
+                    logger.warning(f"   ❌ Could not create index: {idx_err}")
+                    logger.warning("   Returning empty set - all pages will be processed")
+                    return set()
+            else:
+                logger.warning(f"⚠️ Error checking existing IDs: {e}")
+                return set()
 
         return existing_ids
 

visual_rag/retrieval/multi_vector.py

@@ -2,6 +2,25 @@ import os
 from typing import Any, Dict, List, Optional
 from urllib.parse import urlparse
 
+import numpy as np
+import torch
+
+try:
+    from dotenv import load_dotenv
+
+    DOTENV_AVAILABLE = True
+except ImportError:
+    DOTENV_AVAILABLE = False
+    load_dotenv = None
+
+try:
+    from qdrant_client import QdrantClient
+
+    QDRANT_AVAILABLE = True
+except ImportError:
+    QDRANT_AVAILABLE = False
+    QdrantClient = None
+
 from visual_rag.embedding.visual_embedder import VisualEmbedder
 from visual_rag.retrieval.single_stage import SingleStageRetriever
 from visual_rag.retrieval.three_stage import ThreeStageRetriever
@@ -11,9 +30,7 @@ from visual_rag.retrieval.two_stage import TwoStageRetriever
 class MultiVectorRetriever:
     @staticmethod
     def _maybe_load_dotenv() -> None:
-        try:
-            from dotenv import load_dotenv
-        except ImportError:
+        if not DOTENV_AVAILABLE:
             return
         if os.path.exists(".env"):
             load_dotenv(".env")
@@ -33,87 +50,84 @@ class MultiVectorRetriever:
     ):
         if qdrant_client is None:
             self._maybe_load_dotenv()
-            try:
-                from qdrant_client import QdrantClient
-            except ImportError as e:
+            if not QDRANT_AVAILABLE:
                 raise ImportError(
                     "Qdrant client not installed. Install with: pip install visual-rag-toolkit[qdrant]"
-                ) from e
+                )
 
             qdrant_url = (
                 qdrant_url
-                or os.getenv("SIGIR_QDRANT_URL")
-                or os.getenv("DEST_QDRANT_URL")
                 or os.getenv("QDRANT_URL")
+                or os.getenv("SIGIR_QDRANT_URL")  # legacy
             )
             if not qdrant_url:
                 raise ValueError(
-                    "QDRANT_URL is required (pass qdrant_url or set env var). "
-                    "You can also set DEST_QDRANT_URL to override."
+                    "QDRANT_URL is required (pass qdrant_url or set env var)."
                 )
 
             qdrant_api_key = (
                 qdrant_api_key
-                or os.getenv("SIGIR_QDRANT_KEY")
-                or os.getenv("SIGIR_QDRANT_API_KEY")
-                or os.getenv("DEST_QDRANT_API_KEY")
                 or os.getenv("QDRANT_API_KEY")
+                or os.getenv("SIGIR_QDRANT_KEY")  # legacy
             )
 
             grpc_port = None
             if prefer_grpc:
                 try:
-                    if urlparse(qdrant_url).port == 6333:
+                    parsed = urlparse(qdrant_url)
+                    port = parsed.port
+                    if port == 6333:
                         grpc_port = 6334
                 except Exception:
-                    grpc_port = None
+                    pass
 
             def _make_client(use_grpc: bool):
                 return QdrantClient(
                     url=qdrant_url,
                     api_key=qdrant_api_key,
+                    timeout=request_timeout,
                     prefer_grpc=bool(use_grpc),
                     grpc_port=grpc_port,
-                    timeout=int(request_timeout),
                     check_compatibility=False,
                 )
 
-            qdrant_client = _make_client(prefer_grpc)
+            client = _make_client(prefer_grpc)
             if prefer_grpc:
                 try:
-                    _ = qdrant_client.get_collections()
+                    _ = client.get_collections()
                 except Exception as e:
                     msg = str(e)
-                    if (
-                        "StatusCode.PERMISSION_DENIED" in msg
-                        or "http2 header with status: 403" in msg
-                    ):
-                        qdrant_client = _make_client(False)
+                    if "StatusCode.PERMISSION_DENIED" in msg or "http2 header with status: 403" in msg:
+                        client = _make_client(False)
                     else:
                         raise
+            qdrant_client = client
 
         self.client = qdrant_client
         self.collection_name = collection_name
+
         self.embedder = embedder or VisualEmbedder(model_name=model_name)
 
         self._two_stage = TwoStageRetriever(
-            self.client,
-            collection_name=self.collection_name,
-            request_timeout=int(request_timeout),
-            max_retries=int(max_retries),
-            retry_sleep=float(retry_sleep),
+            qdrant_client=qdrant_client,
+            collection_name=collection_name,
+            request_timeout=request_timeout,
+            max_retries=max_retries,
+            retry_sleep=retry_sleep,
         )
         self._three_stage = ThreeStageRetriever(
-            self.client,
-            collection_name=self.collection_name,
-            request_timeout=int(request_timeout),
-            max_retries=int(max_retries),
-            retry_sleep=float(retry_sleep),
+            qdrant_client=qdrant_client,
+            collection_name=collection_name,
+            request_timeout=request_timeout,
+            max_retries=max_retries,
+            retry_sleep=retry_sleep,
         )
         self._single_stage = SingleStageRetriever(
-            self.client,
-            collection_name=self.collection_name,
-            request_timeout=int(request_timeout),
+            qdrant_client=qdrant_client,
+            collection_name=collection_name,
+            request_timeout=request_timeout,
+            max_retries=max_retries,
+            retry_sleep=retry_sleep,
         )
 
     def build_filter(
@@ -143,14 +157,10 @@ class MultiVectorRetriever:
         return_embeddings: bool = False,
     ) -> List[Dict[str, Any]]:
         q = self.embedder.embed_query(query)
-        try:
-            import torch
-        except ImportError:
-            torch = None
-        if torch is not None and isinstance(q, torch.Tensor):
+        if isinstance(q, torch.Tensor):
             query_embedding = q.detach().cpu().numpy()
         else:
-            query_embedding = q.numpy()
+            query_embedding = np.asarray(q)
 
         return self.search_embedded(
             query_embedding=query_embedding,
@@ -179,27 +189,17 @@ class MultiVectorRetriever:
             return self._single_stage.search(
                 query_embedding=query_embedding,
                 top_k=top_k,
-                strategy="multi_vector",
                 filter_obj=filter_obj,
+                using="initial",
             )
-
-        if mode == "single_tiles":
+        elif mode == "single_pooled":
             return self._single_stage.search(
                 query_embedding=query_embedding,
                 top_k=top_k,
-                strategy="tiles_maxsim",
                 filter_obj=filter_obj,
+                using="mean_pooling",
             )
-
-        if mode == "single_global":
-            return self._single_stage.search(
-                query_embedding=query_embedding,
-                top_k=top_k,
-                strategy="pooled_global",
-                filter_obj=filter_obj,
-            )
-
-        if mode == "two_stage":
+        elif mode == "two_stage":
             return self._two_stage.search_server_side(
                 query_embedding=query_embedding,
                 top_k=top_k,
@@ -207,16 +207,14 @@ class MultiVectorRetriever:
                 filter_obj=filter_obj,
                 stage1_mode=stage1_mode,
             )
-
-        if mode == "three_stage":
-            s1 = int(stage1_k) if stage1_k is not None else 1000
-            s2 = int(stage2_k) if stage2_k is not None else 300
+        elif mode == "three_stage":
             return self._three_stage.search_server_side(
                 query_embedding=query_embedding,
                 top_k=top_k,
-                stage1_k=s1,
-                stage2_k=s2,
+                stage1_k=stage1_k,
+                stage2_k=stage2_k,
                 filter_obj=filter_obj,
+                stage1_mode=stage1_mode,
             )
-
-        raise ValueError(f"Unknown mode: {mode}")
+        else:
+            raise ValueError(f"Unknown mode: {mode}")

visual_rag/retrieval/single_stage.py

@@ -30,6 +30,9 @@ class SingleStageRetriever:
     Args:
         qdrant_client: Connected Qdrant client
         collection_name: Name of the Qdrant collection
+        request_timeout: Timeout for Qdrant requests (seconds)
+        max_retries: Number of retry attempts on failure
+        retry_sleep: Sleep time between retries (seconds)
 
     Example:
         >>> retriever = SingleStageRetriever(client, "my_collection")
@@ -41,10 +44,14 @@ class SingleStageRetriever:
         qdrant_client,
         collection_name: str,
         request_timeout: int = 120,
+        max_retries: int = 3,
+        retry_sleep: float = 1.0,
     ):
         self.client = qdrant_client
         self.collection_name = collection_name
         self.request_timeout = int(request_timeout)
+        self.max_retries = max_retries
+        self.retry_sleep = retry_sleep
 
     def search(
         self,

visual_rag/retrieval/two_stage.py

@@ -17,11 +17,17 @@ Research Context:
 """
 
 import logging
+import time
 from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 import torch
 
+from qdrant_client.http import models as qdrant_models
+from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
+
+from visual_rag.embedding.pooling import compute_maxsim_score
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +88,6 @@ class TwoStageRetriever:
         self.retry_sleep = float(retry_sleep)
 
     def _retry_call(self, fn):
-        import time
-
         last_err = None
         for attempt in range(self.max_retries):
             try:
@@ -120,8 +124,6 @@ class TwoStageRetriever:
         Returns:
             List of results with scores
         """
-        from qdrant_client.http import models
-
         query_np = self._to_numpy(query_embedding)
 
         if prefetch_k is None:
@@ -155,9 +157,9 @@ class TwoStageRetriever:
                 limit=top_k,
                 query_filter=filter_obj,
                 with_payload=True,
-                search_params=models.SearchParams(exact=True),
+                search_params=qdrant_models.SearchParams(exact=True),
                 prefetch=[
-                    models.Prefetch(
+                    qdrant_models.Prefetch(
                         query=prefetch_query,
                         using=prefetch_using,
                         limit=prefetch_k,
@@ -363,8 +365,6 @@ class TwoStageRetriever:
         return_embeddings: bool = False,
     ) -> List[Dict[str, Any]]:
         """Stage 2: Rerank with full multi-vector MaxSim scoring."""
-        from visual_rag.embedding.pooling import compute_maxsim_score
-
         # Fetch full embeddings for candidates
         candidate_ids = [c["id"] for c in candidates]
 
@@ -435,8 +435,6 @@ class TwoStageRetriever:
 
         Supports single values or lists (using MatchAny).
         """
-        from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
-
         conditions = []
 
         if year is not None: