visual-rag-toolkit 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

demo/ui/playground.py

@@ -9,6 +9,7 @@ from demo.qdrant_utils import (
     sample_points_cached,
     search_collection,
 )
+from visual_rag.retrieval import MultiVectorRetriever
 
 
 def render_playground_tab():
@@ -46,7 +47,6 @@ def render_playground_tab():
         if not st.session_state.get("model_loaded"):
             with st.spinner(f"Loading {model_short}..."):
                 try:
-                    from visual_rag.retrieval import MultiVectorRetriever
                     _ = MultiVectorRetriever(collection_name=active_collection, model_name=model_name)
                     st.session_state["model_loaded"] = True
                     st.session_state["loaded_model_key"] = cache_key

demo/ui/sidebar.py

@@ -3,6 +3,8 @@
 import os
 import streamlit as st
 
+from qdrant_client.models import VectorParamsDiff
+
 from demo.qdrant_utils import (
     get_qdrant_credentials,
     init_qdrant_client_with_creds,
@@ -14,11 +16,33 @@ from demo.qdrant_utils import (
 
 
 def render_sidebar():
+    # CSS to make sidebar metrics smaller
+    st.markdown("""
+    <style>
+    /* Smaller metrics in sidebar */
+    [data-testid="stSidebar"] [data-testid="stMetricValue"] {
+        font-size: 1.2rem !important;
+    }
+    [data-testid="stSidebar"] [data-testid="stMetricLabel"] {
+        font-size: 0.75rem !important;
+    }
+    /* Smaller expander headers in sidebar */
+    [data-testid="stSidebar"] [data-testid="stExpander"] summary {
+        font-size: 0.9rem !important;
+    }
+    /* Compact subheaders */
+    [data-testid="stSidebar"] h3 {
+        font-size: 1rem !important;
+        margin-bottom: 0.5rem !important;
+    }
+    </style>
+    """, unsafe_allow_html=True)
+    
     with st.sidebar:
         st.subheader("🔑 Qdrant Credentials")
         
-        env_url = os.getenv("SIGIR_QDRANT_URL") or os.getenv("DEST_QDRANT_URL") or os.getenv("QDRANT_URL") or ""
-        env_key = os.getenv("SIGIR_QDRANT_KEY") or os.getenv("SIGIR_QDRANT_API_KEY") or os.getenv("DEST_QDRANT_API_KEY") or os.getenv("QDRANT_API_KEY") or ""
+        env_url = os.getenv("QDRANT_URL") or os.getenv("SIGIR_QDRANT_URL") or ""
+        env_key = os.getenv("QDRANT_API_KEY") or os.getenv("SIGIR_QDRANT_KEY") or ""
         
         if "qdrant_url_input" not in st.session_state:
             st.session_state["qdrant_url_input"] = env_url
@@ -136,7 +160,6 @@ def render_sidebar():
                         if target_in_ram != current_in_ram:
                             if st.button("💾 Apply Change", key="admin_apply"):
                                 try:
-                                    from qdrant_client.models import VectorParamsDiff
                                     client.update_collection(
                                         collection_name=active,
                                         vectors_config={sel_vec: VectorParamsDiff(on_disk=not target_in_ram)}

demo/ui/upload.py

@@ -9,6 +9,7 @@ import inspect
 from datetime import datetime
 from pathlib import Path
 
+import numpy as np
 import streamlit as st
 
 from demo.config import AVAILABLE_MODELS
@@ -17,6 +18,10 @@ from demo.qdrant_utils import (
     get_collection_stats,
     sample_points_cached,
 )
+from visual_rag.embedding.visual_embedder import VisualEmbedder
+from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
+from visual_rag.indexing.pipeline import ProcessingPipeline
 
 
 VECTOR_TYPES = ["initial", "mean_pooling", "experimental_pooling", "global_pooling"]
@@ -251,10 +256,6 @@ def process_pdfs(uploaded_files, config):
             model_short = model_name.split("/")[-1]
             model_status.info(f"Loading `{model_short}`...")
             
-            import numpy as np
-            from visual_rag import VisualEmbedder
-            from visual_rag.indexing import QdrantIndexer, CloudinaryUploader, ProcessingPipeline
-            
             output_dtype = np.float16 if vector_dtype == "float16" else np.float32
             embedder_key = f"{model_name}::{vector_dtype}"
             embedder = None
@@ -448,7 +449,7 @@ def process_pdfs(uploaded_files, config):
         
         if total_uploaded > 0:
             st.session_state["upload_success"] = f"Uploaded {total_uploaded} pages to {collection_name}"
-            st.balloons()
+            st.rerun()  # Immediately refresh to show success toast + balloons
             
     except Exception as e:
         st.error(f"❌ Processing error: {e}")

visual_rag/__init__.py

@@ -31,7 +31,47 @@ Quick Start:
 Each component works independently - use only what you need.
 """
 
-__version__ = "0.1.0"
+import logging
+
+__version__ = "0.1.4"
+
+
+def setup_logging(level: str = "INFO", format: str = None) -> None:
+    """
+    Configure logging for visual_rag package.
+
+    Args:
+        level: Log level ("DEBUG", "INFO", "WARNING", "ERROR")
+        format: Custom format string. Default shows time, level, and message.
+
+    Example:
+        >>> import visual_rag
+        >>> visual_rag.setup_logging("INFO")
+        >>> # Now you'll see processing logs
+    """
+    if format is None:
+        format = "[%(asctime)s] %(levelname)s - %(message)s"
+
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format=format,
+        datefmt="%H:%M:%S",
+    )
+
+    # Also set the visual_rag logger specifically
+    logger = logging.getLogger("visual_rag")
+    logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+
+
+# Enable INFO logging by default for visual_rag package and all submodules
+# This ensures logs like "Processing PDF...", "Embedding pages..." are visible
+_logger = logging.getLogger("visual_rag")
+if not _logger.handlers:
+    _handler = logging.StreamHandler()
+    _handler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s", datefmt="%H:%M:%S"))
+    _logger.addHandler(_handler)
+_logger.setLevel(logging.INFO)
+_logger.propagate = False  # Don't duplicate to root logger
 
 # Import main classes at package level for convenience
 # These are optional - if dependencies aren't installed, we catch the error
@@ -71,13 +111,16 @@ try:
 except ImportError:
     QdrantAdmin = None
 
-try:
-    from visual_rag.demo_runner import demo
-except ImportError:
-    demo = None
+# demo is lazily imported to avoid RuntimeWarning when running as __main__
+# Access via visual_rag.demo() which triggers __getattr__
 
 # Config utilities (always available)
-from visual_rag.config import get, get_section, load_config
+try:
+    from visual_rag.config import get, get_section, load_config
+except ImportError:
+    get = None
+    get_section = None
+    load_config = None
 
 __all__ = [
     # Version
@@ -95,4 +138,18 @@ __all__ = [
     "load_config",
     "get",
     "get_section",
+    # Logging
+    "setup_logging",
 ]
+
+
+def __getattr__(name: str):
+    """Lazy import for demo to avoid RuntimeWarning when running as __main__."""
+    if name == "demo":
+        try:
+            from visual_rag.demo_runner import demo
+
+            return demo
+        except ImportError:
+            return None
+    raise AttributeError(f"module 'visual_rag' has no attribute {name!r}")

visual_rag/config.py

@@ -21,16 +21,13 @@ _raw_config_cache_path: Optional[str] = None
 
 
 def _env_qdrant_url() -> Optional[str]:
-    return os.getenv("SIGIR_QDRANT_URL") or os.getenv("DEST_QDRANT_URL") or os.getenv("QDRANT_URL")
+    """Get Qdrant URL from environment. Prefers QDRANT_URL."""
+    return os.getenv("QDRANT_URL") or os.getenv("SIGIR_QDRANT_URL")  # legacy fallback
 
 
 def _env_qdrant_api_key() -> Optional[str]:
-    return (
-        os.getenv("SIGIR_QDRANT_KEY")
-        or os.getenv("SIGIR_QDRANT_API_KEY")
-        or os.getenv("DEST_QDRANT_API_KEY")
-        or os.getenv("QDRANT_API_KEY")
-    )
+    """Get Qdrant API key from environment. Prefers QDRANT_API_KEY."""
+    return os.getenv("QDRANT_API_KEY") or os.getenv("SIGIR_QDRANT_KEY")  # legacy fallback
 
 
 def load_config(

visual_rag/demo_runner.py

@@ -52,13 +52,11 @@ def demo(
     cmd = [sys.executable, "-m", "streamlit", "run", str(app_path)]
     cmd += ["--server.address", str(host)]
     cmd += ["--server.port", str(int(port))]
-    cmd += ["--server.headless", "true" if headless else "false"]
+    # headless=true prevents browser from auto-opening; open_browser overrides
+    should_be_headless = headless and not open_browser
+    cmd += ["--server.headless", "true" if should_be_headless else "false"]
     cmd += ["--browser.gatherUsageStats", "false"]
     cmd += ["--server.runOnSave", "false"]
-    cmd += ["--browser.serverAddress", str(host)]
-    if not open_browser:
-        cmd += ["--browser.serverPort", str(int(port))]
-        cmd += ["--browser.open", "false"]
 
     if extra_args:
         cmd += list(extra_args)

visual_rag/indexing/__init__.py

@@ -8,10 +8,27 @@ Components:
 - ProcessingPipeline: End-to-end PDF → Qdrant pipeline
 """
 
-from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
-from visual_rag.indexing.pdf_processor import PDFProcessor
-from visual_rag.indexing.pipeline import ProcessingPipeline
-from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+# Lazy imports to avoid failures when optional dependencies aren't installed
+
+try:
+    from visual_rag.indexing.pdf_processor import PDFProcessor
+except ImportError:
+    PDFProcessor = None
+
+try:
+    from visual_rag.indexing.qdrant_indexer import QdrantIndexer
+except ImportError:
+    QdrantIndexer = None
+
+try:
+    from visual_rag.indexing.cloudinary_uploader import CloudinaryUploader
+except ImportError:
+    CloudinaryUploader = None
+
+try:
+    from visual_rag.indexing.pipeline import ProcessingPipeline
+except ImportError:
+    ProcessingPipeline = None
 
 __all__ = [
     "PDFProcessor",

visual_rag/indexing/qdrant_indexer.py

@@ -19,6 +19,23 @@ from urllib.parse import urlparse
 
 import numpy as np
 
+try:
+    from qdrant_client import QdrantClient
+    from qdrant_client.http import models as qdrant_models
+    from qdrant_client.http.models import Distance, VectorParams
+    from qdrant_client.models import FieldCondition, Filter, MatchValue
+
+    QDRANT_AVAILABLE = True
+except ImportError:
+    QDRANT_AVAILABLE = False
+    QdrantClient = None
+    qdrant_models = None
+    Distance = None
+    VectorParams = None
+    FieldCondition = None
+    Filter = None
+    MatchValue = None
+
 logger = logging.getLogger(__name__)
 
 
@@ -58,9 +75,7 @@ class QdrantIndexer:
         prefer_grpc: bool = False,
         vector_datatype: str = "float32",
     ):
-        try:
-            from qdrant_client import QdrantClient
-        except ImportError:
+        if not QDRANT_AVAILABLE:
             raise ImportError(
                 "Qdrant client not installed. "
                 "Install with: pip install visual-rag-toolkit[qdrant]"
@@ -139,9 +154,6 @@ class QdrantIndexer:
         Returns:
             True if created, False if already existed
         """
-        from qdrant_client.http import models
-        from qdrant_client.http.models import Distance, VectorParams
-
         if self.collection_exists():
             if force_recreate:
                 logger.info(f"🗑️ Deleting existing collection: {self.collection_name}")
@@ -153,15 +165,15 @@ class QdrantIndexer:
         logger.info(f"📦 Creating collection: {self.collection_name}")
 
         # Multi-vector config for ColBERT-style MaxSim
-        multivector_config = models.MultiVectorConfig(
-            comparator=models.MultiVectorComparator.MAX_SIM
+        multivector_config = qdrant_models.MultiVectorConfig(
+            comparator=qdrant_models.MultiVectorComparator.MAX_SIM
         )
 
         # Vector configs - simplified for compatibility
         datatype = (
-            models.Datatype.FLOAT16
+            qdrant_models.Datatype.FLOAT16
             if self.vector_datatype == "float16"
-            else models.Datatype.FLOAT32
+            else qdrant_models.Datatype.FLOAT32
         )
         vectors_config = {
             "initial": VectorParams(
@@ -198,6 +210,18 @@ class QdrantIndexer:
             vectors_config=vectors_config,
         )
 
+        # Create required payload index for skip_existing functionality
+        # This index is needed for filtering by filename when checking existing docs
+        try:
+            self.client.create_payload_index(
+                collection_name=self.collection_name,
+                field_name="filename",
+                field_schema=qdrant_models.PayloadSchemaType.KEYWORD,
+            )
+            logger.info("   📇 Created payload index: filename")
+        except Exception as e:
+            logger.warning(f"   ⚠️ Could not create filename index: {e}")
+
         logger.info(f"✅ Collection created: {self.collection_name}")
         return True
 
@@ -212,14 +236,12 @@ class QdrantIndexer:
             fields: List of {field, type} dicts
                    type can be: integer, keyword, bool, float, text
         """
-        from qdrant_client.http import models
-
         type_mapping = {
-            "integer": models.PayloadSchemaType.INTEGER,
-            "keyword": models.PayloadSchemaType.KEYWORD,
-            "bool": models.PayloadSchemaType.BOOL,
-            "float": models.PayloadSchemaType.FLOAT,
-            "text": models.PayloadSchemaType.TEXT,
+            "integer": qdrant_models.PayloadSchemaType.INTEGER,
+            "keyword": qdrant_models.PayloadSchemaType.KEYWORD,
+            "bool": qdrant_models.PayloadSchemaType.BOOL,
+            "float": qdrant_models.PayloadSchemaType.FLOAT,
+            "text": qdrant_models.PayloadSchemaType.TEXT,
         }
 
         if not fields:
@@ -230,7 +252,7 @@ class QdrantIndexer:
         for field_config in fields:
             field_name = field_config["field"]
             field_type_str = field_config.get("type", "keyword")
-            field_type = type_mapping.get(field_type_str, models.PayloadSchemaType.KEYWORD)
+            field_type = type_mapping.get(field_type_str, qdrant_models.PayloadSchemaType.KEYWORD)
 
             try:
                 self.client.create_payload_index(
@@ -271,8 +293,6 @@ class QdrantIndexer:
         Returns:
             Number of successfully uploaded points
         """
-        from qdrant_client.http import models
-
         if not points:
             return 0
 
@@ -315,8 +335,10 @@ class QdrantIndexer:
                 return val.tolist()
             return val
 
-        def _build_qdrant_points(batch_points: List[Dict[str, Any]]) -> List[models.PointStruct]:
-            qdrant_points: List[models.PointStruct] = []
+        def _build_qdrant_points(
+            batch_points: List[Dict[str, Any]],
+        ) -> List[qdrant_models.PointStruct]:
+            qdrant_points: List[qdrant_models.PointStruct] = []
             for p in batch_points:
                 global_pooled = p.get("global_pooled_embedding")
                 if global_pooled is None:
@@ -336,7 +358,7 @@ class QdrantIndexer:
                 global_pooling = global_pooled.astype(self._np_vector_dtype, copy=False)
 
                 qdrant_points.append(
-                    models.PointStruct(
+                    qdrant_models.PointStruct(
                         id=p["id"],
                         vector={
                             "initial": _to_list(initial),
@@ -361,6 +383,8 @@ class QdrantIndexer:
                     wait=wait,
                 )
 
+                logger.info(f"   ✅ Uploaded {len(points)} points to Qdrant")
+
                 if delay_between_batches > 0:
                     if _is_cancelled():
                         return 0
@@ -413,32 +437,60 @@ class QdrantIndexer:
             return False
 
     def get_existing_ids(self, filename: str) -> Set[str]:
-        """Get all point IDs for a specific file."""
-        from qdrant_client.models import FieldCondition, Filter, MatchValue
+        """Get all point IDs for a specific file.
 
+        Requires a payload index on 'filename' field. If the index doesn't exist,
+        this method will attempt to create it automatically.
+        """
         existing_ids = set()
         offset = None
 
-        while True:
-            results = self.client.scroll(
-                collection_name=self.collection_name,
-                scroll_filter=Filter(
-                    must=[FieldCondition(key="filename", match=MatchValue(value=filename))]
-                ),
-                limit=100,
-                offset=offset,
-                with_payload=["page_number"],
-                with_vectors=False,
-            )
+        try:
+            while True:
+                results = self.client.scroll(
+                    collection_name=self.collection_name,
+                    scroll_filter=Filter(
+                        must=[FieldCondition(key="filename", match=MatchValue(value=filename))]
+                    ),
+                    limit=100,
+                    offset=offset,
+                    with_payload=["page_number"],
+                    with_vectors=False,
+                )
 
-            points, next_offset = results
+                points, next_offset = results
 
-            for point in points:
-                existing_ids.add(str(point.id))
+                for point in points:
+                    existing_ids.add(str(point.id))
 
-            if next_offset is None or len(points) == 0:
-                break
-            offset = next_offset
+                if next_offset is None or len(points) == 0:
+                    break
+                offset = next_offset
+
+        except Exception as e:
+            error_msg = str(e).lower()
+            if "index required" in error_msg or "index" in error_msg and "filename" in error_msg:
+                # Missing payload index - try to create it
+                logger.warning(
+                    "⚠️ Missing 'filename' payload index. Creating it now... "
+                    "(skip_existing requires this index for filtering)"
+                )
+                try:
+                    self.client.create_payload_index(
+                        collection_name=self.collection_name,
+                        field_name="filename",
+                        field_schema=qdrant_models.PayloadSchemaType.KEYWORD,
+                    )
+                    logger.info("   ✅ Created 'filename' index. Retrying query...")
+                    # Retry the query
+                    return self.get_existing_ids(filename)
+                except Exception as idx_err:
+                    logger.warning(f"   ❌ Could not create index: {idx_err}")
+                    logger.warning("   Returning empty set - all pages will be processed")
+                    return set()
+            else:
+                logger.warning(f"⚠️ Error checking existing IDs: {e}")
+                return set()
 
         return existing_ids