visual-rag-toolkit 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

visual_rag/retrieval/two_stage.py

@@ -17,11 +17,17 @@ Research Context:
 """
 
 import logging
+import time
 from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 import torch
 
+from qdrant_client.http import models as qdrant_models
+from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
+
+from visual_rag.embedding.pooling import compute_maxsim_score
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +88,6 @@ class TwoStageRetriever:
         self.retry_sleep = float(retry_sleep)
 
     def _retry_call(self, fn):
-        import time
-
         last_err = None
         for attempt in range(self.max_retries):
             try:
@@ -120,8 +124,6 @@ class TwoStageRetriever:
         Returns:
             List of results with scores
         """
-        from qdrant_client.http import models
-
         query_np = self._to_numpy(query_embedding)
 
         if prefetch_k is None:
@@ -155,9 +157,9 @@ class TwoStageRetriever:
                 limit=top_k,
                 query_filter=filter_obj,
                 with_payload=True,
-                search_params=models.SearchParams(exact=True),
+                search_params=qdrant_models.SearchParams(exact=True),
                 prefetch=[
-                    models.Prefetch(
+                    qdrant_models.Prefetch(
                         query=prefetch_query,
                         using=prefetch_using,
                         limit=prefetch_k,
@@ -363,8 +365,6 @@ class TwoStageRetriever:
         return_embeddings: bool = False,
     ) -> List[Dict[str, Any]]:
         """Stage 2: Rerank with full multi-vector MaxSim scoring."""
-        from visual_rag.embedding.pooling import compute_maxsim_score
-
         # Fetch full embeddings for candidates
         candidate_ids = [c["id"] for c in candidates]
 
@@ -435,8 +435,6 @@ class TwoStageRetriever:
 
         Supports single values or lists (using MatchAny).
         """
-        from qdrant_client.models import FieldCondition, Filter, MatchAny, MatchValue
-
         conditions = []
 
         if year is not None:

{visual_rag_toolkit-0.1.2.dist-info → visual_rag_toolkit-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: visual-rag-toolkit
-Version: 0.1.2
+Version: 0.1.3
 Summary: End-to-end visual document retrieval with ColPali, featuring two-stage pooling for scalable search
 Project-URL: Homepage, https://github.com/Ara-Yeroyan/visual-rag-toolkit
 Project-URL: Documentation, https://github.com/Ara-Yeroyan/visual-rag-toolkit#readme
@@ -88,11 +88,6 @@ Description-Content-Type: text/markdown
 [![PyPI](https://img.shields.io/pypi/v/visual-rag-toolkit)](https://pypi.org/project/visual-rag-toolkit/)
 [![Python](https://img.shields.io/pypi/pyversions/visual-rag-toolkit)](https://pypi.org/project/visual-rag-toolkit/)
 [![License](https://img.shields.io/pypi/l/visual-rag-toolkit)](LICENSE)
-[![CI](https://img.shields.io/github/actions/workflow/status/Ara-Yeroyan/visual-rag-toolkit/ci.yaml?branch=main)](https://github.com/Ara-Yeroyan/visual-rag-toolkit/actions/workflows/ci.yaml)
-
-Note:
-- The **PyPI badge** shows “not found” until the first release is published.
-- The **CI badge** requires the GitHub repo to be **public** (GitHub does not serve Actions badges for private repos).
 
 End-to-end visual document retrieval toolkit featuring **fast multi-stage retrieval** (prefetch with pooled vectors + exact MaxSim reranking).
 
@@ -162,7 +157,7 @@ for r in results[:3]:
 
 ### End-to-end: ingest PDFs (with cropping) → index in Qdrant
 
-This is the “SDK-style” pipeline: PDF → images → optional crop → embed → store vectors + payload in Qdrant.
+This is the "SDK-style" pipeline: PDF → images → optional crop → embed → store vectors + payload in Qdrant.
 
 ```python
 import os
@@ -174,8 +169,8 @@ import torch
 from visual_rag import VisualEmbedder
 from visual_rag.indexing import ProcessingPipeline, QdrantIndexer
 
-QDRANT_URL = os.environ["SIGIR_QDRANT_URL"]  # or QDRANT_URL
-QDRANT_KEY = os.getenv("SIGIR_QDRANT_KEY", "")  # or QDRANT_API_KEY
+QDRANT_URL = os.environ["QDRANT_URL"]
+QDRANT_KEY = os.getenv("QDRANT_API_KEY", "")
 
 collection = "my_visual_docs"
 
@@ -193,6 +188,8 @@ indexer = QdrantIndexer(
     prefer_grpc=True,
     vector_datatype="float16",
 )
+
+# Creates collection + required payload indexes (e.g., "filename" for skip_existing)
 indexer.create_collection(force_recreate=False)
 
 pipeline = ProcessingPipeline(
@@ -208,19 +205,32 @@ pipeline = ProcessingPipeline(
 
 pdfs = [Path("docs/a.pdf"), Path("docs/b.pdf")]
 for pdf_path in pdfs:
-    pipeline.process_pdf(
+    result = pipeline.process_pdf(
         pdf_path,
-        skip_existing=True,
+        skip_existing=True,  # Skip pages already in Qdrant (uses filename index)
         upload_to_cloudinary=False,
         upload_to_qdrant=True,
     )
+    # Logs automatically shown:
+    # [10:23:45] 📚 Processing PDF: a.pdf
+    # [10:23:45] 🖼️ Converting PDF to images...
+    # [10:23:46]    ✅ Converted 12 pages
+    # [10:23:46] 📦 Processing pages 1-8/12
+    # [10:23:46] 🤖 Generating embeddings for 8 pages...
+    # [10:23:48] 📤 Uploading batch of 8 pages...
+    # [10:23:48]    ✅ Uploaded 8 points to Qdrant
+    # [10:23:48] 📦 Processing pages 9-12/12
+    # [10:23:48] 🤖 Generating embeddings for 4 pages...
+    # [10:23:50] 📤 Uploading batch of 4 pages...
+    # [10:23:50]    ✅ Uploaded 4 points to Qdrant
+    # [10:23:50] ✅ Completed a.pdf: 12 uploaded, 0 skipped, 0 failed
 ```
 
 CLI equivalent:
 
 ```bash
-export SIGIR_QDRANT_URL="https://YOUR_QDRANT"
-export SIGIR_QDRANT_KEY="YOUR_KEY"
+export QDRANT_URL="https://YOUR_QDRANT"
+export QDRANT_API_KEY="YOUR_KEY"
 
 visual-rag process \
   --reports-dir ./docs \
@@ -263,7 +273,7 @@ Stage 2: Exact MaxSim reranking on candidates
          └── Return top-k results (e.g., 10)
 ```
 
-Three-stage extends this with an additional “cheap prefetch” stage before stage 2.
+Three-stage extends this with an additional "cheap prefetch" stage before stage 2.
 
 ## 📁 Package Structure
 
@@ -374,4 +384,3 @@ MIT License - see [LICENSE](LICENSE) for details.
 - [Qdrant](https://qdrant.tech/) - Vector database with multi-vector support
 - [ColPali](https://github.com/illuin-tech/colpali) - Visual document retrieval models
 - [ViDoRe](https://huggingface.co/spaces/vidore/vidore-leaderboard) - Benchmark dataset
-

{visual_rag_toolkit-0.1.2.dist-info → visual_rag_toolkit-0.1.3.dist-info}/RECORD

@@ -12,24 +12,24 @@ benchmarks/vidore_tatdqa_test/metrics.py,sha256=cLdYbRt5VcxInO1cN79ve6ZLP3kaSxRk
 benchmarks/vidore_tatdqa_test/run_qdrant.py,sha256=_PikeqIYpWPim-KEQOwvT-aqwYoAWASjqJVisi8PfQg,28681
 benchmarks/vidore_tatdqa_test/sweep_eval.py,sha256=d_kbyNTJ1LoFfIVnsZyiRO1nKyMqmRB5jEweZL6kYd4,12688
 demo/__init__.py,sha256=jVzjsVKZl5ZZuFxawA8Pxj3yuIKL7llkao3rBpde-aQ,204
-demo/app.py,sha256=1GZJ_JhVWvqoBewngc8tHeiuM1fNbxddEO6ZsEdwBfg,1029
+demo/app.py,sha256=nZbCz1mpRK-GZTgOHyz4m4AfgKFgsH-09JwXeL3d3ng,1405
 demo/commands.py,sha256=qxRE2x610yZvcjwEfSKiR9CyFonX-vRxFqQNJCUKfyA,13690
 demo/config.py,sha256=BNkV4NSEEMIV9e6Z-cxds2v247uVmTPCgL-M5ItPzMg,757
 demo/download_models.py,sha256=J10qQt2TpEshVOxvCX_ZSbV7YozIBqDATZnt8fUKFHs,2868
-demo/evaluation.py,sha256=wiVxzRu3UZ5wAwHlpSKQ6srZjnSR06dgQw3G0OOV2Eg,28954
+demo/evaluation.py,sha256=4ixJGg50KAVNiZ_mr5FMVv-QKCrZRooJ80LbrjKXM1s,27467
 demo/example_metadata_mapping_sigir.json,sha256=UCgqZtr6Wnq_vS7zxPxpvuokk9gxOVgKydC7f1lauw8,824
-demo/indexing.py,sha256=u80FSQo5ahHfry7fQNWa3kvKDngURRkLhaze3zWro1o,14077
-demo/qdrant_utils.py,sha256=VWEC7BwhMjjB7iIS5iaVDMGt_CMh9mQG4F94k1Pt0yA,7677
+demo/indexing.py,sha256=qUVEB3QrIolS53Ggxurccbh-QyeLLbzcY5TLyVBVKME,10620
+demo/qdrant_utils.py,sha256=Xh-thLIrACrYkFCrqazYNH0p3vS8_yMCaTbvt4HAy98,7778
 demo/results.py,sha256=dprvxnyHwxJvkAQuh4deaCsiEG1wm0n9svPyxI37vJg,1050
 demo/test_qdrant_connection.py,sha256=hkbyl3zGsw_GdBBp5MkW_3SBKTHXbwH3Sr_pUE54_po,3866
 demo/ui/__init__.py,sha256=EyBCvnXYfPbdyxJzyp9TjQBeJJUgmOY1yRHkUeC6JFQ,412
 demo/ui/benchmark.py,sha256=HiGCN4HrqeOC7L6t2kuzIiyWdcVE_cP2JTxoewrmPSo,14218
 demo/ui/header.py,sha256=J2hXr_nNyg1H9rmrd-EGx3WUl7lYo-Ca30ptgzBCfBs,806
-demo/ui/playground.py,sha256=Z3OgCWOzzTld1I3eN1IcTadaSzsqDQf7MiHwTbxbvJA,13692
-demo/ui/sidebar.py,sha256=muVCnvoeMOm1rHx7UPt68yLXlG3OERdXvJ3QqIXAUoc,7839
-demo/ui/upload.py,sha256=BHJmbIQOAYdMF_svxlRSYIe163Y5UX5P_gilJ09YHSA,20372
-visual_rag/__init__.py,sha256=UkGFXjPmjbO6Iad8ty1uJOMQsVMpV_s63ihchHltLx8,2555
-visual_rag/config.py,sha256=pd48M3j3n8ZV1HhaabMmP_uoEJnqhBC-Bma9vuvc8V4,7368
+demo/ui/playground.py,sha256=yRlWWzJgsc596vALn5f0PHhmhtJCMmfv61nYakW75GQ,13672
+demo/ui/sidebar.py,sha256=DLVhEj-8xAJCXUwOhndNv8ZFT4K3u8iE6FVOoH-jRuA,7699
+demo/ui/upload.py,sha256=6iv4xDsacMtUF1FrquRBE_xNb92HevgxCMS0LBK4Ay0,20455
+visual_rag/__init__.py,sha256=4NksVCaN_p32ezMF1N-oxpPFKeOm8xRo70VC4OSa2a0,3911
+visual_rag/config.py,sha256=qqSQk2lM5MiRji-6xQNGS2gSiXA4NgyJnCbgGx7uGJQ,7395
 visual_rag/demo_runner.py,sha256=wi0Wz3gZ39l4aovMd6zURq_CKUSgma4kGjF6hpQHwGY,2793
 visual_rag/qdrant_admin.py,sha256=NNczko2S5-K3qATNUxgYn51hNWgWb6boheL7vlCQGpM,7055
 visual_rag/cli/__init__.py,sha256=WgBRXm0VACfLltvVlLcSs3FTM1uQ7Uuw3CVD4-zWZwc,46
@@ -37,22 +37,22 @@ visual_rag/cli/main.py,sha256=QmpnQ0lbC6Q9lwxaSCDh6paEEzI78IPY1jwc3_9y7VI,21083
 visual_rag/embedding/__init__.py,sha256=7QIENmxwRnwnUzsYKRY3VQTyF3HJkRiL1D7Au9XHF0w,682
 visual_rag/embedding/pooling.py,sha256=x8uY4VHbxEnsJRM2JeOkzPHDiwOkbi5NK4XW21U1hAc,11401
 visual_rag/embedding/visual_embedder.py,sha256=he9JpVHmo_szOiXCwtJdrCseGmf2y5Gi0UEFjwazzVY,23198
-visual_rag/indexing/__init__.py,sha256=pMLuinCIERbwWechn176nMrtlmTp0ySfuj8gdkNvRks,679
+visual_rag/indexing/__init__.py,sha256=rloBEBt3x8BQut1Tj1n8fuaQ3iXMS3pm64o8n-NlSAw,985
 visual_rag/indexing/cloudinary_uploader.py,sha256=e-G5du4D7z6mWWl2lahMidG-Wdc-baImFFILTojebpA,8826
 visual_rag/indexing/pdf_processor.py,sha256=V3RAKpwgIFicqUaXzaaljePxh_oP4UV5W0aiJyfv0BY,10247
 visual_rag/indexing/pipeline.py,sha256=1ScpVRlLCq2FWi3IPvlQcIfDCQQ2F64IlRd9ZZHiTaA,25037
-visual_rag/indexing/qdrant_indexer.py,sha256=uUOA-6Qkd_vEeP1LdgGyoh1FHu1ZNEyYKuNxJAqetBU,17121
+visual_rag/indexing/qdrant_indexer.py,sha256=Q0e8JCr9B1OxgOMW7BWeg7MlWiLPaBUmjoFof4gZFYo,19519
 visual_rag/preprocessing/__init__.py,sha256=rCzfBO0jaVKp6MpPRRused_4gasHfobAbG-139Y806E,121
 visual_rag/preprocessing/crop_empty.py,sha256=iHXITFkRlF40VPJ4k9d432RUAi_89BhAEvK4wOEn96Q,5211
 visual_rag/retrieval/__init__.py,sha256=J9pnbeB83Fqs9n4g3GcIp1VR9dnuyAlcsIDVsf0lSb8,601
-visual_rag/retrieval/multi_vector.py,sha256=m5PKjkj0TFeWNccKNmCqghTM5b9ARr43Lq3sRhOxnjw,7381
-visual_rag/retrieval/single_stage.py,sha256=TSndnh4Kz9aT_0kKhNyLEvokbDLkgq--lXuyldzP5sU,4105
+visual_rag/retrieval/multi_vector.py,sha256=ZZ_O4x7MZbhF--kRp8T4UJG5GuenfjJ91FKicklhK3Q,7006
+visual_rag/retrieval/single_stage.py,sha256=Ba06V-KRSFSZm0xzbjFR3EBEWaQkDo7U_pWNx25W8H0,4425
 visual_rag/retrieval/three_stage.py,sha256=YC0CVEohxTT5zhilcQHI7nYAk08E5jC3zkQ3-rNdLMw,5951
-visual_rag/retrieval/two_stage.py,sha256=_RnEgIx_qY4yu2iIk0a3w47D7WiKHlmBivm5gLEpyI4,16779
+visual_rag/retrieval/two_stage.py,sha256=JJ6rXv_3_3WLIjAcxOY7NuhSyuPzIMyHf3ooiGFTp9k,16776
 visual_rag/visualization/__init__.py,sha256=SITKNvBEseDp7F3K6UzLPA-6OQFqYfY5azS5nlDdihQ,447
 visual_rag/visualization/saliency.py,sha256=F3Plc18Sf3tzWcyncuaruTmENm1IfW5j9NFGEQR93cY,11248
-visual_rag_toolkit-0.1.2.dist-info/METADATA,sha256=LrZ-EUezUsmUJpnNofm5TTM5IJIUerznFDcBSucI7rc,12830
-visual_rag_toolkit-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-visual_rag_toolkit-0.1.2.dist-info/entry_points.txt,sha256=6Tob1GPg_ILGELjYTPsAnNMZ1W0NS939nfI7xyW2DIY,102
-visual_rag_toolkit-0.1.2.dist-info/licenses/LICENSE,sha256=hEg_weKnHXJakQRR3sw2ygcZ101zCI00zMhBOPb3yfA,1069
-visual_rag_toolkit-0.1.2.dist-info/RECORD,,
+visual_rag_toolkit-0.1.3.dist-info/METADATA,sha256=IQaXJV0GkBuRZG5JTkFA-Zv6pboaAeoWoQIeWzu7-Z4,13180
+visual_rag_toolkit-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+visual_rag_toolkit-0.1.3.dist-info/entry_points.txt,sha256=6Tob1GPg_ILGELjYTPsAnNMZ1W0NS939nfI7xyW2DIY,102
+visual_rag_toolkit-0.1.3.dist-info/licenses/LICENSE,sha256=hEg_weKnHXJakQRR3sw2ygcZ101zCI00zMhBOPb3yfA,1069
+visual_rag_toolkit-0.1.3.dist-info/RECORD,,