PyPI - retrievalbase - Versions diffs - 2.1.3__tar.gz → 2.2.0__tar.gz - Mend

@@ -5,7 +5,9 @@ from typing import TypedDict, cast
 import polars as pl
 import pytest
+from minio.error import S3Error
 from pydantic import SecretStr
+from urllib3.response import BaseHTTPResponse
 from retrievalbase.connector.minio import MinioDatasetConnector
 from retrievalbase.connector.parquet import ParquetDatasetConnector
@@ -82,6 +84,18 @@ def test_parquet_connector_round_trips_dataframe(tmp_path, lazy: bool) -> None:
     assert loaded.polars.to_dict(as_series=False) == sample_text_df.to_dict(as_series=False)
+def test_parquet_connector_reports_target_existence(tmp_path) -> None:
+    existing_path = tmp_path / "dataset.parquet"
+    missing_path = tmp_path / "missing.parquet"
+    existing_path.write_bytes(b"not a real parquet file")
+    connector = ParquetDatasetConnector(
+        ParquetDatasetConnectorSettings(module_path="x", path=str(existing_path), lazy=False)
+    )
+    assert connector.target_exists(str(existing_path)) is True
+    assert connector.target_exists(str(missing_path)) is False
 def test_minio_connector_reads_and_writes_parquet_payloads(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -160,3 +174,36 @@ def test_minio_connector_reads_and_writes_parquet_payloads(
     assert loaded.polars.to_dict(as_series=False) == sample_text_df.to_dict(as_series=False)
     assert writes["closed"] is True
     assert writes["released"] is True
+def test_minio_connector_reports_target_existence(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls: list[tuple[str, str]] = []
+    class FakeMinio:
+        def __init__(self, endpoint: str, access_key: str, secret_key: str, secure: bool) -> None:
+            pass
+        def stat_object(self, bucket: str, key: str) -> object:
+            calls.append((bucket, key))
+            if key == "missing.parquet":
+                raise S3Error(
+                    cast(BaseHTTPResponse, None), "NoSuchKey", "not found", key, "request-id", "host-id", bucket, key
+                )
+            return object()
+    monkeypatch.setattr("retrievalbase.connector.minio.Minio", FakeMinio)
+    connector = MinioDatasetConnector(
+        MinioDatasetConnectorSettings(
+            module_path="x",
+            endpoint="https://minio.local",
+            bucket="datasets",
+            key="sample.parquet",
+            access_key=SecretStr("access"),
+            secret_key=SecretStr("secret"),
+        )
+    )
+    assert connector.target_exists("sample.parquet") is True
+    assert connector.target_exists("missing.parquet") is False
+    assert calls == [("datasets", "sample.parquet"), ("datasets", "missing.parquet")]

@@ -1,3 +1,10 @@
+# [2.2.0](https://gitlab.com/efysent/agentic-core/retrievalbase/compare/v2.1.3...v2.2.0) (2026-05-22)
+### Features
+* **connector:** add target existence checks ([80e739f](https://gitlab.com/efysent/agentic-core/retrievalbase/commit/80e739fddeca65a66d326c1bb673e1ff6a674ba6))
 ## [2.1.3](https://gitlab.com/efysent/agentic-core/retrievalbase/compare/v2.1.2...v2.1.3) (2026-05-19)

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: retrievalbase
-Version: 2.1.3
+Version: 2.2.0
 Author-email: jalal <jalalkhaldi3@gmail.com>
 Requires-Python: <3.13,>=3.11
 Requires-Dist: faiss-cpu<2.0.0,>=1.13.2

@@ -1,6 +1,6 @@
 [project]
 name = "retrievalbase"
-version = "2.1.3"
+version = "2.2.0"
 description = ""
 authors = [
     { name = "jalal", email = "jalalkhaldi3@gmail.com" }

@@ -33,6 +33,10 @@ class DatasetConnector[TCDatasetConnector: DatasetConnectorSettings](
     def to(self, ds: "Dataset[Any]") -> None:
         raise NotImplementedError
+    @abstractmethod
+    def target_exists(self, target: str) -> bool:
+        raise NotImplementedError
     def load(self) -> "Dataset[pl.DataFrame | pl.LazyFrame]":
         from retrievalbase.dataset.polars import PolarsDataset

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any
 import polars as pl
 from minio import Minio
+from minio.error import S3Error
 from retrievalbase.connector import DatasetConnector
 from retrievalbase.connector.settings import MinioDatasetConnectorSettings
@@ -43,3 +44,12 @@ class MinioDatasetConnector(DatasetConnector[MinioDatasetConnectorSettings]):
             length=buffer.getbuffer().nbytes,
             content_type="application/octet-stream",
         )
+    def target_exists(self, target: str) -> bool:
+        try:
+            self.client.stat_object(self.config.bucket, target)
+        except S3Error as error:
+            if error.code in {"NoSuchBucket", "NoSuchKey"}:
+                return False
+            raise
+        return True

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 import polars as pl
@@ -18,3 +19,6 @@ class ParquetDatasetConnector(DatasetConnector[ParquetDatasetConnectorSettings])
     def to(self, ds: "Dataset[Any]") -> None:
         ds.polars.write_parquet(self.config.path)
+    def target_exists(self, target: str) -> bool:
+        return Path(target).exists()

@@ -79,6 +79,9 @@ class FakeDatasetConnector(DatasetConnector[FakeDatasetConnectorSettings]):
     def to(self, ds: Any) -> None:
         self.__class__.last_written = ds.polars
+    def target_exists(self, target: str) -> bool:
+        return target == "exists"
 class FakeTextPreprocessorSettings(TextPreprocessorSettings):
     kind: str = "suffix"

@@ -33,6 +33,9 @@ class RaisingConnector(DatasetConnector[DatasetConnectorSettings]):
     def to(self, ds) -> None:
         super().to(ds)
+    def target_exists(self, target: str) -> bool:
+        return super().target_exists(target)
 class RaisingPreprocessor(TextPreprocessor[TextPreprocessorSettings]):
     def apply(self, ds):
@@ -94,6 +97,8 @@ def test_base_contracts_raise_not_implemented_and_noops() -> None:
         connector._load()
     with pytest.raises(NotImplementedError):
         connector.to(PolarsTextDataset.from_records([("x", {"doc_id": "1"})]))
+    with pytest.raises(NotImplementedError):
+        connector.target_exists("x")
     with pytest.raises(NotImplementedError):
         preprocessor.apply(PolarsTextDataset.from_records([("x", {"doc_id": "1"})]))
     with pytest.raises(NotImplementedError):

@@ -2584,7 +2584,7 @@ wheels = [
 [[package]]
 name = "retrievalbase"
-version = "2.1.3"
+version = "2.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "faiss-cpu" },

retrievalbase 2.1.3__tar.gz → 2.2.0__tar.gz

retrievalbase 2.1.3tar.gz → 2.2.0tar.gz