retrievalbase 2.1.2__tar.gz → 2.2.0__tar.gz

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [2.2.0](https://gitlab.com/efysent/agentic-core/retrievalbase/compare/v2.1.3...v2.2.0) (2026-05-22)
+
+
+### Features
+
+* **connector:** add target existence checks ([80e739f](https://gitlab.com/efysent/agentic-core/retrievalbase/commit/80e739fddeca65a66d326c1bb673e1ff6a674ba6))
+
+## [2.1.3](https://gitlab.com/efysent/agentic-core/retrievalbase/compare/v2.1.2...v2.1.3) (2026-05-19)
+
+
+### Bug Fixes
+
+* resolve env vars loading errors ([061d549](https://gitlab.com/efysent/agentic-core/retrievalbase/commit/061d549dea621dc2ab34b4af7f23c090f6d70350))
+
 ## [2.1.2](https://gitlab.com/efysent/agentic-core/retrievalbase/compare/v2.1.1...v2.1.2) (2026-05-19)
 
 

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: retrievalbase
-Version: 2.1.2
+Version: 2.2.0
 Author-email: jalal <jalalkhaldi3@gmail.com>
 Requires-Python: <3.13,>=3.11
 Requires-Dist: faiss-cpu<2.0.0,>=1.13.2

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "retrievalbase"
-version = "2.1.2"
+version = "2.2.0"
 description = ""
 authors = [
     { name = "jalal", email = "jalalkhaldi3@gmail.com" }

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/src/retrievalbase/connector/__init__.py

@@ -33,6 +33,10 @@ class DatasetConnector[TCDatasetConnector: DatasetConnectorSettings](
     def to(self, ds: "Dataset[Any]") -> None:
         raise NotImplementedError
 
+    @abstractmethod
+    def target_exists(self, target: str) -> bool:
+        raise NotImplementedError
+
     def load(self) -> "Dataset[pl.DataFrame | pl.LazyFrame]":
         from retrievalbase.dataset.polars import PolarsDataset
 

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/src/retrievalbase/connector/minio.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any
 
 import polars as pl
 from minio import Minio
+from minio.error import S3Error
 
 from retrievalbase.connector import DatasetConnector
 from retrievalbase.connector.settings import MinioDatasetConnectorSettings
@@ -43,3 +44,12 @@ class MinioDatasetConnector(DatasetConnector[MinioDatasetConnectorSettings]):
             length=buffer.getbuffer().nbytes,
             content_type="application/octet-stream",
         )
+
+    def target_exists(self, target: str) -> bool:
+        try:
+            self.client.stat_object(self.config.bucket, target)
+        except S3Error as error:
+            if error.code in {"NoSuchBucket", "NoSuchKey"}:
+                return False
+            raise
+        return True

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/src/retrievalbase/connector/parquet.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 import polars as pl
@@ -18,3 +19,6 @@ class ParquetDatasetConnector(DatasetConnector[ParquetDatasetConnectorSettings])
 
     def to(self, ds: "Dataset[Any]") -> None:
         ds.polars.write_parquet(self.config.path)
+
+    def target_exists(self, target: str) -> bool:
+        return Path(target).exists()

retrievalbase-2.2.0/src/retrievalbase/utils.py

@@ -0,0 +1,107 @@
+import importlib
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+import yaml
+from pydantic import SecretStr
+
+from retrievalbase.mixins import FromConfigMixin
+
+if TYPE_CHECKING:
+    from retrievalbase.connector.minio import MinioDatasetConnector
+    from retrievalbase.connector.parquet import ParquetDatasetConnector
+
+
+def load_class(path: str) -> Any:
+    module_path, class_name = path.rsplit(".", 1)
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)
+
+
+def _get_minio_connector(
+    bucket: str, key: str, endpoint: str, access_key: str, secret_key: str
+) -> "MinioDatasetConnector":
+    from retrievalbase.connector.minio import MinioDatasetConnector
+    from retrievalbase.connector.settings import MinioDatasetConnectorSettings
+
+    config = MinioDatasetConnectorSettings(
+        module_path="",
+        endpoint=endpoint,
+        bucket=bucket,
+        key=key,
+        access_key=SecretStr(access_key),
+        secret_key=SecretStr(secret_key),
+    )
+    return MinioDatasetConnector(config)
+
+
+def _get_parquet_connector(path: str, *, lazy: bool) -> "ParquetDatasetConnector":
+    from retrievalbase.connector.parquet import ParquetDatasetConnector
+    from retrievalbase.connector.settings import ParquetDatasetConnectorSettings
+
+    config = ParquetDatasetConnectorSettings(module_path="", path=path, lazy=lazy)
+    return ParquetDatasetConnector(config)
+
+
+def extract_schema_columns(
+    schema: dict[str, pl.DataType] | list[pl.Field],
+    prefix: str = "",
+) -> list[str]:
+    cols: list[str] = []
+    if isinstance(schema, dict):
+        items = schema.items()
+    else:
+        items = ((field.name, field.dtype) for field in schema)  # type: ignore[assignment]
+    for name, dtype in items:
+        full_name = f"{prefix}.{name}" if prefix else name
+
+        if isinstance(dtype, pl.Struct):
+            nested = extract_schema_columns(dtype.fields, full_name)
+            cols.extend(nested)
+        else:
+            cols.append(full_name)
+    return cols
+
+
+def resolve_column(path: str) -> pl.Expr:
+    parts = path.split(".")
+
+    expr = pl.col(parts[0])
+    for p in parts[1:]:
+        expr = expr.struct.field(p)
+
+    return expr
+
+
+def build_schema(
+    df: pl.DataFrame,
+    schema: dict[str, str],
+) -> pl.DataFrame:
+    exprs = []
+
+    for target_col, source_path in schema.items():
+        expr = resolve_column(source_path).alias(target_col)
+        exprs.append(expr)
+
+    return df.select(exprs)
+
+
+def comp(
+    path: str,
+    key: str | None = None,
+) -> FromConfigMixin[Any]:
+    """
+    Generic factory:
+    - loads YAML
+    - resolves module_path
+    - instantiates component via FromConfigMixin
+    """
+    yaml_path = Path(path)
+    with yaml_path.open("r") as f:
+        raw: dict[str, Any] = yaml.safe_load(f)
+    if key is not None:
+        raw = raw[key]
+    module_path = raw["module_path"]
+    cls: type[FromConfigMixin[Any]] = load_class(module_path)
+    return cls.from_kwargs(**raw)

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/tests/fixtures/components.py

@@ -79,6 +79,9 @@ class FakeDatasetConnector(DatasetConnector[FakeDatasetConnectorSettings]):
     def to(self, ds: Any) -> None:
         self.__class__.last_written = ds.polars
 
+    def target_exists(self, target: str) -> bool:
+        return target == "exists"
+
 
 class FakeTextPreprocessorSettings(TextPreprocessorSettings):
     kind: str = "suffix"

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/tests/unit/test_connector/test_connectors.py

@@ -5,7 +5,9 @@ from typing import TypedDict, cast
 
 import polars as pl
 import pytest
+from minio.error import S3Error
 from pydantic import SecretStr
+from urllib3.response import BaseHTTPResponse
 
 from retrievalbase.connector.minio import MinioDatasetConnector
 from retrievalbase.connector.parquet import ParquetDatasetConnector
@@ -82,6 +84,18 @@ def test_parquet_connector_round_trips_dataframe(tmp_path, lazy: bool) -> None:
     assert loaded.polars.to_dict(as_series=False) == sample_text_df.to_dict(as_series=False)
 
 
+def test_parquet_connector_reports_target_existence(tmp_path) -> None:
+    existing_path = tmp_path / "dataset.parquet"
+    missing_path = tmp_path / "missing.parquet"
+    existing_path.write_bytes(b"not a real parquet file")
+    connector = ParquetDatasetConnector(
+        ParquetDatasetConnectorSettings(module_path="x", path=str(existing_path), lazy=False)
+    )
+
+    assert connector.target_exists(str(existing_path)) is True
+    assert connector.target_exists(str(missing_path)) is False
+
+
 def test_minio_connector_reads_and_writes_parquet_payloads(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -160,3 +174,36 @@ def test_minio_connector_reads_and_writes_parquet_payloads(
     assert loaded.polars.to_dict(as_series=False) == sample_text_df.to_dict(as_series=False)
     assert writes["closed"] is True
     assert writes["released"] is True
+
+
+def test_minio_connector_reports_target_existence(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls: list[tuple[str, str]] = []
+
+    class FakeMinio:
+        def __init__(self, endpoint: str, access_key: str, secret_key: str, secure: bool) -> None:
+            pass
+
+        def stat_object(self, bucket: str, key: str) -> object:
+            calls.append((bucket, key))
+            if key == "missing.parquet":
+                raise S3Error(
+                    cast(BaseHTTPResponse, None), "NoSuchKey", "not found", key, "request-id", "host-id", bucket, key
+                )
+            return object()
+
+    monkeypatch.setattr("retrievalbase.connector.minio.Minio", FakeMinio)
+
+    connector = MinioDatasetConnector(
+        MinioDatasetConnectorSettings(
+            module_path="x",
+            endpoint="https://minio.local",
+            bucket="datasets",
+            key="sample.parquet",
+            access_key=SecretStr("access"),
+            secret_key=SecretStr("secret"),
+        )
+    )
+
+    assert connector.target_exists("sample.parquet") is True
+    assert connector.target_exists("missing.parquet") is False
+    assert calls == [("datasets", "sample.parquet"), ("datasets", "missing.parquet")]

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/tests/unit/test_evaluation/test_base_contracts.py

@@ -33,6 +33,9 @@ class RaisingConnector(DatasetConnector[DatasetConnectorSettings]):
     def to(self, ds) -> None:
         super().to(ds)
 
+    def target_exists(self, target: str) -> bool:
+        return super().target_exists(target)
+
 
 class RaisingPreprocessor(TextPreprocessor[TextPreprocessorSettings]):
     def apply(self, ds):
@@ -94,6 +97,8 @@ def test_base_contracts_raise_not_implemented_and_noops() -> None:
         connector._load()
     with pytest.raises(NotImplementedError):
         connector.to(PolarsTextDataset.from_records([("x", {"doc_id": "1"})]))
+    with pytest.raises(NotImplementedError):
+        connector.target_exists("x")
     with pytest.raises(NotImplementedError):
         preprocessor.apply(PolarsTextDataset.from_records([("x", {"doc_id": "1"})]))
     with pytest.raises(NotImplementedError):

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/tests/unit/test_utils/test_utils.py

@@ -25,7 +25,7 @@ def test_load_class_and_comp_create_runtime_from_module_path(tmp_path) -> None:
     assert instance.config.value == 17
 
 
-def test_comp_resolves_nested_settings_from_environment(tmp_path, monkeypatch) -> None:
+def test_comp_loads_missing_nested_settings_fields_from_environment(tmp_path, monkeypatch) -> None:
     monkeypatch.setenv("FAKE_CHILD_TOKEN", "token-from-env")
     config_path = tmp_path / "component.yaml"
     config_path.write_text(
@@ -35,7 +35,6 @@ def test_comp_resolves_nested_settings_from_environment(tmp_path, monkeypatch) -
                 f"  module_path: {FAKE_PARENT_RUNTIME_PATH}",
                 "  child:",
                 f"    module_path: {FAKE_ENV_CHILD_RUNTIME_PATH}",
-                "    token: ${FAKE_CHILD_TOKEN}",
                 "",
             ]
         ),

{retrievalbase-2.1.2 → retrievalbase-2.2.0}/uv.lock

@@ -2584,7 +2584,7 @@ wheels = [
 
 [[package]]
 name = "retrievalbase"
-version = "2.1.2"
+version = "2.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "faiss-cpu" },

retrievalbase-2.1.2/src/retrievalbase/utils.py

@@ -1,182 +0,0 @@
-import importlib
-from pathlib import Path
-from types import UnionType
-from typing import TYPE_CHECKING, Any, get_args, get_origin
-
-import polars as pl
-import yaml
-from pydantic import SecretStr
-
-from retrievalbase.mixins import FromConfigMixin
-from retrievalbase.settings import FromConfigMixinSettings
-
-if TYPE_CHECKING:
-    from retrievalbase.connector.minio import MinioDatasetConnector
-    from retrievalbase.connector.parquet import ParquetDatasetConnector
-
-
-def load_class(path: str) -> Any:
-    module_path, class_name = path.rsplit(".", 1)
-    module = importlib.import_module(module_path)
-    return getattr(module, class_name)
-
-
-def _get_minio_connector(
-    bucket: str, key: str, endpoint: str, access_key: str, secret_key: str
-) -> "MinioDatasetConnector":
-    from retrievalbase.connector.minio import MinioDatasetConnector
-    from retrievalbase.connector.settings import MinioDatasetConnectorSettings
-
-    config = MinioDatasetConnectorSettings(
-        module_path="",
-        endpoint=endpoint,
-        bucket=bucket,
-        key=key,
-        access_key=SecretStr(access_key),
-        secret_key=SecretStr(secret_key),
-    )
-    return MinioDatasetConnector(config)
-
-
-def _get_parquet_connector(path: str, *, lazy: bool) -> "ParquetDatasetConnector":
-    from retrievalbase.connector.parquet import ParquetDatasetConnector
-    from retrievalbase.connector.settings import ParquetDatasetConnectorSettings
-
-    config = ParquetDatasetConnectorSettings(module_path="", path=path, lazy=lazy)
-    return ParquetDatasetConnector(config)
-
-
-def extract_schema_columns(
-    schema: dict[str, pl.DataType] | list[pl.Field],
-    prefix: str = "",
-) -> list[str]:
-    cols: list[str] = []
-    if isinstance(schema, dict):
-        items = schema.items()
-    else:
-        items = ((field.name, field.dtype) for field in schema)  # type: ignore[assignment]
-    for name, dtype in items:
-        full_name = f"{prefix}.{name}" if prefix else name
-
-        if isinstance(dtype, pl.Struct):
-            nested = extract_schema_columns(dtype.fields, full_name)
-            cols.extend(nested)
-        else:
-            cols.append(full_name)
-    return cols
-
-
-def resolve_column(path: str) -> pl.Expr:
-    parts = path.split(".")
-
-    expr = pl.col(parts[0])
-    for p in parts[1:]:
-        expr = expr.struct.field(p)
-
-    return expr
-
-
-def build_schema(
-    df: pl.DataFrame,
-    schema: dict[str, str],
-) -> pl.DataFrame:
-    exprs = []
-
-    for target_col, source_path in schema.items():
-        expr = resolve_column(source_path).alias(target_col)
-        exprs.append(expr)
-
-    return df.select(exprs)
-
-
-def _is_env_placeholder(value: Any) -> bool:
-    return isinstance(value, str) and value.startswith("${") and value.endswith("}")
-
-
-def _remove_env_placeholders(raw: dict[str, Any]) -> dict[str, Any]:
-    return {key: value for key, value in raw.items() if not _is_env_placeholder(value)}
-
-
-def _is_settings_class(value: Any) -> bool:
-    return isinstance(value, type) and issubclass(value, FromConfigMixinSettings)
-
-
-def _settings_classes_from_annotation(annotation: Any) -> list[type[FromConfigMixinSettings]]:
-    origin = get_origin(annotation)
-    if _is_settings_class(annotation):
-        return [annotation]
-    if _is_settings_class(origin):
-        return [origin]
-    if origin in (UnionType,):
-        return [settings_cls for arg in get_args(annotation) for settings_cls in _settings_classes_from_annotation(arg)]
-    return [settings_cls for arg in get_args(annotation) for settings_cls in _settings_classes_from_annotation(arg)]
-
-
-def _settings_class_from_module_path(module_path: str) -> type[FromConfigMixinSettings] | None:
-    try:
-        cls = load_class(module_path)
-    except (ImportError, AttributeError, ValueError):
-        return None
-    if isinstance(cls, type) and issubclass(cls, FromConfigMixin):
-        return cls.get_config_class()
-    return None
-
-
-def _settings_class_for_field(annotation: Any, value: dict[str, Any]) -> type[FromConfigMixinSettings] | None:
-    module_path = value.get("module_path")
-    if isinstance(module_path, str):
-        settings_cls = _settings_class_from_module_path(module_path)
-        if settings_cls is not None:
-            return settings_cls
-
-    settings_classes = _settings_classes_from_annotation(annotation)
-    if len(settings_classes) == 1:
-        return settings_classes[0]
-    return None
-
-
-def _resolve_settings_value(annotation: Any, value: Any) -> Any:
-    if isinstance(value, list):
-        return [_resolve_settings_value(Any, item) for item in value]
-    if not isinstance(value, dict):
-        return value
-
-    settings_cls = _settings_class_for_field(annotation, value)
-    if settings_cls is None:
-        return {
-            key: _resolve_settings_value(Any, nested_value)
-            for key, nested_value in _remove_env_placeholders(value).items()
-        }
-    return _build_settings(settings_cls, value)
-
-
-def _build_settings(
-    settings_cls: type[FromConfigMixinSettings],
-    raw: dict[str, Any],
-) -> FromConfigMixinSettings:
-    resolved = _remove_env_placeholders(raw)
-    for field_name, field_info in settings_cls.model_fields.items():
-        if field_name in resolved:
-            resolved[field_name] = _resolve_settings_value(field_info.annotation, resolved[field_name])
-    return settings_cls(**resolved)
-
-
-def comp(
-    path: str,
-    key: str | None = None,
-) -> FromConfigMixin[Any]:
-    """
-    Generic factory:
-    - loads YAML
-    - resolves module_path
-    - instantiates component via FromConfigMixin
-    """
-    yaml_path = Path(path)
-    with yaml_path.open("r") as f:
-        raw: dict[str, Any] = yaml.safe_load(f)
-    if key is not None:
-        raw = raw[key]
-    module_path = raw["module_path"]
-    cls: type[FromConfigMixin[Any]] = load_class(module_path)
-    settings = _build_settings(cls.get_config_class(), raw)
-    return cls.from_config(settings)