retrievalbase 1.0.0__tar.gz

retrievalbase-1.0.0/.gitignore

@@ -0,0 +1,38 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# env files
+.env
+
+# Pycharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Testing
+.coverage
+coverage.xml
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+
+# Ruff
+.ruff_cache/
+
+# Pre-commit
+.pre-commit-cache/
+
+# MacOS
+.DS_Store
+
+# Claude
+.claude/

retrievalbase-1.0.0/.gitlab-ci.yml

@@ -0,0 +1,110 @@
+image: python:3.12
+
+workflow:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev"'
+    - if: '$CI_COMMIT_BRANCH == "dev"'
+    - when: never
+
+stages:
+  - checks
+  - test
+  - deploy
+
+variables:
+  UV_SYSTEM_PYTHON: "1"
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  GIT_DEPTH: 0
+
+
+cache:
+  key:
+    files:
+      - uv.lock
+  paths:
+    - .venv/
+    - ~/.cache/uv
+
+
+before_script:
+  - pip install uv
+  - uv sync --group dev --all-extras
+
+ruff:
+  stage: checks
+  script:
+    - uv run ruff check .
+    - uv run ruff format --check .
+
+ty:
+  stage: checks
+  script:
+    - uv run ty check .
+
+bandit:
+  stage: checks
+  script:
+    - uv run bandit -c pyproject.toml -r .
+
+pytest:
+  stage: test
+  script:
+    - |
+      if [ "$CI_PIPELINE_SOURCE" = "merge_request_event" ]; then
+        uv run pytest --cov=src --cov-report=xml --cov-report=term --cov-fail-under=80
+      else
+        uv run pytest
+      fi
+  artifacts:
+    paths:
+      - coverage.xml
+
+check-package:
+  stage: test
+  needs: ["pytest"]
+  script:
+    - uv build
+    - pip install dist/*.whl
+
+codecov:
+  stage: test
+  needs: ["pytest"]
+  script:
+    - curl -Os https://cli.codecov.io/latest/linux/codecov
+    - chmod +x codecov
+    - ./codecov upload-process -t $CODECOV_TOKEN
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+release:
+  stage: deploy
+  image: node:22
+  needs: ["check-package"]
+  before_script:
+    - apt-get update && apt-get install -y python3 python3-pip
+    - pip install --break-system-packages uv
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "dev"'
+  script:
+    - npm install -g semantic-release @semantic-release/gitlab @semantic-release/changelog @semantic-release/git @semantic-release/exec
+    - semantic-release
+
+conventional-commits:
+  stage: checks
+  image: node:20
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+  before_script: []
+  script:
+    - npm install -g @commitlint/cli @commitlint/config-conventional
+    - echo "$CI_MERGE_REQUEST_TITLE" > pr_title.txt
+    - |
+      npx commitlint --config commitlint.config.cjs --edit pr_title.txt || {
+        echo "❌ PR title must follow Conventional Commits format (e.g. feat:, fix:)"
+        exit 1
+      }
+
+default:
+  interruptible: true
+  retry: 1
+  timeout: 10m

retrievalbase-1.0.0/.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+repos:
+  # Ruff for linting and formatting
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.13.3
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix]
+      # Run the formatter
+      - id: ruff-format
+
+  # Type checking with ty (disabled in pre-commit, enabled in CI)
+  # - repo: https://github.com/astral-sh/ty-pre-commit
+  #   rev: v0.0.17
+  #   hooks:
+  #     - id: ty
+  #       exclude: ^(tests/|alembic/)
+
+  # Conventional commits enforcement
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v3.6.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+        args: [--maxkb=1000]
+      - id: check-json
+      - id: check-toml
+      - id: detect-private-key
+      - id: mixed-line-ending
+      - id: check-merge-conflict
+
+  # Security checks with bandit
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.8.0
+    hooks:
+      - id: bandit
+        args: [-c, pyproject.toml]
+        additional_dependencies: ["bandit[toml]"]
+        exclude: ^tests/

retrievalbase-1.0.0/.releaserc.json

@@ -0,0 +1,28 @@
+{
+    "branches": [
+        "dev"
+    ],
+    "plugins": [
+        "@semantic-release/commit-analyzer",
+        "@semantic-release/release-notes-generator",
+        "@semantic-release/changelog",
+        [
+            "@semantic-release/exec",
+            {
+                "prepareCmd": "uv version ${nextRelease.version}",
+                "publishCmd": "uv build && uv publish --token $PYPI_TOKEN"
+            }
+        ],
+        [
+            "@semantic-release/git",
+            {
+                "assets": [
+                    "CHANGELOG.md",
+                    "pyproject.toml"
+                ],
+                "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+            }
+        ],
+        "@semantic-release/gitlab"
+    ]
+}

retrievalbase-1.0.0/CHANGELOG.md

@@ -0,0 +1,11 @@
+# 1.0.0 (2026-04-19)
+
+
+### Bug Fixes
+
+* fix deps in ci ([dc69d67](https://gitlab.com/efysent/agentic-core/retrievalbase/commit/dc69d67cc33794c6cd477c7391576db3cf317800))
+
+
+### Features
+
+* first commit, setup ci ([47b9e29](https://gitlab.com/efysent/agentic-core/retrievalbase/commit/47b9e29b747b07eef2689a6c332c7107d84abc2d))

retrievalbase-1.0.0/Makefile

@@ -0,0 +1,87 @@
+.PHONY: help install dev-install setup-hooks \
+        format lint type-check security \
+        test test-cov coverage-diff \
+        ci ci-fast clean run
+
+# ------------------------
+# 📚 Help
+# ------------------------
+help:
+	@echo "Available commands:"
+	@echo "  make install         - Install production dependencies"
+	@echo "  make dev-install     - Install all dependencies + hooks"
+	@echo "  make setup-hooks     - Install pre-commit hooks"
+	@echo "  make format          - Format code (ruff)"
+	@echo "  make lint            - Lint code (ruff)"
+	@echo "  make type-check      - Run type checking (ty)"
+	@echo "  make security        - Run security checks (bandit)"
+	@echo "  make test            - Run tests"
+	@echo "  make test-cov        - Run tests with coverage"
+	@echo "  make ci              - Full CI locally"
+	@echo "  make ci-fast         - CI without slow steps"
+	@echo "  make clean           - Clean cache files"
+
+# ------------------------
+# 📦 Install
+# ------------------------
+install:
+	uv sync --no-dev
+
+dev-install:
+	uv sync --group dev --all-extras
+	uv run pre-commit install
+	uv run pre-commit install --hook-type commit-msg
+	@echo "✅ Dev environment ready"
+
+setup-hooks:
+	uv run pre-commit install
+	uv run pre-commit install --hook-type commit-msg
+
+# ------------------------
+# 🧹 Code quality
+# ------------------------
+format:
+	uv run ruff format .
+	uv run ruff check --fix .
+
+lint:
+	uv run ruff check .
+
+type-check:
+	uv run ty check .
+
+security:
+	uv run bandit -c pyproject.toml -r .
+
+# ------------------------
+# 🧪 Tests
+# ------------------------
+test:
+	uv run pytest
+
+test-cov:
+	uv run pytest \
+		--cov=src \
+		--cov-report=term \
+		--cov-report=xml:coverage.xml \
+		--cov-fail-under=80
+
+# ------------------------
+# 🚀 CI equivalent
+# ------------------------
+ci: lint type-check security test-cov
+	@echo ""
+	@echo "✅ Full CI checks passed"
+
+# Faster version (no bandit blocking)
+ci-fast: lint type-check test
+	@echo ""
+	@echo "⚡ Fast CI checks passed"
+
+# ------------------------
+# 🧹 Cleanup
+# ------------------------
+clean:
+	find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+	find . -type f -name "*.pyc" -delete 2>/dev/null || true
+	rm -rf .pytest_cache .ruff_cache .ty_cache htmlcov coverage.xml 2>/dev/null || true

retrievalbase-1.0.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: retrievalbase
+Version: 1.0.0
+Author-email: jalal <jalalkhaldi3@gmail.com>
+Requires-Python: <3.13,>=3.11
+Requires-Dist: faiss-cpu<2.0.0,>=1.13.2
+Requires-Dist: langchain<2.0.0,>=1.2.10
+Requires-Dist: minio<8.0.0,>=7.2.20
+Requires-Dist: numpy<3.0.0,>=2.4.2
+Requires-Dist: openai<3.0.0,>=2.21.0
+Requires-Dist: polars<2.0.0,>=1.38.1
+Requires-Dist: pydantic-settings<3.0.0,>=2.13.0
+Requires-Dist: qdrant-client<2.0.0,>=1.16.2
+Requires-Dist: rank-bm25<0.3.0,>=0.2.2
+Provides-Extra: torch
+Requires-Dist: datasets<5.0.0,>=4.5.0; extra == 'torch'
+Requires-Dist: sentence-transformers<6.0.0,>=5.1.2; extra == 'torch'
+Requires-Dist: torch<3.0.0,>=2.10.0; extra == 'torch'
+Requires-Dist: transformers<6.0.0,>=5.3.0; extra == 'torch'
+Provides-Extra: transformers
+Requires-Dist: datasets<5.0.0,>=4.5.0; extra == 'transformers'
+Requires-Dist: sentence-transformers<6.0.0,>=5.1.2; extra == 'transformers'
+Requires-Dist: transformers<6.0.0,>=5.3.0; extra == 'transformers'

retrievalbase-1.0.0/codecov.yml

@@ -0,0 +1,30 @@
+coverage:
+  # Controls how the coverage badge is colored
+  range: 50..80
+  round: down
+  precision: 2
+
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 2%
+    patch:
+      default:
+        target: 80%
+
+comment:
+  # Compact PR comment
+  layout: "condensed_header, condensed_files, condensed_footer"
+
+  # Always comment, even if coverage didn’t change
+  require_changes: false
+
+  # Don’t block comment if baseline is missing
+  require_base: false
+
+  # Ensure we have coverage on the PR commit
+  require_head: true
+
+  # Hide global coverage in PR (focus on patch)
+  hide_project_coverage: true

retrievalbase-1.0.0/commitlint.config.cjs

@@ -0,0 +1,22 @@
+module.exports = {
+    extends: ['@commitlint/config-conventional'],
+    rules: {
+        'type-enum': [
+            2,
+            'always',
+            [
+                'feat',
+                'fix',
+                'docs',
+                'style',
+                'refactor',
+                'perf',
+                'test',
+                'chore',
+                'ci',
+                'build',
+            ],
+        ],
+        'subject-empty': [2, 'never'],
+    },
+};

retrievalbase-1.0.0/pyproject.toml

@@ -0,0 +1,105 @@
+[project]
+name = "retrievalbase"
+version = "1.0.0"
+description = ""
+authors = [
+    { name = "jalal", email = "jalalkhaldi3@gmail.com" }
+]
+readme = "README.md"
+requires-python = ">=3.11,<3.13"
+
+dependencies = [
+    "polars>=1.38.1,<2.0.0",
+    "langchain>=1.2.10,<2.0.0",
+    "numpy>=2.4.2,<3.0.0",
+    "rank-bm25>=0.2.2,<0.3.0",
+    "openai>=2.21.0,<3.0.0",
+    "qdrant-client>=1.16.2,<2.0.0",
+    "pydantic-settings>=2.13.0,<3.0.0",
+    "faiss-cpu>=1.13.2,<2.0.0",
+    "minio>=7.2.20,<8.0.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/retrievalbase"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "E",     # pycodestyle errors
+    "W",     # pycodestyle warnings
+    "F",     # pyflakes
+    "I",     # isort
+    "B",     # flake8-bugbear
+    "C4",    # flake8-comprehensions
+    "UP",    # pyupgrade
+    "ARG",   # flake8-unused-arguments
+    "SIM",   # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "B008",   # do not perform function calls in argument defaults
+    "B904",   # Within an except clause, raise exceptions with `raise ... from err`
+    "ARG001", # Unused function argument
+    "ARG002", # Unused method argument
+    "ARG005", # Unused lambda argument
+    "SIM108", # Use ternary operator (less readable in many cases)
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["retrievalbase"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+]
+
+[tool.ty.environment]
+python-version = "3.12"
+
+[tool.bandit]
+exclude_dirs = ["tests", ".venv"]
+skips = [
+    "B101",  # assert statements
+    "B404",  # import subprocess (needed for CLI)
+    "B603",  # subprocess without shell=True (needed for CLI)
+    "B607",  # partial executable path (needed for CLI to run docker, systemctl, etc.)
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+    "pytest-mock>=3.15.1",
+    "pytest-cov>=6.0.0",
+    "ruff>=0.13.0",
+    "ty>=0.0.17,<0.1.0",
+    "bandit>=1.8.0",
+    "types-pyyaml>=6.0.12.20250915,<7.0.0.0",
+    "types-tqdm>=4.67.3.20260205,<5.0.0.0",
+    "jupyter>=1.1.1,<2.0.0",
+    "transformers>=5.2.0,<6.0.0",
+    "torch>=2.10.0,<3.0.0",
+]
+
+[project.optional-dependencies]
+torch = [
+    "transformers>=5.3.0,<6.0.0",
+    "datasets>=4.5.0,<5.0.0",
+    "sentence-transformers>=5.1.2,<6.0.0",
+    "torch>=2.10.0,<3.0.0",
+
+]
+transformers = [
+    "transformers>=5.3.0,<6.0.0",
+    "datasets>=4.5.0,<5.0.0",
+    "sentence-transformers>=5.1.2,<6.0.0",
+]

retrievalbase-1.0.0/src/retrievalbase/connector/__init__.py

@@ -0,0 +1,69 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+
+from retrievalbase.connector.settings import DatasetConnectorSettings
+from retrievalbase.mixins import FromConfigMixin
+from retrievalbase.types import TCDatasetConnector as TCDatasetConnector
+
+if TYPE_CHECKING:
+    from retrievalbase.dataset import Dataset, TextDataset
+
+_logger = logging.getLogger(__name__)
+
+
+class DatasetConnector[TCDatasetConnector: DatasetConnectorSettings](
+    FromConfigMixin[TCDatasetConnector],
+    ABC,
+):
+    def __init__(self, config: TCDatasetConnector):
+        super().__init__(config)
+
+        _logger.info(
+            f"Initializing dataset connector | class={self.__class__.__name__} | module={self.__class__.__module__}"
+        )
+
+    @abstractmethod
+    def _load(self) -> pl.DataFrame | pl.LazyFrame:
+        """
+        Load raw data as Polars DataFrame or LazyFrame.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def to(self, ds: "Dataset[Any]") -> None:
+        raise NotImplementedError()
+
+    def load(self) -> "Dataset[pl.DataFrame | pl.LazyFrame]":
+        from retrievalbase.dataset.polars import PolarsDataset
+
+        _logger.info(f"Loading dataset | connector={self.__class__.__name__}")
+
+        df = self._load()
+        self._log_polars_info(df)
+
+        return PolarsDataset.from_polars(df)
+
+    def load_text(self) -> "TextDataset[pl.DataFrame | pl.LazyFrame]":
+        from retrievalbase.dataset.polars import PolarsTextDataset
+
+        _logger.info(f"Loading text dataset | connector={self.__class__.__name__}")
+
+        df = self._load()
+        self._log_polars_info(df)
+
+        return PolarsTextDataset.from_polars(df)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _log_polars_info(self, df: pl.DataFrame | pl.LazyFrame) -> None:
+        """
+        Log dataset structure without forcing materialization.
+        """
+        if isinstance(df, pl.DataFrame):
+            schema = df.schema
+            _logger.info(f"Loaded DataFrame | columns={len(schema)} | schema={list(schema.items())}")

retrievalbase-1.0.0/src/retrievalbase/connector/minio.py

@@ -0,0 +1,45 @@
+import io
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+from minio import Minio
+
+from retrievalbase.connector import DatasetConnector
+from retrievalbase.connector.settings import MinioDatasetConnectorSettings
+
+if TYPE_CHECKING:
+    from retrievalbase.dataset import Dataset
+
+
+class MinioDatasetConnector(DatasetConnector[MinioDatasetConnectorSettings]):
+    def __init__(self, config: MinioDatasetConnectorSettings):
+        super().__init__(config)
+        self.client = Minio(
+            self.config.endpoint.replace("http://", "").replace("https://", ""),
+            access_key=self.config.access_key.get_secret_value(),
+            secret_key=self.config.secret_key.get_secret_value(),
+            secure=self.config.endpoint.startswith("https://"),
+        )
+
+    def _load(self) -> pl.DataFrame | pl.LazyFrame:
+        response = self.client.get_object(self.config.bucket, self.config.key)
+        try:
+            buffer = io.BytesIO(response.read())
+        finally:
+            response.close()
+            response.release_conn()
+        df = pl.read_parquet(buffer)
+        return df
+
+    def to(self, ds: "Dataset[Any]") -> None:
+        df = ds.polars
+        buffer = io.BytesIO()
+        df.write_parquet(buffer)
+        buffer.seek(0)
+        self.client.put_object(
+            bucket_name=self.config.bucket,
+            object_name=self.config.key,
+            data=buffer,
+            length=buffer.getbuffer().nbytes,
+            content_type="application/octet-stream",
+        )

retrievalbase-1.0.0/src/retrievalbase/connector/parquet.py

@@ -0,0 +1,20 @@
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+
+from retrievalbase.connector import DatasetConnector
+from retrievalbase.connector.settings import ParquetDatasetConnectorSettings
+
+if TYPE_CHECKING:
+    from retrievalbase.dataset import Dataset
+
+
+class ParquetDatasetConnector(DatasetConnector[ParquetDatasetConnectorSettings]):
+    def __init__(self, config: ParquetDatasetConnectorSettings):
+        super().__init__(config)
+
+    def _load(self) -> pl.DataFrame | pl.LazyFrame:
+        return pl.scan_parquet(self.config.path) if self.config.lazy else pl.read_parquet(self.config.path)
+
+    def to(self, ds: "Dataset[Any]") -> None:
+        ds.polars.write_parquet(self.config.path)

retrievalbase-1.0.0/src/retrievalbase/connector/settings.py

@@ -0,0 +1,22 @@
+from pydantic import SecretStr
+from pydantic_settings import SettingsConfigDict
+
+from retrievalbase.settings import FromConfigMixinSettings
+
+
+class DatasetConnectorSettings(FromConfigMixinSettings):
+    pass
+
+
+class ParquetDatasetConnectorSettings(DatasetConnectorSettings):
+    path: str
+    lazy: bool
+
+
+class MinioDatasetConnectorSettings(DatasetConnectorSettings):
+    endpoint: str
+    bucket: str
+    key: str
+    access_key: SecretStr
+    secret_key: SecretStr
+    model_config = SettingsConfigDict(env_prefix="MINIO_", extra="ignore")

retrievalbase-1.0.0/src/retrievalbase/constants.py

@@ -0,0 +1 @@
+CONFIG_PATH = "/config/config.yaml"