ragit 0.8__py3-none-any.whl → 0.8.2__py3-none-any.whl

ragit/providers/sentence_transformers.py

@@ -0,0 +1,225 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+SentenceTransformers provider for offline embedding.
+
+This module provides embedding capabilities using the sentence-transformers
+library, enabling fully offline RAG pipelines without API dependencies.
+
+Requires: pip install ragit[transformers]
+"""
+
+from typing import TYPE_CHECKING
+
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    EmbeddingResponse,
+)
+
+if TYPE_CHECKING:
+    from sentence_transformers import SentenceTransformer
+
+# Lazy import flag
+_sentence_transformers_available: bool | None = None
+_model_cache: dict[str, "SentenceTransformer"] = {}
+
+
+def _check_sentence_transformers() -> bool:
+    """Check if sentence-transformers is available."""
+    global _sentence_transformers_available
+    if _sentence_transformers_available is None:
+        try:
+            from sentence_transformers import SentenceTransformer  # noqa: F401
+
+            _sentence_transformers_available = True
+        except ImportError:
+            _sentence_transformers_available = False
+    return _sentence_transformers_available
+
+
+def _get_model(model_name: str, device: str | None = None) -> "SentenceTransformer":
+    """Get or create a cached SentenceTransformer model."""
+    cache_key = f"{model_name}:{device or 'auto'}"
+    if cache_key not in _model_cache:
+        from sentence_transformers import SentenceTransformer
+
+        _model_cache[cache_key] = SentenceTransformer(model_name, device=device)
+    return _model_cache[cache_key]
+
+
+class SentenceTransformersProvider(BaseEmbeddingProvider):
+    """
+    Embedding provider using sentence-transformers for offline operation.
+
+    This provider uses the sentence-transformers library to generate embeddings
+    locally without requiring any API calls. It's ideal for:
+    - Offline/air-gapped environments
+    - Development and testing
+    - Cost-sensitive applications
+    - Privacy-sensitive use cases
+
+    Parameters
+    ----------
+    model_name : str
+        HuggingFace model name. Default: "all-MiniLM-L6-v2" (fast, 384 dims).
+        Other popular options:
+        - "all-mpnet-base-v2" (768 dims, higher quality)
+        - "paraphrase-MiniLM-L6-v2" (384 dims)
+        - "multi-qa-MiniLM-L6-cos-v1" (384 dims, optimized for QA)
+    device : str, optional
+        Device to run on ("cpu", "cuda", "mps"). Auto-detected if None.
+
+    Examples
+    --------
+    >>> # Basic usage
+    >>> from ragit.providers import SentenceTransformersProvider
+    >>> provider = SentenceTransformersProvider()
+    >>>
+    >>> # With RAGAssistant (retrieval-only)
+    >>> assistant = RAGAssistant(docs, provider=provider)
+    >>> results = assistant.retrieve("query")
+    >>>
+    >>> # Custom model
+    >>> provider = SentenceTransformersProvider(model_name="all-mpnet-base-v2")
+
+    Raises
+    ------
+    ImportError
+        If sentence-transformers is not installed.
+
+    Note
+    ----
+    Install with: pip install ragit[transformers]
+    """
+
+    # Known model dimensions for common models
+    MODEL_DIMENSIONS: dict[str, int] = {
+        "all-MiniLM-L6-v2": 384,
+        "all-mpnet-base-v2": 768,
+        "paraphrase-MiniLM-L6-v2": 384,
+        "multi-qa-MiniLM-L6-cos-v1": 384,
+        "all-distilroberta-v1": 768,
+        "paraphrase-multilingual-MiniLM-L12-v2": 384,
+    }
+
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        device: str | None = None,
+    ) -> None:
+        if not _check_sentence_transformers():
+            raise ImportError(
+                "sentence-transformers is required for SentenceTransformersProvider. "
+                "Install with: pip install ragit[transformers]"
+            )
+
+        self._model_name = model_name
+        self._device = device
+        self._model: SentenceTransformer | None = None  # Lazy loaded
+        self._dimensions: int | None = self.MODEL_DIMENSIONS.get(model_name)
+
+    def _ensure_model(self) -> "SentenceTransformer":
+        """Ensure model is loaded (lazy loading)."""
+        if self._model is None:
+            model = _get_model(self._model_name, self._device)
+            self._model = model
+            # Update dimensions from actual model
+            self._dimensions = model.get_sentence_embedding_dimension()
+        return self._model
+
+    @property
+    def provider_name(self) -> str:
+        return "sentence_transformers"
+
+    @property
+    def dimensions(self) -> int:
+        if self._dimensions is None:
+            # Load model to get dimensions
+            self._ensure_model()
+        return self._dimensions or 384  # Fallback
+
+    @property
+    def model_name(self) -> str:
+        """Return the model name being used."""
+        return self._model_name
+
+    def is_available(self) -> bool:
+        """Check if sentence-transformers is installed and model can be loaded."""
+        if not _check_sentence_transformers():
+            return False
+        try:
+            self._ensure_model()
+            return True
+        except Exception:
+            return False
+
+    def embed(self, text: str, model: str = "") -> EmbeddingResponse:
+        """
+        Generate embedding for text.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (ignored, uses model from constructor).
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+        """
+        model_instance = self._ensure_model()
+        embedding = model_instance.encode(text, convert_to_numpy=True)
+
+        # Convert to tuple
+        embedding_tuple = tuple(float(x) for x in embedding)
+
+        return EmbeddingResponse(
+            embedding=embedding_tuple,
+            model=self._model_name,
+            provider=self.provider_name,
+            dimensions=len(embedding_tuple),
+        )
+
+    def embed_batch(self, texts: list[str], model: str = "") -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts efficiently.
+
+        Uses batch encoding for better performance.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier (ignored).
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        if not texts:
+            return []
+
+        model_instance = self._ensure_model()
+
+        # Batch encode for efficiency
+        embeddings = model_instance.encode(texts, convert_to_numpy=True, show_progress_bar=False)
+
+        results = []
+        for embedding in embeddings:
+            embedding_tuple = tuple(float(x) for x in embedding)
+            results.append(
+                EmbeddingResponse(
+                    embedding=embedding_tuple,
+                    model=self._model_name,
+                    provider=self.provider_name,
+                    dimensions=len(embedding_tuple),
+                )
+            )
+
+        return results

ragit/utils/__init__.py

@@ -0,0 +1,105 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Ragit utilities module.
+"""
+
+from collections import deque
+from collections.abc import Hashable, Sequence
+from datetime import datetime
+from math import floor
+from typing import Any
+
+import pandas as pd
+
+
+def get_hashable_repr(dct: dict[str, object]) -> tuple[tuple[str, object, float, int | None], ...]:
+    """
+    Returns a hashable representation of the provided dictionary.
+    """
+    queue: deque[tuple[str, object, float, int | None]] = deque((k, v, 0.0, None) for k, v in dct.items())
+    dict_unpacked: list[tuple[str, object, float, int | None]] = []
+    while queue:
+        key, val, lvl, p_ref = queue.pop()
+        if hasattr(val, "items"):  # we have a nested dict
+            dict_unpacked.append((key, "+", lvl, p_ref))
+            if hash(key) != p_ref:
+                lvl += 1
+            queue.extendleft((k, v, lvl, hash(key)) for k, v in val.items())
+        elif isinstance(val, Hashable):
+            dict_unpacked.append((key, val, lvl, p_ref))
+        elif isinstance(val, Sequence):
+            dict_unpacked.append((key, "+", lvl, p_ref))
+            queue.extendleft((key, vv, floor(lvl) + ind * 0.01, hash(key)) for ind, vv in enumerate(val, 1))
+        else:
+            raise ValueError(f"Unsupported type in dict: {type(val)}")
+
+    return tuple(sorted(dict_unpacked, key=lambda it: (it[2], it[0])))
+
+
+def remove_duplicates(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Deduplicates list of provided dictionary items.
+
+    Parameters
+    ----------
+    items : list[dict]
+        List of items to deduplicate.
+
+    Returns
+    -------
+    list[dict]
+        A deduplicated list of input items.
+    """
+    duplicate_tracker = set()
+    deduplicated_items = []
+    for ind, elem in enumerate(map(get_hashable_repr, items)):
+        if elem not in duplicate_tracker:
+            duplicate_tracker.add(elem)
+            deduplicated_items.append(items[ind])
+    return deduplicated_items
+
+
+def handle_missing_values_in_combinations(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Handle missing values in experiment data combinations.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Experiment data with combinations being explored.
+
+    Returns
+    -------
+    pd.DataFrame
+        Data with NaN values properly replaced.
+    """
+    if "chunk_overlap" in df.columns:
+        df["chunk_overlap"] = df["chunk_overlap"].map(lambda el: 0 if pd.isna(el) else el)
+
+    return df
+
+
+def datetime_str_to_epoch_time(timestamp: str | int) -> str | int:
+    """
+    Convert datetime string to epoch time.
+
+    Parameters
+    ----------
+    timestamp : str | int
+        Either a datetime string or a unix timestamp.
+
+    Returns
+    -------
+    int
+        Unix timestamp or -1 if parsing fails.
+    """
+    if not isinstance(timestamp, str):
+        return timestamp
+    try:
+        iso_parseable = datetime.fromisoformat(timestamp)
+    except ValueError:
+        return -1
+    return int(iso_parseable.timestamp())

ragit/version.py

@@ -0,0 +1,5 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+__version__ = "0.8.2"

ragit-0.8.2.dist-info/METADATA

@@ -0,0 +1,166 @@
+Metadata-Version: 2.4
+Name: ragit
+Version: 0.8.2
+Summary: Automatic RAG Pattern Optimization Engine
+Author: RODMENA LIMITED
+Maintainer-email: RODMENA LIMITED <info@rodmena.co.uk>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/rodmena-limited/ragit
+Project-URL: Repository, https://github.com/rodmena-limited/ragit
+Project-URL: Issues, https://github.com/rodmena-limited/ragit/issues
+Keywords: AI,RAG,LLM,GenAI,Optimization,Ollama
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: <3.14,>=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: scikit-learn>=1.5.0
+Requires-Dist: tqdm>=4.66.0
+Requires-Dist: trio>=0.24.0
+Requires-Dist: httpx>=0.27.0
+Provides-Extra: dev
+Requires-Dist: ragit[test]; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: issuedb[web]; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Provides-Extra: transformers
+Requires-Dist: sentence-transformers>=2.2.0; extra == "transformers"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
+Dynamic: license-file
+
+# ragit
+
+RAG toolkit for Python. Document loading, chunking, vector search, LLM integration.
+
+## Installation
+
+```bash
+pip install ragit
+
+# For offline embedding
+pip install ragit[transformers]
+```
+
+## Quick Start
+
+You must provide an embedding source: custom function, SentenceTransformers, or any provider.
+
+### Custom Embedding Function
+
+```python
+from ragit import RAGAssistant
+
+def my_embed(text: str) -> list[float]:
+    # Use any embedding API: OpenAI, Cohere, HuggingFace, etc.
+    return embedding_vector
+
+assistant = RAGAssistant("docs/", embed_fn=my_embed)
+results = assistant.retrieve("search query")
+```
+
+### With LLM for Q&A
+
+```python
+def my_embed(text: str) -> list[float]:
+    return embedding_vector
+
+def my_generate(prompt: str, system_prompt: str = "") -> str:
+    return llm_response
+
+assistant = RAGAssistant("docs/", embed_fn=my_embed, generate_fn=my_generate)
+answer = assistant.ask("How does authentication work?")
+```
+
+### Offline Embedding (SentenceTransformers)
+
+Models are downloaded automatically on first use (~90MB for default model).
+
+```python
+from ragit import RAGAssistant
+from ragit.providers import SentenceTransformersProvider
+
+# Uses all-MiniLM-L6-v2 by default
+assistant = RAGAssistant("docs/", provider=SentenceTransformersProvider())
+
+# Or specify a model
+assistant = RAGAssistant(
+    "docs/",
+    provider=SentenceTransformersProvider(model_name="all-mpnet-base-v2")
+)
+```
+
+Available models: `all-MiniLM-L6-v2` (384d), `all-mpnet-base-v2` (768d), `paraphrase-MiniLM-L6-v2` (384d)
+
+## Core API
+
+```python
+assistant = RAGAssistant(
+    documents,           # Path, list of Documents, or list of Chunks
+    embed_fn=...,        # Embedding function: (str) -> list[float]
+    generate_fn=...,     # LLM function: (prompt, system_prompt) -> str
+    provider=...,        # Or use a provider instead of functions
+    chunk_size=512,
+    chunk_overlap=50
+)
+
+results = assistant.retrieve(query, top_k=3)      # [(Chunk, score), ...]
+context = assistant.get_context(query, top_k=3)   # Formatted string
+answer = assistant.ask(question, top_k=3)         # Requires generate_fn/LLM
+code = assistant.generate_code(request)           # Requires generate_fn/LLM
+```
+
+## Document Loading
+
+```python
+from ragit import load_text, load_directory, chunk_text
+
+doc = load_text("file.md")
+docs = load_directory("docs/", "*.md")
+chunks = chunk_text(text, chunk_size=512, chunk_overlap=50, doc_id="id")
+```
+
+## Hyperparameter Optimization
+
+```python
+from ragit import RagitExperiment, Document, BenchmarkQuestion
+
+def my_embed(text: str) -> list[float]:
+    return embedding_vector
+
+def my_generate(prompt: str, system_prompt: str = "") -> str:
+    return llm_response
+
+docs = [Document(id="1", content="...")]
+benchmark = [BenchmarkQuestion(question="...", ground_truth="...")]
+
+experiment = RagitExperiment(
+    docs, benchmark,
+    embed_fn=my_embed,
+    generate_fn=my_generate
+)
+results = experiment.run(max_configs=20)
+print(results[0])  # Best config
+```
+
+## License
+
+Apache-2.0 - RODMENA LIMITED

ragit-0.8.2.dist-info/RECORD

@@ -0,0 +1,20 @@
+ragit/__init__.py,sha256=JUkL7ivgr4o4nZak-96P1C-pzKdNuN3Tl0X0WvpeXBU,3142
+ragit/assistant.py,sha256=LNof1zJAQWLIfhd7aPmKCpPQDCShpt9ezeM2nQ8ouyQ,18777
+ragit/config.py,sha256=7XnueNO4h22ibeWd1akHnfVoGSD8xE5vuOCMYeQOOU4,1898
+ragit/loaders.py,sha256=1JXgDLorvmtaDaRpbnKEqQjbQ4O5yfZxlb4QRUdGr58,6415
+ragit/version.py,sha256=WCqbf2oV6eXhq3DvqECcVFop-dseJIExoMxZ4fCtkvs,97
+ragit/core/__init__.py,sha256=j53PFfoSMXwSbK1rRHpMbo8mX2i4R1LJ5kvTxBd7-0w,100
+ragit/core/experiment/__init__.py,sha256=4vAPOOYlY5Dcr2gOolyhBSPGIUxZKwEkgQffxS9BodA,452
+ragit/core/experiment/experiment.py,sha256=aANDJ-XlMB0ijT8SBsPkb2U-lM3cChOuRO3oP9u3XxA,19331
+ragit/core/experiment/results.py,sha256=KHpN3YSLJ83_JUfIMccRPS-q7LEt0S9p8ehDRawk_4k,3487
+ragit/providers/__init__.py,sha256=tKWjUV31OZprD8k9aUUidtDMg7C_dWBXN7igtxeB8Ec,1339
+ragit/providers/base.py,sha256=MJ8mVeXuGWhkX2XGTbkWIY3cVoTOPr4h5XBXw8rAX2Q,3434
+ragit/providers/function_adapter.py,sha256=A-TQhBgBWbuO_w1sy795Dxep1FOCBpAlWpXCKVQD8rc,7778
+ragit/providers/ollama.py,sha256=YJH5a9nQHnP0NrIK7G9PqjV5A53f9JxmEJDAJ6d297M,15410
+ragit/providers/sentence_transformers.py,sha256=tTkd4HpE1MyfFJAwur-a7w-GlBxe93HlyM_dRffDrdY,6996
+ragit/utils/__init__.py,sha256=-UsE5oJSnmEnBDswl-ph0A09Iu8yKNbPhd1-_7Lcb8Y,3051
+ragit-0.8.2.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+ragit-0.8.2.dist-info/METADATA,sha256=wlBpVj_aHxR7ZWy5yzpo2Wt-IoLcVlFGo4oBXGzMajY,4888
+ragit-0.8.2.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+ragit-0.8.2.dist-info/top_level.txt,sha256=pkPbG7yrw61wt9_y_xcLE2vq2a55fzockASD0yq0g4s,6
+ragit-0.8.2.dist-info/RECORD,,

{ragit-0.8.dist-info → ragit-0.8.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any