ragit 0.7.5__py3-none-any.whl → 0.8.1__py3-none-any.whl

ragit/providers/function_adapter.py

@@ -0,0 +1,237 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Function-based provider adapter for pluggable embedding and LLM functions.
+
+This module provides a simple adapter that wraps user-provided functions
+into the provider interface, enabling easy integration with custom
+embedding and LLM implementations.
+"""
+
+import inspect
+from collections.abc import Callable
+
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    BaseLLMProvider,
+    EmbeddingResponse,
+    LLMResponse,
+)
+
+
+class FunctionProvider(BaseLLMProvider, BaseEmbeddingProvider):
+    """
+    Adapter that wraps user-provided embedding and generation functions.
+
+    This provider allows users to bring their own embedding and/or LLM functions
+    without implementing the full provider interface.
+
+    Parameters
+    ----------
+    embed_fn : Callable[[str], list[float]], optional
+        Function that takes text and returns an embedding vector.
+        Example: `lambda text: openai.embeddings.create(input=text).data[0].embedding`
+    generate_fn : Callable, optional
+        Function for text generation. Supports two signatures:
+        - (prompt: str) -> str
+        - (prompt: str, system_prompt: str) -> str
+    embedding_dimensions : int, optional
+        Embedding dimensions. Auto-detected on first call if not provided.
+
+    Examples
+    --------
+    >>> # Simple embedding function
+    >>> def my_embed(text: str) -> list[float]:
+    ...     return openai.embeddings.create(input=text).data[0].embedding
+    >>>
+    >>> # Use with RAGAssistant (retrieval-only)
+    >>> assistant = RAGAssistant(docs, embed_fn=my_embed)
+    >>> results = assistant.retrieve("query")
+    >>>
+    >>> # With LLM for full RAG
+    >>> def my_llm(prompt: str, system_prompt: str = None) -> str:
+    ...     return openai.chat.completions.create(
+    ...         messages=[{"role": "user", "content": prompt}]
+    ...     ).choices[0].message.content
+    >>>
+    >>> assistant = RAGAssistant(docs, embed_fn=my_embed, generate_fn=my_llm)
+    >>> answer = assistant.ask("What is X?")
+    """
+
+    def __init__(
+        self,
+        embed_fn: Callable[[str], list[float]] | None = None,
+        generate_fn: Callable[..., str] | None = None,
+        embedding_dimensions: int | None = None,
+    ) -> None:
+        self._embed_fn = embed_fn
+        self._generate_fn = generate_fn
+        self._embedding_dimensions = embedding_dimensions
+        self._generate_fn_signature: int | None = None  # Number of args (1 or 2)
+
+        # Detect generate_fn signature if provided
+        if generate_fn is not None:
+            self._detect_generate_signature()
+
+    def _detect_generate_signature(self) -> None:
+        """Detect whether generate_fn accepts 1 or 2 arguments."""
+        if self._generate_fn is None:
+            return
+
+        sig = inspect.signature(self._generate_fn)
+        params = [
+            p
+            for p in sig.parameters.values()
+            if p.default is inspect.Parameter.empty and p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
+        ]
+        # Count required parameters
+        required_count = len(params)
+
+        if required_count == 1:
+            self._generate_fn_signature = 1
+        else:
+            # Assume 2 args if more than 1 required or if has optional args
+            self._generate_fn_signature = 2
+
+    @property
+    def provider_name(self) -> str:
+        return "function"
+
+    @property
+    def dimensions(self) -> int:
+        if self._embedding_dimensions is None:
+            raise ValueError("Embedding dimensions not yet determined. Call embed() first or provide dimensions.")
+        return self._embedding_dimensions
+
+    @property
+    def has_embedding(self) -> bool:
+        """Check if embedding function is configured."""
+        return self._embed_fn is not None
+
+    @property
+    def has_llm(self) -> bool:
+        """Check if LLM generation function is configured."""
+        return self._generate_fn is not None
+
+    def is_available(self) -> bool:
+        """Check if the provider has at least one function configured."""
+        return self._embed_fn is not None or self._generate_fn is not None
+
+    def embed(self, text: str, model: str = "") -> EmbeddingResponse:
+        """
+        Generate embedding using the provided function.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (ignored, kept for interface compatibility).
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+
+        Raises
+        ------
+        ValueError
+            If no embedding function was provided.
+        """
+        if self._embed_fn is None:
+            raise ValueError("No embedding function configured. Provide embed_fn to use embeddings.")
+
+        raw_embedding = self._embed_fn(text)
+
+        # Convert to tuple for immutability
+        embedding_tuple: tuple[float, ...] = tuple(raw_embedding)
+
+        # Auto-detect dimensions on first call
+        if self._embedding_dimensions is None:
+            self._embedding_dimensions = len(embedding_tuple)
+
+        return EmbeddingResponse(
+            embedding=embedding_tuple,
+            model=model or "function",
+            provider=self.provider_name,
+            dimensions=len(embedding_tuple),
+        )
+
+    def embed_batch(self, texts: list[str], model: str = "") -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts.
+
+        Iterates over embed_fn for each text. For providers with native batch
+        support, users should implement their own BatchEmbeddingProvider.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier (ignored).
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        return [self.embed(text, model) for text in texts]
+
+    def generate(
+        self,
+        prompt: str,
+        model: str = "",
+        system_prompt: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """
+        Generate text using the provided function.
+
+        Parameters
+        ----------
+        prompt : str
+            The user prompt.
+        model : str
+            Model identifier (ignored, kept for interface compatibility).
+        system_prompt : str, optional
+            System prompt for context.
+        temperature : float
+            Sampling temperature (ignored if function doesn't support it).
+        max_tokens : int, optional
+            Maximum tokens (ignored if function doesn't support it).
+
+        Returns
+        -------
+        LLMResponse
+            The generated response.
+
+        Raises
+        ------
+        NotImplementedError
+            If no generation function was provided.
+        """
+        if self._generate_fn is None:
+            raise NotImplementedError(
+                "No LLM configured. Provide generate_fn or a provider with LLM support "
+                "to use ask(), generate(), or generate_code() methods."
+            )
+
+        # Call with appropriate signature
+        if self._generate_fn_signature == 1:
+            # Single argument - prepend system prompt to prompt if provided
+            full_prompt = f"{system_prompt}\n\n{prompt}" if system_prompt else prompt
+            text = self._generate_fn(full_prompt)
+        else:
+            # Two arguments - pass separately
+            text = self._generate_fn(prompt, system_prompt)
+
+        return LLMResponse(
+            text=text,
+            model=model or "function",
+            provider=self.provider_name,
+            usage=None,
+        )

ragit/providers/ollama.py

@@ -158,7 +158,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
                 f"{self.base_url}/api/tags",
                 timeout=5,
             )
-            return response.status_code == 200
+            return bool(response.status_code == 200)
         except requests.RequestException:
             return False
 

ragit/providers/sentence_transformers.py

@@ -0,0 +1,225 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+SentenceTransformers provider for offline embedding.
+
+This module provides embedding capabilities using the sentence-transformers
+library, enabling fully offline RAG pipelines without API dependencies.
+
+Requires: pip install ragit[transformers]
+"""
+
+from typing import TYPE_CHECKING
+
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    EmbeddingResponse,
+)
+
+if TYPE_CHECKING:
+    from sentence_transformers import SentenceTransformer
+
+# Lazy import flag
+_sentence_transformers_available: bool | None = None
+_model_cache: dict[str, "SentenceTransformer"] = {}
+
+
+def _check_sentence_transformers() -> bool:
+    """Check if sentence-transformers is available."""
+    global _sentence_transformers_available
+    if _sentence_transformers_available is None:
+        try:
+            from sentence_transformers import SentenceTransformer  # noqa: F401
+
+            _sentence_transformers_available = True
+        except ImportError:
+            _sentence_transformers_available = False
+    return _sentence_transformers_available
+
+
+def _get_model(model_name: str, device: str | None = None) -> "SentenceTransformer":
+    """Get or create a cached SentenceTransformer model."""
+    cache_key = f"{model_name}:{device or 'auto'}"
+    if cache_key not in _model_cache:
+        from sentence_transformers import SentenceTransformer
+
+        _model_cache[cache_key] = SentenceTransformer(model_name, device=device)
+    return _model_cache[cache_key]
+
+
+class SentenceTransformersProvider(BaseEmbeddingProvider):
+    """
+    Embedding provider using sentence-transformers for offline operation.
+
+    This provider uses the sentence-transformers library to generate embeddings
+    locally without requiring any API calls. It's ideal for:
+    - Offline/air-gapped environments
+    - Development and testing
+    - Cost-sensitive applications
+    - Privacy-sensitive use cases
+
+    Parameters
+    ----------
+    model_name : str
+        HuggingFace model name. Default: "all-MiniLM-L6-v2" (fast, 384 dims).
+        Other popular options:
+        - "all-mpnet-base-v2" (768 dims, higher quality)
+        - "paraphrase-MiniLM-L6-v2" (384 dims)
+        - "multi-qa-MiniLM-L6-cos-v1" (384 dims, optimized for QA)
+    device : str, optional
+        Device to run on ("cpu", "cuda", "mps"). Auto-detected if None.
+
+    Examples
+    --------
+    >>> # Basic usage
+    >>> from ragit.providers import SentenceTransformersProvider
+    >>> provider = SentenceTransformersProvider()
+    >>>
+    >>> # With RAGAssistant (retrieval-only)
+    >>> assistant = RAGAssistant(docs, provider=provider)
+    >>> results = assistant.retrieve("query")
+    >>>
+    >>> # Custom model
+    >>> provider = SentenceTransformersProvider(model_name="all-mpnet-base-v2")
+
+    Raises
+    ------
+    ImportError
+        If sentence-transformers is not installed.
+
+    Note
+    ----
+    Install with: pip install ragit[transformers]
+    """
+
+    # Known model dimensions for common models
+    MODEL_DIMENSIONS: dict[str, int] = {
+        "all-MiniLM-L6-v2": 384,
+        "all-mpnet-base-v2": 768,
+        "paraphrase-MiniLM-L6-v2": 384,
+        "multi-qa-MiniLM-L6-cos-v1": 384,
+        "all-distilroberta-v1": 768,
+        "paraphrase-multilingual-MiniLM-L12-v2": 384,
+    }
+
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        device: str | None = None,
+    ) -> None:
+        if not _check_sentence_transformers():
+            raise ImportError(
+                "sentence-transformers is required for SentenceTransformersProvider. "
+                "Install with: pip install ragit[transformers]"
+            )
+
+        self._model_name = model_name
+        self._device = device
+        self._model: SentenceTransformer | None = None  # Lazy loaded
+        self._dimensions: int | None = self.MODEL_DIMENSIONS.get(model_name)
+
+    def _ensure_model(self) -> "SentenceTransformer":
+        """Ensure model is loaded (lazy loading)."""
+        if self._model is None:
+            model = _get_model(self._model_name, self._device)
+            self._model = model
+            # Update dimensions from actual model
+            self._dimensions = model.get_sentence_embedding_dimension()
+        return self._model
+
+    @property
+    def provider_name(self) -> str:
+        return "sentence_transformers"
+
+    @property
+    def dimensions(self) -> int:
+        if self._dimensions is None:
+            # Load model to get dimensions
+            self._ensure_model()
+        return self._dimensions or 384  # Fallback
+
+    @property
+    def model_name(self) -> str:
+        """Return the model name being used."""
+        return self._model_name
+
+    def is_available(self) -> bool:
+        """Check if sentence-transformers is installed and model can be loaded."""
+        if not _check_sentence_transformers():
+            return False
+        try:
+            self._ensure_model()
+            return True
+        except Exception:
+            return False
+
+    def embed(self, text: str, model: str = "") -> EmbeddingResponse:
+        """
+        Generate embedding for text.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (ignored, uses model from constructor).
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+        """
+        model_instance = self._ensure_model()
+        embedding = model_instance.encode(text, convert_to_numpy=True)
+
+        # Convert to tuple
+        embedding_tuple = tuple(float(x) for x in embedding)
+
+        return EmbeddingResponse(
+            embedding=embedding_tuple,
+            model=self._model_name,
+            provider=self.provider_name,
+            dimensions=len(embedding_tuple),
+        )
+
+    def embed_batch(self, texts: list[str], model: str = "") -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts efficiently.
+
+        Uses batch encoding for better performance.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier (ignored).
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        if not texts:
+            return []
+
+        model_instance = self._ensure_model()
+
+        # Batch encode for efficiency
+        embeddings = model_instance.encode(texts, convert_to_numpy=True, show_progress_bar=False)
+
+        results = []
+        for embedding in embeddings:
+            embedding_tuple = tuple(float(x) for x in embedding)
+            results.append(
+                EmbeddingResponse(
+                    embedding=embedding_tuple,
+                    model=self._model_name,
+                    provider=self.provider_name,
+                    dimensions=len(embedding_tuple),
+                )
+            )
+
+        return results

ragit/version.py

@@ -2,4 +2,4 @@
 # Copyright RODMENA LIMITED 2025
 # SPDX-License-Identifier: Apache-2.0
 #
-__version__ = "0.7.5"
+__version__ = "0.8.1"

ragit-0.8.1.dist-info/METADATA

@@ -0,0 +1,166 @@
+Metadata-Version: 2.4
+Name: ragit
+Version: 0.8.1
+Summary: Automatic RAG Pattern Optimization Engine
+Author: RODMENA LIMITED
+Maintainer-email: RODMENA LIMITED <info@rodmena.co.uk>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/rodmena-limited/ragit
+Project-URL: Repository, https://github.com/rodmena-limited/ragit
+Project-URL: Issues, https://github.com/rodmena-limited/ragit/issues
+Keywords: AI,RAG,LLM,GenAI,Optimization,Ollama
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: <3.14,>=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: scikit-learn>=1.5.0
+Requires-Dist: tqdm>=4.66.0
+Requires-Dist: trio>=0.24.0
+Requires-Dist: httpx>=0.27.0
+Provides-Extra: dev
+Requires-Dist: ragit[test]; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: issuedb[web]; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Provides-Extra: transformers
+Requires-Dist: sentence-transformers>=2.2.0; extra == "transformers"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
+Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
+Dynamic: license-file
+
+# ragit
+
+RAG toolkit for Python. Document loading, chunking, vector search, LLM integration.
+
+## Installation
+
+```bash
+pip install ragit
+
+# For offline embedding
+pip install ragit[transformers]
+```
+
+## Quick Start
+
+You must provide an embedding source: custom function, SentenceTransformers, or any provider.
+
+### Custom Embedding Function
+
+```python
+from ragit import RAGAssistant
+
+def my_embed(text: str) -> list[float]:
+    # Use any embedding API: OpenAI, Cohere, HuggingFace, etc.
+    return embedding_vector
+
+assistant = RAGAssistant("docs/", embed_fn=my_embed)
+results = assistant.retrieve("search query")
+```
+
+### With LLM for Q&A
+
+```python
+def my_embed(text: str) -> list[float]:
+    return embedding_vector
+
+def my_generate(prompt: str, system_prompt: str = "") -> str:
+    return llm_response
+
+assistant = RAGAssistant("docs/", embed_fn=my_embed, generate_fn=my_generate)
+answer = assistant.ask("How does authentication work?")
+```
+
+### Offline Embedding (SentenceTransformers)
+
+Models are downloaded automatically on first use (~90MB for default model).
+
+```python
+from ragit import RAGAssistant
+from ragit.providers import SentenceTransformersProvider
+
+# Uses all-MiniLM-L6-v2 by default
+assistant = RAGAssistant("docs/", provider=SentenceTransformersProvider())
+
+# Or specify a model
+assistant = RAGAssistant(
+    "docs/",
+    provider=SentenceTransformersProvider(model_name="all-mpnet-base-v2")
+)
+```
+
+Available models: `all-MiniLM-L6-v2` (384d), `all-mpnet-base-v2` (768d), `paraphrase-MiniLM-L6-v2` (384d)
+
+## Core API
+
+```python
+assistant = RAGAssistant(
+    documents,           # Path, list of Documents, or list of Chunks
+    embed_fn=...,        # Embedding function: (str) -> list[float]
+    generate_fn=...,     # LLM function: (prompt, system_prompt) -> str
+    provider=...,        # Or use a provider instead of functions
+    chunk_size=512,
+    chunk_overlap=50
+)
+
+results = assistant.retrieve(query, top_k=3)      # [(Chunk, score), ...]
+context = assistant.get_context(query, top_k=3)   # Formatted string
+answer = assistant.ask(question, top_k=3)         # Requires generate_fn/LLM
+code = assistant.generate_code(request)           # Requires generate_fn/LLM
+```
+
+## Document Loading
+
+```python
+from ragit import load_text, load_directory, chunk_text
+
+doc = load_text("file.md")
+docs = load_directory("docs/", "*.md")
+chunks = chunk_text(text, chunk_size=512, chunk_overlap=50, doc_id="id")
+```
+
+## Hyperparameter Optimization
+
+```python
+from ragit import RagitExperiment, Document, BenchmarkQuestion
+
+def my_embed(text: str) -> list[float]:
+    return embedding_vector
+
+def my_generate(prompt: str, system_prompt: str = "") -> str:
+    return llm_response
+
+docs = [Document(id="1", content="...")]
+benchmark = [BenchmarkQuestion(question="...", ground_truth="...")]
+
+experiment = RagitExperiment(
+    docs, benchmark,
+    embed_fn=my_embed,
+    generate_fn=my_generate
+)
+results = experiment.run(max_configs=20)
+print(results[0])  # Best config
+```
+
+## License
+
+Apache-2.0 - RODMENA LIMITED

ragit-0.8.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+ragit/__init__.py,sha256=JUkL7ivgr4o4nZak-96P1C-pzKdNuN3Tl0X0WvpeXBU,3142
+ragit/assistant.py,sha256=FW8LVqEOA1nemTMdTZhb79aONeHsQM8tHADxCQ47p1Y,14705
+ragit/config.py,sha256=7XnueNO4h22ibeWd1akHnfVoGSD8xE5vuOCMYeQOOU4,1898
+ragit/loaders.py,sha256=keusuPzXPBiLDVj4hKfPCcge-rm-cnzNRk50fGXvTJs,5571
+ragit/version.py,sha256=_qpX4vMVMSqb-_4jdv6EZJ3tkvFsyu_Pj00vRC6T2sg,97
+ragit/core/__init__.py,sha256=j53PFfoSMXwSbK1rRHpMbo8mX2i4R1LJ5kvTxBd7-0w,100
+ragit/core/experiment/__init__.py,sha256=4vAPOOYlY5Dcr2gOolyhBSPGIUxZKwEkgQffxS9BodA,452
+ragit/core/experiment/experiment.py,sha256=WQZWRLbLPuGpG0tpCZCEz3sKgSv4CNimmABbOLR_oKs,19314
+ragit/core/experiment/results.py,sha256=KHpN3YSLJ83_JUfIMccRPS-q7LEt0S9p8ehDRawk_4k,3487
+ragit/providers/__init__.py,sha256=tKWjUV31OZprD8k9aUUidtDMg7C_dWBXN7igtxeB8Ec,1339
+ragit/providers/base.py,sha256=MJ8mVeXuGWhkX2XGTbkWIY3cVoTOPr4h5XBXw8rAX2Q,3434
+ragit/providers/function_adapter.py,sha256=A-TQhBgBWbuO_w1sy795Dxep1FOCBpAlWpXCKVQD8rc,7778
+ragit/providers/ollama.py,sha256=YJH5a9nQHnP0NrIK7G9PqjV5A53f9JxmEJDAJ6d297M,15410
+ragit/providers/sentence_transformers.py,sha256=tTkd4HpE1MyfFJAwur-a7w-GlBxe93HlyM_dRffDrdY,6996
+ragit/utils/__init__.py,sha256=-UsE5oJSnmEnBDswl-ph0A09Iu8yKNbPhd1-_7Lcb8Y,3051
+ragit-0.8.1.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+ragit-0.8.1.dist-info/METADATA,sha256=OaOeM-ujuMlkfjiNcXRUC6JpIApFgkvP536nHsaLW0g,4888
+ragit-0.8.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ragit-0.8.1.dist-info/top_level.txt,sha256=pkPbG7yrw61wt9_y_xcLE2vq2a55fzockASD0yq0g4s,6
+ragit-0.8.1.dist-info/RECORD,,