ragit 0.8__py3-none-any.whl → 0.8.1__py3-none-any.whl

ragit/loaders.py

@@ -0,0 +1,219 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Document loading and chunking utilities.
+
+Provides simple functions to load documents from files and chunk text.
+"""
+
+import re
+from pathlib import Path
+
+from ragit.core.experiment.experiment import Chunk, Document
+
+
+def load_text(path: str | Path) -> Document:
+    """
+    Load a single text file as a Document.
+
+    Parameters
+    ----------
+    path : str or Path
+        Path to the text file (.txt, .md, .rst, etc.)
+
+    Returns
+    -------
+    Document
+        Document with file content and metadata.
+
+    Examples
+    --------
+    >>> doc = load_text("docs/tutorial.rst")
+    >>> print(doc.id, len(doc.content))
+    """
+    path = Path(path)
+    content = path.read_text(encoding="utf-8")
+    return Document(id=path.stem, content=content, metadata={"source": str(path), "filename": path.name})
+
+
+def load_directory(path: str | Path, pattern: str = "*.txt", recursive: bool = False) -> list[Document]:
+    """
+    Load all matching files from a directory as Documents.
+
+    Parameters
+    ----------
+    path : str or Path
+        Directory path.
+    pattern : str
+        Glob pattern for files (default: "*.txt").
+    recursive : bool
+        If True, search recursively (default: False).
+
+    Returns
+    -------
+    list[Document]
+        List of loaded documents.
+
+    Examples
+    --------
+    >>> docs = load_directory("docs/", "*.rst")
+    >>> docs = load_directory("docs/", "**/*.md", recursive=True)
+    """
+    path = Path(path)
+    glob_method = path.rglob if recursive else path.glob
+    documents = []
+
+    for file_path in sorted(glob_method(pattern)):
+        if file_path.is_file():
+            documents.append(load_text(file_path))
+
+    return documents
+
+
+def chunk_text(text: str, chunk_size: int = 512, chunk_overlap: int = 50, doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split text into overlapping chunks.
+
+    Parameters
+    ----------
+    text : str
+        Text to chunk.
+    chunk_size : int
+        Maximum characters per chunk (default: 512).
+    chunk_overlap : int
+        Overlap between chunks (default: 50).
+    doc_id : str
+        Document ID for the chunks (default: "doc").
+
+    Returns
+    -------
+    list[Chunk]
+        List of text chunks.
+
+    Examples
+    --------
+    >>> chunks = chunk_text("Long document...", chunk_size=256, chunk_overlap=50)
+    """
+    if chunk_overlap >= chunk_size:
+        raise ValueError("chunk_overlap must be less than chunk_size")
+
+    chunks = []
+    start = 0
+    chunk_idx = 0
+
+    while start < len(text):
+        end = start + chunk_size
+        chunk_text = text[start:end].strip()
+
+        if chunk_text:
+            chunks.append(Chunk(content=chunk_text, doc_id=doc_id, chunk_index=chunk_idx))
+            chunk_idx += 1
+
+        start = end - chunk_overlap
+        if start >= len(text) - chunk_overlap:
+            break
+
+    return chunks
+
+
+def chunk_document(doc: Document, chunk_size: int = 512, chunk_overlap: int = 50) -> list[Chunk]:
+    """
+    Split a Document into overlapping chunks.
+
+    Parameters
+    ----------
+    doc : Document
+        Document to chunk.
+    chunk_size : int
+        Maximum characters per chunk.
+    chunk_overlap : int
+        Overlap between chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of chunks from the document.
+    """
+    return chunk_text(doc.content, chunk_size, chunk_overlap, doc.id)
+
+
+def chunk_by_separator(text: str, separator: str = "\n\n", doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split text by a separator (e.g., paragraphs, sections).
+
+    Parameters
+    ----------
+    text : str
+        Text to split.
+    separator : str
+        Separator string (default: double newline for paragraphs).
+    doc_id : str
+        Document ID for the chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of chunks.
+
+    Examples
+    --------
+    >>> chunks = chunk_by_separator(text, separator="\\n---\\n")
+    """
+    parts = text.split(separator)
+    chunks = []
+
+    for idx, part in enumerate(parts):
+        content = part.strip()
+        if content:
+            chunks.append(Chunk(content=content, doc_id=doc_id, chunk_index=idx))
+
+    return chunks
+
+
+def chunk_rst_sections(text: str, doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split RST document by section headers.
+
+    Parameters
+    ----------
+    text : str
+        RST document text.
+    doc_id : str
+        Document ID for the chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of section chunks.
+    """
+    # Match RST section headers (title followed by underline of =, -, ~, etc.)
+    pattern = r"\n([^\n]+)\n([=\-~`\'\"^_*+#]+)\n"
+
+    # Find all section positions
+    matches = list(re.finditer(pattern, text))
+
+    if not matches:
+        # No sections found, return whole text as one chunk
+        return [Chunk(content=text.strip(), doc_id=doc_id, chunk_index=0)] if text.strip() else []
+
+    chunks = []
+
+    # Handle content before first section
+    first_pos = matches[0].start()
+    if first_pos > 0:
+        pre_content = text[:first_pos].strip()
+        if pre_content:
+            chunks.append(Chunk(content=pre_content, doc_id=doc_id, chunk_index=0))
+
+    # Extract each section
+    for i, match in enumerate(matches):
+        start = match.start()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+
+        section_content = text[start:end].strip()
+        if section_content:
+            chunks.append(Chunk(content=section_content, doc_id=doc_id, chunk_index=len(chunks)))
+
+    return chunks

ragit/providers/__init__.py

@@ -0,0 +1,47 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Ragit Providers - LLM and Embedding providers for RAG optimization.
+
+Supported providers:
+- OllamaProvider: Connect to local or remote Ollama servers
+- FunctionProvider: Wrap custom embedding/LLM functions
+- SentenceTransformersProvider: Offline embedding (requires ragit[transformers])
+
+Base classes for implementing custom providers:
+- BaseLLMProvider: Abstract base for LLM providers
+- BaseEmbeddingProvider: Abstract base for embedding providers
+"""
+
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    BaseLLMProvider,
+    EmbeddingResponse,
+    LLMResponse,
+)
+from ragit.providers.function_adapter import FunctionProvider
+from ragit.providers.ollama import OllamaProvider
+
+__all__ = [
+    # Base classes
+    "BaseLLMProvider",
+    "BaseEmbeddingProvider",
+    "LLMResponse",
+    "EmbeddingResponse",
+    # Built-in providers
+    "OllamaProvider",
+    "FunctionProvider",
+]
+
+# Conditionally export SentenceTransformersProvider if available
+try:
+    from ragit.providers.sentence_transformers import (
+        SentenceTransformersProvider as SentenceTransformersProvider,
+    )
+
+    __all__ += ["SentenceTransformersProvider"]
+except ImportError:
+    # sentence-transformers not installed, SentenceTransformersProvider not available
+    pass

ragit/providers/base.py

@@ -0,0 +1,147 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Base provider interfaces for LLM and Embedding providers.
+
+These abstract classes define the interface that all providers must implement,
+making it easy to add new providers (Gemini, Claude, OpenAI, etc.)
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class LLMResponse:
+    """Response from an LLM call."""
+
+    text: str
+    model: str
+    provider: str
+    usage: dict[str, int] | None = None
+
+
+@dataclass(frozen=True)
+class EmbeddingResponse:
+    """Response from an embedding call (immutable)."""
+
+    embedding: tuple[float, ...]
+    model: str
+    provider: str
+    dimensions: int
+
+
+class BaseLLMProvider(ABC):
+    """
+    Abstract base class for LLM providers.
+
+    Implement this to add support for new LLM providers like Gemini, Claude, etc.
+    """
+
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Return the provider name (e.g., 'ollama', 'gemini', 'claude')."""
+        pass
+
+    @abstractmethod
+    def generate(
+        self,
+        prompt: str,
+        model: str,
+        system_prompt: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """
+        Generate text from the LLM.
+
+        Parameters
+        ----------
+        prompt : str
+            The user prompt/query.
+        model : str
+            Model identifier (e.g., 'llama3', 'qwen3-vl:235b-instruct-cloud').
+        system_prompt : str, optional
+            System prompt for context/instructions.
+        temperature : float
+            Sampling temperature (0.0 to 1.0).
+        max_tokens : int, optional
+            Maximum tokens to generate.
+
+        Returns
+        -------
+        LLMResponse
+            The generated response.
+        """
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the provider is available and configured."""
+        pass
+
+
+class BaseEmbeddingProvider(ABC):
+    """
+    Abstract base class for embedding providers.
+
+    Implement this to add support for new embedding providers.
+    """
+
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Return the provider name."""
+        pass
+
+    @property
+    @abstractmethod
+    def dimensions(self) -> int:
+        """Return the embedding dimensions for the current model."""
+        pass
+
+    @abstractmethod
+    def embed(self, text: str, model: str) -> EmbeddingResponse:
+        """
+        Generate embedding for text.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (e.g., 'nomic-embed-text').
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+        """
+        pass
+
+    @abstractmethod
+    def embed_batch(self, texts: list[str], model: str) -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier.
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the provider is available and configured."""
+        pass

ragit/providers/function_adapter.py

@@ -0,0 +1,237 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Function-based provider adapter for pluggable embedding and LLM functions.
+
+This module provides a simple adapter that wraps user-provided functions
+into the provider interface, enabling easy integration with custom
+embedding and LLM implementations.
+"""
+
+import inspect
+from collections.abc import Callable
+
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    BaseLLMProvider,
+    EmbeddingResponse,
+    LLMResponse,
+)
+
+
+class FunctionProvider(BaseLLMProvider, BaseEmbeddingProvider):
+    """
+    Adapter that wraps user-provided embedding and generation functions.
+
+    This provider allows users to bring their own embedding and/or LLM functions
+    without implementing the full provider interface.
+
+    Parameters
+    ----------
+    embed_fn : Callable[[str], list[float]], optional
+        Function that takes text and returns an embedding vector.
+        Example: `lambda text: openai.embeddings.create(input=text).data[0].embedding`
+    generate_fn : Callable, optional
+        Function for text generation. Supports two signatures:
+        - (prompt: str) -> str
+        - (prompt: str, system_prompt: str) -> str
+    embedding_dimensions : int, optional
+        Embedding dimensions. Auto-detected on first call if not provided.
+
+    Examples
+    --------
+    >>> # Simple embedding function
+    >>> def my_embed(text: str) -> list[float]:
+    ...     return openai.embeddings.create(input=text).data[0].embedding
+    >>>
+    >>> # Use with RAGAssistant (retrieval-only)
+    >>> assistant = RAGAssistant(docs, embed_fn=my_embed)
+    >>> results = assistant.retrieve("query")
+    >>>
+    >>> # With LLM for full RAG
+    >>> def my_llm(prompt: str, system_prompt: str = None) -> str:
+    ...     return openai.chat.completions.create(
+    ...         messages=[{"role": "user", "content": prompt}]
+    ...     ).choices[0].message.content
+    >>>
+    >>> assistant = RAGAssistant(docs, embed_fn=my_embed, generate_fn=my_llm)
+    >>> answer = assistant.ask("What is X?")
+    """
+
+    def __init__(
+        self,
+        embed_fn: Callable[[str], list[float]] | None = None,
+        generate_fn: Callable[..., str] | None = None,
+        embedding_dimensions: int | None = None,
+    ) -> None:
+        self._embed_fn = embed_fn
+        self._generate_fn = generate_fn
+        self._embedding_dimensions = embedding_dimensions
+        self._generate_fn_signature: int | None = None  # Number of args (1 or 2)
+
+        # Detect generate_fn signature if provided
+        if generate_fn is not None:
+            self._detect_generate_signature()
+
+    def _detect_generate_signature(self) -> None:
+        """Detect whether generate_fn accepts 1 or 2 arguments."""
+        if self._generate_fn is None:
+            return
+
+        sig = inspect.signature(self._generate_fn)
+        params = [
+            p
+            for p in sig.parameters.values()
+            if p.default is inspect.Parameter.empty and p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
+        ]
+        # Count required parameters
+        required_count = len(params)
+
+        if required_count == 1:
+            self._generate_fn_signature = 1
+        else:
+            # Assume 2 args if more than 1 required or if has optional args
+            self._generate_fn_signature = 2
+
+    @property
+    def provider_name(self) -> str:
+        return "function"
+
+    @property
+    def dimensions(self) -> int:
+        if self._embedding_dimensions is None:
+            raise ValueError("Embedding dimensions not yet determined. Call embed() first or provide dimensions.")
+        return self._embedding_dimensions
+
+    @property
+    def has_embedding(self) -> bool:
+        """Check if embedding function is configured."""
+        return self._embed_fn is not None
+
+    @property
+    def has_llm(self) -> bool:
+        """Check if LLM generation function is configured."""
+        return self._generate_fn is not None
+
+    def is_available(self) -> bool:
+        """Check if the provider has at least one function configured."""
+        return self._embed_fn is not None or self._generate_fn is not None
+
+    def embed(self, text: str, model: str = "") -> EmbeddingResponse:
+        """
+        Generate embedding using the provided function.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (ignored, kept for interface compatibility).
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+
+        Raises
+        ------
+        ValueError
+            If no embedding function was provided.
+        """
+        if self._embed_fn is None:
+            raise ValueError("No embedding function configured. Provide embed_fn to use embeddings.")
+
+        raw_embedding = self._embed_fn(text)
+
+        # Convert to tuple for immutability
+        embedding_tuple: tuple[float, ...] = tuple(raw_embedding)
+
+        # Auto-detect dimensions on first call
+        if self._embedding_dimensions is None:
+            self._embedding_dimensions = len(embedding_tuple)
+
+        return EmbeddingResponse(
+            embedding=embedding_tuple,
+            model=model or "function",
+            provider=self.provider_name,
+            dimensions=len(embedding_tuple),
+        )
+
+    def embed_batch(self, texts: list[str], model: str = "") -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts.
+
+        Iterates over embed_fn for each text. For providers with native batch
+        support, users should implement their own BatchEmbeddingProvider.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier (ignored).
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        return [self.embed(text, model) for text in texts]
+
+    def generate(
+        self,
+        prompt: str,
+        model: str = "",
+        system_prompt: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """
+        Generate text using the provided function.
+
+        Parameters
+        ----------
+        prompt : str
+            The user prompt.
+        model : str
+            Model identifier (ignored, kept for interface compatibility).
+        system_prompt : str, optional
+            System prompt for context.
+        temperature : float
+            Sampling temperature (ignored if function doesn't support it).
+        max_tokens : int, optional
+            Maximum tokens (ignored if function doesn't support it).
+
+        Returns
+        -------
+        LLMResponse
+            The generated response.
+
+        Raises
+        ------
+        NotImplementedError
+            If no generation function was provided.
+        """
+        if self._generate_fn is None:
+            raise NotImplementedError(
+                "No LLM configured. Provide generate_fn or a provider with LLM support "
+                "to use ask(), generate(), or generate_code() methods."
+            )
+
+        # Call with appropriate signature
+        if self._generate_fn_signature == 1:
+            # Single argument - prepend system prompt to prompt if provided
+            full_prompt = f"{system_prompt}\n\n{prompt}" if system_prompt else prompt
+            text = self._generate_fn(full_prompt)
+        else:
+            # Two arguments - pass separately
+            text = self._generate_fn(prompt, system_prompt)
+
+        return LLMResponse(
+            text=text,
+            model=model or "function",
+            provider=self.provider_name,
+            usage=None,
+        )