ragit 0.7__py3-none-any.whl → 0.7.2__py3-none-any.whl

ragit/loaders.py

@@ -0,0 +1,219 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Document loading and chunking utilities.
+
+Provides simple functions to load documents from files and chunk text.
+"""
+
+import re
+from pathlib import Path
+
+from ragit.core.experiment.experiment import Chunk, Document
+
+
+def load_text(path: str | Path) -> Document:
+    """
+    Load a single text file as a Document.
+
+    Parameters
+    ----------
+    path : str or Path
+        Path to the text file (.txt, .md, .rst, etc.)
+
+    Returns
+    -------
+    Document
+        Document with file content and metadata.
+
+    Examples
+    --------
+    >>> doc = load_text("docs/tutorial.rst")
+    >>> print(doc.id, len(doc.content))
+    """
+    path = Path(path)
+    content = path.read_text(encoding="utf-8")
+    return Document(id=path.stem, content=content, metadata={"source": str(path), "filename": path.name})
+
+
+def load_directory(path: str | Path, pattern: str = "*.txt", recursive: bool = False) -> list[Document]:
+    """
+    Load all matching files from a directory as Documents.
+
+    Parameters
+    ----------
+    path : str or Path
+        Directory path.
+    pattern : str
+        Glob pattern for files (default: "*.txt").
+    recursive : bool
+        If True, search recursively (default: False).
+
+    Returns
+    -------
+    list[Document]
+        List of loaded documents.
+
+    Examples
+    --------
+    >>> docs = load_directory("docs/", "*.rst")
+    >>> docs = load_directory("docs/", "**/*.md", recursive=True)
+    """
+    path = Path(path)
+    glob_method = path.rglob if recursive else path.glob
+    documents = []
+
+    for file_path in sorted(glob_method(pattern)):
+        if file_path.is_file():
+            documents.append(load_text(file_path))
+
+    return documents
+
+
+def chunk_text(text: str, chunk_size: int = 512, chunk_overlap: int = 50, doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split text into overlapping chunks.
+
+    Parameters
+    ----------
+    text : str
+        Text to chunk.
+    chunk_size : int
+        Maximum characters per chunk (default: 512).
+    chunk_overlap : int
+        Overlap between chunks (default: 50).
+    doc_id : str
+        Document ID for the chunks (default: "doc").
+
+    Returns
+    -------
+    list[Chunk]
+        List of text chunks.
+
+    Examples
+    --------
+    >>> chunks = chunk_text("Long document...", chunk_size=256, chunk_overlap=50)
+    """
+    if chunk_overlap >= chunk_size:
+        raise ValueError("chunk_overlap must be less than chunk_size")
+
+    chunks = []
+    start = 0
+    chunk_idx = 0
+
+    while start < len(text):
+        end = start + chunk_size
+        chunk_text = text[start:end].strip()
+
+        if chunk_text:
+            chunks.append(Chunk(content=chunk_text, doc_id=doc_id, chunk_index=chunk_idx))
+            chunk_idx += 1
+
+        start = end - chunk_overlap
+        if start >= len(text) - chunk_overlap:
+            break
+
+    return chunks
+
+
+def chunk_document(doc: Document, chunk_size: int = 512, chunk_overlap: int = 50) -> list[Chunk]:
+    """
+    Split a Document into overlapping chunks.
+
+    Parameters
+    ----------
+    doc : Document
+        Document to chunk.
+    chunk_size : int
+        Maximum characters per chunk.
+    chunk_overlap : int
+        Overlap between chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of chunks from the document.
+    """
+    return chunk_text(doc.content, chunk_size, chunk_overlap, doc.id)
+
+
+def chunk_by_separator(text: str, separator: str = "\n\n", doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split text by a separator (e.g., paragraphs, sections).
+
+    Parameters
+    ----------
+    text : str
+        Text to split.
+    separator : str
+        Separator string (default: double newline for paragraphs).
+    doc_id : str
+        Document ID for the chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of chunks.
+
+    Examples
+    --------
+    >>> chunks = chunk_by_separator(text, separator="\\n---\\n")
+    """
+    parts = text.split(separator)
+    chunks = []
+
+    for idx, part in enumerate(parts):
+        content = part.strip()
+        if content:
+            chunks.append(Chunk(content=content, doc_id=doc_id, chunk_index=idx))
+
+    return chunks
+
+
+def chunk_rst_sections(text: str, doc_id: str = "doc") -> list[Chunk]:
+    """
+    Split RST document by section headers.
+
+    Parameters
+    ----------
+    text : str
+        RST document text.
+    doc_id : str
+        Document ID for the chunks.
+
+    Returns
+    -------
+    list[Chunk]
+        List of section chunks.
+    """
+    # Match RST section headers (title followed by underline of =, -, ~, etc.)
+    pattern = r"\n([^\n]+)\n([=\-~`\'\"^_*+#]+)\n"
+
+    # Find all section positions
+    matches = list(re.finditer(pattern, text))
+
+    if not matches:
+        # No sections found, return whole text as one chunk
+        return [Chunk(content=text.strip(), doc_id=doc_id, chunk_index=0)] if text.strip() else []
+
+    chunks = []
+
+    # Handle content before first section
+    first_pos = matches[0].start()
+    if first_pos > 0:
+        pre_content = text[:first_pos].strip()
+        if pre_content:
+            chunks.append(Chunk(content=pre_content, doc_id=doc_id, chunk_index=0))
+
+    # Extract each section
+    for i, match in enumerate(matches):
+        start = match.start()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+
+        section_content = text[start:end].strip()
+        if section_content:
+            chunks.append(Chunk(content=section_content, doc_id=doc_id, chunk_index=len(chunks)))
+
+    return chunks

ragit/providers/__init__.py

@@ -0,0 +1,20 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Ragit Providers - LLM and Embedding providers for RAG optimization.
+
+Supported providers:
+- Ollama (local)
+- Future: Gemini, Claude, OpenAI
+"""
+
+from ragit.providers.base import BaseEmbeddingProvider, BaseLLMProvider
+from ragit.providers.ollama import OllamaProvider
+
+__all__ = [
+    "BaseLLMProvider",
+    "BaseEmbeddingProvider",
+    "OllamaProvider",
+]

ragit/providers/base.py

@@ -0,0 +1,147 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Base provider interfaces for LLM and Embedding providers.
+
+These abstract classes define the interface that all providers must implement,
+making it easy to add new providers (Gemini, Claude, OpenAI, etc.)
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class LLMResponse:
+    """Response from an LLM call."""
+
+    text: str
+    model: str
+    provider: str
+    usage: dict[str, int] | None = None
+
+
+@dataclass(frozen=True)
+class EmbeddingResponse:
+    """Response from an embedding call (immutable)."""
+
+    embedding: tuple[float, ...]
+    model: str
+    provider: str
+    dimensions: int
+
+
+class BaseLLMProvider(ABC):
+    """
+    Abstract base class for LLM providers.
+
+    Implement this to add support for new LLM providers like Gemini, Claude, etc.
+    """
+
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Return the provider name (e.g., 'ollama', 'gemini', 'claude')."""
+        pass
+
+    @abstractmethod
+    def generate(
+        self,
+        prompt: str,
+        model: str,
+        system_prompt: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """
+        Generate text from the LLM.
+
+        Parameters
+        ----------
+        prompt : str
+            The user prompt/query.
+        model : str
+            Model identifier (e.g., 'llama3', 'qwen3-vl:235b-instruct-cloud').
+        system_prompt : str, optional
+            System prompt for context/instructions.
+        temperature : float
+            Sampling temperature (0.0 to 1.0).
+        max_tokens : int, optional
+            Maximum tokens to generate.
+
+        Returns
+        -------
+        LLMResponse
+            The generated response.
+        """
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the provider is available and configured."""
+        pass
+
+
+class BaseEmbeddingProvider(ABC):
+    """
+    Abstract base class for embedding providers.
+
+    Implement this to add support for new embedding providers.
+    """
+
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Return the provider name."""
+        pass
+
+    @property
+    @abstractmethod
+    def dimensions(self) -> int:
+        """Return the embedding dimensions for the current model."""
+        pass
+
+    @abstractmethod
+    def embed(self, text: str, model: str) -> EmbeddingResponse:
+        """
+        Generate embedding for text.
+
+        Parameters
+        ----------
+        text : str
+            Text to embed.
+        model : str
+            Model identifier (e.g., 'nomic-embed-text').
+
+        Returns
+        -------
+        EmbeddingResponse
+            The embedding response.
+        """
+        pass
+
+    @abstractmethod
+    def embed_batch(self, texts: list[str], model: str) -> list[EmbeddingResponse]:
+        """
+        Generate embeddings for multiple texts.
+
+        Parameters
+        ----------
+        texts : list[str]
+            Texts to embed.
+        model : str
+            Model identifier.
+
+        Returns
+        -------
+        list[EmbeddingResponse]
+            List of embedding responses.
+        """
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the provider is available and configured."""
+        pass

ragit/providers/ollama.py

@@ -0,0 +1,284 @@
+#
+# Copyright RODMENA LIMITED 2025
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Ollama provider for LLM and Embedding operations.
+
+This provider connects to a local or remote Ollama server.
+Configuration is loaded from environment variables.
+"""
+
+import requests
+
+from ragit.config import config
+from ragit.providers.base import (
+    BaseEmbeddingProvider,
+    BaseLLMProvider,
+    EmbeddingResponse,
+    LLMResponse,
+)
+
+
+class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
+    """
+    Ollama provider for both LLM and Embedding operations.
+
+    Parameters
+    ----------
+    base_url : str, optional
+        Ollama server URL (default: from OLLAMA_BASE_URL env var)
+    api_key : str, optional
+        API key for authentication (default: from OLLAMA_API_KEY env var)
+    timeout : int, optional
+        Request timeout in seconds (default: from OLLAMA_TIMEOUT env var)
+
+    Examples
+    --------
+    >>> provider = OllamaProvider()
+    >>> response = provider.generate("What is RAG?", model="llama3")
+    >>> print(response.text)
+
+    >>> embedding = provider.embed("Hello world", model="nomic-embed-text")
+    >>> print(len(embedding.embedding))
+    """
+
+    # Known embedding model dimensions
+    EMBEDDING_DIMENSIONS = {
+        "nomic-embed-text": 768,
+        "nomic-embed-text:latest": 768,
+        "mxbai-embed-large": 1024,
+        "all-minilm": 384,
+        "snowflake-arctic-embed": 1024,
+        "qwen3-embedding": 4096,
+        "qwen3-embedding:0.6b": 1024,
+        "qwen3-embedding:4b": 2560,
+        "qwen3-embedding:8b": 4096,
+    }
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        embedding_url: str | None = None,
+        api_key: str | None = None,
+        timeout: int | None = None,
+    ) -> None:
+        self.base_url = (base_url or config.OLLAMA_BASE_URL).rstrip("/")
+        self.embedding_url = (embedding_url or config.OLLAMA_EMBEDDING_URL).rstrip("/")
+        self.api_key = api_key or config.OLLAMA_API_KEY
+        self.timeout = timeout or config.OLLAMA_TIMEOUT
+        self._current_embed_model: str | None = None
+        self._current_dimensions: int = 768  # default
+
+    def _get_headers(self, include_auth: bool = True) -> dict[str, str]:
+        """Get request headers including authentication if API key is set."""
+        headers = {"Content-Type": "application/json"}
+        if include_auth and self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    @property
+    def provider_name(self) -> str:
+        return "ollama"
+
+    @property
+    def dimensions(self) -> int:
+        return self._current_dimensions
+
+    def is_available(self) -> bool:
+        """Check if Ollama server is reachable."""
+        try:
+            response = requests.get(
+                f"{self.base_url}/api/tags",
+                headers=self._get_headers(),
+                timeout=5,
+            )
+            return response.status_code == 200
+        except requests.RequestException:
+            return False
+
+    def list_models(self) -> list[dict[str, str]]:
+        """List available models on the Ollama server."""
+        try:
+            response = requests.get(
+                f"{self.base_url}/api/tags",
+                headers=self._get_headers(),
+                timeout=10,
+            )
+            response.raise_for_status()
+            data = response.json()
+            return list(data.get("models", []))
+        except requests.RequestException as e:
+            raise ConnectionError(f"Failed to list Ollama models: {e}") from e
+
+    def generate(
+        self,
+        prompt: str,
+        model: str,
+        system_prompt: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """Generate text using Ollama."""
+        options: dict[str, float | int] = {"temperature": temperature}
+        if max_tokens:
+            options["num_predict"] = max_tokens
+
+        payload: dict[str, str | bool | dict[str, float | int]] = {
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+            "options": options,
+        }
+
+        if system_prompt:
+            payload["system"] = system_prompt
+
+        try:
+            response = requests.post(
+                f"{self.base_url}/api/generate",
+                headers=self._get_headers(),
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return LLMResponse(
+                text=data.get("response", ""),
+                model=model,
+                provider=self.provider_name,
+                usage={
+                    "prompt_tokens": data.get("prompt_eval_count"),
+                    "completion_tokens": data.get("eval_count"),
+                    "total_duration": data.get("total_duration"),
+                },
+            )
+        except requests.RequestException as e:
+            raise ConnectionError(f"Ollama generate failed: {e}") from e
+
+    def embed(self, text: str, model: str) -> EmbeddingResponse:
+        """Generate embedding using Ollama (uses embedding_url, no auth for local)."""
+        self._current_embed_model = model
+        self._current_dimensions = self.EMBEDDING_DIMENSIONS.get(model, 768)
+
+        try:
+            response = requests.post(
+                f"{self.embedding_url}/api/embeddings",
+                headers=self._get_headers(include_auth=False),
+                json={"model": model, "prompt": text},
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            embedding = data.get("embedding", [])
+            if not embedding:
+                raise ValueError("Empty embedding returned from Ollama")
+
+            # Update dimensions from actual response
+            self._current_dimensions = len(embedding)
+
+            return EmbeddingResponse(
+                embedding=tuple(embedding),
+                model=model,
+                provider=self.provider_name,
+                dimensions=len(embedding),
+            )
+        except requests.RequestException as e:
+            raise ConnectionError(f"Ollama embed failed: {e}") from e
+
+    def embed_batch(self, texts: list[str], model: str) -> list[EmbeddingResponse]:
+        """Generate embeddings for multiple texts (uses embedding_url, no auth for local).
+
+        Note: Ollama /api/embeddings only supports single prompts, so we loop.
+        """
+        self._current_embed_model = model
+        self._current_dimensions = self.EMBEDDING_DIMENSIONS.get(model, 768)
+
+        results = []
+        try:
+            for text in texts:
+                response = requests.post(
+                    f"{self.embedding_url}/api/embeddings",
+                    headers=self._get_headers(include_auth=False),
+                    json={"model": model, "prompt": text},
+                    timeout=self.timeout,
+                )
+                response.raise_for_status()
+                data = response.json()
+
+                embedding = data.get("embedding", [])
+                if embedding:
+                    self._current_dimensions = len(embedding)
+
+                results.append(
+                    EmbeddingResponse(
+                        embedding=tuple(embedding),
+                        model=model,
+                        provider=self.provider_name,
+                        dimensions=len(embedding),
+                    )
+                )
+            return results
+        except requests.RequestException as e:
+            raise ConnectionError(f"Ollama batch embed failed: {e}") from e
+
+    def chat(
+        self,
+        messages: list[dict[str, str]],
+        model: str,
+        temperature: float = 0.7,
+        max_tokens: int | None = None,
+    ) -> LLMResponse:
+        """
+        Chat completion using Ollama.
+
+        Parameters
+        ----------
+        messages : list[dict]
+            List of messages with 'role' and 'content' keys.
+        model : str
+            Model identifier.
+        temperature : float
+            Sampling temperature.
+        max_tokens : int, optional
+            Maximum tokens to generate.
+
+        Returns
+        -------
+        LLMResponse
+            The generated response.
+        """
+        options: dict[str, float | int] = {"temperature": temperature}
+        if max_tokens:
+            options["num_predict"] = max_tokens
+
+        payload: dict[str, str | bool | list[dict[str, str]] | dict[str, float | int]] = {
+            "model": model,
+            "messages": messages,
+            "stream": False,
+            "options": options,
+        }
+
+        try:
+            response = requests.post(
+                f"{self.base_url}/api/chat",
+                headers=self._get_headers(),
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            return LLMResponse(
+                text=data.get("message", {}).get("content", ""),
+                model=model,
+                provider=self.provider_name,
+                usage={
+                    "prompt_tokens": data.get("prompt_eval_count"),
+                    "completion_tokens": data.get("eval_count"),
+                },
+            )
+        except requests.RequestException as e:
+            raise ConnectionError(f"Ollama chat failed: {e}") from e