ragit 0.10.1__py3-none-any.whl → 0.11.0__py3-none-any.whl

ragit/assistant.py

@@ -7,10 +7,16 @@ High-level RAG Assistant for document Q&A and code generation.
 
 Provides a simple interface for RAG-based tasks.
 
-Note: This class is NOT thread-safe. Do not share instances across threads.
+Thread Safety:
+    This class uses lock-free atomic operations for thread safety.
+    The IndexState is immutable, and all mutations create a new state
+    that is atomically swapped. Python's GIL ensures reference assignment
+    is atomic, making concurrent reads and writes safe.
 """
 
+import json
 from collections.abc import Callable
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -18,8 +24,9 @@ import numpy as np
 from numpy.typing import NDArray
 
 from ragit.core.experiment.experiment import Chunk, Document
+from ragit.exceptions import IndexingError
 from ragit.loaders import chunk_document, chunk_rst_sections, load_directory, load_text
-from ragit.logging import log_operation
+from ragit.logging import log_operation, logger
 from ragit.providers.base import BaseEmbeddingProvider, BaseLLMProvider
 from ragit.providers.function_adapter import FunctionProvider
 
@@ -27,6 +34,23 @@ if TYPE_CHECKING:
     from numpy.typing import NDArray
 
 
+@dataclass(frozen=True)
+class IndexState:
+    """Immutable snapshot of index state for lock-free thread safety.
+
+    This class holds all mutable index data in a single immutable structure.
+    Updates create a new IndexState instance, and the reference swap is
+    atomic under Python's GIL, ensuring thread-safe reads and writes.
+
+    Attributes:
+        chunks: Tuple of indexed chunks (immutable).
+        embedding_matrix: Pre-normalized numpy array of embeddings, or None if empty.
+    """
+
+    chunks: tuple[Chunk, ...]
+    embedding_matrix: NDArray[np.float64] | None
+
+
 class RAGAssistant:
     """
     High-level RAG assistant for document Q&A and generation.
@@ -63,9 +87,12 @@ class RAGAssistant:
     ValueError
         If neither embed_fn nor provider is provided.
 
-    Note
-    ----
-    This class is NOT thread-safe. Each thread should have its own instance.
+    Thread Safety
+    -------------
+    This class uses lock-free atomic operations for thread safety.
+    Multiple threads can safely call retrieve() while another thread
+    calls add_documents(). The IndexState is immutable, and reference
+    swaps are atomic under Python's GIL.
 
     Examples
     --------
@@ -80,6 +107,10 @@ class RAGAssistant:
     >>> # With Ollama provider (supports nomic-embed-text)
     >>> from ragit.providers import OllamaProvider
     >>> assistant = RAGAssistant(docs, provider=OllamaProvider())
+    >>>
+    >>> # Save and load index for persistence
+    >>> assistant.save_index("/path/to/index")
+    >>> loaded = RAGAssistant.load_index("/path/to/index", provider=OllamaProvider())
     """
 
     def __init__(
@@ -134,9 +165,8 @@ class RAGAssistant:
         # Load documents if path provided
         self.documents = self._load_documents(documents)
 
-        # Index chunks - embeddings stored as pre-normalized numpy matrix for fast search
-        self._chunks: tuple[Chunk, ...] = ()
-        self._embedding_matrix: NDArray[np.float64] | None = None  # Pre-normalized
+        # Thread-safe index state (immutable, atomic reference swap)
+        self._state: IndexState = IndexState(chunks=(), embedding_matrix=None)
         self._build_index()
 
     def _load_documents(self, documents: list[Document] | str | Path) -> list[Document]:
@@ -171,7 +201,11 @@ class RAGAssistant:
         raise ValueError(f"Invalid documents source: {documents}")
 
     def _build_index(self) -> None:
-        """Build vector index from documents using batch embedding."""
+        """Build vector index from documents using batch embedding.
+
+        Raises:
+            IndexingError: If embedding count doesn't match chunk count.
+        """
         all_chunks: list[Chunk] = []
 
         for doc in self.documents:
@@ -183,33 +217,54 @@ class RAGAssistant:
             all_chunks.extend(chunks)
 
         if not all_chunks:
-            self._chunks = ()
-            self._embedding_matrix = None
+            logger.warning("No chunks produced from documents - index will be empty")
+            self._state = IndexState(chunks=(), embedding_matrix=None)
             return
 
         # Batch embed all chunks at once (single API call)
         texts = [chunk.content for chunk in all_chunks]
         responses = self._embedding_provider.embed_batch(texts, self.embedding_model)
 
+        # CRITICAL: Validate embedding count matches chunk count
+        if len(responses) != len(all_chunks):
+            raise IndexingError(
+                f"Embedding count mismatch: expected {len(all_chunks)} embeddings, "
+                f"got {len(responses)}. Index may be corrupted."
+            )
+
         # Build embedding matrix directly (skip storing in chunks to avoid duplication)
         embedding_matrix = np.array([response.embedding for response in responses], dtype=np.float64)
 
+        # Additional validation: matrix shape
+        if embedding_matrix.shape[0] != len(all_chunks):
+            raise IndexingError(
+                f"Matrix row count {embedding_matrix.shape[0]} doesn't match chunk count {len(all_chunks)}"
+            )
+
         # Pre-normalize for fast cosine similarity (normalize once, use many times)
         norms = np.linalg.norm(embedding_matrix, axis=1, keepdims=True)
         norms[norms == 0] = 1  # Avoid division by zero
 
-        # Store as immutable tuple and pre-normalized numpy matrix
-        self._chunks = tuple(all_chunks)
-        self._embedding_matrix = embedding_matrix / norms
+        # Atomic state update (thread-safe under GIL)
+        self._state = IndexState(
+            chunks=tuple(all_chunks),
+            embedding_matrix=embedding_matrix / norms,
+        )
 
     def add_documents(self, documents: list[Document] | str | Path) -> int:
         """Add documents to the existing index incrementally.
 
+        This method is thread-safe. It creates a new IndexState and atomically
+        swaps the reference, ensuring readers always see a consistent state.
+
         Args:
             documents: Documents to add.
 
         Returns:
             Number of chunks added.
+
+        Raises:
+            IndexingError: If embedding count doesn't match chunk count.
         """
         new_docs = self._load_documents(documents)
         if not new_docs:
@@ -233,6 +288,13 @@ class RAGAssistant:
         texts = [chunk.content for chunk in new_chunks]
         responses = self._embedding_provider.embed_batch(texts, self.embedding_model)
 
+        # Validate embedding count
+        if len(responses) != len(new_chunks):
+            raise IndexingError(
+                f"Embedding count mismatch: expected {len(new_chunks)} embeddings, "
+                f"got {len(responses)}. Index update aborted."
+            )
+
         new_matrix = np.array([response.embedding for response in responses], dtype=np.float64)
 
         # Normalize
@@ -240,21 +302,27 @@ class RAGAssistant:
         norms[norms == 0] = 1
         new_matrix_norm = new_matrix / norms
 
-        # Update state
-        current_chunks = list(self._chunks)
-        current_chunks.extend(new_chunks)
-        self._chunks = tuple(current_chunks)
+        # Read current state (atomic read)
+        current_state = self._state
 
-        if self._embedding_matrix is None:
-            self._embedding_matrix = new_matrix_norm
+        # Build new state
+        combined_chunks = current_state.chunks + tuple(new_chunks)
+        if current_state.embedding_matrix is None:
+            combined_matrix = new_matrix_norm
         else:
-            self._embedding_matrix = np.vstack((self._embedding_matrix, new_matrix_norm))
+            combined_matrix = np.vstack((current_state.embedding_matrix, new_matrix_norm))
+
+        # Atomic state swap (thread-safe under GIL)
+        self._state = IndexState(chunks=combined_chunks, embedding_matrix=combined_matrix)
 
         return len(new_chunks)
 
     def remove_documents(self, source_path_pattern: str) -> int:
         """Remove documents matching a source path pattern.
 
+        This method is thread-safe. It creates a new IndexState and atomically
+        swaps the reference.
+
         Args:
             source_path_pattern: Glob pattern to match 'source' metadata.
 
@@ -263,14 +331,17 @@ class RAGAssistant:
         """
         import fnmatch
 
-        if not self._chunks:
+        # Read current state (atomic read)
+        current_state = self._state
+
+        if not current_state.chunks:
             return 0
 
         indices_to_keep = []
         kept_chunks = []
         removed_count = 0
 
-        for i, chunk in enumerate(self._chunks):
+        for i, chunk in enumerate(current_state.chunks):
             source = chunk.metadata.get("source", "")
             if not source or not fnmatch.fnmatch(source, source_path_pattern):
                 indices_to_keep.append(i)
@@ -281,13 +352,14 @@ class RAGAssistant:
         if removed_count == 0:
             return 0
 
-        self._chunks = tuple(kept_chunks)
+        # Build new embedding matrix
+        if current_state.embedding_matrix is not None:
+            new_matrix = None if not kept_chunks else current_state.embedding_matrix[indices_to_keep]
+        else:
+            new_matrix = None
 
-        if self._embedding_matrix is not None:
-            if not kept_chunks:
-                self._embedding_matrix = None
-            else:
-                self._embedding_matrix = self._embedding_matrix[indices_to_keep]
+        # Atomic state swap (thread-safe under GIL)
+        self._state = IndexState(chunks=tuple(kept_chunks), embedding_matrix=new_matrix)
 
         # Also remove from self.documents
         self.documents = [
@@ -330,6 +402,7 @@ class RAGAssistant:
         Retrieve relevant chunks for a query.
 
         Uses vectorized cosine similarity for fast search over all chunks.
+        This method is thread-safe - it reads a consistent snapshot of the index.
 
         Parameters
         ----------
@@ -349,7 +422,10 @@ class RAGAssistant:
         >>> for chunk, score in results:
         ...     print(f"{score:.2f}: {chunk.content[:100]}...")
         """
-        if not self._chunks or self._embedding_matrix is None:
+        # Atomic state read - get consistent snapshot
+        state = self._state
+
+        if not state.chunks or state.embedding_matrix is None:
             return []
 
         # Get query embedding and normalize
@@ -361,7 +437,7 @@ class RAGAssistant:
         query_normalized = query_vec / query_norm
 
         # Fast cosine similarity: matrix is pre-normalized, just dot product
-        similarities = self._embedding_matrix @ query_normalized
+        similarities = state.embedding_matrix @ query_normalized
 
         # Get top_k indices using argpartition (faster than full sort for large arrays)
         if len(similarities) <= top_k:
@@ -372,7 +448,7 @@ class RAGAssistant:
             # Sort the top_k by score
             top_indices = top_indices[np.argsort(similarities[top_indices])[::-1]]
 
-        return [(self._chunks[i], float(similarities[i])) for i in top_indices]
+        return [(state.chunks[i], float(similarities[i])) for i in top_indices]
 
     def retrieve_with_context(
         self,
@@ -415,6 +491,9 @@ class RAGAssistant:
         >>> for chunk, score in results:
         ...     print(f"{score:.2f}: {chunk.content[:50]}...")
         """
+        # Get consistent state snapshot
+        state = self._state
+
         with log_operation("retrieve_with_context", query_len=len(query), top_k=top_k, window_size=window_size) as ctx:
             # Get initial results (more than top_k to account for filtering)
             results = self.retrieve(query, top_k * 2)
@@ -428,7 +507,7 @@ class RAGAssistant:
                 return results[:top_k]
 
             # Build chunk index for fast lookup
-            chunk_to_idx = {id(chunk): i for i, chunk in enumerate(self._chunks)}
+            chunk_to_idx = {id(chunk): i for i, chunk in enumerate(state.chunks)}
 
             expanded_results: list[tuple[Chunk, float]] = []
             seen_indices: set[int] = set()
@@ -441,13 +520,13 @@ class RAGAssistant:
 
                 # Get window of adjacent chunks from same document
                 start_idx = max(0, chunk_idx - window_size)
-                end_idx = min(len(self._chunks), chunk_idx + window_size + 1)
+                end_idx = min(len(state.chunks), chunk_idx + window_size + 1)
 
                 for idx in range(start_idx, end_idx):
                     if idx in seen_indices:
                         continue
 
-                    adjacent_chunk = self._chunks[idx]
+                    adjacent_chunk = state.chunks[idx]
                     # Only include adjacent chunks from same document
                     if adjacent_chunk.doc_id == chunk.doc_id:
                         seen_indices.add(idx)
@@ -456,7 +535,7 @@ class RAGAssistant:
                         expanded_results.append((adjacent_chunk, adj_score))
 
             # Sort by score (highest first)
-            expanded_results.sort(key=lambda x: (-x[1], self._chunks.index(x[0]) if x[0] in self._chunks else 0))
+            expanded_results.sort(key=lambda x: (-x[1], state.chunks.index(x[0]) if x[0] in state.chunks else 0))
             ctx["expanded_chunks"] = len(expanded_results)
 
             return expanded_results
@@ -489,6 +568,9 @@ class RAGAssistant:
         str
             Formatted context string with merged chunks.
         """
+        # Get consistent state snapshot
+        state = self._state
+
         results = self.retrieve_with_context(query, top_k, window_size, min_score)
 
         if not results:
@@ -506,10 +588,10 @@ class RAGAssistant:
 
         for _doc_id, chunks in doc_chunks.items():
             # Sort chunks by their position in the original list
-            chunks.sort(key=lambda x: self._chunks.index(x[0]) if x[0] in self._chunks else 0)
+            chunks.sort(key=lambda x: state.chunks.index(x[0]) if x[0] in state.chunks else 0)
 
             # Merge overlapping text
-            merged_content = []
+            merged_content: list[str] = []
             for chunk, _ in chunks:
                 if merged_content:
                     # Check for overlap with previous chunk
@@ -744,7 +826,17 @@ Generate the {language} code:"""
     @property
     def num_chunks(self) -> int:
         """Return number of indexed chunks."""
-        return len(self._chunks)
+        return len(self._state.chunks)
+
+    @property
+    def chunk_count(self) -> int:
+        """Number of chunks in index (alias for num_chunks)."""
+        return len(self._state.chunks)
+
+    @property
+    def is_indexed(self) -> bool:
+        """Check if index has any documents."""
+        return len(self._state.chunks) > 0
 
     @property
     def num_documents(self) -> int:
@@ -755,3 +847,122 @@ Generate the {language} code:"""
     def has_llm(self) -> bool:
         """Check if LLM is configured."""
         return self._llm_provider is not None
+
+    def save_index(self, path: str | Path) -> None:
+        """Save index to disk for later restoration.
+
+        Saves the index in an efficient format:
+        - chunks.json: Chunk metadata and content
+        - embeddings.npy: Numpy array of embeddings (binary format)
+        - metadata.json: Index configuration
+
+        Args:
+            path: Directory path to save index files.
+
+        Example:
+            >>> assistant.save_index("/path/to/index")
+            >>> # Later...
+            >>> loaded = RAGAssistant.load_index("/path/to/index", provider=provider)
+        """
+        path = Path(path)
+        path.mkdir(parents=True, exist_ok=True)
+
+        state = self._state
+
+        # Save chunks as JSON
+        chunks_data = [
+            {
+                "content": chunk.content,
+                "doc_id": chunk.doc_id,
+                "chunk_index": chunk.chunk_index,
+                "metadata": chunk.metadata,
+            }
+            for chunk in state.chunks
+        ]
+        (path / "chunks.json").write_text(json.dumps(chunks_data, indent=2))
+
+        # Save embeddings as numpy binary (efficient for large arrays)
+        if state.embedding_matrix is not None:
+            np.save(path / "embeddings.npy", state.embedding_matrix)
+
+        # Save metadata for validation and configuration restoration
+        metadata = {
+            "chunk_count": len(state.chunks),
+            "embedding_model": self.embedding_model,
+            "chunk_size": self.chunk_size,
+            "chunk_overlap": self.chunk_overlap,
+            "version": "1.0",
+        }
+        (path / "metadata.json").write_text(json.dumps(metadata, indent=2))
+
+        logger.info(f"Index saved to {path} ({len(state.chunks)} chunks)")
+
+    @classmethod
+    def load_index(
+        cls,
+        path: str | Path,
+        provider: BaseEmbeddingProvider | BaseLLMProvider | None = None,
+    ) -> "RAGAssistant":
+        """Load a previously saved index.
+
+        Args:
+            path: Directory path containing saved index files.
+            provider: Provider for embeddings/LLM (required for new queries).
+
+        Returns:
+            RAGAssistant instance with loaded index.
+
+        Raises:
+            IndexingError: If loaded index is corrupted (count mismatch).
+            FileNotFoundError: If index files don't exist.
+
+        Example:
+            >>> loaded = RAGAssistant.load_index("/path/to/index", provider=OllamaProvider())
+            >>> results = loaded.retrieve("query")
+        """
+        path = Path(path)
+
+        # Load metadata
+        metadata = json.loads((path / "metadata.json").read_text())
+
+        # Load chunks
+        chunks_data = json.loads((path / "chunks.json").read_text())
+        chunks = tuple(
+            Chunk(
+                content=c["content"],
+                doc_id=c.get("doc_id", ""),
+                chunk_index=c.get("chunk_index", 0),
+                metadata=c.get("metadata", {}),
+            )
+            for c in chunks_data
+        )
+
+        # Load embeddings
+        embeddings_path = path / "embeddings.npy"
+        embedding_matrix: NDArray[np.float64] | None = None
+        if embeddings_path.exists():
+            embedding_matrix = np.load(embeddings_path)
+
+        # Validate consistency
+        if embedding_matrix is not None and embedding_matrix.shape[0] != len(chunks):
+            raise IndexingError(
+                f"Loaded index corrupted: {embedding_matrix.shape[0]} embeddings but {len(chunks)} chunks"
+            )
+
+        # Create instance without calling __init__ (skip indexing)
+        instance = object.__new__(cls)
+
+        # Initialize required attributes
+        instance._state = IndexState(chunks=chunks, embedding_matrix=embedding_matrix)
+        instance.embedding_model = metadata.get("embedding_model", "default")
+        instance.llm_model = metadata.get("llm_model", "default")
+        instance.chunk_size = metadata.get("chunk_size", 512)
+        instance.chunk_overlap = metadata.get("chunk_overlap", 50)
+        instance.documents = []  # Original docs not saved
+
+        # Set up providers
+        instance._embedding_provider = provider if isinstance(provider, BaseEmbeddingProvider) else None  # type: ignore
+        instance._llm_provider = provider if isinstance(provider, BaseLLMProvider) else None
+
+        logger.info(f"Index loaded from {path} ({len(chunks)} chunks)")
+        return instance

ragit/config.py

@@ -153,16 +153,15 @@ def _safe_get_env(key: str, default: str | None = None) -> str | None:
     return value
 
 
-def _safe_get_int_env(key: str, default: int) -> int | str:
-    """Get environment variable as int, returning raw string if invalid."""
+def _safe_get_int_env(key: str, default: int) -> int:
+    """Get environment variable as int, raising on invalid values."""
     value = os.getenv(key)
     if value is None:
         return default
     try:
         return int(value)
     except ValueError:
-        # Return the raw string so Pydantic can give a better error message
-        return value
+        raise ConfigValidationError(f"Invalid integer value for {key}: {value!r}") from None
 
 
 def load_config() -> RagitConfig:

ragit/exceptions.py

@@ -24,7 +24,7 @@ class RagitError(Exception):
     ----------
     message : str
         Human-readable error message.
-    original_exception : Exception, optional
+    original_exception : BaseException, optional
         The underlying exception that caused this error.
 
     Examples
@@ -37,7 +37,7 @@ class RagitError(Exception):
     ...         print(f"Caused by: {e.original_exception}")
     """
 
-    def __init__(self, message: str, original_exception: Exception | None = None):
+    def __init__(self, message: str, original_exception: BaseException | None = None):
         self.message = message
         self.original_exception = original_exception
         super().__init__(self._format_message())

ragit/loaders.py

@@ -282,7 +282,7 @@ def chunk_by_separator(
     """
     effective_doc_id = doc_id or generate_document_id(text)
     parts = text.split(separator)
-    chunks = []
+    chunks: list[Chunk] = []
     current_pos = 0
 
     for _idx, part in enumerate(parts):

ragit/providers/ollama.py

@@ -10,7 +10,7 @@ Configuration is loaded from environment variables.
 
 Performance optimizations:
 - Connection pooling via requests.Session()
-- Async parallel embedding via trio + httpx
+- Async parallel embedding via httpx
 - LRU cache for repeated embedding queries
 
 Resilience features (via resilient-circuit):
@@ -216,22 +216,42 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
 
     @property
     def session(self) -> requests.Session:
-        """Lazy-initialized session for connection pooling."""
+        """Lazy-initialized session for connection pooling.
+
+        Note: API key is NOT stored in session headers to prevent
+        potential exposure in logs or error messages. Authentication
+        is handled per-request via _get_headers().
+        """
         if self._session is None:
             self._session = requests.Session()
             self._session.headers.update({"Content-Type": "application/json"})
-            if self.api_key:
-                self._session.headers.update({"Authorization": f"Bearer {self.api_key}"})
+            # Security: API key is injected per-request via _get_headers()
+            # rather than stored in session headers to prevent log exposure
         return self._session
 
     def close(self) -> None:
         """Close the session and release resources."""
-        if self._session is not None:
-            self._session.close()
+        session = getattr(self, "_session", None)
+        if session is not None:
+            session.close()
             self._session = None
 
+    def __enter__(self) -> "OllamaProvider":
+        """Context manager entry - returns self for use in 'with' statements.
+
+        Example:
+            with OllamaProvider() as provider:
+                result = provider.generate("Hello", model="llama3")
+            # Session automatically closed here
+        """
+        return self
+
+    def __exit__(self, exc_type: type | None, exc_val: Exception | None, exc_tb: object) -> None:
+        """Context manager exit - ensures cleanup regardless of exceptions."""
+        self.close()
+
     def __del__(self) -> None:
-        """Cleanup on garbage collection."""
+        """Cleanup on garbage collection (fallback, prefer context manager)."""
         self.close()
 
     def _get_headers(self, include_auth: bool = True) -> dict[str, str]:
@@ -254,6 +274,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
         try:
             response = self.session.get(
                 f"{self.base_url}/api/tags",
+                headers=self._get_headers(),
                 timeout=self._timeouts["health"],
             )
             return bool(response.status_code == 200)
@@ -265,6 +286,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
         try:
             response = self.session.get(
                 f"{self.base_url}/api/tags",
+                headers=self._get_headers(),
                 timeout=self._timeouts["list_models"],
             )
             response.raise_for_status()
@@ -326,6 +348,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
             try:
                 response = self.session.post(
                     f"{self.base_url}/api/generate",
+                    headers=self._get_headers(),
                     json=payload,
                     timeout=self._timeouts["generate"],
                 )
@@ -385,6 +408,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
                     # Direct call without cache
                     response = self.session.post(
                         f"{self.embedding_url}/api/embed",
+                        headers=self._get_headers(),
                         json={"model": model, "input": truncated_text},
                         timeout=self._timeouts["embed"],
                     )
@@ -446,6 +470,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
             try:
                 response = self.session.post(
                     f"{self.embedding_url}/api/embed",
+                    headers=self._get_headers(),
                     json={"model": model, "input": truncated_texts},
                     timeout=self._timeouts["embed_batch"],
                 )
@@ -504,8 +529,8 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
 
         Examples
         --------
-        >>> import trio
-        >>> embeddings = trio.run(provider.embed_batch_async, texts, "mxbai-embed-large")
+        >>> import asyncio
+        >>> embeddings = asyncio.run(provider.embed_batch_async(texts, "mxbai-embed-large"))
         """
         self._current_embed_model = model
         self._current_dimensions = self.EMBEDDING_DIMENSIONS.get(model, 768)
@@ -611,6 +636,7 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
             try:
                 response = self.session.post(
                     f"{self.base_url}/api/chat",
+                    headers=self._get_headers(),
                     json=payload,
                     timeout=self._timeouts["chat"],
                 )
@@ -638,16 +664,24 @@ class OllamaProvider(BaseLLMProvider, BaseEmbeddingProvider):
         if not self.use_resilience or self._generate_policy is None:
             return "disabled"
         # Access the circuit protector (second policy in SafetyNet)
-        circuit = self._generate_policy._policies[1]
-        return circuit.status.name
+        policies = getattr(self._generate_policy, "policies", None)
+        if policies is None or len(policies) < 2:
+            return "unknown"
+        circuit = policies[1]
+        status = getattr(circuit, "status", None)
+        return str(getattr(status, "name", "unknown"))
 
     @property
     def embed_circuit_status(self) -> str:
         """Get embed circuit breaker status (CLOSED, OPEN, HALF_OPEN, or 'disabled')."""
         if not self.use_resilience or self._embed_policy is None:
             return "disabled"
-        circuit = self._embed_policy._policies[1]
-        return circuit.status.name
+        policies = getattr(self._embed_policy, "policies", None)
+        if policies is None or len(policies) < 2:
+            return "unknown"
+        circuit = policies[1]
+        status = getattr(circuit, "status", None)
+        return str(getattr(status, "name", "unknown"))
 
     @staticmethod
     def clear_embedding_cache() -> None:

ragit/utils/__init__.py

@@ -12,8 +12,6 @@ from datetime import datetime
 from math import floor
 from typing import Any
 
-import pandas as pd
-
 
 def get_hashable_repr(dct: dict[str, object]) -> tuple[tuple[str, object, float, int | None], ...]:
     """
@@ -62,26 +60,6 @@ def remove_duplicates(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return deduplicated_items
 
 
-def handle_missing_values_in_combinations(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Handle missing values in experiment data combinations.
-
-    Parameters
-    ----------
-    df : pd.DataFrame
-        Experiment data with combinations being explored.
-
-    Returns
-    -------
-    pd.DataFrame
-        Data with NaN values properly replaced.
-    """
-    if "chunk_overlap" in df.columns:
-        df["chunk_overlap"] = df["chunk_overlap"].map(lambda el: 0 if pd.isna(el) else el)
-
-    return df
-
-
 def datetime_str_to_epoch_time(timestamp: str | int) -> str | int:
     """
     Convert datetime string to epoch time.

ragit/version.py

@@ -2,4 +2,4 @@
 # Copyright RODMENA LIMITED 2025
 # SPDX-License-Identifier: Apache-2.0
 #
-__version__ = "0.10.1"
+__version__ = "0.11.0"

{ragit-0.10.1.dist-info → ragit-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragit
-Version: 0.10.1
+Version: 0.11.0
 Summary: Automatic RAG Pattern Optimization Engine
 Author: RODMENA LIMITED
 Maintainer-email: RODMENA LIMITED <info@rodmena.co.uk>
@@ -21,12 +21,9 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
 Requires-Dist: numpy>=1.26.0
-Requires-Dist: pandas>=2.2.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dotenv>=1.0.0
-Requires-Dist: scikit-learn>=1.5.0
 Requires-Dist: tqdm>=4.66.0
-Requires-Dist: trio>=0.24.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: resilient-circuit>=0.4.7
 Provides-Extra: dev
@@ -115,6 +112,45 @@ answer = assistant.ask(question, top_k=3)         # Requires generate_fn/LLM
 code = assistant.generate_code(request)           # Requires generate_fn/LLM
 ```
 
+## Index Persistence
+
+Save and load indexes to avoid re-computing embeddings:
+
+```python
+# Save index to disk
+assistant.save_index("./my_index")
+
+# Load index later (much faster than re-indexing)
+loaded = RAGAssistant.load_index("./my_index", provider=OllamaProvider())
+results = loaded.retrieve("query")
+```
+
+## Thread Safety
+
+RAGAssistant is thread-safe. Multiple threads can safely read while another writes:
+
+```python
+import threading
+
+assistant = RAGAssistant("docs/", provider=OllamaProvider())
+
+# Safe: concurrent reads and writes
+threading.Thread(target=lambda: assistant.retrieve("query")).start()
+threading.Thread(target=lambda: assistant.add_documents([new_doc])).start()
+```
+
+## Resource Management
+
+Use context managers for automatic cleanup:
+
+```python
+from ragit.providers import OllamaProvider
+
+with OllamaProvider() as provider:
+    response = provider.generate("Hello", model="llama3")
+# Session automatically closed
+```
+
 ## Document Loading
 
 ```python

{ragit-0.10.1.dist-info → ragit-0.11.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ragit/__init__.py,sha256=54z3-xCkEa4_P4eonrweSu3Lbig1BWLIGOGT3QUJ4N8,3263
-ragit/assistant.py,sha256=JqgXF0i-UdENR8Uz-aEaOMOwpU9ARGOkhg7GUVXJ748,25266
-ragit/config.py,sha256=0K81taFTKnbqCnEprLYqAeJFESzuxO1IqyVnsylbzP4,6415
-ragit/exceptions.py,sha256=RfY1tY6Bdpvmt--U-nm09SPoa6emczJyo_-0-FNk5Nk,7207
-ragit/loaders.py,sha256=RCzha8Ns8fUmxUwcM2ZqfdtjVHtZKt9AT_rx014yEwo,11029
+ragit/assistant.py,sha256=pjB58KyHGD7PwpwLE-lDyXxMhaehDe3IFiO9j7yewxk,33252
+ragit/config.py,sha256=M3YCyogalJ-_cNbY3vAnKIknNsBmqeUFH6lhknuPKV4,6399
+ragit/exceptions.py,sha256=2nBdAWbeLxTkykmwJBTn6BFBNib2dgPfr_Z58p1IwlY,7215
+ragit/loaders.py,sha256=r9hDPTpnVHs9-nMeL2IhEfjIda-TCwYmG3RvnpDcs70,11042
 ragit/logging.py,sha256=YnvhOfnOE3nTd-fR9LKPUHrWdh8fcSHIBEBS5iWDMs8,5739
 ragit/monitor.py,sha256=ajYTdQKM4QlYhlzjiKbSiks4kQj94v0pOhW4q16vJWY,10272
-ragit/version.py,sha256=U4XpLGbCMs36f22-jyIEAkggAHOhIuMFX64utC0xk0Q,98
+ragit/version.py,sha256=e-rBQeeVkLzfQCMzS0MEjneUF2NDFJmoWYFtrbdq75c,98
 ragit/core/__init__.py,sha256=j53PFfoSMXwSbK1rRHpMbo8mX2i4R1LJ5kvTxBd7-0w,100
 ragit/core/experiment/__init__.py,sha256=4vAPOOYlY5Dcr2gOolyhBSPGIUxZKwEkgQffxS9BodA,452
 ragit/core/experiment/experiment.py,sha256=Ydf3jz5AXbttc2xcvIMecfc3lh4MKgCtCtyNCsFsn9c,19573
@@ -13,10 +13,10 @@ ragit/core/experiment/results.py,sha256=KHpN3YSLJ83_JUfIMccRPS-q7LEt0S9p8ehDRawk
 ragit/providers/__init__.py,sha256=DSdv2-N9kJwrF6PymKYiktKbjc7g22J_7MD1Rm2ep4g,919
 ragit/providers/base.py,sha256=MJ8mVeXuGWhkX2XGTbkWIY3cVoTOPr4h5XBXw8rAX2Q,3434
 ragit/providers/function_adapter.py,sha256=A-TQhBgBWbuO_w1sy795Dxep1FOCBpAlWpXCKVQD8rc,7778
-ragit/providers/ollama.py,sha256=pi580O55fVN6YWDMaozIIW0cWSK5VG_YZyD-Hbz50M8,26086
-ragit/utils/__init__.py,sha256=-UsE5oJSnmEnBDswl-ph0A09Iu8yKNbPhd1-_7Lcb8Y,3051
-ragit-0.10.1.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-ragit-0.10.1.dist-info/METADATA,sha256=yQ8hREYiXNnvIsTVc4rD6d_yEdYVWGht6izPYTnmBlY,4451
-ragit-0.10.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ragit-0.10.1.dist-info/top_level.txt,sha256=pkPbG7yrw61wt9_y_xcLE2vq2a55fzockASD0yq0g4s,6
-ragit-0.10.1.dist-info/RECORD,,
+ragit/providers/ollama.py,sha256=oV6_FojbMrxYyh-g5x77EM1vhzFT4aF98aj2TybWrlw,27600
+ragit/utils/__init__.py,sha256=6oQm2KwXFWIMtAE-0TgcDB6WwKyMy736UPnhG3bFFK4,2531
+ragit-0.11.0.dist-info/licenses/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+ragit-0.11.0.dist-info/METADATA,sha256=msgmpc2zt4zWkLbKN0XSiIxvQ5Nt4f-nU5HnVLtoc4c,5300
+ragit-0.11.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ragit-0.11.0.dist-info/top_level.txt,sha256=pkPbG7yrw61wt9_y_xcLE2vq2a55fzockASD0yq0g4s,6
+ragit-0.11.0.dist-info/RECORD,,