ragit 0.8.1__tar.gz → 0.10.1__tar.gz

{ragit-0.8.1/ragit.egg-info → ragit-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragit
-Version: 0.8.1
+Version: 0.10.1
 Summary: Automatic RAG Pattern Optimization Engine
 Author: RODMENA LIMITED
 Maintainer-email: RODMENA LIMITED <info@rodmena.co.uk>
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: POSIX :: Linux
-Requires-Python: <3.14,>=3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
@@ -28,6 +28,7 @@ Requires-Dist: scikit-learn>=1.5.0
 Requires-Dist: tqdm>=4.66.0
 Requires-Dist: trio>=0.24.0
 Requires-Dist: httpx>=0.27.0
+Requires-Dist: resilient-circuit>=0.4.7
 Provides-Extra: dev
 Requires-Dist: ragit[test]; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
@@ -39,8 +40,6 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
-Provides-Extra: transformers
-Requires-Dist: sentence-transformers>=2.2.0; extra == "transformers"
 Provides-Extra: docs
 Requires-Dist: sphinx>=7.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme>=2.0; extra == "docs"
@@ -55,14 +54,11 @@ RAG toolkit for Python. Document loading, chunking, vector search, LLM integrati
 
 ```bash
 pip install ragit
-
-# For offline embedding
-pip install ragit[transformers]
 ```
 
 ## Quick Start
 
-You must provide an embedding source: custom function, SentenceTransformers, or any provider.
+You must provide an embedding source: custom function, Ollama, or any provider.
 
 ### Custom Embedding Function
 
@@ -90,26 +86,17 @@ assistant = RAGAssistant("docs/", embed_fn=my_embed, generate_fn=my_generate)
 answer = assistant.ask("How does authentication work?")
 ```
 
-### Offline Embedding (SentenceTransformers)
-
-Models are downloaded automatically on first use (~90MB for default model).
+### With Ollama (nomic-embed-text)
 
 ```python
 from ragit import RAGAssistant
-from ragit.providers import SentenceTransformersProvider
+from ragit.providers import OllamaProvider
 
-# Uses all-MiniLM-L6-v2 by default
-assistant = RAGAssistant("docs/", provider=SentenceTransformersProvider())
-
-# Or specify a model
-assistant = RAGAssistant(
-    "docs/",
-    provider=SentenceTransformersProvider(model_name="all-mpnet-base-v2")
-)
+# Uses nomic-embed-text for embeddings (768d)
+assistant = RAGAssistant("docs/", provider=OllamaProvider())
+results = assistant.retrieve("search query")
 ```
 
-Available models: `all-MiniLM-L6-v2` (384d), `all-mpnet-base-v2` (768d), `paraphrase-MiniLM-L6-v2` (384d)
-
 ## Core API
 
 ```python

{ragit-0.8.1 → ragit-0.10.1}/README.md

@@ -6,14 +6,11 @@ RAG toolkit for Python. Document loading, chunking, vector search, LLM integrati
 
 ```bash
 pip install ragit
-
-# For offline embedding
-pip install ragit[transformers]
 ```
 
 ## Quick Start
 
-You must provide an embedding source: custom function, SentenceTransformers, or any provider.
+You must provide an embedding source: custom function, Ollama, or any provider.
 
 ### Custom Embedding Function
 
@@ -41,26 +38,17 @@ assistant = RAGAssistant("docs/", embed_fn=my_embed, generate_fn=my_generate)
 answer = assistant.ask("How does authentication work?")
 ```
 
-### Offline Embedding (SentenceTransformers)
-
-Models are downloaded automatically on first use (~90MB for default model).
+### With Ollama (nomic-embed-text)
 
 ```python
 from ragit import RAGAssistant
-from ragit.providers import SentenceTransformersProvider
+from ragit.providers import OllamaProvider
 
-# Uses all-MiniLM-L6-v2 by default
-assistant = RAGAssistant("docs/", provider=SentenceTransformersProvider())
-
-# Or specify a model
-assistant = RAGAssistant(
-    "docs/",
-    provider=SentenceTransformersProvider(model_name="all-mpnet-base-v2")
-)
+# Uses nomic-embed-text for embeddings (768d)
+assistant = RAGAssistant("docs/", provider=OllamaProvider())
+results = assistant.retrieve("search query")
 ```
 
-Available models: `all-MiniLM-L6-v2` (384d), `all-mpnet-base-v2` (768d), `paraphrase-MiniLM-L6-v2` (384d)
-
 ## Core API
 
 ```python

{ragit-0.8.1 → ragit-0.10.1}/pyproject.toml

@@ -10,7 +10,7 @@ maintainers = [
     { name = "RODMENA LIMITED", email = "info@rodmena.co.uk" },
 ]
 readme = "README.md"
-requires-python = ">=3.12,<3.14"
+requires-python = ">=3.12"
 classifiers = [
     "Development Status :: 2 - Pre-Alpha",
     "Natural Language :: English",
@@ -40,6 +40,7 @@ dependencies = [
     "tqdm>=4.66.0",
     "trio>=0.24.0",
     "httpx>=0.27.0",
+    "resilient-circuit>=0.4.7",
 ]
 
 [project.urls]
@@ -59,8 +60,6 @@ dev = [
 
 test = ["pytest", "pytest-cov", "pytest-mock"]
 
-transformers = ["sentence-transformers>=2.2.0"]
-
 docs = [
     "sphinx>=7.0",
     "sphinx-rtd-theme>=2.0",

{ragit-0.8.1 → ragit-0.10.1}/ragit/__init__.py

@@ -16,11 +16,7 @@ Quick Start
 >>> assistant = RAGAssistant("docs/", embed_fn=my_embed)
 >>> results = assistant.retrieve("How do I create a REST API?")
 >>>
->>> # With SentenceTransformers (offline, requires ragit[transformers])
->>> from ragit.providers import SentenceTransformersProvider
->>> assistant = RAGAssistant("docs/", provider=SentenceTransformersProvider())
->>>
->>> # With Ollama (explicit)
+>>> # With Ollama
 >>> from ragit.providers import OllamaProvider
 >>> assistant = RAGAssistant("docs/", provider=OllamaProvider())
 >>> answer = assistant.ask("How do I create a REST API?")
@@ -63,14 +59,27 @@ from ragit.core.experiment.experiment import (  # noqa: E402
     RagitExperiment,
 )
 from ragit.core.experiment.results import EvaluationResult, ExperimentResults  # noqa: E402
+from ragit.exceptions import (  # noqa: E402
+    ConfigurationError,
+    EvaluationError,
+    ExceptionAggregator,
+    GenerationError,
+    IndexingError,
+    ProviderError,
+    RagitError,
+    RetrievalError,
+)
 from ragit.loaders import (  # noqa: E402
     chunk_by_separator,
     chunk_document,
     chunk_rst_sections,
     chunk_text,
+    deduplicate_documents,
+    generate_document_id,
     load_directory,
     load_text,
 )
+from ragit.monitor import ExecutionMonitor  # noqa: E402
 from ragit.providers import (  # noqa: E402
     BaseEmbeddingProvider,
     BaseLLMProvider,
@@ -89,6 +98,8 @@ __all__ = [
     "chunk_document",
     "chunk_by_separator",
     "chunk_rst_sections",
+    "generate_document_id",
+    "deduplicate_documents",
     # Core classes
     "Document",
     "Chunk",
@@ -97,6 +108,17 @@ __all__ = [
     "FunctionProvider",
     "BaseLLMProvider",
     "BaseEmbeddingProvider",
+    # Exceptions
+    "RagitError",
+    "ConfigurationError",
+    "ProviderError",
+    "IndexingError",
+    "RetrievalError",
+    "GenerationError",
+    "EvaluationError",
+    "ExceptionAggregator",
+    # Monitoring
+    "ExecutionMonitor",
     # Optimization
     "RagitExperiment",
     "BenchmarkQuestion",
@@ -104,13 +126,3 @@ __all__ = [
     "EvaluationResult",
     "ExperimentResults",
 ]
-
-# Conditionally add SentenceTransformersProvider if available
-try:
-    from ragit.providers import (  # noqa: E402
-        SentenceTransformersProvider as SentenceTransformersProvider,
-    )
-
-    __all__ += ["SentenceTransformersProvider"]
-except ImportError:
-    pass

{ragit-0.8.1 → ragit-0.10.1}/ragit/assistant.py

@@ -19,6 +19,7 @@ from numpy.typing import NDArray
 
 from ragit.core.experiment.experiment import Chunk, Document
 from ragit.loaders import chunk_document, chunk_rst_sections, load_directory, load_text
+from ragit.logging import log_operation
 from ragit.providers.base import BaseEmbeddingProvider, BaseLLMProvider
 from ragit.providers.function_adapter import FunctionProvider
 
@@ -76,13 +77,9 @@ class RAGAssistant:
     >>> assistant = RAGAssistant(docs, embed_fn=my_embed, generate_fn=my_llm)
     >>> answer = assistant.ask("What is X?")
     >>>
-    >>> # With explicit provider
+    >>> # With Ollama provider (supports nomic-embed-text)
     >>> from ragit.providers import OllamaProvider
     >>> assistant = RAGAssistant(docs, provider=OllamaProvider())
-    >>>
-    >>> # With SentenceTransformers (offline)
-    >>> from ragit.providers import SentenceTransformersProvider
-    >>> assistant = RAGAssistant(docs, provider=SentenceTransformersProvider())
     """
 
     def __init__(
@@ -116,8 +113,7 @@ class RAGAssistant:
             # Use explicit provider
             if not isinstance(provider, BaseEmbeddingProvider):
                 raise ValueError(
-                    "Provider must implement BaseEmbeddingProvider for embeddings. "
-                    "Alternatively, provide embed_fn."
+                    "Provider must implement BaseEmbeddingProvider for embeddings. Alternatively, provide embed_fn."
                 )
             self._embedding_provider = provider
             if isinstance(provider, BaseLLMProvider):
@@ -127,8 +123,7 @@ class RAGAssistant:
                 "Must provide embed_fn or provider for embeddings. "
                 "Examples:\n"
                 "  RAGAssistant(docs, embed_fn=my_embed_function)\n"
-                "  RAGAssistant(docs, provider=OllamaProvider())\n"
-                "  RAGAssistant(docs, provider=SentenceTransformersProvider())"
+                "  RAGAssistant(docs, provider=OllamaProvider())"
             )
 
         self.embedding_model = embedding_model or "default"
@@ -156,7 +151,20 @@ class RAGAssistant:
 
         if path.is_dir():
             docs: list[Document] = []
-            for pattern in ("*.txt", "*.md", "*.rst"):
+            for pattern in (
+                "*.txt",
+                "*.md",
+                "*.rst",
+                "*.py",
+                "*.js",
+                "*.ts",
+                "*.go",
+                "*.java",
+                "*.c",
+                "*.cpp",
+                "*.h",
+                "*.hpp",
+            ):
                 docs.extend(load_directory(path, pattern))
             return docs
 
@@ -194,6 +202,129 @@ class RAGAssistant:
         self._chunks = tuple(all_chunks)
         self._embedding_matrix = embedding_matrix / norms
 
+    def add_documents(self, documents: list[Document] | str | Path) -> int:
+        """Add documents to the existing index incrementally.
+
+        Args:
+            documents: Documents to add.
+
+        Returns:
+            Number of chunks added.
+        """
+        new_docs = self._load_documents(documents)
+        if not new_docs:
+            return 0
+
+        self.documents.extend(new_docs)
+
+        # Chunk new docs
+        new_chunks: list[Chunk] = []
+        for doc in new_docs:
+            if doc.metadata.get("filename", "").endswith(".rst"):
+                chunks = chunk_rst_sections(doc.content, doc.id)
+            else:
+                chunks = chunk_document(doc, self.chunk_size, self.chunk_overlap)
+            new_chunks.extend(chunks)
+
+        if not new_chunks:
+            return 0
+
+        # Embed new chunks
+        texts = [chunk.content for chunk in new_chunks]
+        responses = self._embedding_provider.embed_batch(texts, self.embedding_model)
+
+        new_matrix = np.array([response.embedding for response in responses], dtype=np.float64)
+
+        # Normalize
+        norms = np.linalg.norm(new_matrix, axis=1, keepdims=True)
+        norms[norms == 0] = 1
+        new_matrix_norm = new_matrix / norms
+
+        # Update state
+        current_chunks = list(self._chunks)
+        current_chunks.extend(new_chunks)
+        self._chunks = tuple(current_chunks)
+
+        if self._embedding_matrix is None:
+            self._embedding_matrix = new_matrix_norm
+        else:
+            self._embedding_matrix = np.vstack((self._embedding_matrix, new_matrix_norm))
+
+        return len(new_chunks)
+
+    def remove_documents(self, source_path_pattern: str) -> int:
+        """Remove documents matching a source path pattern.
+
+        Args:
+            source_path_pattern: Glob pattern to match 'source' metadata.
+
+        Returns:
+            Number of chunks removed.
+        """
+        import fnmatch
+
+        if not self._chunks:
+            return 0
+
+        indices_to_keep = []
+        kept_chunks = []
+        removed_count = 0
+
+        for i, chunk in enumerate(self._chunks):
+            source = chunk.metadata.get("source", "")
+            if not source or not fnmatch.fnmatch(source, source_path_pattern):
+                indices_to_keep.append(i)
+                kept_chunks.append(chunk)
+            else:
+                removed_count += 1
+
+        if removed_count == 0:
+            return 0
+
+        self._chunks = tuple(kept_chunks)
+
+        if self._embedding_matrix is not None:
+            if not kept_chunks:
+                self._embedding_matrix = None
+            else:
+                self._embedding_matrix = self._embedding_matrix[indices_to_keep]
+
+        # Also remove from self.documents
+        self.documents = [
+            doc for doc in self.documents if not fnmatch.fnmatch(doc.metadata.get("source", ""), source_path_pattern)
+        ]
+
+        return removed_count
+
+    def update_documents(self, documents: list[Document] | str | Path) -> int:
+        """Update existing documents (remove old, add new).
+
+        Uses document source path to identify what to remove.
+
+        Args:
+            documents: New versions of documents.
+
+        Returns:
+            Number of chunks added.
+        """
+        new_docs = self._load_documents(documents)
+        if not new_docs:
+            return 0
+
+        # Identify sources to remove
+        sources_to_remove = set()
+        for doc in new_docs:
+            source = doc.metadata.get("source")
+            if source:
+                sources_to_remove.add(source)
+
+        # Remove old versions
+        for source in sources_to_remove:
+            self.remove_documents(source)
+
+        # Add new versions
+        return self.add_documents(new_docs)
+
     def retrieve(self, query: str, top_k: int = 3) -> list[tuple[Chunk, float]]:
         """
         Retrieve relevant chunks for a query.
@@ -243,6 +374,190 @@ class RAGAssistant:
 
         return [(self._chunks[i], float(similarities[i])) for i in top_indices]
 
+    def retrieve_with_context(
+        self,
+        query: str,
+        top_k: int = 3,
+        window_size: int = 1,
+        min_score: float = 0.0,
+    ) -> list[tuple[Chunk, float]]:
+        """
+        Retrieve chunks with adjacent context expansion (window search).
+
+        For each retrieved chunk, also includes adjacent chunks from the
+        same document to provide more context. This is useful when relevant
+        information spans multiple chunks.
+
+        Pattern inspired by ai4rag window_search.
+
+        Parameters
+        ----------
+        query : str
+            Search query.
+        top_k : int
+            Number of initial chunks to retrieve (default: 3).
+        window_size : int
+            Number of adjacent chunks to include on each side (default: 1).
+            Set to 0 to disable window expansion.
+        min_score : float
+            Minimum similarity score threshold (default: 0.0).
+
+        Returns
+        -------
+        list[tuple[Chunk, float]]
+            List of (chunk, similarity_score) tuples, sorted by relevance.
+            Adjacent chunks have slightly lower scores.
+
+        Examples
+        --------
+        >>> # Get chunks with 1 adjacent chunk on each side
+        >>> results = assistant.retrieve_with_context("query", window_size=1)
+        >>> for chunk, score in results:
+        ...     print(f"{score:.2f}: {chunk.content[:50]}...")
+        """
+        with log_operation("retrieve_with_context", query_len=len(query), top_k=top_k, window_size=window_size) as ctx:
+            # Get initial results (more than top_k to account for filtering)
+            results = self.retrieve(query, top_k * 2)
+
+            # Apply minimum score threshold
+            if min_score > 0:
+                results = [(chunk, score) for chunk, score in results if score >= min_score]
+
+            if window_size == 0 or not results:
+                ctx["expanded_chunks"] = len(results)
+                return results[:top_k]
+
+            # Build chunk index for fast lookup
+            chunk_to_idx = {id(chunk): i for i, chunk in enumerate(self._chunks)}
+
+            expanded_results: list[tuple[Chunk, float]] = []
+            seen_indices: set[int] = set()
+
+            for chunk, score in results[:top_k]:
+                chunk_idx = chunk_to_idx.get(id(chunk))
+                if chunk_idx is None:
+                    expanded_results.append((chunk, score))
+                    continue
+
+                # Get window of adjacent chunks from same document
+                start_idx = max(0, chunk_idx - window_size)
+                end_idx = min(len(self._chunks), chunk_idx + window_size + 1)
+
+                for idx in range(start_idx, end_idx):
+                    if idx in seen_indices:
+                        continue
+
+                    adjacent_chunk = self._chunks[idx]
+                    # Only include adjacent chunks from same document
+                    if adjacent_chunk.doc_id == chunk.doc_id:
+                        seen_indices.add(idx)
+                        # Original chunk keeps full score, adjacent get 80%
+                        adj_score = score if idx == chunk_idx else score * 0.8
+                        expanded_results.append((adjacent_chunk, adj_score))
+
+            # Sort by score (highest first)
+            expanded_results.sort(key=lambda x: (-x[1], self._chunks.index(x[0]) if x[0] in self._chunks else 0))
+            ctx["expanded_chunks"] = len(expanded_results)
+
+            return expanded_results
+
+    def get_context_with_window(
+        self,
+        query: str,
+        top_k: int = 3,
+        window_size: int = 1,
+        min_score: float = 0.0,
+    ) -> str:
+        """
+        Get formatted context with adjacent chunk expansion.
+
+        Merges overlapping text from adjacent chunks intelligently.
+
+        Parameters
+        ----------
+        query : str
+            Search query.
+        top_k : int
+            Number of initial chunks to retrieve.
+        window_size : int
+            Number of adjacent chunks on each side.
+        min_score : float
+            Minimum similarity score threshold.
+
+        Returns
+        -------
+        str
+            Formatted context string with merged chunks.
+        """
+        results = self.retrieve_with_context(query, top_k, window_size, min_score)
+
+        if not results:
+            return ""
+
+        # Group chunks by document to merge properly
+        doc_chunks: dict[str, list[tuple[Chunk, float]]] = {}
+        for chunk, score in results:
+            doc_id = chunk.doc_id or "unknown"
+            if doc_id not in doc_chunks:
+                doc_chunks[doc_id] = []
+            doc_chunks[doc_id].append((chunk, score))
+
+        merged_sections: list[str] = []
+
+        for _doc_id, chunks in doc_chunks.items():
+            # Sort chunks by their position in the original list
+            chunks.sort(key=lambda x: self._chunks.index(x[0]) if x[0] in self._chunks else 0)
+
+            # Merge overlapping text
+            merged_content = []
+            for chunk, _ in chunks:
+                if merged_content:
+                    # Check for overlap with previous chunk
+                    prev_content = merged_content[-1]
+                    non_overlapping = self._get_non_overlapping_text(prev_content, chunk.content)
+                    if non_overlapping != chunk.content:
+                        # Found overlap, extend previous chunk
+                        merged_content[-1] = prev_content + non_overlapping
+                    else:
+                        # No overlap, add as new section
+                        merged_content.append(chunk.content)
+                else:
+                    merged_content.append(chunk.content)
+
+            merged_sections.append("\n".join(merged_content))
+
+        return "\n\n---\n\n".join(merged_sections)
+
+    def _get_non_overlapping_text(self, str1: str, str2: str) -> str:
+        """
+        Find non-overlapping portion of str2 when appending after str1.
+
+        Detects overlap where the end of str1 matches the beginning of str2,
+        and returns only the non-overlapping portion of str2.
+
+        Pattern from ai4rag vector_store/utils.py.
+
+        Parameters
+        ----------
+        str1 : str
+            First string (previous content).
+        str2 : str
+            Second string (content to potentially append).
+
+        Returns
+        -------
+        str
+            Non-overlapping portion of str2, or full str2 if no overlap.
+        """
+        # Limit overlap search to avoid O(n^2) for large strings
+        max_overlap = min(len(str1), len(str2), 200)
+
+        for i in range(max_overlap, 0, -1):
+            if str1[-i:] == str2[:i]:
+                return str2[i:]
+
+        return str2
+
     def get_context(self, query: str, top_k: int = 3) -> str:
         """
         Get formatted context string from retrieved chunks.