piragi 0.1.0__py3-none-any.whl

piragi-0.1.0.dist-info/METADATA

@@ -0,0 +1,149 @@
+Metadata-Version: 2.4
+Name: piragi
+Version: 0.1.0
+Summary: The best RAG interface yet - Built-in vector store, embeddings, citations, and auto-updates
+Project-URL: Homepage, https://github.com/hemanth/ragi
+Project-URL: Documentation, https://github.com/hemanth/ragi#readme
+Project-URL: Repository, https://github.com/hemanth/ragi
+Project-URL: Issues, https://github.com/hemanth/ragi/issues
+Author-email: Hemanth HM <hemanth.hm@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,embeddings,llm,rag,retrieval,vector-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: lancedb>=0.3.0
+Requires-Dist: markitdown[all]>=0.0.1
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=4.51.0
+Provides-Extra: dev
+Requires-Dist: black>=23.0.0; extra == 'dev'
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Ragi
+
+**The best RAG interface yet.**
+
+```python
+from ragi import Ragi
+
+kb = Ragi(["./docs", "./code/**/*.py", "https://api.example.com/docs"])
+answer = kb.ask("How do I deploy this?")
+```
+
+That's it. Built-in vector store, embeddings, citations, and auto-updates. Free & local by default.
+
+---
+
+## Installation
+
+```bash
+pip install piragi
+
+# Optional: Install Ollama for local LLM
+curl -fsSL https://ollama.com/install.sh | sh
+ollama pull llama3.2
+```
+
+---
+
+## Features
+
+- **Simple Setup** - Works with free local models out of the box
+- **All Formats** - PDF, Word, Excel, Markdown, Code, URLs, Images, Audio
+- **Auto-Updates** - Background refresh, queries never blocked
+- **Smart Citations** - Every answer includes sources
+- **OpenAI Compatible** - Drop-in support for any OpenAI-compatible API
+
+---
+
+## Examples
+
+```python
+# Basic
+kb = Ragi("./docs")
+answer = kb("What is this?")
+
+# Multiple sources
+kb = Ragi(["./docs/*.pdf", "https://api.docs.com", "./code/**/*.py"])
+
+# OpenAI
+kb = Ragi("./docs", config={
+    "llm": {"model": "gpt-4o-mini", "api_key": "sk-..."},
+    "embedding": {"model": "text-embedding-3-small", "api_key": "sk-..."}
+})
+
+# Filter
+answer = kb.filter(file_type="pdf").ask("What's in the PDFs?")
+```
+
+---
+
+## Configuration
+
+```python
+# Defaults (all optional)
+config = {
+    "llm": {
+        "model": "llama3.2",
+        "base_url": "http://localhost:11434/v1"
+    },
+    "embedding": {
+        "model": "nvidia/llama-embed-nemotron-8b"
+    },
+    "auto_update": {
+        "enabled": True,
+        "interval": 300  # seconds
+    }
+}
+```
+
+---
+
+## Auto-Updates
+
+Changes detected and refreshed automatically in background. Zero query latency.
+
+```python
+kb = Ragi(["./docs", "https://api.docs.com"])
+# That's it - auto-updates enabled by default
+
+# Disable if needed
+kb = Ragi("./docs", config={"auto_update": {"enabled": False}})
+```
+
+---
+
+## API
+
+```python
+kb = Ragi(sources, persist_dir=".ragi", config=None)
+kb.add("./more-docs")
+kb.ask(query, top_k=5)
+kb(query)  # Shorthand
+kb.filter(**metadata).ask(query)
+kb.count()
+kb.clear()
+```
+
+Full docs: [API.md](API.md)
+
+---
+
+MIT License | **Ragi** = **R**etrieval **A**ugmented **G**eneration **I**nterface

piragi-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+ragi/__init__.py,sha256=BKXHlsfnnCP9bIc-WfbztwKAUbQUAI38oOy8iEHMO9k,781
+ragi/async_updater.py,sha256=rjiaMUPSJ0cDzCdKRdifZ1yU6c70D_N0t9zkj281Jws,11218
+ragi/change_detection.py,sha256=m2-bMywK9WDVrjniKJyTrrkUzzsOvGXAWf5ZFIFVLiU,6863
+ragi/chunking.py,sha256=L9tRiLL0Hz10bD4gaLc-eYyviTTfXka2_QdPpZ_FBmQ,4525
+ragi/core.py,sha256=IT558mpiUhE7JpeTczC3f05vtSs8U_MOEk5MPiO1btQ,10194
+ragi/embeddings.py,sha256=tPVB8fc73BWXMvwAbOfOmiyVYrrs84d9lJfeKf5l00M,5094
+ragi/loader.py,sha256=WBiDn6iuJLGHq31O0Q2nuiDNVtc3hzbCOIxwevqOtMo,3978
+ragi/retrieval.py,sha256=uTMqLCZwpjI03_xVq8OIGmtKyrbCc6-sVpoNH2eh8r4,3788
+ragi/store.py,sha256=7mYDnD9DW065nklrUjgngVDeY97rF8G4i-tLfI469wc,5176
+ragi/types.py,sha256=5EY50FJNB7CceGQkpX-5O3vLkWRsElKvpf0oxizxPOs,2142
+piragi-0.1.0.dist-info/METADATA,sha256=NDSacGNkfd4Ms_Yit9CKIGFtWpq_2TGoeM7F3BRrP9M,3747
+piragi-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+piragi-0.1.0.dist-info/licenses/LICENSE,sha256=p8azJxT7o36aVnN6FDXGGTs0JjfOMab1J2REesqgoJM,1074
+piragi-0.1.0.dist-info/RECORD,,

piragi-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

piragi-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ragi Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ragi/__init__.py

@@ -0,0 +1,28 @@
+"""
+Ragi - Zero-setup RAG library with auto-chunking, embeddings, and smart citations.
+
+Example:
+    >>> from ragi import Ragi
+    >>>
+    >>> # One-liner setup and query
+    >>> kb = Ragi("./docs")
+    >>> answer = kb.ask("How do I install this?")
+    >>>
+    >>> # Access answer and citations
+    >>> print(answer.text)
+    >>> for citation in answer.citations:
+    ...     print(f"Source: {citation.source}")
+    ...     print(f"Relevance: {citation.score:.2f}")
+    >>>
+    >>> # Callable shorthand
+    >>> answer = kb("What's the API?")
+    >>>
+    >>> # Filter by metadata
+    >>> answer = kb.filter(type="documentation").ask("How to configure?")
+"""
+
+from .core import Ragi
+from .types import Answer, Citation
+
+__version__ = "0.1.0"
+__all__ = ["Ragi", "Answer", "Citation"]

ragi/async_updater.py

@@ -0,0 +1,345 @@
+"""Async background updater for automatic document refresh."""
+
+import asyncio
+import threading
+import time
+from typing import Any, Callable, Dict, List, Optional, Union
+from queue import Queue, Empty
+import logging
+
+from .change_detection import ChangeDetector
+
+logger = logging.getLogger(__name__)
+
+
+class UpdateTask:
+    """Represents an update task."""
+
+    def __init__(self, source: str, priority: int = 0):
+        self.source = source
+        self.priority = priority
+        self.created_at = time.time()
+
+
+class AsyncUpdater:
+    """
+    Background updater that checks and refreshes sources asynchronously.
+
+    This prevents queries from being blocked while checking for updates.
+    Updates happen in a background thread/task.
+    """
+
+    def __init__(
+        self,
+        refresh_callback: Callable[[Union[str, List[str]]], None],
+        check_interval: float = 300.0,
+        max_workers: int = 2,
+    ):
+        """
+        Initialize the async updater.
+
+        Args:
+            refresh_callback: Function to call when refresh is needed (e.g., kb.refresh)
+            check_interval: Default interval between checks in seconds
+            max_workers: Number of background workers
+        """
+        self.refresh_callback = refresh_callback
+        self.check_interval = check_interval
+        self.max_workers = max_workers
+
+        # Source metadata tracking
+        self.sources_metadata: Dict[str, Dict[str, Any]] = {}
+
+        # Update queue and control
+        self.update_queue: Queue = Queue()
+        self.running = False
+        self.workers: List[threading.Thread] = []
+        self._lock = threading.RLock()
+
+        # Statistics
+        self.stats = {
+            "checks_performed": 0,
+            "updates_performed": 0,
+            "last_check_time": None,
+        }
+
+    def register_source(
+        self,
+        source: str,
+        content: str,
+        check_interval: Optional[float] = None,
+    ) -> None:
+        """
+        Register a source for automatic update checking.
+
+        Args:
+            source: File path or URL
+            content: Current content
+            check_interval: Custom check interval (uses default if None)
+        """
+        with self._lock:
+            if ChangeDetector.is_url(source):
+                metadata = ChangeDetector.get_url_metadata(source, content)
+            else:
+                metadata = ChangeDetector.get_file_metadata(source, content)
+
+            if check_interval is not None:
+                metadata["check_interval"] = check_interval
+
+            self.sources_metadata[source] = metadata
+            logger.info(f"Registered source for auto-update: {source}")
+
+    def unregister_source(self, source: str) -> None:
+        """Remove a source from auto-update tracking."""
+        with self._lock:
+            if source in self.sources_metadata:
+                del self.sources_metadata[source]
+                logger.info(f"Unregistered source: {source}")
+
+    def start(self) -> None:
+        """Start background update workers."""
+        if self.running:
+            logger.warning("Updater already running")
+            return
+
+        self.running = True
+
+        # Start worker threads
+        for i in range(self.max_workers):
+            worker = threading.Thread(
+                target=self._worker_loop,
+                name=f"AsyncUpdater-Worker-{i}",
+                daemon=True,
+            )
+            worker.start()
+            self.workers.append(worker)
+
+        # Start scheduler thread
+        scheduler = threading.Thread(
+            target=self._scheduler_loop,
+            name="AsyncUpdater-Scheduler",
+            daemon=True,
+        )
+        scheduler.start()
+        self.workers.append(scheduler)
+
+        logger.info(f"Started {self.max_workers} update workers + scheduler")
+
+    def stop(self) -> None:
+        """Stop background workers."""
+        self.running = False
+        logger.info("Stopping async updater...")
+
+        # Wait for workers to finish current tasks
+        for worker in self.workers:
+            if worker.is_alive():
+                worker.join(timeout=5.0)
+
+        self.workers.clear()
+        logger.info("Async updater stopped")
+
+    def _scheduler_loop(self) -> None:
+        """
+        Main scheduler loop that checks which sources need updating.
+        Runs in background thread.
+        """
+        while self.running:
+            try:
+                current_time = time.time()
+
+                with self._lock:
+                    sources_to_check = []
+
+                    for source, metadata in self.sources_metadata.items():
+                        # Check if enough time has passed
+                        if ChangeDetector.should_check_now(
+                            metadata["last_checked"], metadata["check_interval"]
+                        ):
+                            sources_to_check.append(source)
+
+                # Queue checks
+                for source in sources_to_check:
+                    self.update_queue.put(UpdateTask(source))
+
+                self.stats["last_check_time"] = current_time
+
+                # Sleep until next scheduled check (check every 10s for any due sources)
+                time.sleep(10)
+
+            except Exception as e:
+                logger.error(f"Scheduler error: {e}")
+                time.sleep(10)
+
+    def _worker_loop(self) -> None:
+        """
+        Worker loop that processes update tasks.
+        Runs in background thread.
+        """
+        while self.running:
+            try:
+                # Get task from queue (timeout to allow checking self.running)
+                try:
+                    task = self.update_queue.get(timeout=1.0)
+                except Empty:
+                    continue
+
+                # Process the update task
+                self._check_and_refresh(task.source)
+
+                self.update_queue.task_done()
+
+            except Exception as e:
+                logger.error(f"Worker error: {e}")
+
+    def _check_and_refresh(self, source: str) -> None:
+        """
+        Check if source changed and refresh if needed.
+
+        Args:
+            source: Source to check
+        """
+        try:
+            with self._lock:
+                if source not in self.sources_metadata:
+                    return
+
+                metadata = self.sources_metadata[source]
+
+            self.stats["checks_performed"] += 1
+
+            # Check for changes
+            changed = False
+            new_metadata = {}
+
+            if ChangeDetector.is_url(source):
+                result = ChangeDetector.check_url_changed(
+                    source,
+                    metadata.get("etag"),
+                    metadata.get("last_modified"),
+                )
+                changed = result.get("changed", False)
+                if "etag" in result:
+                    new_metadata["etag"] = result["etag"]
+                if "last_modified" in result:
+                    new_metadata["last_modified"] = result["last_modified"]
+            else:
+                changed = ChangeDetector.check_file_changed(
+                    source, metadata.get("mtime"), metadata["content_hash"]
+                )
+
+            # Update last_checked timestamp
+            with self._lock:
+                self.sources_metadata[source]["last_checked"] = time.time()
+                if new_metadata:
+                    self.sources_metadata[source].update(new_metadata)
+
+            # Refresh if changed
+            if changed:
+                logger.info(f"Change detected in {source}, refreshing...")
+                self.refresh_callback(source)
+                self.stats["updates_performed"] += 1
+
+                # Update metadata after refresh
+                # Note: In production, you'd want to read the new content and hash
+                # For now, we just update the timestamp
+
+        except Exception as e:
+            logger.error(f"Error checking source {source}: {e}")
+
+    def queue_update(self, source: str, priority: int = 0) -> None:
+        """
+        Manually queue a source for update check.
+
+        Args:
+            source: Source to check
+            priority: Priority (higher = more urgent)
+        """
+        self.update_queue.put(UpdateTask(source, priority))
+        logger.debug(f"Queued update check for {source}")
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get updater statistics."""
+        with self._lock:
+            return {
+                **self.stats,
+                "registered_sources": len(self.sources_metadata),
+                "queue_size": self.update_queue.qsize(),
+                "running": self.running,
+            }
+
+
+class AsyncUpdaterMixin:
+    """
+    Mixin to add async update capabilities to Ragi.
+
+    Usage:
+        class AutoUpdateRagi(AsyncUpdaterMixin, Ragi):
+            pass
+
+        kb = AutoUpdateRagi("./docs", auto_update=True)
+    """
+
+    def __init__(self, *args, auto_update: bool = False, **kwargs):
+        """
+        Initialize with optional auto-update.
+
+        Args:
+            auto_update: Enable background auto-updates
+            **kwargs: Additional config including:
+                - auto_update_interval: Check interval in seconds
+                - auto_update_workers: Number of background workers
+        """
+        super().__init__(*args, **kwargs)
+
+        self.auto_update_enabled = auto_update
+        self.updater: Optional[AsyncUpdater] = None
+
+        if auto_update:
+            # Extract auto-update config
+            config = kwargs.get("config", {})
+            auto_config = config.get("auto_update", {})
+
+            interval = auto_config.get("interval", 300.0)
+            workers = auto_config.get("workers", 2)
+
+            # Initialize updater
+            self.updater = AsyncUpdater(
+                refresh_callback=self.refresh,
+                check_interval=interval,
+                max_workers=workers,
+            )
+
+            # Start background workers
+            self.updater.start()
+
+    def add(self, sources, **kwargs):
+        """Override add to register sources with auto-updater."""
+        result = super().add(sources, **kwargs)
+
+        # Register sources with updater if auto-update enabled
+        if self.auto_update_enabled and self.updater:
+            # Note: Would need access to loaded documents to register properly
+            # This is a simplified version
+            source_list = sources if isinstance(sources, list) else [sources]
+            for source in source_list:
+                # Register with default metadata
+                # In production, you'd track the actual content
+                self.updater.register_source(source, "", check_interval=None)
+
+        return result
+
+    def get_update_stats(self) -> Dict[str, Any]:
+        """Get auto-update statistics."""
+        if self.updater:
+            return self.updater.get_stats()
+        return {"auto_update_enabled": False}
+
+    def stop_auto_update(self) -> None:
+        """Stop auto-update background workers."""
+        if self.updater:
+            self.updater.stop()
+
+    def __del__(self):
+        """Cleanup on deletion."""
+        if hasattr(self, "updater") and self.updater:
+            self.updater.stop()