ragxo 0.1.2__py3-none-any.whl → 0.1.5__py3-none-any.whl

ragxo/__init__.py

@@ -1,3 +1,3 @@
-from ragx import Ragxo, Document
 
+from .client import Ragxo, Document
 __all__ = ["Ragxo", "Document"]

ragxo/client.py

@@ -0,0 +1,253 @@
+import time
+from typing import Self, Callable
+from pymilvus import MilvusClient
+from pydantic import BaseModel
+import boto3
+import dill
+import os
+import shutil
+import logging
+import tempfile
+from botocore.exceptions import ClientError
+import openai
+from openai import ChatCompletion
+
+logger = logging.getLogger(__name__)
+
+class Document(BaseModel):
+    text: str
+    metadata: dict
+    id: int
+
+class Ragxo:
+    def __init__(self, dimension: int) -> None:
+        self.dimension = dimension
+        self.collection_name = "ragx"
+        os.makedirs("ragx_artifacts", exist_ok=True)
+
+        self.db_path = f"ragx_artifacts/milvus_{int(time.time())}.db"
+        self.client = MilvusClient(self.db_path)
+        self.client.create_collection(self.collection_name, dimension=dimension)
+        self.processing_fn = []
+        self.embedding_fn = None
+        self.system_prompt = None
+        self.model = "gpt-4o-mini"
+    
+    def add_preprocess(self, fn: Callable) -> Self:
+        self.processing_fn.append(fn)
+        return self
+    
+    def add_embedding_fn(self, fn: Callable) -> Self:
+        if not fn:
+            raise ValueError("Embedding function cannot be None")
+        self.embedding_fn = fn
+        return self
+    
+    def add_system_prompt(self, prompt: str) -> Self:
+        self.system_prompt = prompt
+        return self
+    
+    def add_model(self, model: str) -> Self:
+        self.model = model
+        return self
+    
+    def index(self, data: list[Document]) -> Self:
+        if not self.embedding_fn:
+            raise ValueError("Embedding function not set")
+            
+        processed_text = []
+        for item in data:
+            current_text = item.text
+            for fn in self.processing_fn:
+                current_text = fn(current_text)
+            processed_text.append(current_text)
+            
+        embeddings = [
+            self.embedding_fn(text)
+            for text in processed_text
+        ]
+        
+        self.client.insert(self.collection_name, [
+            {
+                "text": item.text,
+                "metadata": item.metadata,
+                "id": item.id,
+                "vector": embedding
+            }
+            for item, embedding in zip(data, embeddings)
+        ])
+        return self
+    
+    def query(self, query: str, output_fields: list[str] = ['text', 'metadata']) -> list[list[dict]]:
+        if not self.embedding_fn:
+            raise ValueError("Embedding function not set. Please call add_embedding_fn first.")
+            
+        preprocessed_query = query
+        for fn in self.processing_fn:
+            preprocessed_query = fn(preprocessed_query)
+        
+        embedding = self.embedding_fn(preprocessed_query)
+        
+        return self.client.search(
+            collection_name=self.collection_name,
+            data=[embedding],
+            limit=10,
+            output_fields=output_fields
+        )
+
+    def export(self, destination: str, s3_bucket: str = None) -> Self:
+        """
+        Export the Ragx instance to either local filesystem or S3.
+        
+        Args:
+            destination: str - Local path or S3 key prefix
+            s3_bucket: str, optional - S3 bucket name. If provided, export to S3
+        """
+        try:
+            # If s3_bucket is provided, export to S3
+            if s3_bucket:
+                return self._export_to_s3(destination, s3_bucket)
+            
+            # Otherwise, export to local filesystem
+            os.makedirs(destination, exist_ok=True)
+            
+            # Save using dill
+            pickle_path = os.path.join(destination, "ragx.pkl")
+            with open(pickle_path, "wb") as f:
+                dill.dump(self, f)
+            
+            # Copy database
+            db_dest = os.path.join(destination, "milvus.db")
+            shutil.copy(self.db_path, db_dest)
+            
+            return self
+            
+        except Exception as e:
+            logger.error(f"Error in export: {e}")
+            raise
+
+    def _export_to_s3(self, prefix: str, bucket: str) -> Self:
+        """
+        Internal method to handle S3 export.
+        """
+        try:
+            s3_client = boto3.client('s3')
+            
+            # Create a temporary directory for the files
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Save pickle file
+                pickle_path = os.path.join(temp_dir, "ragx.pkl")
+                with open(pickle_path, "wb") as f:
+                    dill.dump(self, f)
+                
+                # Copy database
+                db_path = os.path.join(temp_dir, "milvus.db")
+                shutil.copy(self.db_path, db_path)
+                
+                # Upload to S3
+                s3_client.upload_file(
+                    pickle_path,
+                    bucket,
+                    f"{prefix}/ragx.pkl"
+                )
+                s3_client.upload_file(
+                    db_path,
+                    bucket,
+                    f"{prefix}/milvus.db"
+                )
+            
+            return self
+            
+        except ClientError as e:
+            logger.error(f"Error uploading to S3: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Error in S3 export: {e}")
+            raise
+
+    @classmethod
+    def load(cls, source: str, s3_bucket: str = None) -> Self:
+        """
+        Load a Ragx instance from either local filesystem or S3.
+        
+        Args:
+            source: str - Local path or S3 key prefix
+            s3_bucket: str, optional - S3 bucket name. If provided, load from S3
+        """
+        try:
+            # If s3_bucket is provided, load from S3
+            if s3_bucket:
+                return cls._load_from_s3(source, s3_bucket)
+            
+            # Otherwise, load from local filesystem
+            pickle_path = os.path.join(source, "ragx.pkl")
+            
+            with open(pickle_path, "rb") as f:
+                instance = dill.load(f)
+            
+            # Restore client
+            instance.client = MilvusClient(os.path.join(source, "milvus.db"))
+            
+            return instance
+            
+        except Exception as e:
+            logger.error(f"Error in load: {e}")
+            raise
+
+    @classmethod
+    def _load_from_s3(cls, prefix: str, bucket: str) -> 'Ragx':
+        """
+        Internal classmethod to handle S3 loading.
+        """
+        try:
+            s3_client = boto3.client('s3')
+            
+            # Create a temporary directory for the files
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Download files from S3
+                pickle_path = os.path.join(temp_dir, "ragx.pkl")
+                db_path = os.path.join(temp_dir, "milvus.db")
+                
+                s3_client.download_file(
+                    bucket,
+                    f"{prefix}/ragx.pkl",
+                    pickle_path
+                )
+                s3_client.download_file(
+                    bucket,
+                    f"{prefix}/milvus.db",
+                    db_path
+                )
+                
+                # Load the pickle file
+                with open(pickle_path, "rb") as f:
+                    instance = dill.load(f)
+                
+                # Restore client with the downloaded database
+                instance.client = MilvusClient(db_path)
+                
+                return instance
+                
+        except ClientError as e:
+            logger.error(f"Error downloading from S3: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Error in S3 load: {e}")
+            raise
+    
+    def generate_llm_response(self, query: str, data: list[dict] = None) -> ChatCompletion:
+        if data is None:
+            data = self.query(query)[0]
+        
+        if not self.system_prompt:
+            raise ValueError("System prompt not set. Please call add_system_prompt first.")
+        
+        response = openai.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": "query: {} data: {}".format(query, data)}
+            ]
+        )
+        
+        return response

ragxo-0.1.5.dist-info/METADATA

@@ -0,0 +1,167 @@
+Metadata-Version: 2.1
+Name: ragxo
+Version: 0.1.5
+Summary: A RAG (Retrieval-Augmented Generation) toolkit with Milvus integration
+Home-page: https://github.com/yourusername/ragx
+License: MIT
+Keywords: rag,milvus,nlp,embeddings,openai
+Author: Mohamed Sadek
+Author-email: mohamedfawzydes@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: boto3 (>=1.36.14,<2.0.0)
+Requires-Dist: dill (>=0.3.9,<0.4.0)
+Requires-Dist: milvus (>=2.3.9,<3.0.0)
+Requires-Dist: openai (>=1.61.1,<2.0.0)
+Requires-Dist: pydantic (>=2.10.6,<3.0.0)
+Requires-Dist: pymilvus (>=2.5.4,<3.0.0)
+Project-URL: Repository, https://github.com/yourusername/ragx
+Description-Content-Type: text/markdown
+
+# RagXO 🚀
+
+[![PyPI version](https://badge.fury.io/py/ragxo.svg)](https://badge.fury.io/py/ragxo)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/release/python-380/)
+
+RagXO extends the capabilities of traditional RAG (Retrieval-Augmented Generation) systems by providing a unified way to package, version, and deploy your entire RAG pipeline with LLM integration. Export your complete system—including embedding functions, preprocessing steps, vector store, and LLM configurations—into a single, portable artifact.
+
+## Features ✨
+
+- **Complete RAG Pipeline**: Package your entire RAG system into a versioned artifact
+- **LLM Integration**: Built-in support for OpenAI models
+- **Flexible Embedding**: Compatible with any embedding function (Sentence Transformers, OpenAI, etc.)
+- **Custom Preprocessing**: Chain multiple preprocessing steps
+- **Vector Store Integration**: Built-in Milvus support
+- **System Prompts**: Include and version your system prompts
+
+## Installation 🛠️
+
+```bash
+pip install ragxo
+```
+
+## Usage Guide 📚
+
+### Import
+
+```python
+from ragxo import Ragxo, Document
+
+ragxo_client = Ragxo(dimension=768)
+
+```
+
+### Adding Preprocessing Steps
+
+```python
+import re
+
+def remove_special_chars(text: str) -> str:
+    return re.sub(r'[^a-zA-Z0-9\s]', '', text)
+
+def lowercase(text: str) -> str:
+    return text.lower()
+
+ragxo_client.add_preprocess(remove_special_chars)
+ragxo_client.add_preprocess(lowercase)
+```
+
+### Custom Embedding Functions
+
+```python
+# Using SentenceTransformers
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer('all-MiniLM-L6-v2')
+
+def get_embeddings(text: str) -> list[float]:
+    return model.encode(text).tolist()
+
+ragxo.add_embedding_fn(get_embeddings)
+
+# Or using OpenAI
+from openai import OpenAI
+client = OpenAI()
+
+def get_openai_embeddings(text: str) -> list[float]:
+    response = client.embeddings.create(
+        input=text,
+        model="text-embedding-ada-002"
+    )
+    return response.data[0].embedding
+
+ragxo.add_embedding_fn(get_openai_embeddings)
+```
+
+
+### Creating Documents
+
+```python
+from ragxo import Document
+
+doc = Document(
+    text="Your document content here",
+    metadata={"source": "wiki", "category": "science"},
+    id=1
+)
+
+ragxo_client.index([doc])
+
+```
+
+### LLM Configuration
+
+```python
+# Set system prompt
+ragxo_client.add_system_prompt("""
+You are a helpful assistant. Use the provided context to answer questions accurately.
+If you're unsure about something, please say so.
+""")
+
+# Set LLM model
+ragxo_client.add_model("gpt-4")
+```
+
+### Export and Load
+
+```python
+# Export your RAG pipeline
+ragxo_client.export("rag_pipeline_v1")
+
+# Load it elsewhere
+loaded_ragxo_client = Ragxo.load("rag_pipeline_v1")
+```
+
+## Best Practices 💡
+
+1. **Version Your Exports**: Use semantic versioning for your exports:
+```python
+ragxo.export("my_rag_v1.0.0")
+```
+
+2. **S3**: Use S3 to store your exports
+
+```shell
+export AWS_ACCESS_KEY_ID=your_access_key
+export AWS_SECRET_ACCESS_KEY=your_secret_key
+```
+
+```python
+ragxo_client.export("my_rag_v1.0.0", s3_bucket="my_bucket")
+```
+
+## License 📝
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Contributing 🤝
+
+Contributions are welcome! Please feel free to submit a Pull Request.

ragxo-0.1.5.dist-info/RECORD

@@ -0,0 +1,5 @@
+ragxo/__init__.py,sha256=0VVe-z4XkkGQLQIG0hF0Hyf87_RgX0E4T9TRwwTkbmE,68
+ragxo/client.py,sha256=smS3vt7k0k1p1mDOT8Taa4vpHl0fiuvbo3RdRY_D01k,8300
+ragxo-0.1.5.dist-info/METADATA,sha256=0oS9x3tRULcvKZNQQOyIlKpQ_baDzt1pHB0wTubsb7I,4600
+ragxo-0.1.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+ragxo-0.1.5.dist-info/RECORD,,

ragxo/ragx.py

@@ -1,141 +0,0 @@
-from typing import Self, Callable
-from pymilvus import MilvusClient
-from pydantic import BaseModel
-import dill
-import os
-import shutil
-import logging
-import openai
-from openai import ChatCompletion
-
-logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-class Document(BaseModel):
-    text: str
-    metadata: dict
-    id: int
-
-class Ragxo:
-    def __init__(self, dimension: int) -> None:
-        self.dimension = dimension
-        self.collection_name = "ragx"
-        self.db_path = "milvus.db"
-        self.client = MilvusClient(self.db_path)
-        self.client.create_collection(self.collection_name, dimension=dimension)
-        self.processing_fn = []
-        self.embedding_fn = None
-        self.system_prompt = None
-        self.model = "gpt-4o-mini"
-    
-    def add_preprocess(self, fn: Callable) -> Self:
-        self.processing_fn.append(fn)
-        return self
-    
-    def add_embedding_fn(self, fn: Callable) -> Self:
-        if not fn:
-            raise ValueError("Embedding function cannot be None")
-        self.embedding_fn = fn
-        return self
-    
-    def add_system_prompt(self, prompt: str) -> Self:
-        self.system_prompt = prompt
-        return self
-    
-    def add_model(self, model: str) -> Self:
-        self.model = model
-        return self
-    
-    def index(self, data: list[Document]) -> Self:
-        if not self.embedding_fn:
-            raise ValueError("Embedding function not set")
-            
-        processed_text = []
-        for item in data:
-            current_text = item.text
-            for fn in self.processing_fn:
-                current_text = fn(current_text)
-            processed_text.append(current_text)
-            
-        embeddings = [
-            self.embedding_fn(text)
-            for text in processed_text
-        ]
-        
-        self.client.insert(self.collection_name, [
-            {
-                "text": item.text,
-                "metadata": item.metadata,
-                "id": item.id,
-                "vector": embedding
-            }
-            for item, embedding in zip(data, embeddings)
-        ])
-        return self
-    
-    def query(self, query: str, output_fields: list[str] = ['text', 'metadata']) -> list[list[dict]]:
-        if not self.embedding_fn:
-            raise ValueError("Embedding function not set. Please call add_embedding_fn first.")
-            
-        preprocessed_query = query
-        for fn in self.processing_fn:
-            preprocessed_query = fn(preprocessed_query)
-        
-        embedding = self.embedding_fn(preprocessed_query)
-        
-        return self.client.search(
-            collection_name=self.collection_name,
-            data=[embedding],
-            limit=10,
-            output_fields=output_fields
-        )
-    
-    def export(self, folder_path: str) -> Self:
-        try:
-            os.makedirs(folder_path, exist_ok=True)
-            
-            # Save using dill
-            pickle_path = os.path.join(folder_path, "ragx.pkl")
-            with open(pickle_path, "wb") as f:
-                dill.dump(self, f)
-            
-            # Copy database
-            db_dest = os.path.join(folder_path, "milvus.db")
-            shutil.copy(self.db_path, db_dest)
-            
-            return self
-            
-        except Exception as e:
-            logger.error(f"Error in export: {e}")
-            raise
-    
-    @classmethod
-    def load(cls, folder_path: str) -> 'Ragx':
-        try:
-            pickle_path = os.path.join(folder_path, "ragx.pkl")
-            
-            with open(pickle_path, "rb") as f:
-                instance = dill.load(f)
-            
-            # Restore client
-            instance.client = MilvusClient(os.path.join(folder_path, "milvus.db"))
-            
-            return instance
-            
-        except Exception as e:
-            logger.error(f"Error in load: {e}")
-            raise
-    
-    def generate_llm_response(self, query: str) -> ChatCompletion:
-        if not self.system_prompt:
-            raise ValueError("System prompt not set. Please call add_system_prompt first.")
-        
-        response = openai.chat.completions.create(
-            model=self.model,
-            messages=[
-                {"role": "system", "content": self.system_prompt},
-                {"role": "user", "content": query}
-            ]
-        )
-        
-        return response

ragxo-0.1.2.dist-info/METADATA

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: ragxo
-Version: 0.1.2
-Summary: A RAG (Retrieval-Augmented Generation) toolkit with Milvus integration
-Home-page: https://github.com/yourusername/ragx
-License: MIT
-Keywords: rag,milvus,nlp,embeddings,openai
-Author: Mohamed Sadek
-Author-email: mohamedfawzydes@gmail.com
-Requires-Python: >=3.11,<4.0
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: dill (>=0.3.9,<0.4.0)
-Requires-Dist: milvus (>=2.3.9,<3.0.0)
-Requires-Dist: openai (>=1.61.1,<2.0.0)
-Requires-Dist: pydantic (>=2.10.6,<3.0.0)
-Requires-Dist: pymilvus (>=2.5.4,<3.0.0)
-Project-URL: Repository, https://github.com/yourusername/ragx
-Description-Content-Type: text/markdown
-
-

ragxo-0.1.2.dist-info/RECORD

@@ -1,5 +0,0 @@
-ragxo/__init__.py,sha256=jI_6iulTUQk9JUDft-jM6NHESpZSmJVPIaVOmd4-jWw,65
-ragxo/ragx.py,sha256=KotppZuO9U1aQG0CSbvRTVOXBN0BpaKMlS1IrqBmigk,4394
-ragxo-0.1.2.dist-info/METADATA,sha256=xZ3DcX6lsLCBLnwcn13J5pufLilNnvjL8nk8ccGlyGo,1111
-ragxo-0.1.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-ragxo-0.1.2.dist-info/RECORD,,