rag_server 0.0.1__py3-none-any.whl

rag_server/__init__.py

@@ -0,0 +1,9 @@
+from . import server
+import asyncio
+
+def main():
+    """Main entry point for the package."""
+    asyncio.run(server.main())
+
+# Optionally expose other important items at package level
+__all__ = ['main', 'server']

rag_server/server.py

@@ -0,0 +1,50 @@
+# server.py
+import uuid
+from typing import Optional
+
+from fastmcp import FastMCP
+
+from rag_server.utils.vector.misc import chunk_text, extract_text_from_url
+from rag_server.utils.vector.store import VectorStore
+
+# Initialize the MCP server
+mcp = FastMCP(name="syne_rag_server", instructions= "You are a helpful assistant that can answer questions about the documents in the session.")
+
+# In-memory sessions: mapping session_id -> VectorStore
+_sessions = {}
+
+@mcp.tool(
+    description="Ingest a list of public URLs (PDF, DOCX, DOC) into an ephemeral session. Returns a session_id to use for querying. You can pass in a session_id to ingest into a specific session."
+)
+def ingest_urls(urls: list[str], session: Optional[str] = None) -> str:
+    """
+    Ingest a list of public URLs (PDF, DOCX, DOC) into an ephemeral session.
+    Returns a session_id to use for querying.
+    """
+    session_id = str(uuid.uuid4() if session is None else session)
+    vs = VectorStore()
+    for url in urls:
+        text = extract_text_from_url(url)
+        chunks = chunk_text(text)
+        vs.add(chunks)
+    _sessions[session_id] = vs
+    return session_id
+
+@mcp.tool(
+    description="Query the ingested documents in the given session using RAG. Returns a generated answer."
+)
+def query_knowledge(session_id: str, question: str) -> str:
+    """
+    Query the ingested documents in the given session using RAG.
+    Returns a generated answer.
+    """
+    vs = _sessions.get(session_id)
+    if not vs:
+        return f"Session ID {session_id} not found. Please call ingest_urls first."
+    docs = vs.search(question)
+    context = "\n\n".join(docs)
+    return context
+
+def main():
+    # Run the server
+    mcp.run()

rag_server/utils/llm.py

@@ -0,0 +1,9 @@
+import os
+
+from openai import OpenAI
+
+openai_client = OpenAI(
+    api_key=os.environ.get("OPENAI_API_KEY"),
+    base_url=os.environ.get("OPENAI_API_URL"),
+)
+

rag_server/utils/vector/misc.py

@@ -0,0 +1,50 @@
+import io
+
+import docx
+import requests
+import textract
+from PyPDF2 import PdfReader
+from openai.types import CreateEmbeddingResponse
+
+from rag_server.utils.llm import openai_client
+
+
+def extract_text_from_url(url: str) -> str:
+    """Download the file at the given URL and extract its text."""
+    resp = requests.get(url)
+    resp.raise_for_status()
+    content = resp.content
+    ext = url.split(".")[-1].lower()
+    if ext == "pdf":
+        reader = PdfReader(io.BytesIO(content))
+        return "\n".join(p.extract_text() or "" for p in reader.pages)
+    elif ext == "docx":
+        doc = docx.Document(io.BytesIO(content))
+        return "\n".join(p.text for p in doc.paragraphs)
+    elif ext == "doc":
+        return textract.process(io.BytesIO(content), extension="doc").decode("utf-8", errors="ignore")
+    else:
+        return content.decode("utf-8", errors="ignore")
+
+
+def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> list[str]:
+    """Split text into chunks of approximately chunk_size words with overlap."""
+    words = text.split()
+    chunks = []
+    start = 0
+    while start < len(words):
+        chunk = " ".join(words[start: start + chunk_size])
+        chunks.append(chunk)
+        start += chunk_size - overlap
+    return chunks
+
+
+def embed_texts(texts: list[str]) -> list[list[float]]:
+    """Embed a list of texts using OpenAI embeddings."""
+    resp : CreateEmbeddingResponse = openai_client.embeddings.create(input=texts, model="text-embedding-ada-002")
+    return [d.embedding for d in resp.data]
+
+def get_embedding(text: str) -> list[float]:
+    """Embed a single text."""
+    resp : CreateEmbeddingResponse = openai_client.embeddings.create(input=text, model="text-embedding-ada-002")
+    return resp.data[0].embedding

rag_server/utils/vector/store.py

@@ -0,0 +1,67 @@
+import faiss
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from rag_server.utils.vector.misc import get_embedding, embed_texts
+
+
+class VectorStore:
+    """Simple in-memory vector store using FAISS."""
+    def __init__(self, dim: int = 1536):
+        self.dim = dim
+        # Use an HNSW approximate nearest neighbor index (no training needed)
+        self.index = faiss.index_factory(dim, "HNSW32")
+        # Configure HNSW parameters for construction and search quality
+        try:
+            self.index.hnsw.efConstruction = 200
+            self.index.hnsw.efSearch = 128
+        except AttributeError:
+            pass
+        self.texts: list[str] = []
+        # Initialize TF-IDF vectorizer and matrix
+        self.vectorizer = TfidfVectorizer()
+        self.tfidf_matrix = None
+
+    def add(self, chunks: list[str]) -> None:
+        embeddings = embed_texts(chunks)
+        arr = np.array(embeddings, dtype="float32")
+        self.index.add(arr)
+        self.texts.extend(chunks)
+        # Update TF-IDF matrix
+        self.tfidf_matrix = self.vectorizer.fit_transform(self.texts)
+
+    def search(self, query: str, top_k: int = 5, alpha: float = 0.5) -> list[str]:
+        """Perform hybrid search combining semantic (FAISS) and lexical (TF-IDF) scores."""
+        # Semantic search via FAISS
+        q_emb = np.array([get_embedding(query)], dtype="float32")
+        D, I = self.index.search(q_emb, top_k)
+        vect_ids = I[0].tolist()
+        vect_scores = [-d for d in D[0]]
+        # Lexical search via TF-IDF
+        if self.tfidf_matrix is None:
+            self.tfidf_matrix = self.vectorizer.fit_transform(self.texts)
+        q_tfidf = self.vectorizer.transform([query])
+        tfidf_scores_all = q_tfidf.dot(self.tfidf_matrix.T).toarray()[0]
+        tfidf_top = np.argsort(-tfidf_scores_all)[:top_k].tolist()
+        # Combine candidate document indices
+        candidate_ids = set(vect_ids + tfidf_top)
+        vect_min = min(vect_scores) if vect_scores else 0.0
+        scores = []
+        for idx in candidate_ids:
+            vs = vect_scores[vect_ids.index(idx)] if idx in vect_ids else vect_min
+            ts = float(tfidf_scores_all[idx])
+            scores.append((idx, vs, ts))
+        # Normalize and blend scores
+        vs_vals = [v for _, v, _ in scores]
+        ts_vals = [t for _, _, t in scores]
+        vmin, vmax = min(vs_vals), max(vs_vals)
+        tmin, tmax = min(ts_vals), max(ts_vals)
+        blended = []
+        for idx, vs, ts in scores:
+            vn = (vs - vmin) / (vmax - vmin) if vmax > vmin else 0.0
+            tn = (ts - tmin) / (tmax - tmin) if tmax > tmin else 0.0
+            combined = alpha * vn + (1 - alpha) * tn
+            blended.append((idx, combined))
+        # Sort by blended score and return top_k chunks
+        top = sorted(blended, key=lambda x: x[1], reverse=True)[:top_k]
+        return [self.texts[i] for i, _ in top]

rag_server-0.0.1.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: rag_server
+Version: 0.0.1
+Summary: A FastMCP-based RAG server for dynamic document ingestion
+Project-URL: Homepage, https://github.com/synehq/mcp-hybrid-rag
+Project-URL: Bug Tracker, https://github.com/synehq/mcp-hybrid-rag/issues
+Author-email: SyneHQ <human@synehq.com>
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: faiss-cpu
+Requires-Dist: fastmcp
+Requires-Dist: numpy
+Requires-Dist: openai
+Requires-Dist: pypdf2
+Requires-Dist: python-docx
+Requires-Dist: requests
+Requires-Dist: scikit-learn
+Requires-Dist: textract
+Description-Content-Type: text/markdown
+
+# RAG Server
+
+A FastMCP-based Retrieval-Augmented Generation server for dynamically ingesting public documents and querying them on-the-fly.
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+Ensure you set your OpenAI API key:
+
+```bash
+export OPENAI_API_KEY=your_key_here
+```
+
+## Running the server
+
+```bash
+python -m rag_server.server
+```
+
+## API Tools
+
+- ingest_urls(urls: List[str], session_id: Optional[str]) -> session_id
+- query_knowledge(session_id: str, question: str) -> answer

rag_server-0.0.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+rag_server/__init__.py,sha256=KNZ1bD9ZGfyZwlv91Ueeega_1lsRDLs2fYQDgNbBdtc,212
+rag_server/server.py,sha256=75IV2Ggowcx30LEtFy1stRbJGodsgvsD-CKObhbCeg4,1699
+rag_server/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rag_server/utils/llm.py,sha256=yEmxoRQ750LGu8ufWu38RoX0umBRWw8q0GQxzFmqAy8,158
+rag_server/utils/vector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rag_server/utils/vector/misc.py,sha256=NbBRzU6RBc4A5Pu0cl76dutuZZfj_abwuAkKjM-LD6k,1768
+rag_server/utils/vector/store.py,sha256=b7GtzjnXuqDVpQHMZ4Otms4wIY4zB0y6aLBCu58DSNE,2929
+rag_server-0.0.1.dist-info/METADATA,sha256=i9DFzwVljGdfABtAK21WFGM9JoxE0hdAPloJlpE0za0,1104
+rag_server-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+rag_server-0.0.1.dist-info/entry_points.txt,sha256=sWdH-o-5Mge0fcw28bZ-lAMdlVq3PJOsXTZSzZy_ndc,76
+rag_server-0.0.1.dist-info/RECORD,,

rag_server-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

rag_server-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+rag-server = rag_server:main
+rag_server = rag_server:main