rag_server 0.0.1__tar.gz

rag_server-0.0.1/.github/workflows/publish-to-pypi.yaml

@@ -0,0 +1,48 @@
+name: Publish Python distribution to PyPI
+
+on:
+  push:
+    branches:
+      - sudo
+    tags:
+      - 'v*'
+
+jobs:
+  run:
+    name: "Build and publish release"
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Get latest tag
+        id: latest_tag
+        run: |
+          TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+          echo "tag=$TAG" >> $GITHUB_ENV
+      - name: Update version
+        run: |
+          TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+          if [[ $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            VERSION=${TAG#v} # Remove the 'v' prefix
+            sed -i "s/^version = .*/version = \"$VERSION\"/" pyproject.toml
+          else
+            echo "Invalid tag format. Expected format: v<MAJOR>.<MINOR>.<PATCH>"
+            exit 1
+          fi
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        run: uv publish -t ${{ secrets.PYPI_TOKEN }}

rag_server-0.0.1/.gitignore

@@ -0,0 +1,19 @@
+.venv
+
+*egg_info
+.pycache
+
+.idea/*
+
+.ruff_cache/*
+
+.mypy_cache/*
+
+*.pyc
+
+*.pyo
+
+*.pyd
+
+*.pyw
+

rag_server-0.0.1/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: rag_server
+Version: 0.0.1
+Summary: A FastMCP-based RAG server for dynamic document ingestion
+Project-URL: Homepage, https://github.com/synehq/mcp-hybrid-rag
+Project-URL: Bug Tracker, https://github.com/synehq/mcp-hybrid-rag/issues
+Author-email: SyneHQ <human@synehq.com>
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: faiss-cpu
+Requires-Dist: fastmcp
+Requires-Dist: numpy
+Requires-Dist: openai
+Requires-Dist: pypdf2
+Requires-Dist: python-docx
+Requires-Dist: requests
+Requires-Dist: scikit-learn
+Requires-Dist: textract
+Description-Content-Type: text/markdown
+
+# RAG Server
+
+A FastMCP-based Retrieval-Augmented Generation server for dynamically ingesting public documents and querying them on-the-fly.
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+Ensure you set your OpenAI API key:
+
+```bash
+export OPENAI_API_KEY=your_key_here
+```
+
+## Running the server
+
+```bash
+python -m rag_server.server
+```
+
+## API Tools
+
+- ingest_urls(urls: List[str], session_id: Optional[str]) -> session_id
+- query_knowledge(session_id: str, question: str) -> answer

rag_server-0.0.1/README.md

@@ -0,0 +1,26 @@
+# RAG Server
+
+A FastMCP-based Retrieval-Augmented Generation server for dynamically ingesting public documents and querying them on-the-fly.
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+Ensure you set your OpenAI API key:
+
+```bash
+export OPENAI_API_KEY=your_key_here
+```
+
+## Running the server
+
+```bash
+python -m rag_server.server
+```
+
+## API Tools
+
+- ingest_urls(urls: List[str], session_id: Optional[str]) -> session_id
+- query_knowledge(session_id: str, question: str) -> answer

rag_server-0.0.1/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "rag_server"
+version = "0.0.1"
+description = "A FastMCP-based RAG server for dynamic document ingestion"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+dependencies = [
+"fastmcp",
+"openai",
+"requests",
+"numpy",
+"faiss-cpu",
+"PyPDF2",
+"python-docx",
+"textract", "scikit-learn"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/synehq/mcp-hybrid-rag"
+"Bug Tracker" = "https://github.com/synehq/mcp-hybrid-rag/issues"
+
+[[project.authors]]
+name = "SyneHQ"
+email = "human@synehq.com"
+
+[build-system]
+requires = [ "hatchling",]
+build-backend = "hatchling.build"
+
+[project.scripts]
+rag_server = "rag_server:main"
+"rag-server" = "rag_server:main"
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]

rag_server-0.0.1/src/rag_server/__init__.py

@@ -0,0 +1,9 @@
+from . import server
+import asyncio
+
+def main():
+    """Main entry point for the package."""
+    asyncio.run(server.main())
+
+# Optionally expose other important items at package level
+__all__ = ['main', 'server']

rag_server-0.0.1/src/rag_server/server.py

@@ -0,0 +1,50 @@
+# server.py
+import uuid
+from typing import Optional
+
+from fastmcp import FastMCP
+
+from rag_server.utils.vector.misc import chunk_text, extract_text_from_url
+from rag_server.utils.vector.store import VectorStore
+
+# Initialize the MCP server
+mcp = FastMCP(name="syne_rag_server", instructions= "You are a helpful assistant that can answer questions about the documents in the session.")
+
+# In-memory sessions: mapping session_id -> VectorStore
+_sessions = {}
+
+@mcp.tool(
+    description="Ingest a list of public URLs (PDF, DOCX, DOC) into an ephemeral session. Returns a session_id to use for querying. You can pass in a session_id to ingest into a specific session."
+)
+def ingest_urls(urls: list[str], session: Optional[str] = None) -> str:
+    """
+    Ingest a list of public URLs (PDF, DOCX, DOC) into an ephemeral session.
+    Returns a session_id to use for querying.
+    """
+    session_id = str(uuid.uuid4() if session is None else session)
+    vs = VectorStore()
+    for url in urls:
+        text = extract_text_from_url(url)
+        chunks = chunk_text(text)
+        vs.add(chunks)
+    _sessions[session_id] = vs
+    return session_id
+
+@mcp.tool(
+    description="Query the ingested documents in the given session using RAG. Returns a generated answer."
+)
+def query_knowledge(session_id: str, question: str) -> str:
+    """
+    Query the ingested documents in the given session using RAG.
+    Returns a generated answer.
+    """
+    vs = _sessions.get(session_id)
+    if not vs:
+        return f"Session ID {session_id} not found. Please call ingest_urls first."
+    docs = vs.search(question)
+    context = "\n\n".join(docs)
+    return context
+
+def main():
+    # Run the server
+    mcp.run()

rag_server-0.0.1/src/rag_server/utils/llm.py

@@ -0,0 +1,9 @@
+import os
+
+from openai import OpenAI
+
+openai_client = OpenAI(
+    api_key=os.environ.get("OPENAI_API_KEY"),
+    base_url=os.environ.get("OPENAI_API_URL"),
+)
+

rag_server-0.0.1/src/rag_server/utils/vector/misc.py

@@ -0,0 +1,50 @@
+import io
+
+import docx
+import requests
+import textract
+from PyPDF2 import PdfReader
+from openai.types import CreateEmbeddingResponse
+
+from rag_server.utils.llm import openai_client
+
+
+def extract_text_from_url(url: str) -> str:
+    """Download the file at the given URL and extract its text."""
+    resp = requests.get(url)
+    resp.raise_for_status()
+    content = resp.content
+    ext = url.split(".")[-1].lower()
+    if ext == "pdf":
+        reader = PdfReader(io.BytesIO(content))
+        return "\n".join(p.extract_text() or "" for p in reader.pages)
+    elif ext == "docx":
+        doc = docx.Document(io.BytesIO(content))
+        return "\n".join(p.text for p in doc.paragraphs)
+    elif ext == "doc":
+        return textract.process(io.BytesIO(content), extension="doc").decode("utf-8", errors="ignore")
+    else:
+        return content.decode("utf-8", errors="ignore")
+
+
+def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> list[str]:
+    """Split text into chunks of approximately chunk_size words with overlap."""
+    words = text.split()
+    chunks = []
+    start = 0
+    while start < len(words):
+        chunk = " ".join(words[start: start + chunk_size])
+        chunks.append(chunk)
+        start += chunk_size - overlap
+    return chunks
+
+
+def embed_texts(texts: list[str]) -> list[list[float]]:
+    """Embed a list of texts using OpenAI embeddings."""
+    resp : CreateEmbeddingResponse = openai_client.embeddings.create(input=texts, model="text-embedding-ada-002")
+    return [d.embedding for d in resp.data]
+
+def get_embedding(text: str) -> list[float]:
+    """Embed a single text."""
+    resp : CreateEmbeddingResponse = openai_client.embeddings.create(input=text, model="text-embedding-ada-002")
+    return resp.data[0].embedding

rag_server-0.0.1/src/rag_server/utils/vector/store.py

@@ -0,0 +1,67 @@
+import faiss
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from rag_server.utils.vector.misc import get_embedding, embed_texts
+
+
+class VectorStore:
+    """Simple in-memory vector store using FAISS."""
+    def __init__(self, dim: int = 1536):
+        self.dim = dim
+        # Use an HNSW approximate nearest neighbor index (no training needed)
+        self.index = faiss.index_factory(dim, "HNSW32")
+        # Configure HNSW parameters for construction and search quality
+        try:
+            self.index.hnsw.efConstruction = 200
+            self.index.hnsw.efSearch = 128
+        except AttributeError:
+            pass
+        self.texts: list[str] = []
+        # Initialize TF-IDF vectorizer and matrix
+        self.vectorizer = TfidfVectorizer()
+        self.tfidf_matrix = None
+
+    def add(self, chunks: list[str]) -> None:
+        embeddings = embed_texts(chunks)
+        arr = np.array(embeddings, dtype="float32")
+        self.index.add(arr)
+        self.texts.extend(chunks)
+        # Update TF-IDF matrix
+        self.tfidf_matrix = self.vectorizer.fit_transform(self.texts)
+
+    def search(self, query: str, top_k: int = 5, alpha: float = 0.5) -> list[str]:
+        """Perform hybrid search combining semantic (FAISS) and lexical (TF-IDF) scores."""
+        # Semantic search via FAISS
+        q_emb = np.array([get_embedding(query)], dtype="float32")
+        D, I = self.index.search(q_emb, top_k)
+        vect_ids = I[0].tolist()
+        vect_scores = [-d for d in D[0]]
+        # Lexical search via TF-IDF
+        if self.tfidf_matrix is None:
+            self.tfidf_matrix = self.vectorizer.fit_transform(self.texts)
+        q_tfidf = self.vectorizer.transform([query])
+        tfidf_scores_all = q_tfidf.dot(self.tfidf_matrix.T).toarray()[0]
+        tfidf_top = np.argsort(-tfidf_scores_all)[:top_k].tolist()
+        # Combine candidate document indices
+        candidate_ids = set(vect_ids + tfidf_top)
+        vect_min = min(vect_scores) if vect_scores else 0.0
+        scores = []
+        for idx in candidate_ids:
+            vs = vect_scores[vect_ids.index(idx)] if idx in vect_ids else vect_min
+            ts = float(tfidf_scores_all[idx])
+            scores.append((idx, vs, ts))
+        # Normalize and blend scores
+        vs_vals = [v for _, v, _ in scores]
+        ts_vals = [t for _, _, t in scores]
+        vmin, vmax = min(vs_vals), max(vs_vals)
+        tmin, tmax = min(ts_vals), max(ts_vals)
+        blended = []
+        for idx, vs, ts in scores:
+            vn = (vs - vmin) / (vmax - vmin) if vmax > vmin else 0.0
+            tn = (ts - tmin) / (tmax - tmin) if tmax > tmin else 0.0
+            combined = alpha * vn + (1 - alpha) * tn
+            blended.append((idx, combined))
+        # Sort by blended score and return top_k chunks
+        top = sorted(blended, key=lambda x: x[1], reverse=True)[:top_k]
+        return [self.texts[i] for i, _ in top]