ragpy-core 1.0.0__py3-none-any.whl

ragpy-core/AzureOpenAIRelay.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+"""
+Azure OpenAI relay utilities for the RAGpy pipeline.
+
+This module provides a thin, deterministic wrapper around the Azure OpenAI
+Python SDK. It exposes simple embedding and chat‑completion functions used
+throughout the RAG workflow, while keeping configuration isolated and
+monkeypatch‑friendly for unit testing.
+
+Two independent clients are supported:
+    • Embedding client — for generating text embeddings.
+    • Chat completion client — for LLM‑based scoring, compression, and
+      answer generation.
+
+The relay is intentionally minimal to ensure predictable behavior and easy
+replacement during offline tests.
+
+"""
+from openai import AzureOpenAI
+
+#new Azure OpenAI SDK
+embedding_Client = None
+chat_Client = None
+embedding_Deployment = None
+completion_Deployment = None
+
+def SetCompletionEndpointInfo(completion_Endpoint, completion_Deployment_Name, completion_Api_Version, api_Key):
+    """
+    Configure the Azure OpenAI chat completion client.
+
+    This sets up the client used for all LLM‑based operations in the RAG
+    pipeline, including reranking, compression, and final answer generation.
+
+    Args:
+        completion_Endpoint (str):
+            The Azure endpoint URL for chat completions.
+        completion_Deployment_Name (str):
+            The name of the deployed chat model.
+        completion_Api_Version (str):
+            The API version to use.
+        api_Key (str):
+            The Azure OpenAI API key.
+
+    Returns:
+        None
+    """
+    global chat_Client, completion_Deployment
+    completion_Deployment = completion_Deployment_Name
+
+    chat_Client = AzureOpenAI(
+        azure_endpoint=completion_Endpoint,
+        api_version=completion_Api_Version,
+        api_key=api_Key
+            
+    )
+    
+    return
+
+def SetEmbeddingEndpointInfo(embedding_Endpoint, embedding_Deployment_Name, embedding_Api_Version, azure_Api_Key):
+    """
+    Configure the Azure OpenAI embedding client.
+
+    This sets up the client used for generating embeddings during ingestion
+    and query processing.
+
+    Args:
+        embedding_Endpoint (str):
+            The Azure endpoint URL for embeddings.
+        embedding_Deployment_Name (str):
+            The name of the deployed embedding model.
+        embedding_Api_Version (str):
+            The API version to use.
+        azure_Api_Key (str):
+            The Azure OpenAI API key.
+
+    Returns:
+        None
+    """
+    global embedding_Client, embedding_Deployment
+    embedding_Deployment = embedding_Deployment_Name
+    
+    embedding_Client = AzureOpenAI(
+        azure_endpoint=embedding_Endpoint,
+        api_version=embedding_Api_Version,
+        api_key = azure_Api_Key
+        )
+    
+    return
+    
+def EmbedText(text):
+    """
+    Generate an embedding vector for a single text string.
+
+    Args:
+        text (str):
+            The input text to embed.
+
+    Returns:
+        list[float]:
+            A 3072‑dimensional embedding vector produced by the configured
+            Azure OpenAI embedding model.
+
+    Raises:
+        ValueError:
+            If the returned embedding dimension is unexpected.
+
+    Notes:
+        - This function is intentionally simple to support monkeypatching
+          during unit tests.
+        - The embedding client must be configured before calling this
+          function.
+    """
+    vector = embedding_Client.embeddings.create(
+        model=embedding_Deployment,
+        input=text
+    ).data[0].embedding
+
+    if len(vector) != 3072:
+        raise ValueError(f"Unexpected embedding dimension: {len(vector)}")
+
+    return vector 
+
+def EmbedChunksInBatches(batched_chunks, batch_size=16):
+    """
+    Embed multiple batches of text chunks.
+
+    Each batch is sent to the embedding model as a single request, and all
+    resulting vectors are returned in a flat list (one embedding per chunk).
+
+    Args:
+        batched_chunks (list[list[str]]):
+            A list of batches, each containing text chunks.
+        batch_size (int):
+            Optional batch size hint (unused but kept for compatibility).
+
+    Returns:
+        list[list[float]]:
+            A flat list of embedding vectors, one per chunk.
+
+    Notes:
+        - Ordering is preserved: embeddings appear in the same order as
+          the input chunks.
+        - This function is intentionally simple to support monkeypatching.
+    """
+    all_vectors = []
+
+    for batch in batched_chunks:
+        response = embedding_Client.embeddings.create(
+            model=embedding_Deployment,
+            input=batch
+        )
+
+        for item in response.data:
+            all_vectors.append(item.embedding)
+
+    return all_vectors
+
+def ChatCompletion(prompt):
+    """
+    Generate a chat completion response using the configured Azure OpenAI client.
+
+    Args:
+        prompt (str):
+            The user prompt to send to the model.
+
+    Returns:
+        str:
+            The model's response text. Returns an empty string if no chat
+            client is configured.
+
+    Notes:
+        - This function is intentionally simple to support monkeypatching
+          during unit tests.
+        - No additional formatting or metadata is returned.
+    """
+    client = GetChatClient()
+    if client is None:
+        # In tests, FakeChat will override this entirely.
+        return ""
+
+    response = client.chat.completions.create(
+        model=GetChatModel(),
+        messages=[{"role": "user", "content": prompt}]
+    )
+
+    return response.choices[0].message.content
+
+def GetChatClient():
+    """
+    Retrieve the configured Azure OpenAI chat client.
+
+    Returns:
+        AzureOpenAI | None:
+            The chat client instance, or None if not configured.
+    """
+    return chat_Client
+
+def GetChatModel():
+    """
+    Retrieve the configured chat model deployment name.
+
+    Returns:
+        str | None:
+            The model deployment name, or None if not configured.
+    """
+    return completion_Deployment

ragpy-core/ChunkCompressor.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+LLM-based context compressor for the RAGpy pipeline.
+
+This module provides a lightweight, deterministic interface for reducing
+multiple retrieved chunks into a single compact summary. The compressor
+uses AzureOpenAIRelay to generate a concise, query‑aware context block
+that preserves meaning while reducing token usage. The design is
+intentionally simple to support both production usage and easy
+monkeypatching during unit tests.
+
+"""
+from ragpy import AzureOpenAIRelay as AI
+
+def CompressChunks(query, chunks):
+    """
+    Compress a list of retrieved chunks into a shorter, unified context block.
+
+    This function sends the provided chunks and user query to an LLM,
+    requesting a concise summary that preserves the information most
+    relevant to the query. It is primarily used to reduce token usage
+    during prompt construction while maintaining grounding in the
+    retrieved content.
+
+    Args:
+        query (str):
+            The user query that determines what information is relevant.
+        chunks (list[dict]):
+            A list of chunk objects, each containing at least a "chunk"
+            field (or "text") holding the raw text to be compressed.
+
+    Returns:
+        str:
+            A compressed context string. Returns an empty string if the
+            compression model is not configured or returns no output.
+
+    Notes:
+        - The function is intentionally simple to allow monkeypatching
+          during unit tests.
+        - The compression prompt is defined inline for clarity and
+          isolation.
+        - If AzureOpenAIRelay is not configured, the function safely
+          returns an empty string.
+    """
+    # If monkeypatched, FakeCompress will run instead of this function.
+    if AI.GetChatClient() is None:
+        return ""
+
+    text = "\n\n".join(c.get("chunk") or c.get("text", "") for c in chunks)
+
+    prompt = f"""
+    You are a compression model. Your job is to compress the following text
+    into a concise summary that preserves meaning and focuses on information
+    relevant to the user query:
+
+    Query:
+    {query}
+
+    {text}
+
+    Return ONLY the compressed summary.
+    """
+
+    return AI.ChatCompletion(prompt)

ragpy-core/DatabaseRetriever.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+Retrieval Layer for RAG
+Created on Tue Jun 2 2026
+@author: William
+"""
+
+from ragpy import VectorDatabase as VD
+import logging
+from typing import List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+def RetrieveTopK(embedded_query: List[float], k: int = 5) -> List[Dict[str, Any]]:
+    """
+    Retrieve the top‑K most relevant chunks from the active ChromaDB collection.
+
+    This function performs a vector similarity search against the currently
+    opened database collection using the provided query embedding. Results
+    are normalized into a consistent structure expected by downstream
+    components such as the reranker, compressor, and orchestrator.
+
+    Args:
+        embedded_query (list[float]):
+            The embedding vector representing the user query. Must match the
+            dimensionality of the embeddings stored in the database.
+        k (int):
+            Number of chunks to retrieve from the vector database.
+
+    Returns:
+        list[dict]:
+            A list of retrieved chunk objects, each containing:
+                - "text": The raw chunk text stored in the database.
+                - "metadata": Metadata associated with the chunk, including:
+                    • "source": The originating filename.
+                    • "chunk_id": The index of the chunk within that file.
+                - "distance": The vector distance score returned by ChromaDB.
+                  Lower values indicate closer matches.
+
+    Raises:
+        RuntimeError:
+            If no database collection is currently open.
+
+    Notes:
+        - Distance values are returned directly from ChromaDB. If the backend
+          does not provide distances, they default to 0.0.
+        - Output format is intentionally normalized to ensure compatibility
+          with Reranker, ChunkCompressor, and RAGOrchestrator.
+    """
+    if VD.collection is None:
+        raise RuntimeError("No database open. Call CreateDatabase() or OpenDatabase() first.")
+    
+    if len(embedded_query) != 3072:
+        raise ValueError("Query embedding dimension mismatch.")
+
+    results = VD.collection.query(
+        query_embeddings=[embedded_query],
+        n_results=k
+    )
+
+    docs = results["documents"][0]
+    metas = results["metadatas"][0]
+    dists = results.get("distances", [[0.0] * len(docs)])[0]
+    logger.debug(f"Distance range: {min(dists)} to {max(dists)}")
+
+    normalized = []
+    for doc, meta, dist in zip(docs, metas, dists):
+        normalized.append({
+            "text": doc,
+            "metadata": meta,
+            "distance": dist
+            
+        })
+    
+    logger.info(f"Retrieved {len(normalized)} chunks from database.")
+    
+    return normalized

ragpy-core/RAGOrchestrator.py

@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+"""
+RAG Orchestrator for the RAGpy pipeline.
+
+This module coordinates the full Retrieval‑Augmented Generation workflow,
+including file ingestion, chunking, batching, embedding, retrieval,
+reranking, context construction, prompt building, and final answer
+generation. It acts as the high‑level interface that ties together all
+RAGpy components:
+
+    • VectorDatabase — persistent storage of embedded chunks
+    • FileLoader — file loading and text extraction
+    • TextChunker — chunking of raw text
+    • ChunkBatcher — batching for efficient embedding
+    • AzureOpenAIRelay — embedding and LLM completion
+    • DatabaseRetriever — vector search
+    • Reranker — LLM‑based and deterministic reranking
+    • ChunkCompressor — context compression for efficient prompting
+
+All functions in this module operate at the orchestration level and are
+intended to be used directly by applications integrating RAGpy.
+"""
+import os
+import logging
+
+from ragpy.VectorDatabase import OpenDatabase, CheckIfInDatabase, AddToDatabase
+from ragpy.loaders.FileLoader import LoadFile
+from ragpy.loaders.TextChunker import TextToChunk
+from ragpy.batching.ChunkBatcher import BatchChunks
+
+import ragpy.AzureOpenAIRelay as AI
+import ragpy.Reranker as RR
+import ragpy.DatabaseRetriever as DR
+import ragpy.ChunkCompressor as CC
+
+logger = logging.getLogger(__name__)
+
+def IngestFile(path, databaseName, databaseLoc = "./vectorDB"):
+    """
+    Ingest a file into the vector database.
+
+    Performs the full ingestion pipeline:
+        1. Load the file from disk.
+        2. Chunk the extracted text.
+        3. Batch chunks for efficient embedding.
+        4. Embed all batches using AzureOpenAIRelay.
+        5. Store embedded chunks in the persistent vector database.
+
+    If the file has already been ingested, the operation is skipped.
+
+    Args:
+        path (str):
+            Path to the file to ingest.
+        databaseName (str):
+            Name of the vector database collection.
+        databaseLoc (str):
+            Directory where the database is stored.
+
+    Returns:
+        None
+    """
+    filename = os.path.basename(path)
+    logger.info(f"Found the file: {filename}")
+    
+    #Open the Database
+    OpenDatabase(databaseName, databaseLoc)
+    
+    # Check if already embedded
+    if CheckIfInDatabase(path):
+        logger.info(f"Skipping {filename}: already embedded.")
+        return
+    else:
+        logger.info("File did not exist in DB, Embedding.")
+        text = LoadFile(path)
+        chunks = TextToChunk(text)
+        batches = list(BatchChunks(chunks))
+        vectors = AI.EmbedChunksInBatches(batches)
+
+    AddToDatabase(chunks, vectors, filename)
+    
+    logger.info(f"Index operation complete. {filename} was split into {len(chunks)} chunks, batched into {len(batches)} batches, and indexed.")
+
+def BuildContext(retrieved_chunks):
+    """
+    Construct a readable context block from retrieved chunks.
+
+    Each chunk is formatted with a citation marker and includes metadata
+    such as the source filename and chunk index. This context block is
+    used directly in RAG prompt construction.
+
+    Args:
+        retrieved_chunks (list[dict]):
+            Retrieved chunk objects containing:
+                - "text": The chunk text.
+                - "metadata": Metadata including "source" and "chunk_id".
+
+    Returns:
+        str:
+            A formatted context block with citation markers.
+    """
+    context = []
+
+    for i, r in enumerate(retrieved_chunks):
+        source = r["metadata"].get("source", "unknown")
+        chunk_id = r["metadata"].get("chunk_id", "N/A")
+        text = r.get("text")
+
+        context.append(f"[CITATION {i}] Source: {source} (chunk {chunk_id})\n{text}\n")
+
+    return "\n".join(context)
+
+def BuildPrompt(context, query, use_citations=True):
+    """
+    Build a full RAG prompt including context, optional citation instructions,
+    and the user query.
+
+    Args:
+        context (str):
+            The context block to include in the prompt.
+        query (str):
+            The user question.
+        use_citations (bool):
+            Whether to include citation instructions for the LLM.
+
+    Returns:
+        str:
+            The constructed prompt ready for LLM completion.
+    """
+    if use_citations:
+        citation_instructions = """
+        When you use information from a chunk, cite it like this: [CITATION X].
+        Do not invent citations. Only cite chunks that appear in the context.
+        """
+    else:
+        citation_instructions = ""
+    
+    prompt = f"""
+    {citation_instructions}
+
+    ### Context:
+    {context}
+    
+    ### Focus specifically on information relevant to the user query:
+    {query}
+    
+    ### Your Answer (friendly, readable, grounded):
+    """
+    return prompt
+
+def GenerateAnswer(query, k=5):
+    """
+    Generate a grounded answer using the full RAG pipeline.
+
+    Steps:
+        1. Embed the user query.
+        2. Retrieve top‑K candidate chunks from the vector database.
+        3. Rerank candidates using the LLM-based reranker.
+        4. Compress the top chunks into a compact context summary.
+        5. Build a RAG prompt using the compressed context.
+        6. Generate the final answer using AzureOpenAIRelay.
+
+    Args:
+        query (str):
+            The user question.
+        k (int):
+            Number of chunks to retrieve before reranking.
+
+    Returns:
+        str:
+            The final grounded answer generated by the LLM.
+
+    Raises:
+        ValueError:
+            If the query embedding has an unexpected dimensionality.
+    """
+    embedded_Query = AI.EmbedText(query)
+    
+    if len(embedded_Query) != 3072:
+        raise ValueError("Query embedding dimension mismatch.")
+
+    # Step 1: Retrieve more chunks
+    retrieved = DR.RetrieveTopK(embedded_Query, k)
+
+    # Step 2: Rerank them
+    reranked = RR.RerankLLM(query, retrieved)
+
+    # Step 3: Keep top chunks for compression
+    top_for_compression = [
+    {"chunk": item["text"], "metadata": item["metadata"]}
+    for item in reranked[:10]
+    ]
+    
+    # Step 4: Compress them
+    compressed_context = CC.CompressChunks(query, top_for_compression)
+
+    # Step 5: Build prompt using compressed context
+    prompt = BuildPrompt(compressed_context, query, use_citations=False)
+
+    # Step 6: Generate answer
+    raw_answer = AI.ChatCompletion(prompt)
+
+    # Step 7: Resolve citations (optional with compression)
+    # You can skip citations here OR keep them if you want
+    # but compressed context loses chunk-level mapping.
+    # So we skip citations for compressed mode.
+
+    return raw_answer

ragpy-core/Reranker.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+"""
+LLM-based and deterministic rerankers for RAG pipelines.
+
+This module provides two reranking strategies:
+
+    • RerankLLM — the primary reranker used during real RAGpy execution.
+      It evaluates chunk relevance using an LLM scoring prompt and returns
+      chunks sorted by descending relevance score while preserving metadata.
+
+    • Rerank — a deterministic, test-friendly fallback used exclusively
+      for unit testing. It sorts chunks by text length to ensure stable,
+      reproducible ordering without requiring an LLM.
+
+Both functions accept retrieved chunk objects produced by the retrieval
+layer and return normalized dictionaries suitable for downstream
+compression and prompt construction.
+"""
+
+from ragpy import AzureOpenAIRelay as AI
+
+def RerankLLM(query, retrieved_chunks):
+    """
+    Rerank retrieved chunks using an LLM-based scoring model.
+
+    Each chunk is evaluated for relevance to the user query by sending a
+    scoring prompt to the LLM via AzureOpenAIRelay. The model is expected
+    to return a floating‑point number between 0 and 1. Invalid or malformed
+    output safely defaults to 0.0. Scores are clamped to the range [0, 1].
+
+    Args:
+        query (str):
+            The user query used to evaluate relevance.
+        retrieved_chunks (list[dict]):
+            A list of retrieved chunk objects containing:
+                - "text": The chunk text.
+                - "metadata": Metadata including "source" and "chunk_id".
+                - "distance": The vector distance score.
+
+    Returns:
+        list[dict]:
+            A list of dictionaries with keys:
+                - "text": The chunk text.
+                - "metadata": Original metadata preserved.
+                - "score": The relevance score (float).
+            Sorted in descending score order.
+
+    Notes:
+        - This function is used during real RAG pipeline execution.
+        - It is intentionally monkeypatch‑friendly for offline testing.
+        - Metadata is preserved to support compression and citation workflows.
+    """
+    scored = []
+
+    for r in retrieved_chunks:
+        chunk_text = r.get("chunk") or r.get("text")
+
+        prompt = f"""
+        You are a scoring model. Rate the relevance of the chunk to the query from 0 to 1.
+        Return ONLY a number.
+        
+        Query:
+        {query}
+        
+        Chunk:
+        {chunk_text}
+        """
+
+        score_text = AI.ChatCompletion(prompt).strip()
+
+        try:
+            score = float(score_text)
+            score = max(0.0, min(score, 1.0))
+        except:
+            score = 0.0
+
+        scored.append({
+            "text": chunk_text,
+            "metadata": r.get("metadata", {}),
+            "score": score
+        })
+
+    scored.sort(key=lambda x: x["score"], reverse=True)
+    return scored
+
+def Rerank(query, retrieved_chunks):
+    """
+Deterministic, test-friendly reranker used only for unit testing.
+
+This fallback reranker provides predictable behavior without relying
+on an LLM. It normalizes chunk objects and sorts them by text length,
+placing shorter chunks first. All metadata is preserved.
+
+Args:
+    query (str):
+        The user query (unused in this heuristic but included for API consistency).
+    retrieved_chunks (list[dict]):
+        A list of retrieved chunk objects containing "text" and "metadata".
+
+Returns:
+    list[dict]:
+        A list of normalized chunk dictionaries sorted by ascending text length.
+
+Notes:
+    - This function is used exclusively for testing.
+    - Tests expect shorter chunks to appear first.
+    - Metadata is preserved to maintain compatibility with downstream steps.
+"""
+    normalized = []
+    for r in retrieved_chunks:
+        text = r.get("text") or r.get("chunk") or ""
+        normalized.append({
+            "text": text,
+            **{k: v for k, v in r.items() if k not in ["text", "chunk"]}
+        })
+
+    # Heuristic: shorter text is usually more focused → tests expect this ordering.
+    return sorted(normalized, key=lambda c: len(c["text"]))

ragpy-core/VectorDatabase.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+# Global state:
+# - `client` holds the active ChromaDB PersistentClient
+# - `collection` holds the active ChromaDB collection
+"""
+ChromaDB persistence utilities for the RAGpy pipeline.
+
+This module provides a thin wrapper around ChromaDB's PersistentClient,
+offering simple functions for creating, opening, and populating a vector
+database. The design is intentionally minimal and deterministic to support
+production usage and easy monkeypatching during unit tests.
+
+Global State:
+    client     — the active ChromaDB PersistentClient
+    collection — the active ChromaDB collection
+"""
+import chromadb
+import os
+import logging
+
+client = None 
+collection = None
+logger = logging.getLogger(__name__)
+
+
+def CreateDatabase(dbName, path="./vectorDB"):
+    """
+    Create a new persistent ChromaDB collection.
+
+    Initializes a PersistentClient at the given path and unconditionally
+    creates a new collection with the specified name. If the collection
+    already exists, ChromaDB will raise an error.
+
+    Args:
+        dbName (str):
+            Name of the collection to create.
+        path (str):
+            Filesystem path for the persistent database.
+
+    Returns:
+        chromadb.api.models.Collection.Collection:
+            The newly created collection.
+
+    Side Effects:
+        - Sets the global `client` and `collection` variables.
+    """
+    global client, collection
+    
+    client = chromadb.PersistentClient(path)
+   
+    collection = client.create_collection(dbName)
+
+    return collection
+
+
+def OpenDatabase(dbName, path="./vectorDB"):
+    """
+    Open an existing ChromaDB collection, creating it if necessary.
+
+    Initializes a PersistentClient at the given path and attempts to load
+    an existing collection. If the collection does not exist, it is created
+    automatically.
+
+    Args:
+        dbName (str):
+            Name of the collection to open.
+        path (str):
+            Filesystem path for the persistent database.
+
+    Returns:
+        chromadb.api.models.Collection.Collection:
+            The opened or newly created collection.
+
+    Side Effects:
+        - Sets the global `client` and `collection` variables.
+    """
+    global client, collection
+    
+    client = chromadb.PersistentClient(path)
+    
+    try:
+        collection = client.get_collection(dbName)
+    except chromadb.errors.NotFoundError:
+        logger.info("Database does not exist. Creating...")
+        collection = CreateDatabase(dbName, path)
+    
+    return collection
+
+def AddToDatabase(chunks, vectors, source_name):
+    """
+    Add embedded text chunks to the active ChromaDB collection.
+
+    Stores text chunks, their embedding vectors, and associated metadata
+    in the currently opened collection. Each chunk receives a globally
+    unique ID based on the source filename and its index.
+
+    Args:
+        chunks (list[str]):
+            Text chunks extracted from a document.
+        vectors (list[list[float]]):
+            Embedding vectors corresponding to each chunk.
+        source_name (str):
+            The name of the source document (e.g., filename).
+
+    Returns:
+        None
+
+    Raises:
+        RuntimeError:
+            If no database collection is currently open.
+        ValueError:
+            If chunk/vector counts differ or an embedding has an unexpected
+            dimensionality.
+
+    Notes:
+        - Embedding vectors are validated to ensure a 3072‑dimensional shape.
+        - Metadata stores only lightweight identifiers (`source`, `chunk_id`)
+          to avoid duplicating chunk text.
+    """
+    global collection
+    if collection is None:
+        raise RuntimeError("No database open. Call CreateDatabase() or OpenDatabase() first.")
+        
+    if len(chunks) != len(vectors):
+        raise ValueError("Chunks and vectors must have the same length.") 
+    
+    ids = []
+    embeddings = []
+    documents = []
+    metadatas = []
+
+    for i, (chunk, vector) in enumerate(zip(chunks, vectors)):
+        
+        if len(vector) != 3072:
+            raise ValueError(f"Invalid embedding length: {len(vector)}")
+
+        ids.append(f"{source_name}_{i}")
+        embeddings.append(vector)
+        documents.append(chunk)
+        metadatas.append({"source": source_name, "chunk_id": i})
+
+    collection.add(
+        ids=ids,
+        embeddings=embeddings,
+
+        documents=documents,
+        metadatas=metadatas
+    )
+    
+def CheckIfInDatabase(filepath):
+    """
+    Determine whether a file has already been ingested into the database.
+
+    Uses metadata filtering to check whether any stored chunk originates
+    from the given file.
+
+    Args:
+        filepath (str):
+            Path to the file being checked.
+
+    Returns:
+        bool:
+            True if any metadata entry references the file, False otherwise.
+
+    Raises:
+        RuntimeError:
+            If no database collection is currently open.
+
+    Notes:
+        - Matching is performed using the filename only (basename).
+        - Uses ChromaDB's `where` filtering for efficient lookup.
+    """
+    if collection is None:
+       raise RuntimeError("No database open. Call CreateDatabase() or OpenDatabase() first.")
+
+    filename = os.path.basename(filepath)
+
+    results = collection.get(where={"source": filename})
+    
+    for m in results["metadatas"]:
+        if m["source"] == filename:
+            return True
+
+    return False

ragpy-core/__init__.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun 24 17:10:10 2026
+
+@author: klusm
+"""
+
+from .AzureOpenAIRelay import (
+    SetEmbeddingEndpointInfo,
+    SetCompletionEndpointInfo
+)
+
+from .RAGOrchestrator import (
+    IngestFile,
+    GenerateAnswer,
+    BuildContext,
+    BuildPrompt
+)
+
+from .VectorDatabase import (
+    OpenDatabase,
+    AddToDatabase,
+    CheckIfInDatabase
+)
+
+from .utils.CitationHelper import (
+    ResolveCitations,
+    BuildReferences
+)
+
+__all__ = [
+    "SetEmbeddingEndpointInfo",
+    "SetCompletionEndpointInfo",
+    "IngestFile",
+    "GenerateAnswer",
+    "BuildContext",
+    "BuildPrompt",
+    "OpenDatabase",
+    "AddToDatabase",
+    "CheckIfInDatabase",
+    "ResolveCitations",
+    "BuildReferences",
+]

ragpy_core-1.0.0.dist-info/METADATA

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: ragpy-core
+Version: 1.0.0
+Summary: A modular Retrieval-Augmented Generation (RAG) pipeline for Python.
+Author: William Klusman
+Author-email: William Klusman <klusmannwilliam@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 William Klusman
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the “Software”), to deal
+        in the Software without restriction, including without limitation the rights  
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell  
+        copies of the Software, and to permit persons to whom the Software is  
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in  
+        all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR  
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,  
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE  
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER  
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING  
+        FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER  
+        DEALINGS IN THE SOFTWARE.
+Keywords: RAG,retrieval-augmented-generation,LLM,vector-database,azure-openai,machine-learning,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: chromadb>=0.4.0
+Requires-Dist: numpy>=1.20
+Requires-Dist: tiktoken>=0.5.0
+Requires-Dist: requests>=2.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pypdf>=4.0.0
+Dynamic: author
+Dynamic: license-file
+Dynamic: requires-python
+
+RAGpy
+	RAGpy is a lightweight, modular Retrieval-Augmented Generation (RAG) pipeline for Python. It provides a clear and testable architecture for document ingestion, chunking, embedding, retrieval, reranking, context compression, and grounded answer generation using Azure OpenAI and ChromaDB.
+
+	RAGpy is designed for developers who want a transparent, hackable RAG system without the complexity of large frameworks.
+
+Features
+	Modular ingestion pipeline for text and PDF documents
+
+	Chunking and batching utilities for efficient embedding
+
+	Azure OpenAI embeddings and chat completions
+
+ChromaDB vector database integration
+
+LLM-based reranking for improved retrieval quality
+
+Context compression to reduce token usage
+
+Fully monkeypatch-friendly design for offline testing
+
+Clean architecture suitable for extension and customization
+
+Installation
+Once published to PyPI:
+
+Code
+pip install ragpy
+For development:
+
+Code
+git clone https://github.com/yourusername/ragpy
+cd ragpy
+pip install -e .
+Quickstart Example
+python
+from ragpy.RAGOrchestrator import IngestFile, GenerateAnswer
+from ragpy.VectorDatabase import OpenDatabase
+
+OpenDatabase("AeroDB", "./vectorDB")
+IngestFile("engine_vibration.pdf", "AeroDB")
+
+answer = GenerateAnswer("What causes engine vibration?", "AeroDB")
+print(answer)
+How RAGpy Works
+1. Ingestion
+Load text or PDF using FileLoader
+
+Chunk text using TextChunker
+
+Batch chunks using ChunkBatcher
+
+Generate embeddings with Azure OpenAI
+
+Store vectors and metadata in ChromaDB
+
+2. Retrieval
+Embed the user query
+
+Retrieve top-K candidates from the vector database
+
+3. Reranking
+Use an LLM-based reranker to reorder retrieved chunks by relevance
+
+4. Compression
+Summarize top chunks into a compact context block
+
+5. Answer Generation
+Build a prompt using compressed context
+
+Generate a grounded answer using Azure OpenAI
+
+Project Structure
+Code
+ragpy/
+    AzureOpenAIRelay.py
+    RAGOrchestrator.py
+    VectorDatabase.py
+    Reranker.py
+    ChunkCompressor.py
+    loaders/
+        FileLoader.py
+        TextChunker.py
+    batching/
+        ChunkBatcher.py
+tests/
+docs/
+Requirements
+Python 3.9+
+
+ChromaDB
+
+numpy
+
+tiktoken
+
+pypdf
+
+openai (Azure OpenAI SDK)
+
+Testing
+RAGpy includes a full pytest suite. All Azure calls are monkeypatch-friendly, allowing offline testing with mock LLMs.
+
+Run tests:
+
+Code
+pytest -q
+Contributing
+Contributions are welcome.
+Please open an issue or submit a pull request on GitHub.
+
+Planned enhancements include:
+
+Local embedding support (sentence-transformers)
+
+Hybrid retrieval (vector + keyword)
+
+Multimodal RAG (image + text)
+
+Evaluation tools for relevance and faithfulness
+
+Agentic RAG extensions
+
+License
+RAGpy is released under the MIT License.

ragpy_core-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+ragpy-core/AzureOpenAIRelay.py,sha256=IqD_zkMnDlmMdyKfTEtJsYS3s7yXR71WrBewkYkHfVI,6147
+ragpy-core/ChunkCompressor.py,sha256=iVAgW7Ku5FQx8ntjEregP8O2J0Qc9OFNxOv-iO5lX-0,2277
+ragpy-core/DatabaseRetriever.py,sha256=jvbN9esU5RzQW98uA6qYE7SQvaGx87XGQdslZ9yIxu8,2793
+ragpy-core/RAGOrchestrator.py,sha256=lR1GldVnF0xVGCfUijETYh4kdWHS8Zd_-FitxuIbV7k,6857
+ragpy-core/Reranker.py,sha256=_AMvwtQp1xulBq_7iInXRuiian39tfkGxzNOLyCdCIg,4167
+ragpy-core/VectorDatabase.py,sha256=xado-9gobVEs14tcJfFe7NmPVCT6Y3P168RVcit8w-0,5576
+ragpy-core/__init__.py,sha256=Vih4zqBsj9rJKXSrdhdCAU9ryEBr-ZHCmbdjvrX_42k,769
+ragpy_core-1.0.0.dist-info/licenses/LICENSE,sha256=N0pk1ZLa00VYhRM9xzwThPMdfL_8acMtFLhMZj_fSeA,1119
+ragpy_core-1.0.0.dist-info/METADATA,sha256=HDopMelIJgs-WSSb4KHNjpSlExBvnBvrBnl4g7FSzyc,5313
+ragpy_core-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+ragpy_core-1.0.0.dist-info/top_level.txt,sha256=2nLuavOLaI-fiobSkBx_CDqjD1kWlO2_2TCgOKT_rhY,11
+ragpy_core-1.0.0.dist-info/RECORD,,

ragpy_core-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ragpy_core-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 William Klusman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights  
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell  
+copies of the Software, and to permit persons to whom the Software is  
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in  
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR  
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,  
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE  
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER  
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING  
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER  
+DEALINGS IN THE SOFTWARE.

ragpy_core-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ragpy-core