prevectorchunks-core 0.1.1__tar.gz

prevectorchunks_core-0.1.1/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: prevectorchunks-core
+Version: 0.1.1
+Summary: A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database
+Author-email: Zul Al-Kabir <zul.developer.2023@gmail.com>
+Project-URL: Homepage, https://github.com/yourusername/mydep
+Description-Content-Type: text/markdown
+Requires-Dist: Django==5.1
+Requires-Dist: packaging~=24.1
+Requires-Dist: requests~=2.32.3
+Requires-Dist: openai~=1.37.1
+Requires-Dist: httpx~=0.27.0
+Requires-Dist: python-dotenv~=1.0.1
+Requires-Dist: django-cors-headers~=4.4.0
+Requires-Dist: PyJWT~=2.7.0
+Requires-Dist: fastapi~=0.112.2
+Requires-Dist: datasets~=4.1.0
+Requires-Dist: pinecone~=7.3.0
+Requires-Dist: pytesseract~=0.3.13
+Requires-Dist: python-docx~=1.2.0
+Requires-Dist: PyPDF2~=3.0.1
+Requires-Dist: pillow~=11.3.0
+
+# 📚 PreVectorChunks
+
+> A lightweight utility for **document chunking** and **vector database upserts** — designed for developers building **RAG (Retrieval-Augmented Generation)** solutions.
+
+---
+
+## ✨ Who Needs This Module?
+Any developer working with:
+- **RAG pipelines**
+- **Vector Databases** (like Pinecone, Weaviate, etc.)
+- **AI applications** requiring **similar content retrieval**
+
+---
+
+
+## 🎯 What Does This Module Do?
+This module helps you:
+- **Chunk documents** into smaller fragments  
+- **Insert (upsert) fragments** into a vector database  
+- **Fetch & update** existing chunks from a vector database  
+
+---
+
+## 📦 Installation
+```bash
+pip install prevectorchunks
+````
+How to import in a file:  
+```python
+from PreVectorChunks.services import chunk_documents_crud_vdb
+
+#How to use Pinecone and OpenAI:
+#Use a .env file in your project root to configure API keys:
+
+PINECONE_API_KEY=YOUR_API_KEY
+OPENAI_API_KEY=YOUR_API_KEY
+
+#how to call relevant functions:
+#Four key functions that you can call are below: 
+#function that chunks any document 
+chunk_documents(instructions,file_path="content_playground/content.json"): 
+#function that chunks any document as well as inserts into vdb - you need an index name inside index_n
+chunk_and_upsert_to_vdb(index_n,instructions,file_path="content_playground/content.json"): 
+#function that loads existing chunks from vdb by document name - you need an index name inside index_n 
+fetch_vdb_chunks_grouped_by_document_name(index_n): 
+#function that updates existing chunks - you need an index name inside index_n 
+update_vdb_chunks_grouped_by_document_name(index_n,dataset):
+```

prevectorchunks_core-0.1.1/README.md

@@ -0,0 +1,48 @@
+# 📚 PreVectorChunks
+
+> A lightweight utility for **document chunking** and **vector database upserts** — designed for developers building **RAG (Retrieval-Augmented Generation)** solutions.
+
+---
+
+## ✨ Who Needs This Module?
+Any developer working with:
+- **RAG pipelines**
+- **Vector Databases** (like Pinecone, Weaviate, etc.)
+- **AI applications** requiring **similar content retrieval**
+
+---
+
+
+## 🎯 What Does This Module Do?
+This module helps you:
+- **Chunk documents** into smaller fragments  
+- **Insert (upsert) fragments** into a vector database  
+- **Fetch & update** existing chunks from a vector database  
+
+---
+
+## 📦 Installation
+```bash
+pip install prevectorchunks
+````
+How to import in a file:  
+```python
+from PreVectorChunks.services import chunk_documents_crud_vdb
+
+#How to use Pinecone and OpenAI:
+#Use a .env file in your project root to configure API keys:
+
+PINECONE_API_KEY=YOUR_API_KEY
+OPENAI_API_KEY=YOUR_API_KEY
+
+#how to call relevant functions:
+#Four key functions that you can call are below: 
+#function that chunks any document 
+chunk_documents(instructions,file_path="content_playground/content.json"): 
+#function that chunks any document as well as inserts into vdb - you need an index name inside index_n
+chunk_and_upsert_to_vdb(index_n,instructions,file_path="content_playground/content.json"): 
+#function that loads existing chunks from vdb by document name - you need an index name inside index_n 
+fetch_vdb_chunks_grouped_by_document_name(index_n): 
+#function that updates existing chunks - you need an index name inside index_n 
+update_vdb_chunks_grouped_by_document_name(index_n,dataset):
+```

prevectorchunks_core-0.1.1/prevectorchunks_core/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

prevectorchunks_core-0.1.1/prevectorchunks_core/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class LmsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'lms'

prevectorchunks_core-0.1.1/prevectorchunks_core/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for PreVectorChunks project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'PreVectorChunks.settings')
+
+application = get_asgi_application()

prevectorchunks_core-0.1.1/prevectorchunks_core/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

prevectorchunks_core-0.1.1/prevectorchunks_core/services/chunk_documents_crud_vdb.py

@@ -0,0 +1,417 @@
+import json
+import uuid
+
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+from dotenv import load_dotenv
+from openai import OpenAI
+from pinecone import Pinecone, ServerlessSpec
+from datasets import load_dataset
+import os
+from django.http import JsonResponse, HttpResponse
+
+
+from ..utils.file_loader import prepare_chunked_text, extract_file_details
+from ..utils.llm_wrapper import LLMClientWrapper
+
+from pinecone import Pinecone, ServerlessSpec
+from collections import defaultdict
+from itertools import chain
+from dotenv import load_dotenv
+# create an index if not already existing
+load_dotenv(override=True)
+index_name = "dl-doc-search"
+EMBED_DIM = 1536
+pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+
+class openAIWrapperLMSContent:
+    def __init__(self, openai_client):
+        self.openai_client = openai_client
+
+    def chatWithOpenAI(self, ragcontent, actualQuery):
+        query = f"""
+        {ragcontent}
+        Question:{actualQuery}
+        """
+
+        response = self.openai_client.chat.completions.create(
+            messages=[
+                {'role': 'system',
+                 'content': (
+                     "You are an experienced and patient TAFE trainer for a Certificate III in Electrotechnology. "
+                     "Your primary goal is to guide apprentices to understand their learning materials. "
+                     "You must **NEVER** give direct answers to questions from assessments (UKT or UST). "
+                     "Instead, use Socratic questioning to prompt critical thinking and encourage students to find the answers in their provided learning content. "
+                     "When appropriate, offer hints by referencing specific sections or modules (e.g., 'Review the section on Ohm's Law in Module 2'). "
+                     "Maintain a helpful, encouraging, and respectful tone. Do not solve problems for the student."
+                     "{}"
+                 ).format(query)}
+            ],
+            model="gpt-4o-mini",
+            temperature=0,
+        )
+
+        return response.choices[0].message.content
+
+
+# load the dataset
+def loadDataset():
+    dataset = load_dataset("pinecone/dl-doc-search", split="train")
+    return dataset
+
+
+def loadDatasetFromJsonFile(file_path="PreVectorChunks/content_playground/content.json"):
+    print(os.getcwd())
+    # File path to your content.json
+
+
+    # Load the contents of the JSON file into a Python variable
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            data = json.load(file)
+        return data
+    except FileNotFoundError:
+        print(f"Error: The file at {file_path} was not found.")
+    except json.JSONDecodeError as e:
+        print(f"Error decoding JSON: {e}")
+    return None
+
+
+# Function to create embeddings with OpenAI
+def get_embedding(text, model="text-embedding-3-small"):
+    response = client.embeddings.create(
+        input=text,
+        model=model
+    )
+    return response.data[0].embedding
+
+
+def createIndexForPineCone():
+    # connect to Pinecone (make sure you set your API key in env: PINECONE_API_KEY)
+
+    # Use the correct spec parameter
+    spec = ServerlessSpec(
+        cloud="aws",  # Replace with your cloud provider ('aws', 'azure', etc.)
+        region="us-east-1"  # Replace with your region ('us-west-2', etc.)
+    )
+
+    # Check if the index exists. If not, create it.
+    if index_name not in [index.name for index in pc.list_indexes()]:
+        pc.create_index(
+            name=index_name,
+            dimension=EMBED_DIM,  # Replace with the dimensionality of your vectors
+
+            spec=spec  # Pass the spec object here
+        )
+
+    index = pc.Index(index_name)
+    return index
+
+
+def upsertRecord(index_n,dataset,document_name=None):
+    # To get the unique host for an index,
+    # see https://docs.pinecone.io/guides/manage-data/target-an-index
+    index = pc.Index(index_n)
+
+    # Upsert records into a namespace
+    # `chunk_text` fields are converted to dense vectors
+    # `category` fields are stored as metadata
+    if dataset==None:
+        dataset = loadDatasetFromJsonFile()
+
+    try:
+        rows = dataset.get("rows", [])  # Try to get "rows" from the dataset
+        if rows is None:  # Check if the value is explicitly None
+            rows = dataset  # Fallback to dataset
+    except (AttributeError, TypeError):
+        # An error occurred (e.g., `dataset` is not a dictionary or `.get()` is unavailable)
+        rows = dataset  # Fallback to dataset
+
+    # Prepare batch for upsert (do small chunks to avoid rate limits)
+    vectors = []
+
+    for i, record in enumerate(rows[:100]):  # limit to 100 for demo
+
+        print(record)
+        # Attempt to access `record["row"]["text"]` if "row" exists and is a dictionary
+        text = record["row"].get("text") if isinstance(record.get("row"), dict) else record.get("text")
+
+        title = record["row"].get("title") if isinstance(record.get("row"), dict) else record.get("title")
+        id= record.get("id")
+        embedding = get_embedding(text)
+        value = id if id else i
+        document_name_retrieved=document_name if document_name else record.get("document_name")
+        vectors.append((
+            str(value),  # unique ID
+            embedding,
+            {"title": title, "text": text,"id":id,"document_name":document_name_retrieved}  # metadata
+        ))
+
+        # Upsert in batches of 20
+        if (i + 1) % 20 == 0:
+            index.upsert(vectors=vectors)
+            print(f"Upserted {i + 1} vectors")
+            vectors = []
+
+    # Flush remaining
+    if vectors:
+        index.upsert(vectors=vectors)
+        print(f"Upserted final {len(vectors)} vectors")
+
+
+def searchOrQueryOnVDB(queryEmbedding):
+    # To get the unique host for an index,
+    # see https://docs.pinecone.io/guides/manage-data/target-an-index
+    index = pc.Index(index_name)
+
+    # Search the dense index
+    results = index.query(
+        vector=queryEmbedding,
+
+        top_k=10,  # Number of top results to retrieve
+        include_metadata=True
+    )
+
+    # Collect top 5 results
+    top_5_results = sorted(results.matches, key=lambda result: result.score, reverse=True)[:5]
+
+    # Initialize an empty list to store the top results
+    top_results_list = []
+
+    # Append the metadata 'text' of each top result to the list
+    for result in top_5_results:
+        top_results_list.append(result.metadata['text'])
+
+    # Check if there are any results
+    if not top_results_list:
+        print("No results found!")
+    else:
+        # Print out the top 5 results (if any exist)
+        print("Top 5 Results:")
+        for text in top_results_list:
+            print(text)
+
+    # Optionally, return the top results list
+    return top_results_list
+
+
+def queryToLLM(query):
+    # Define the query
+    # query = "the underlying principles of workplace health and safety is to:"
+    query_emb = get_embedding(query)
+    ragContent = searchOrQueryOnVDB(query_emb)
+    system_prompt=(
+                     "You are an experienced and patient TAFE trainer for a Certificate III in Electrotechnology. "
+                     "Your primary goal is to guide apprentices to understand their learning materials. "
+                     "You must **NEVER** give direct answers to questions from assessments (UKT or UST). "
+                     "Instead, use Socratic questioning to prompt critical thinking and encourage students to find the answers in their provided learning content. "
+                     "When appropriate, offer hints by referencing specific sections or copy of the section or modules (e.g., 'Review the section on Ohm's Law in Module 2'). "
+                     "Maintain a helpful, encouraging, and respectful tone. Do not solve problems for the student."
+
+                 )
+    openaiwrapper = LLMClientWrapper(client=client, system_prompt=system_prompt)
+    openAiResponse = openaiwrapper.chat(ragContent, query)
+    return openAiResponse
+
+
+
+# function to
+# upload a particular document
+# takes LLM instruction about how to process/chunk the document
+# prepares chunked json objects
+def upload_and_prepare_file_content_in_chunks(request,instructions):
+    try:
+
+        uploaded_file = uploaded_file_ref(request)
+        file_name, file_bytes = extract_file_details(
+            uploaded_file)
+        chunked_text = chunk_documents(instructions,file_name,file_bytes)
+        return chunked_text,file_name
+    except Exception as e:
+        return JsonResponse({"error": f"An unexpected error occurred: {str(e)}"}, status=500)
+
+# function to
+# upload a particular document
+def uploaded_file_ref(request):
+    # Check if the request contains a file
+    if 'file' not in request.FILES:
+        return JsonResponse({"error": "No file part in the request"}, status=400)
+
+    # Retrieve the file from the POST request
+    uploaded_file = request.FILES['file']
+    return uploaded_file
+
+def upload_file(request):
+    """
+       Endpoint to handle POST requests containing a JSON file.
+       The JSON file is used to load and parse data.
+       """
+    try:
+        uploaded_file = uploaded_file_ref(request)
+
+        if uploaded_file.name == '':
+            return JsonResponse({"error": "No file selected for uploading"}, status=400)
+
+        # Read and parse file content as JSON
+        data = None
+        try:
+            file_data = uploaded_file.read().decode('utf-8')  # Decode file content
+            data = json.loads(file_data)
+            return data # Parse content as JSON
+        except json.JSONDecodeError as e:
+            return JsonResponse({"error": f"Failed to parse JSON: {e}"}, status=400)
+
+
+    except Exception as e:
+        return JsonResponse({"error": f"An unexpected error occurred: {str(e)}"}, status=500)
+
+
+@csrf_exempt
+@require_http_methods(["POST"])
+def fetch_records_grouped_by_document_name(request):
+    index = pc.Index(index_name)
+    records_by_doc = qfetch_records_grouped_by_document_name(index)
+    print(records_by_doc.keys())
+    return JsonResponse(records_by_doc, safe=False)
+
+
+
+@csrf_exempt
+@require_http_methods(["POST"])
+def update_records_grouped_by_document_name_in_vdb(request):
+    dataset=upload_file(request)
+    index_n = request.POST.get("index_name")
+    transformed_records = transform_groupd_by_doc_name_to_vdb_metadata_structure(dataset)
+    upsertRecord(index_n,transformed_records)
+    return JsonResponse(transformed_records, safe=False)
+
+##queries and update records within vdb
+
+def transform_groupd_by_doc_name_to_vdb_metadata_structure(input_json):
+    # Initialize an empty list to store the transformed data
+    output_list = []
+
+    # Iterate through the dictionary
+    for document_name, entries in input_json.items():
+        for entry in entries:
+            # Add the document name to each item and append to the new list
+            transformed_entry = {
+                "text": entry["text"],
+                "title": entry["title"],
+                "id": entry["id"],
+                "document_name": document_name
+            }
+            output_list.append(transformed_entry)
+
+    return output_list
+
+
+
+
+def qfetch_records_grouped_by_document_name(index, batch_size=100,limit=100):
+    """
+    Fetch all records from Pinecone grouped by document_name in metadata,
+    processing 100 unique document_name values at a time.
+
+    Args:
+        index: Pinecone Index object
+        batch_size: Number of unique document_name to process per batch
+    Returns:
+        dict: {document_name: [records]}
+    """
+    if limit <= 0 or limit > 100:
+        raise ValueError("The `limit` parameter must be between 1 and 100.")
+    # Step 1: Get all IDs with metadata
+    all_ids = []
+    next_token = None
+    consecutive_none_count=0
+    while True:
+        res = list(index.list(limit=100, next_token=next_token) if next_token else index.list(limit=limit))
+        if not res:
+            break  # Exit loop if there are no more results
+
+        all_ids.extend(chain.from_iterable(res))
+
+                # Update `next_token` for pagination
+        next_token = res[-1].get('next_token') if isinstance(res[-1], dict) and 'next_token' in res[-1] else None
+
+        # Handle cases when `next_token` is None
+        if not next_token:
+            consecutive_none_count += 1
+        else:
+            consecutive_none_count = 0  # Reset counter if next_token is valid again
+
+        # Break loop if `next_token` is None more than once consecutively
+        if consecutive_none_count > 1:  # Adjust count based on logic
+            break
+    # Step 2: Fetch metadata in batches and collect document_names
+    id_to_docname = {}
+    for i in range(0, len(all_ids), batch_size):
+        batch_ids = all_ids[i:i + batch_size]
+        res = index.fetch(ids=batch_ids)
+        vectors = res.vectors
+        for vector in vectors.items():
+            metadata = vector[1].metadata if vector[1] and vector[1].metadata else None
+            doc_name = metadata['document_name'] if metadata and 'document_name' in metadata else None
+            id = metadata['id'] if metadata and 'id' in metadata else None
+            if doc_name and id:
+                id_to_docname[id] = doc_name
+
+    # Step 3: Group records by document_name in batches of 100 unique names
+    grouped_records = defaultdict(list)
+    unique_docnames = list(set(id_to_docname.values()))
+
+    for i in range(0, len(unique_docnames), 100):
+        batch_docnames = unique_docnames[i:i + 100]
+        # Get all IDs for these document_names
+        batch_ids = [rid for rid, dname in id_to_docname.items() if dname in batch_docnames]
+        # Fetch full records
+        res = index.fetch(ids=batch_ids)
+        vectors = res.vectors
+        for vector in vectors.items():
+            metadata = vector[1].metadata if vector[1] and vector[1].metadata else None
+            id = metadata['id'] if metadata and 'id' in metadata else None
+            doc_name = id_to_docname[id] if id in id_to_docname else None
+            text = metadata['text'] if metadata and 'text' in metadata else None
+            title = metadata['title'] if metadata and 'title' in metadata else None
+            id = metadata['id'] if metadata and 'id' in metadata else None
+
+            # Create a dictionary
+            json_object = {
+                "text": text,
+                "title": title,
+                "id": id
+            }
+            grouped_records[doc_name].append(json_object)
+
+
+    return dict(grouped_records)
+
+
+
+#function that chunks any document
+def chunk_documents(instructions,file_name,file_path="content_playground/content.json"):
+    return prepare_chunked_text(file_path, file_name,instructions)
+
+#function that chunks any document as well as inserts into vdb
+def chunk_and_upsert_to_vdb(index_n,instructions,file_name,file_path="content_playground/content.json"):
+    chunked_dataset = prepare_chunked_text(file_path, file_name, instructions)
+    document_name = file_name if file_name else os.path.basename(file_path)   + uuid.uuid4().hex
+    
+    upsertRecord(index_n,chunked_dataset,document_name)
+    return chunked_dataset, document_name
+
+#function that loads existing chunks from vdb by document name
+def fetch_vdb_chunks_grouped_by_document_name(index_n):
+    index = pc.Index(index_n)
+    records_by_doc = qfetch_records_grouped_by_document_name(index)
+    return records_by_doc
+
+#function that updates existing chunks
+def update_vdb_chunks_grouped_by_document_name(index_n,dataset):
+    index = pc.Index(index_n)
+    transformed_records = transform_groupd_by_doc_name_to_vdb_metadata_structure(dataset)
+    upsertRecord(index_n,transformed_records)
+    return transformed_records

prevectorchunks_core-0.1.1/prevectorchunks_core/settings.py

@@ -0,0 +1,176 @@
+"""
+Django settings for djproj project.
+
+Generated by 'django-admin startproject' using Django 5.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.1/ref/settings/
+"""
+
+from pathlib import Path
+from corsheaders.defaults import default_headers
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-xwxywn=k$51m=o4ahmlybvaekxp@j-z!qpoyog-fh7)20u8+ps'
+CORS_ALLOW_ALL_ORIGINS = True
+CSRF_COOKIE_SECURE = False
+SESSION_COOKIE_SECURE = False
+CORS_ALLOW_CREDENTIALS = True
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+CORS_ALLOW_HEADERS = list(default_headers) + [
+    'x-csrf-token',
+]
+CSRF_TRUSTED_ORIGINS = [
+    'http://localhost:3000',
+    'http://127.0.0.1:3000',
+    'http://3.106.90.99:3000',
+    'http://3.106.90.99:80',
+    'http://3.106.90.99',
+    'http://effiez.com.au',
+    'http://effiez.com.au:80',
+    'http://www.effiez.com.au',
+    'http://www.effiez.com.au:80',
+    'https://effiez.com.au',
+    'https://py.effiez.com.au',
+    'https://effiez.com.au:80',
+    'https://www.effiez.com.au',
+    'https://www.effiez.com.au:80',
+
+    'http://*'
+]
+ALLOWED_HOSTS = ['*']
+CORS_ORIGIN_ALLOW_ALL = True
+CORS_ALLOWED_ORIGINS = [
+    'http://localhost:3000',
+    'http://127.0.0.1:3000',
+    'http://3.106.90.99:3000',
+    'http://3.106.90.99:80',
+    'http://3.106.90.99',
+    'http://effiez.com.au',
+    'http://effiez.com.au:80',
+    'http://www.effiez.com.au',
+    'http://www.effiez.com.au:80',
+    'https://effiez.com.au',
+    'https://effiez.com.au:80',
+    'https://www.effiez.com.au',
+    'https://www.effiez.com.au:80',
+    'https://py.effiez.com.au',
+    'http://*'
+]
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    #'reimburse.apps.ReimburseConfig',
+
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'PreVectorChunks-Web.urls'
+
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [BASE_DIR / 'templates']
+        ,
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'PreVectorChunks.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.1/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.1/howto/static-files/
+
+STATIC_URL = 'static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

prevectorchunks_core-0.1.1/prevectorchunks_core/test_loader.py

@@ -0,0 +1,23 @@
+import json
+import pytest
+from PreVectorChunks.prevectorchunks_core.services import chunk_documents_crud_vdb
+
+
+
+# Create a temporary JSON file to test with
+@pytest.fixture
+def temp_json_file(tmp_path):
+    file_path = tmp_path / "test.json"
+    content = [{"id": 1, "text": "hello world"}]
+    with open(file_path, "w") as f:
+        json.dump(content, f)
+    return file_path
+
+
+def test_load_file_and_upsert_chunks_to_vdb(temp_json_file):
+    #dataset = chunk_and_upsert_to_vdb("dl-doc-search","instructions", file_path="content_playground/content.json")
+    #dataset=chunk_documents_crud_vdb.fetch_vdb_chunks_grouped_by_document_name("dl-doc-search")
+    dataset=chunk_documents_crud_vdb.chunk_documents("extract", file_name=None, file_path=temp_json_file)
+    #dataset=chunk_documents_crud_vdb.chunk_documents("Extract doco",file_path="content_playground/content.json",file_name=None)
+    # Assertions
+

prevectorchunks_core-0.1.1/prevectorchunks_core/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

prevectorchunks_core-0.1.1/prevectorchunks_core/utils/file_loader.py

@@ -0,0 +1,264 @@
+import json
+import warnings  # Correct module for warnings, including PendingDeprecationWarning
+import os
+from pathlib import Path
+
+from docx import Document
+from PyPDF2 import PdfReader
+from PIL import Image
+import pytesseract
+import uuid
+from openai import OpenAI
+from openai import OpenAI
+from .llm_wrapper import LLMClientWrapper  # Relative import
+from dotenv import load_dotenv
+import tempfile
+load_dotenv(override=True)
+# Initialize OpenAI client
+client =  OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+from django.core.files.uploadedfile import UploadedFile
+
+
+def extract_content_agnostic(file, filename=None):
+    """
+    Extract text content from a file.
+
+    Supports:
+    - PDF (.pdf)
+    - Word (.docx)
+    - Text (.txt)
+    - Images (.png, .jpg, .jpeg, .tiff, .bmp)
+
+    Parameters:
+    - file: either a file path (str) or bytes (binary content)
+    - filename: required if `file` is bytes, to determine extension
+    """
+    # Determine if file is path or binary
+    if isinstance(file, str):
+        filepath = file
+        ext = os.path.splitext(filepath)[1].lower()
+    elif isinstance(file, Path):
+        filepath = str(file)
+        ext = os.path.splitext(filepath)[1].lower()
+    elif isinstance(file, bytes):
+        if not filename:
+            raise ValueError("filename must be provided if passing binary content")
+        ext = os.path.splitext(filename)[1].lower()
+        # Write bytes to a temporary file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp:
+            tmp.write(file)
+            filepath = tmp.name
+    else:
+        raise TypeError("file must be a string path or bytes")
+
+    # At this point, `filepath` is a valid file path on disk
+    # TODO: implement your extraction logic based on `ext` and `filepath`
+    # Example:
+    # if ext == ".pdf":
+    #     content = extract_pdf(filepath)
+    # elif ext == ".docx":
+    #     content = extract_docx(filepath)
+    # ...
+
+    text = load_file_by_type(ext, filepath)
+
+    # If we created a temporary file, optionally delete it
+    if isinstance(file, UploadedFile) and not hasattr(file, 'temporary_file_path'):
+        try:
+            os.remove(filepath)
+        except Exception:
+            pass
+
+    return text.strip()
+
+
+def extract_content(file):
+    """
+    Extract text content from a file.
+    Supports:
+    - PDF (.pdf)
+    - Word (.docx)
+    - Text (.txt)
+    - Images (.png, .jpg, .jpeg, .tiff, .bmp)
+
+    file: either a file path (str) or a Django UploadedFile object (request.FILES['file'])
+    """
+    # Determine if input is file path or UploadedFile
+    if isinstance(file, UploadedFile):
+        filename = file.name
+        ext = os.path.splitext(filename)[1].lower()
+
+        # Check if file is already on disk
+        if hasattr(file, 'temporary_file_path'):
+            filepath = file.temporary_file_path()
+        else:
+            # Save in-memory file to temporary file
+            with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp:
+                for chunk in file.chunks():
+                    tmp.write(chunk)
+                filepath = tmp.name
+    else:
+        # It's a file path
+        filepath = file
+        ext = os.path.splitext(filepath)[1].lower()
+
+    text = load_file_by_type(ext, filepath)
+
+    # If we created a temporary file, optionally delete it
+    if isinstance(file, UploadedFile) and not hasattr(file, 'temporary_file_path'):
+        try:
+            os.remove(filepath)
+        except Exception:
+            pass
+
+    return text.strip()
+
+
+def load_file_by_type(ext, filepath):
+    text = ""
+    if ext == ".pdf":
+        reader = PdfReader(filepath)
+        text = "\n".join([p.extract_text() or "" for p in reader.pages])
+
+    elif ext == ".docx":
+        doc = Document(filepath)
+        text = "\n".join([p.text for p in doc.paragraphs])
+
+    elif ext in [".png", ".jpg", ".jpeg", ".tiff", ".bmp"]:
+        img = Image.open(filepath)
+        text = pytesseract.image_to_string(img)
+
+    elif ext == ".txt":
+        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+            text = f.read()
+    elif ext == ".json":
+        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+            data = json.load(f)
+            # Convert JSON to text (pretty print or flatten)
+            text = json.dumps(data, ensure_ascii=False, indent=2)
+    else:
+        raise ValueError(f"Unsupported file type: {ext}")
+    return text
+
+
+def split_text_by_words(text, chunk_size=200):
+    """Split text into chunks of N words."""
+    words = text.split()
+    return [" ".join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+
+
+def process_with_llm(chunk,instructions):
+    """
+    Send a chunk to LLM and return structured JSON array.
+    Expected format: [{"id": ..., "title": ..., "text": ...}, ...]
+    """
+    context = f"""
+    Take the following text and split it into sections based on the most important category headings (ignore lower level headings).
+    For each section, return a JSON object with - no extra words other than the json and remove ```json:
+    - "id" (a UUID you generate),
+    - "title" (the most important heading),
+    - "text" (the remaining text under that heading).
+
+    Text:
+    {chunk}
+    """
+    instructions=instructions or "Exract sections"
+    system_prompt="You are a helpful assistant that structures text into JSON sections."
+    # Create an instance of your LLM wrapper
+    llm = LLMClientWrapper(client, model="gpt-4o-mini", temperature=0, system_prompt=system_prompt)
+    response=llm.chat(context,instructions)
+
+    # Parse JSON safely
+    try:
+        structured_data = eval(response)
+    except Exception:
+        structured_data = []
+
+    return structured_data
+
+
+def process_large_text(text, instructions,chunk_size=200):
+    """Main function: split -> send to LLM -> collect results."""
+    chunks = split_text_by_words(text, chunk_size)
+    all_results = []
+
+    for chunk in chunks:
+        structured = process_with_llm(chunk,instructions)
+        # Ensure UUIDs exist
+        for obj in structured:
+            if "id" not in obj:
+                obj["id"] = str(uuid.uuid4())
+        all_results.extend(structured)
+
+    return all_results
+
+
+
+def prepare_chunked_text(file_path,file_name,instructions):
+    content =extract_content_agnostic(file_path,file_name)
+    results=process_large_text(content,instructions, chunk_size=200)
+    print (results)
+    return results
+
+#this function takes a django file and extracts filename and byte content
+def extract_file_details(uploaded_file):
+    # 1. Get the filename
+    filename = uploaded_file.name
+
+    # 2. Get the file content as bytes
+    file_bytes = uploaded_file.read()  # reads entire file into memory
+
+    # Now you can call your extract_content function
+    return filename, file_bytes
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

prevectorchunks_core-0.1.1/prevectorchunks_core/utils/llm_wrapper.py

@@ -0,0 +1,34 @@
+import openai
+
+class LLMClientWrapper:
+    def __init__(self, client, model="gpt-4o-mini", temperature=0, system_prompt=None):
+        """
+        client: OpenAI client instance
+        model: LLM model name
+        temperature: randomness/creativity
+        system_prompt: default system instructions for the LLM
+        """
+        self.client = client
+        self.model = model
+        self.temperature = temperature
+        # Use default system prompt if not provided
+        self.system_prompt = system_prompt or ""
+
+    def chat(self, context, user_query):
+        """
+        context: content retrieved from RAG or other sources
+        user_query: actual question from user
+        """
+        full_query = f"{context}\nQuestion: {user_query}"
+
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": full_query}
+            ],
+            temperature=self.temperature
+        )
+
+        return response.choices[0].message.content
+

prevectorchunks_core-0.1.1/prevectorchunks_core/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for PreVectorChunks project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'PreVectorChunks.settings')
+
+application = get_wsgi_application()

prevectorchunks_core-0.1.1/prevectorchunks_core.egg-info/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: prevectorchunks-core
+Version: 0.1.1
+Summary: A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database
+Author-email: Zul Al-Kabir <zul.developer.2023@gmail.com>
+Project-URL: Homepage, https://github.com/yourusername/mydep
+Description-Content-Type: text/markdown
+Requires-Dist: Django==5.1
+Requires-Dist: packaging~=24.1
+Requires-Dist: requests~=2.32.3
+Requires-Dist: openai~=1.37.1
+Requires-Dist: httpx~=0.27.0
+Requires-Dist: python-dotenv~=1.0.1
+Requires-Dist: django-cors-headers~=4.4.0
+Requires-Dist: PyJWT~=2.7.0
+Requires-Dist: fastapi~=0.112.2
+Requires-Dist: datasets~=4.1.0
+Requires-Dist: pinecone~=7.3.0
+Requires-Dist: pytesseract~=0.3.13
+Requires-Dist: python-docx~=1.2.0
+Requires-Dist: PyPDF2~=3.0.1
+Requires-Dist: pillow~=11.3.0
+
+# 📚 PreVectorChunks
+
+> A lightweight utility for **document chunking** and **vector database upserts** — designed for developers building **RAG (Retrieval-Augmented Generation)** solutions.
+
+---
+
+## ✨ Who Needs This Module?
+Any developer working with:
+- **RAG pipelines**
+- **Vector Databases** (like Pinecone, Weaviate, etc.)
+- **AI applications** requiring **similar content retrieval**
+
+---
+
+
+## 🎯 What Does This Module Do?
+This module helps you:
+- **Chunk documents** into smaller fragments  
+- **Insert (upsert) fragments** into a vector database  
+- **Fetch & update** existing chunks from a vector database  
+
+---
+
+## 📦 Installation
+```bash
+pip install prevectorchunks
+````
+How to import in a file:  
+```python
+from PreVectorChunks.services import chunk_documents_crud_vdb
+
+#How to use Pinecone and OpenAI:
+#Use a .env file in your project root to configure API keys:
+
+PINECONE_API_KEY=YOUR_API_KEY
+OPENAI_API_KEY=YOUR_API_KEY
+
+#how to call relevant functions:
+#Four key functions that you can call are below: 
+#function that chunks any document 
+chunk_documents(instructions,file_path="content_playground/content.json"): 
+#function that chunks any document as well as inserts into vdb - you need an index name inside index_n
+chunk_and_upsert_to_vdb(index_n,instructions,file_path="content_playground/content.json"): 
+#function that loads existing chunks from vdb by document name - you need an index name inside index_n 
+fetch_vdb_chunks_grouped_by_document_name(index_n): 
+#function that updates existing chunks - you need an index name inside index_n 
+update_vdb_chunks_grouped_by_document_name(index_n,dataset):
+```

prevectorchunks_core-0.1.1/prevectorchunks_core.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+README.md
+pyproject.toml
+prevectorchunks_core/__init__.py
+prevectorchunks_core/admin.py
+prevectorchunks_core/apps.py
+prevectorchunks_core/asgi.py
+prevectorchunks_core/models.py
+prevectorchunks_core/settings.py
+prevectorchunks_core/test_loader.py
+prevectorchunks_core/tests.py
+prevectorchunks_core/wsgi.py
+prevectorchunks_core.egg-info/PKG-INFO
+prevectorchunks_core.egg-info/SOURCES.txt
+prevectorchunks_core.egg-info/dependency_links.txt
+prevectorchunks_core.egg-info/requires.txt
+prevectorchunks_core.egg-info/top_level.txt
+prevectorchunks_core/migrations/__init__.py
+prevectorchunks_core/services/__init__.py
+prevectorchunks_core/services/chunk_documents_crud_vdb.py
+prevectorchunks_core/utils/__init__.py
+prevectorchunks_core/utils/file_loader.py
+prevectorchunks_core/utils/llm_wrapper.py

prevectorchunks_core-0.1.1/prevectorchunks_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

prevectorchunks_core-0.1.1/prevectorchunks_core.egg-info/requires.txt

@@ -0,0 +1,15 @@
+Django==5.1
+packaging~=24.1
+requests~=2.32.3
+openai~=1.37.1
+httpx~=0.27.0
+python-dotenv~=1.0.1
+django-cors-headers~=4.4.0
+PyJWT~=2.7.0
+fastapi~=0.112.2
+datasets~=4.1.0
+pinecone~=7.3.0
+pytesseract~=0.3.13
+python-docx~=1.2.0
+PyPDF2~=3.0.1
+pillow~=11.3.0

prevectorchunks_core-0.1.1/prevectorchunks_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+prevectorchunks_core

prevectorchunks_core-0.1.1/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "prevectorchunks-core"
+version = "0.1.1"
+description = "A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Zul Al-Kabir", email = "zul.developer.2023@gmail.com" }
+]
+
+dependencies = [
+    "Django==5.1",
+    "packaging~=24.1",
+    "requests~=2.32.3",
+    "openai~=1.37.1",
+    "httpx~=0.27.0",
+    "python-dotenv~=1.0.1",
+    "django-cors-headers~=4.4.0",
+    "PyJWT~=2.7.0",
+    "fastapi~=0.112.2",
+    "datasets~=4.1.0",
+    "pinecone~=7.3.0",
+    "pytesseract~=0.3.13",
+    "python-docx~=1.2.0",
+    "PyPDF2~=3.0.1",
+    "pillow~=11.3.0"
+]
+
+[tool.setuptools.packages.find]
+include = ["prevectorchunks_core*"]
+
+[project.urls]
+Homepage = "https://github.com/yourusername/mydep"

prevectorchunks_core-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+