prevectorchunks-core 0.1.36__tar.gz → 0.1.37__tar.gz

{prevectorchunks_core-0.1.36/prevectorchunks_core.egg-info → prevectorchunks_core-0.1.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prevectorchunks-core
-Version: 0.1.36
+Version: 0.1.37
 Summary: A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database
 Author-email: Zul Al-Kabir <zul.developer.2023@gmail.com>
 License: MIT License
@@ -23,7 +23,7 @@ Requires-Dist: pytesseract~=0.3.13
 Requires-Dist: python-docx~=1.2.0
 Requires-Dist: PyPDF2~=3.0.1
 Requires-Dist: pillow~=11.3.0
-Requires-Dist: torch~=2.6.0
+Requires-Dist: torch~=2.2.2
 Requires-Dist: torchvision~=0.21.0
 Requires-Dist: torchaudio~=2.6.0
 Requires-Dist: sentence-transformers~=5.1.1

{prevectorchunks_core-0.1.36 → prevectorchunks_core-0.1.37}/prevectorchunks_core/services/markdown_and_chunk_documents.py

@@ -244,6 +244,7 @@ class MarkdownAndChunkDocuments:
             input_bytes: bytes = None,
             include_image: bool = None,
             file_name: str = None,
+            embedding_client=None
     ):
         """Generator version of markdown_and_chunk_documents that yields progress JSON events"""
 
@@ -268,8 +269,16 @@ class MarkdownAndChunkDocuments:
 
         # 4️⃣ Chunking
         yield from report(55, "Chunking text...")
-        chunk_client = OpenAI(api_key=self.api_key)
-        cm = ChunkMapper(chunk_client, markdown_output, embedding_model="text-embedding-3-small")
+
+        # Chunking and mapping
+        # chunk_client = OpenAI(api_key=self.api_key)
+        if embedding_client is None:
+            embedding_client = OpenAIEmbeddings(
+                model="text-embedding-3-small",
+                api_key=self.api_key
+            )
+
+        cm = ChunkMapper(embedding_client, markdown_output, embedding_model="text-embedding-3-small")
 
         splitter_config = SplitterConfig(
             chunk_size=300,

{prevectorchunks_core-0.1.36 → prevectorchunks_core-0.1.37}/prevectorchunks_core/test_loader.py

@@ -20,7 +20,7 @@ def temp_json_file(tmp_path):
     return file_path
 
 
-def test_load_file_and_upsert_chunks_to_vdb(temp_json_file):
+def test_load_file_and_upsert_chunks_to_vdb():
     splitter_config = SplitterConfig(chunk_size=300, chunk_overlap=0, separators=["\n"],
                                      split_type=SplitType.R_PRETRAINED_PROPOSITION.value, min_rl_chunk_size=5,
                                      max_rl_chunk_size=50, enableLLMTouchUp=True,llm_structured_output_type=LLM_Structured_Output_Type.STRUCTURED_WITH_VECTOR_DB_ID_GENERATED)
@@ -38,7 +38,7 @@ def test_load_file_and_upsert_chunks_to_vdb(temp_json_file):
         print(f"Chunk {i + 1}: {c}")
     print(chunks)
 
-def test_markdown(temp_json_file):
+def test_markdown():
 
     client = init_chat_model(
         model="gpt-4o-mini",

{prevectorchunks_core-0.1.36 → prevectorchunks_core-0.1.37/prevectorchunks_core.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prevectorchunks-core
-Version: 0.1.36
+Version: 0.1.37
 Summary: A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database
 Author-email: Zul Al-Kabir <zul.developer.2023@gmail.com>
 License: MIT License
@@ -23,7 +23,7 @@ Requires-Dist: pytesseract~=0.3.13
 Requires-Dist: python-docx~=1.2.0
 Requires-Dist: PyPDF2~=3.0.1
 Requires-Dist: pillow~=11.3.0
-Requires-Dist: torch~=2.6.0
+Requires-Dist: torch~=2.2.2
 Requires-Dist: torchvision~=0.21.0
 Requires-Dist: torchaudio~=2.6.0
 Requires-Dist: sentence-transformers~=5.1.1

{prevectorchunks_core-0.1.36 → prevectorchunks_core-0.1.37}/prevectorchunks_core.egg-info/requires.txt

@@ -9,7 +9,7 @@ pytesseract~=0.3.13
 python-docx~=1.2.0
 PyPDF2~=3.0.1
 pillow~=11.3.0
-torch~=2.6.0
+torch~=2.2.2
 torchvision~=0.21.0
 torchaudio~=2.6.0
 sentence-transformers~=5.1.1

{prevectorchunks_core-0.1.36 → prevectorchunks_core-0.1.37}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "prevectorchunks-core"
-version = "0.1.36"
+version = "0.1.37"
 description = "A Python module that allows conversion of a document into chunks to be inserted into Pinecone vector database"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -25,7 +25,7 @@ dependencies = [
     "python-docx~=1.2.0",
     "PyPDF2~=3.0.1",
     "pillow~=11.3.0",
-    "torch~=2.6.0",
+    "torch~=2.2.2",
     "torchvision~=0.21.0",
     "torchaudio~=2.6.0",
     "sentence-transformers~=5.1.1",