ddi-fw 0.0.108__tar.gz → 0.0.109__tar.gz

{ddi_fw-0.0.108 → ddi_fw-0.0.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.108
+Version: 0.0.109
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.108 → ddi_fw-0.0.109}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.108"
+version = "0.0.109"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.108 → ddi_fw-0.0.109}/src/ddi_fw/langchain/storage.py

@@ -57,6 +57,42 @@ def split_dataframe(df, min_size=512):
     
     return partial_dfs
 
+def split_dataframe_indices(df, min_size=512):
+    total_size = len(df)
+    
+    # If the dataframe is smaller than min_size, return the entire range
+    if total_size <= min_size:
+        return [(0, total_size - 1)]
+    
+    # List to store the start and end indices of each chunk
+    chunk_indices = []
+    start_idx = 0
+    
+    # Calculate the minimum number of chunks needed to ensure each chunk has at least min_size
+    num_chunks = total_size // min_size
+    remaining_rows = total_size
+    
+    # Split into chunks
+    for i in range(num_chunks):
+        chunk_size = min_size
+        if (remaining_rows - chunk_size) < min_size:
+            chunk_size = remaining_rows  # Last chunk takes all remaining rows
+        
+        # Calculate the ending index of the chunk (exclusive, hence chunk_size - 1)
+        end_idx = start_idx + chunk_size - 1
+        chunk_indices.append((start_idx, end_idx))
+        
+        # Update the start index and remaining rows
+        start_idx += chunk_size
+        remaining_rows -= chunk_size
+    
+    # If there are any remaining rows after the loop, they should form the last chunk
+    if remaining_rows > 0:
+        end_idx = start_idx + remaining_rows - 1
+        chunk_indices.append((start_idx, end_idx))
+    
+    return chunk_indices
+
 class DataFrameToVectorDB:
     def __init__(self,
                  collection_name,
@@ -78,8 +114,9 @@ class DataFrameToVectorDB:
         return docs
 
     def __split_list(self, input_list, batch_size):
-        for i in range(0, len(input_list), batch_size):
-            yield input_list[i:i + batch_size]
+        # for i in range(0, len(input_list), batch_size):
+        for s,e in split_dataframe_indices(input_list):
+            yield input_list[s:e+1]
 
     def store_documents(self, df, columns, page_content_columns):
         """

{ddi_fw-0.0.108 → ddi_fw-0.0.109}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.108
+Version: 0.0.109
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>