ddi-fw 0.0.254__py3-none-any.whl → 0.0.256__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -379,7 +379,7 @@ class TextDatasetMixin(BaseModel):
             if self.vector_store_manager is not None:
                 self.embedding_dict = self.vector_store_manager.initialize_embedding_dict()
             # self.embedding_dict = self.__initialize_embedding_dict()    
-            self.__calculate_embedding_size()
+        self.__calculate_embedding_size()
          
 
 

ddi_fw/langchain/faiss_storage.py

@@ -37,7 +37,7 @@ class BaseVectorStoreManager(BaseModel):
         raise NotImplementedError("This method should be implemented by subclasses.")
 
 class FaissVectorStoreManager(BaseVectorStoreManager):
-    persist_directory: str = Field(default="./embeddings")
+    persist_directory: str = Field(default="./embeddings/faiss")
     index: Any = None
     vector_store: Optional[FAISS] | None = None
     class Config:
@@ -360,7 +360,7 @@ def generate_embeddings(
     config_file:Optional[str],
     new_model_names:Optional[List],
     collections:Optional[Dict],
-    persist_directory="embeddings",
+    persist_directory="./embeddings",
 ):
     """
     Generate embeddings for collections based on a configuration file.
@@ -390,7 +390,7 @@ def generate_embeddings(
         partial_df_size = collection_config.get('partial_dataframe_size')
         columns = collection_config.get('columns')
         page_content_columns = collection_config.get('page_content_columns')
-        persist_dir = f'{persist_directory}/{id}'
+        
 
         # Load embedding model
         try:
@@ -417,10 +417,11 @@ def generate_embeddings(
                 text_splitters_suffixes.append(suffix)
             except Exception as e:
                 raise Exception(f"Unknown text splitter: {text_splitter_type}") from e
-
+        
         for text_splitter, suffix in zip(text_splitters, text_splitters_suffixes):
             print(f"{id}_{suffix}")
-
+            # persist_dir = f'{persist_directory}/{id}/{suffix}'
+            persist_dir = f'{persist_directory}/{suffix}'
             # Prepare manager parameters
             manager_params = {
                 "collection_name": f"{id}_{suffix}",

ddi_fw/vectorization/feature_vector_generation.py

@@ -3,7 +3,6 @@ import numpy as np
 import pandas as pd
 from scipy.spatial.distance import pdist, squareform
 from sklearn.preprocessing import MultiLabelBinarizer
-import cupy as cp
 
 # todo pd.unique kullan
 def find_distinct_elements(frame):
@@ -97,6 +96,12 @@ class SimilarityMatrixGenerator:
         Returns:
         cp.ndarray: A 2D CuPy array containing the pairwise Jaccard similarity.
         """
+        try:
+            import cupy as cp
+        except ImportError:
+            raise ImportError("cupy is required for GPU Jaccard similarity computation. Please install it with 'pip install cupy'.")
+                
+                
         if not ((matrix == 0) | (matrix == 1)).all():
             raise ValueError("Input matrix must be binary (contain only 0s and 1s).")
         matrix = cp.asarray(matrix)

{ddi_fw-0.0.254.dist-info → ddi_fw-0.0.256.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.254
+Version: 0.0.256
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.254.dist-info → ddi_fw-0.0.256.dist-info}/RECORD

@@ -1,12 +1,12 @@
 ddi_fw/datasets/__init__.py,sha256=NozQvXPYIS01U0srZmcKhiqJgRDkD-C-VXHL6sKrFSw,166
-ddi_fw/datasets/core.py,sha256=4KtFYjf7uLsbto2_InY2DWNN0AUoAb8Yr0jYn0to4ME,17413
+ddi_fw/datasets/core.py,sha256=UnbCDoWXdxeiAb0e0anhDqXiVFGUi02VA9sKl6NVBZU,17409
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
 ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
 ddi_fw/langchain/chroma_storage.py,sha256=7LSUhdiPdQHZvKC_NapOeVbHtS71iE5ABZVTrI0YQ-A,15520
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=e6WbjKAlCbFZUq9P3LTjTeXmWTAh6Oxp_NMVbvfK-kc,17727
+ddi_fw/langchain/faiss_storage.py,sha256=LG2cf6upCEHBBF4Jixnq_diA_t3A0OQZBqPc9DwmILc,17825
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -36,9 +36,9 @@ ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,
 ddi_fw/utils/utils.py,sha256=PY-zDawREKoXQfzX7lVkxBLVFQPkfvr9385kHCjaNXo,4391
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
-ddi_fw/vectorization/feature_vector_generation.py,sha256=QQQGhCti653BdU343Ag1bH_g1fzi2hlic7dgNy7otjE,7694
+ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.254.dist-info/METADATA,sha256=i_KV8aGISMG8quQkllnwlqFi8lkGe9eEQgmVAjBEdDw,2623
-ddi_fw-0.0.254.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.254.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.254.dist-info/RECORD,,
+ddi_fw-0.0.256.dist-info/METADATA,sha256=4Gwtn1oiXo2cJ8osjOXRRu7JIQuP8i3PWjOsE7B56t0,2623
+ddi_fw-0.0.256.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.256.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.256.dist-info/RECORD,,