ddi-fw 0.0.253__py3-none-any.whl → 0.0.255__py3-none-any.whl

ddi_fw/langchain/__init__.py

@@ -1,5 +1,5 @@
 from ..langchain.embeddings import PoolingStrategy,SumPoolingStrategy,MeanPoolingStrategy,SentenceTransformerDecorator,PretrainedEmbeddings,SBertEmbeddings
 from .sentence_splitter import SentenceSplitter
-from .storage import DataFrameToVectorDB, generate_embeddings
-from .faiss_storage import BaseVectorStoreManager, FaissVectorStoreManager
+# from .storage import DataFrameToVectorDB, generate_embeddings
+from .faiss_storage import BaseVectorStoreManager, FaissVectorStoreManager,generate_embeddings
 from .chroma_storage import ChromaVectorStoreManager

ddi_fw/langchain/faiss_storage.py

@@ -390,7 +390,7 @@ def generate_embeddings(
         partial_df_size = collection_config.get('partial_dataframe_size')
         columns = collection_config.get('columns')
         page_content_columns = collection_config.get('page_content_columns')
-        persist_dir = f'{persist_directory}/{id}'
+        
 
         # Load embedding model
         try:
@@ -417,10 +417,10 @@ def generate_embeddings(
                 text_splitters_suffixes.append(suffix)
             except Exception as e:
                 raise Exception(f"Unknown text splitter: {text_splitter_type}") from e
-
+        
         for text_splitter, suffix in zip(text_splitters, text_splitters_suffixes):
             print(f"{id}_{suffix}")
-
+            persist_dir = f'{persist_directory}/{id}/{suffix}'
             # Prepare manager parameters
             manager_params = {
                 "collection_name": f"{id}_{suffix}",

ddi_fw/vectorization/feature_vector_generation.py

@@ -3,7 +3,6 @@ import numpy as np
 import pandas as pd
 from scipy.spatial.distance import pdist, squareform
 from sklearn.preprocessing import MultiLabelBinarizer
-import cupy as cp
 
 # todo pd.unique kullan
 def find_distinct_elements(frame):
@@ -97,6 +96,12 @@ class SimilarityMatrixGenerator:
         Returns:
         cp.ndarray: A 2D CuPy array containing the pairwise Jaccard similarity.
         """
+        try:
+            import cupy as cp
+        except ImportError:
+            raise ImportError("cupy is required for GPU Jaccard similarity computation. Please install it with 'pip install cupy'.")
+                
+                
         if not ((matrix == 0) | (matrix == 1)).all():
             raise ValueError("Input matrix must be binary (contain only 0s and 1s).")
         matrix = cp.asarray(matrix)

{ddi_fw-0.0.253.dist-info → ddi_fw-0.0.255.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.253
+Version: 0.0.255
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.253.dist-info → ddi_fw-0.0.255.dist-info}/RECORD

@@ -3,10 +3,10 @@ ddi_fw/datasets/core.py,sha256=4KtFYjf7uLsbto2_InY2DWNN0AUoAb8Yr0jYn0to4ME,17413
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/langchain/__init__.py,sha256=xGNaTEZCUxyc_aT1zvzVWGRfsj-9VXqMvPKtV_G7ChA,399
+ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
 ddi_fw/langchain/chroma_storage.py,sha256=7LSUhdiPdQHZvKC_NapOeVbHtS71iE5ABZVTrI0YQ-A,15520
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=e6WbjKAlCbFZUq9P3LTjTeXmWTAh6Oxp_NMVbvfK-kc,17727
+ddi_fw/langchain/faiss_storage.py,sha256=3AihARDZgdsE89trnTWodWDm-yF2NJyfLULmgJ_80Tg,17756
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -36,9 +36,9 @@ ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,
 ddi_fw/utils/utils.py,sha256=PY-zDawREKoXQfzX7lVkxBLVFQPkfvr9385kHCjaNXo,4391
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
-ddi_fw/vectorization/feature_vector_generation.py,sha256=QQQGhCti653BdU343Ag1bH_g1fzi2hlic7dgNy7otjE,7694
+ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.253.dist-info/METADATA,sha256=vYOtl4WgNa-ydlgj2dcuC2eRuFfMRlT-9OubOT3ep2U,2623
-ddi_fw-0.0.253.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.253.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.253.dist-info/RECORD,,
+ddi_fw-0.0.255.dist-info/METADATA,sha256=YiHU3_WiQ7FhmmBuGWWJCO9ZvHl84O9jbxu6XVP02_4,2623
+ddi_fw-0.0.255.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.255.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.255.dist-info/RECORD,,