ddi-fw 0.0.258__py3-none-any.whl → 0.0.260__py3-none-any.whl

ddi_fw/langchain/faiss_storage.py

@@ -1,4 +1,5 @@
 import faiss
+from langchain_text_splitters import TextSplitter
 import pandas as pd
 from uuid import uuid4
 from langchain_community.vectorstores.faiss import FAISS
@@ -45,6 +46,7 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
     persist_directory: str = Field(default="./embeddings/faiss")
     index: Any = None
     vector_store: Optional[FAISS] | None = None
+    text_splitter: Optional[TextSplitter] = None
     class Config:
         arbitrary_types_allowed = True
     # def generate_vector_store(self, docs):
@@ -75,6 +77,7 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
             dict: A dictionary with the structure {type: {drugbank_id: [embedding]}}.
         """
         self.load(self.persist_directory)
+        # df = self.as_dataframe(formatter_fn=custom_formatter)
         df = self.as_dataframe(formatter_fn=custom_formatter)
         type_dict = {}
         for drug_type, group in df.groupby('type'):
@@ -111,7 +114,9 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
 
         valid_docs = []
         valid_ids = []
-
+        if self.text_splitter:
+            docs = self.text_splitter.split_documents(docs)
+            
         for doc in docs:
             content = doc.page_content if hasattr(doc, 'page_content') else ""
             if content and content.strip():
@@ -254,7 +259,7 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
 
 def custom_formatter(document: Document, vector: np.ndarray) -> Dict[str, Any]:
     return {
-        "id": document.metadata.get("drugbank_id", None),
+        "id": document.metadata.get("id", None),
         "type": document.metadata.get("type", None),
         "embedding": vector
     }

ddi_fw/utils/kaggle.py

@@ -22,7 +22,8 @@ def create_kaggle_dataset(base_path: str, collections: list):
         # Step 2: Get metadata for the current folder
         model_info = next((c for c in collections if c['id'] == folder_name), None)
         if model_info is None:
-            continue  # Skip if model info is not found
+            raise FileNotFoundError(f"Model info for {folder_name} not exists")
+            # continue  # Skip if model info is not found
         
         title = model_info['kaggle_title']
         

{ddi_fw-0.0.258.dist-info → ddi_fw-0.0.260.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.258
+Version: 0.0.260
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.258.dist-info → ddi_fw-0.0.260.dist-info}/RECORD

@@ -6,7 +6,7 @@ ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,10
 ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
 ddi_fw/langchain/chroma_storage.py,sha256=Xh0p2d8bI9U-NYkhuu7fLtsgbExP7phkqPE17PBYWd4,15634
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=kFnvsRCE1Kdv77XgC2jLlYrAHZ9Qoz-ygs-IPB2Sg6g,18275
+ddi_fw/langchain/faiss_storage.py,sha256=5AJDXiIwIsQ4Q2xMguoFJdsNC88V-m785gwlr1XxzbQ,18537
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -29,7 +29,7 @@ ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/json_helper.py,sha256=BVU6wmJgdXPxyqLPu3Ck_9Es5RrP1PDanKvE-OSj1D4,571
-ddi_fw/utils/kaggle.py,sha256=wKRJ18KpQ6P-CubpZklEgsDtyFpR9RUL1_HyyF6ttEE,2425
+ddi_fw/utils/kaggle.py,sha256=ZlKS4kZVsNV_JNYO4IvqY4GkhQCvl-_5saaz_bZzgrQ,2508
 ddi_fw/utils/numpy_utils.py,sha256=gd1WNq5NpWD2MBEMTtFuS5I0h8B6FAUNcq6BVOlxdhY,797
 ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5I,1056
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.258.dist-info/METADATA,sha256=3r-JrOhNUwt3sUXiIj0Ua8a92c67V324tVHDxTBMM10,2623
-ddi_fw-0.0.258.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.258.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.258.dist-info/RECORD,,
+ddi_fw-0.0.260.dist-info/METADATA,sha256=06Ua7AYqn4rAGW1beCyefFO4rT5Cem-C026tg5FqTBw,2623
+ddi_fw-0.0.260.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.260.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.260.dist-info/RECORD,,