ddi-fw 0.0.100__py3-none-any.whl → 0.0.102__py3-none-any.whl

ddi_fw/langchain/storage.py

@@ -122,7 +122,7 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
         #     print(f"Configuration for collection {id} not found.")
         #     continue
 
-        embedding_model_type = collection_config['embedding_model_tpe']
+        embedding_model_type = collection_config['embedding_model_type']
         text_splitters_types = collection_config['text_splitters_types']
         batch_size = collection_config['batch_size']
         columns = collection_config['columns']
@@ -135,7 +135,7 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
         # Assuming the classes for the embeddings and splitters are available
         try:
             model = get_import(embedding_model_type)(
-                model_name=name, model_kwargs=c['model_kwargs'])
+                model_name=name, model_kwargs=collection_config['model_kwargs'])
         except:
             # print(f"Unknown embedding model: {embedding_model_type}")
             raise Exception(f"Unknown embedding model: {embedding_model_type}")

ddi_fw/utils/kaggle.py

@@ -17,13 +17,19 @@ def create_kaggle_dataset(base_path: str, collections: list):
     # Step 1: Loop through each folder in base_path
     for folder_name in os.listdir(base_path):
         folder_path = os.path.join(base_path, folder_name)
-
+        metadata_file_path = os.path.join(folder_path, 'dataset-metadata.json')
+       
         # Step 2: Get metadata for the current folder
         model_info = next((c for c in collections if c['id'] == folder_name), None)
         if model_info is None:
             continue  # Skip if model info is not found
         
         title = model_info['kaggle_title']
+        
+        if os.path.exists(metadata_file_path):
+            print(f"{title} has dataset-metadata.json file")
+            continue
+        
         id = model_info['kaggle_id'].lower().replace(' ', '-')
         licenses = model_info['kaggle_licenses']
         description = model_info['kaggle_description']
@@ -41,7 +47,6 @@ def create_kaggle_dataset(base_path: str, collections: list):
         }
 
         # Step 4: Write the metadata to a JSON file in the folder
-        metadata_file_path = os.path.join(folder_path, 'dataset-metadata.json')
         with open(metadata_file_path, 'w') as f:
             json.dump(metadata, f, indent=4)
 

{ddi_fw-0.0.100.dist-info → ddi_fw-0.0.102.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.100
+Version: 0.0.102
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.100.dist-info → ddi_fw-0.0.102.dist-info}/RECORD

@@ -58,7 +58,7 @@ ddi_fw/drugbank/event_extractor.py,sha256=6odoZohhK7OdLF-LF0l-5BFq0_NMG_5jrFJbHr
 ddi_fw/langchain/__init__.py,sha256=zS0CQrakWEP19biSRewFJGcBT8WBZq4899HrEKiMqUY,269
 ddi_fw/langchain/embeddings.py,sha256=b9BUG73Ayx3Wy8MQrfsVeZ-qBB41vjVECSp2YhH-CIY,7514
 ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
-ddi_fw/langchain/storage.py,sha256=-QRlzHsfQ7yj0OEFShRDb2A0H1iMReHiD4absxoYwGU,7415
+ddi_fw/langchain/storage.py,sha256=CDJuUSvUlUgHL8gKb3bo4RHLtI6gY8N2sNyJXg4TBJQ,7432
 ddi_fw/ml/__init__.py,sha256=tIxiW0g6q1VsmDYVXR_ovvHQR3SCir8g2bKxx_CrS7s,221
 ddi_fw/ml/evaluation_helper.py,sha256=o4-w5Xa3t4olLW4ymx_8L-Buhe5wfQEmT2bh4Zz544c,13066
 ddi_fw/ml/ml_helper.py,sha256=fySjIAFzkeEOvaLJhDwtCOgRhgYQ7H106eqaP16GhDY,4489
@@ -86,12 +86,12 @@ ddi_fw/test/torch_cuda_test.py,sha256=R-4VGVErl_Ufk54DoZbgL_YXWoCYFyanIVWd6P39IE
 ddi_fw/test/type_guarding_test.py,sha256=KxjyBxohDu7lwpejalCj-REjtJ-k1S1wQbOB6TGY0O8,766
 ddi_fw/utils/__init__.py,sha256=77563ikqAtdzjjgRlLp5OAsJBbpLA1Cao8iecGaVUXQ,354
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
-ddi_fw/utils/kaggle.py,sha256=FjWR1ncOEif6XCCzDYpErLDz_9fxAQub0L7X4aVPw24,2266
+ddi_fw/utils/kaggle.py,sha256=wKRJ18KpQ6P-CubpZklEgsDtyFpR9RUL1_HyyF6ttEE,2425
 ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5I,1056
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
 ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.100.dist-info/METADATA,sha256=TyW7tsHIuFEoXe8d2tpZ02iCE3HEGqBAKnwKIrpPmgs,1967
-ddi_fw-0.0.100.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-ddi_fw-0.0.100.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.100.dist-info/RECORD,,
+ddi_fw-0.0.102.dist-info/METADATA,sha256=f8-pfgayNvtlcMCO9E-e4Mza4QEddI417SO60XbdwMo,1967
+ddi_fw-0.0.102.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+ddi_fw-0.0.102.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.102.dist-info/RECORD,,