ddi-fw 0.0.100__tar.gz → 0.0.102__tar.gz

{ddi_fw-0.0.100 → ddi_fw-0.0.102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.100
+Version: 0.0.102
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.100 → ddi_fw-0.0.102}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.100"
+version = "0.0.102"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.100 → ddi_fw-0.0.102}/src/ddi_fw/langchain/storage.py

@@ -122,7 +122,7 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
         #     print(f"Configuration for collection {id} not found.")
         #     continue
 
-        embedding_model_type = collection_config['embedding_model_tpe']
+        embedding_model_type = collection_config['embedding_model_type']
         text_splitters_types = collection_config['text_splitters_types']
         batch_size = collection_config['batch_size']
         columns = collection_config['columns']
@@ -135,7 +135,7 @@ def generate_embeddings(df, config_file, new_model_names, collections=None, pers
         # Assuming the classes for the embeddings and splitters are available
         try:
             model = get_import(embedding_model_type)(
-                model_name=name, model_kwargs=c['model_kwargs'])
+                model_name=name, model_kwargs=collection_config['model_kwargs'])
         except:
             # print(f"Unknown embedding model: {embedding_model_type}")
             raise Exception(f"Unknown embedding model: {embedding_model_type}")

{ddi_fw-0.0.100 → ddi_fw-0.0.102}/src/ddi_fw/utils/kaggle.py

@@ -17,13 +17,19 @@ def create_kaggle_dataset(base_path: str, collections: list):
     # Step 1: Loop through each folder in base_path
     for folder_name in os.listdir(base_path):
         folder_path = os.path.join(base_path, folder_name)
-
+        metadata_file_path = os.path.join(folder_path, 'dataset-metadata.json')
+       
         # Step 2: Get metadata for the current folder
         model_info = next((c for c in collections if c['id'] == folder_name), None)
         if model_info is None:
             continue  # Skip if model info is not found
         
         title = model_info['kaggle_title']
+        
+        if os.path.exists(metadata_file_path):
+            print(f"{title} has dataset-metadata.json file")
+            continue
+        
         id = model_info['kaggle_id'].lower().replace(' ', '-')
         licenses = model_info['kaggle_licenses']
         description = model_info['kaggle_description']
@@ -41,7 +47,6 @@ def create_kaggle_dataset(base_path: str, collections: list):
         }
 
         # Step 4: Write the metadata to a JSON file in the folder
-        metadata_file_path = os.path.join(folder_path, 'dataset-metadata.json')
         with open(metadata_file_path, 'w') as f:
             json.dump(metadata, f, indent=4)
 

{ddi_fw-0.0.100 → ddi_fw-0.0.102}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.100
+Version: 0.0.102
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>