ddi-fw 0.0.262__py3-none-any.whl → 0.0.264__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -94,7 +94,7 @@ class BaseDataset(BaseModel, abc.ABC):
 
             train_data,test_data = np.stack(train_data.flatten().tolist()), np.stack(test_data.flatten().tolist())
             column = self.columns[0] if self.columns else 'default'
-            items.append([f'default', np.nan_to_num(train_data),
+            items.append([f'{column}', np.nan_to_num(train_data),
                           y_train_label, np.nan_to_num(test_data), y_test_label])
         else:
             for index, column in enumerate(self.columns):

ddi_fw/langchain/faiss_storage.py

@@ -414,8 +414,9 @@ def generate_embeddings(
         # Load embedding model
         try:
             model_kwargs = collection_config.get('model_kwargs')
+            kwargs = {"model_kwargs":model_kwargs}
             model = get_import(embedding_model_type)(
-                model_name=name, **model_kwargs)
+                model_name=name, **kwargs)
         except Exception as e:
             raise Exception(f"Unknown embedding model: {embedding_model_type}") from e
 
@@ -459,4 +460,48 @@ def generate_embeddings(
             vector_store_manager.generate_vector_store(docs)
 
             # Optionally persist/save
-            vector_store_manager.save(persist_dir)
+            vector_store_manager.save(persist_dir)
+            
+            
+import os
+import json
+
+def generate_embeddings_for_json_object(
+    obj_json: dict,
+    vector_store_manager_type: Type[BaseVectorStoreManager],
+    persist_root: str = "./embeddings",
+    new_model_names: Optional[List] = None,
+    docs=None
+):
+    """
+    Generate embeddings for all collections in the given JSON object, storing them in a container folder.
+
+    Args:
+        obj_json: JSON object with 'id', 'name', and 'collections' keys.
+        vector_store_manager_type: The vector store manager class to use.
+        persist_root: Root directory for all embeddings.
+        new_model_names: Optional list of model names to filter collections.
+        docs: Documents to embed (if needed).
+    """
+    obj_id = obj_json.get("id")
+    obj_name = obj_json.get("name")
+    collections = obj_json.get("collections", [])
+
+    if not obj_id:
+        raise ValueError("JSON object must have an 'id' field.")
+    if not collections:
+        raise ValueError("No collections found in the given JSON object.")
+
+    # Create container directory for this object
+    container_dir = os.path.join(persist_root, str(obj_id))
+    os.makedirs(container_dir, exist_ok=True)
+
+    # Call your existing function
+    generate_embeddings(
+        docs=docs,
+        vector_store_manager_type=vector_store_manager_type,
+        config_file=None,
+        new_model_names=new_model_names,
+        collections=collections,
+        persist_directory=container_dir
+    )

ddi_fw/pipeline/multi_pipeline.py

@@ -149,8 +149,9 @@ class MultiPipeline():
 
         # Default model configuration
         default_model = config.get("default_model", {})
-        default_model_type = get_import(default_model.get("model_type"))
-        default_model_params = default_model.get("params", {})
+        if default_model:
+            default_model_type = get_import(default_model.get("model_type"))
+            default_model_params = default_model.get("params", {})
         
         multi_modal = config.get("multi_modal")
         

ddi_fw/utils/kaggle.py

@@ -37,6 +37,7 @@ def create_kaggle_dataset(base_path: str, collections: list):
         
         # Ensure title is between 6 and 50 characters
         if not (6 <= len(title) <= 50):
+            raise ValueError(f"Title length for {title} must be between 6 and 50 characters.")
             continue  # Skip if title length is out of the expected range
 
         # Step 3: Define the metadata content

{ddi_fw-0.0.262.dist-info → ddi_fw-0.0.264.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.262
+Version: 0.0.264
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.262.dist-info → ddi_fw-0.0.264.dist-info}/RECORD

@@ -1,12 +1,12 @@
 ddi_fw/datasets/__init__.py,sha256=NozQvXPYIS01U0srZmcKhiqJgRDkD-C-VXHL6sKrFSw,166
-ddi_fw/datasets/core.py,sha256=UnbCDoWXdxeiAb0e0anhDqXiVFGUi02VA9sKl6NVBZU,17409
+ddi_fw/datasets/core.py,sha256=Nc0OnaYi0hIYuGCdxOCyT2X4mdWK0wyVSxUw6836fKk,17410
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
 ddi_fw/langchain/__init__.py,sha256=97Y4lYuxShWqx5hfDbzf8VyV0HrM76fDlNp5xXusKQU,445
 ddi_fw/langchain/chroma_storage.py,sha256=fOxoJoaqqyOKqtfUtlq2zJd-XY03rARTDvrPE_9nY2I,15855
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=M_i1WSb66u14pIDnQ9YXTw1XNgPzsy2vchxaSzc6vY4,18669
+ddi_fw/langchain/faiss_storage.py,sha256=M-pogVtmESi_sXsBCEcTItz1-NDILllCAB41Pg54kNo,20235
 ddi_fw/langchain/sentence_splitter.py,sha256=NCcDdDWDnwZTZDqarg-5gSbcDFoAM_sxcgH9ZCu97IA,597
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -21,7 +21,7 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=jHjSfQmRQ-zEwh_5ZPdG4MBVYMrRRzlqYgFAMbDZN0g,10206
+ddi_fw/pipeline/multi_pipeline.py,sha256=ck6VhWF4dDTqeJu7Z0VYBYSxIcRUJLAYj01P6AplQgg,10241
 ddi_fw/pipeline/multi_pipeline_org.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
 ddi_fw/pipeline/ner_pipeline.py,sha256=1gBk81LeZlU1rhjJ1qBgHbFt_HqOeJ5WLnJ4AkYku4s,8188
 ddi_fw/pipeline/pipeline.py,sha256=m6pZrhoBK2lUr7PwpmJl6-WEpYcPGGc9N9C1LNJ78NQ,6974
@@ -29,7 +29,7 @@ ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/json_helper.py,sha256=BVU6wmJgdXPxyqLPu3Ck_9Es5RrP1PDanKvE-OSj1D4,571
-ddi_fw/utils/kaggle.py,sha256=ZlKS4kZVsNV_JNYO4IvqY4GkhQCvl-_5saaz_bZzgrQ,2508
+ddi_fw/utils/kaggle.py,sha256=itisQ5nffYMZz6gFYMdmbrpo2qaQvFVmLiRCC73MB1U,2604
 ddi_fw/utils/numpy_utils.py,sha256=gd1WNq5NpWD2MBEMTtFuS5I0h8B6FAUNcq6BVOlxdhY,797
 ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5I,1056
 ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.262.dist-info/METADATA,sha256=HDaqqJBBz5EFNJ8ggBj4Si6Qib0mSlih4jqdeUEnU-Q,2623
-ddi_fw-0.0.262.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.262.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.262.dist-info/RECORD,,
+ddi_fw-0.0.264.dist-info/METADATA,sha256=PKLhkkd6zsEA_YtV_4vVLh0K-pgLTzpf6IDH_ETlsek,2623
+ddi_fw-0.0.264.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.264.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.264.dist-info/RECORD,,