ddi-fw 0.0.263__py3-none-any.whl → 0.0.264__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -94,7 +94,7 @@ class BaseDataset(BaseModel, abc.ABC):
 
             train_data,test_data = np.stack(train_data.flatten().tolist()), np.stack(test_data.flatten().tolist())
             column = self.columns[0] if self.columns else 'default'
-            items.append([f'default', np.nan_to_num(train_data),
+            items.append([f'{column}', np.nan_to_num(train_data),
                           y_train_label, np.nan_to_num(test_data), y_test_label])
         else:
             for index, column in enumerate(self.columns):

ddi_fw/langchain/faiss_storage.py

@@ -460,4 +460,48 @@ def generate_embeddings(
             vector_store_manager.generate_vector_store(docs)
 
             # Optionally persist/save
-            vector_store_manager.save(persist_dir)
+            vector_store_manager.save(persist_dir)
+            
+            
+import os
+import json
+
+def generate_embeddings_for_json_object(
+    obj_json: dict,
+    vector_store_manager_type: Type[BaseVectorStoreManager],
+    persist_root: str = "./embeddings",
+    new_model_names: Optional[List] = None,
+    docs=None
+):
+    """
+    Generate embeddings for all collections in the given JSON object, storing them in a container folder.
+
+    Args:
+        obj_json: JSON object with 'id', 'name', and 'collections' keys.
+        vector_store_manager_type: The vector store manager class to use.
+        persist_root: Root directory for all embeddings.
+        new_model_names: Optional list of model names to filter collections.
+        docs: Documents to embed (if needed).
+    """
+    obj_id = obj_json.get("id")
+    obj_name = obj_json.get("name")
+    collections = obj_json.get("collections", [])
+
+    if not obj_id:
+        raise ValueError("JSON object must have an 'id' field.")
+    if not collections:
+        raise ValueError("No collections found in the given JSON object.")
+
+    # Create container directory for this object
+    container_dir = os.path.join(persist_root, str(obj_id))
+    os.makedirs(container_dir, exist_ok=True)
+
+    # Call your existing function
+    generate_embeddings(
+        docs=docs,
+        vector_store_manager_type=vector_store_manager_type,
+        config_file=None,
+        new_model_names=new_model_names,
+        collections=collections,
+        persist_directory=container_dir
+    )

ddi_fw/pipeline/multi_pipeline.py

@@ -149,8 +149,9 @@ class MultiPipeline():
 
         # Default model configuration
         default_model = config.get("default_model", {})
-        default_model_type = get_import(default_model.get("model_type"))
-        default_model_params = default_model.get("params", {})
+        if default_model:
+            default_model_type = get_import(default_model.get("model_type"))
+            default_model_params = default_model.get("params", {})
         
         multi_modal = config.get("multi_modal")
         

{ddi_fw-0.0.263.dist-info → ddi_fw-0.0.264.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.263
+Version: 0.0.264
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.263.dist-info → ddi_fw-0.0.264.dist-info}/RECORD

@@ -1,12 +1,12 @@
 ddi_fw/datasets/__init__.py,sha256=NozQvXPYIS01U0srZmcKhiqJgRDkD-C-VXHL6sKrFSw,166
-ddi_fw/datasets/core.py,sha256=UnbCDoWXdxeiAb0e0anhDqXiVFGUi02VA9sKl6NVBZU,17409
+ddi_fw/datasets/core.py,sha256=Nc0OnaYi0hIYuGCdxOCyT2X4mdWK0wyVSxUw6836fKk,17410
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
 ddi_fw/langchain/__init__.py,sha256=97Y4lYuxShWqx5hfDbzf8VyV0HrM76fDlNp5xXusKQU,445
 ddi_fw/langchain/chroma_storage.py,sha256=fOxoJoaqqyOKqtfUtlq2zJd-XY03rARTDvrPE_9nY2I,15855
 ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
-ddi_fw/langchain/faiss_storage.py,sha256=Lu68vuXv3_BhGW7Jf4QDf-eyyN2ykrXTahJcueRSO7c,18715
+ddi_fw/langchain/faiss_storage.py,sha256=M-pogVtmESi_sXsBCEcTItz1-NDILllCAB41Pg54kNo,20235
 ddi_fw/langchain/sentence_splitter.py,sha256=NCcDdDWDnwZTZDqarg-5gSbcDFoAM_sxcgH9ZCu97IA,597
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -21,7 +21,7 @@ ddi_fw/ner/mmlrestclient.py,sha256=NZta7m2Qm6I_qtVguMZhqtAUjVBmmXn0-TMnsNp0jpg,6
 ddi_fw/ner/ner.py,sha256=FHyyX53Xwpdw8Hec261dyN88yD7Z9LmJua2mIrQLguI,17967
 ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,212
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
-ddi_fw/pipeline/multi_pipeline.py,sha256=jHjSfQmRQ-zEwh_5ZPdG4MBVYMrRRzlqYgFAMbDZN0g,10206
+ddi_fw/pipeline/multi_pipeline.py,sha256=ck6VhWF4dDTqeJu7Z0VYBYSxIcRUJLAYj01P6AplQgg,10241
 ddi_fw/pipeline/multi_pipeline_org.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
 ddi_fw/pipeline/ner_pipeline.py,sha256=1gBk81LeZlU1rhjJ1qBgHbFt_HqOeJ5WLnJ4AkYku4s,8188
 ddi_fw/pipeline/pipeline.py,sha256=m6pZrhoBK2lUr7PwpmJl6-WEpYcPGGc9N9C1LNJ78NQ,6974
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.263.dist-info/METADATA,sha256=CIRb9UVKCTd-6aXXLaeK4I_LR1Wn2TghfQjqJ3CiWAU,2623
-ddi_fw-0.0.263.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.263.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.263.dist-info/RECORD,,
+ddi_fw-0.0.264.dist-info/METADATA,sha256=PKLhkkd6zsEA_YtV_4vVLh0K-pgLTzpf6IDH_ETlsek,2623
+ddi_fw-0.0.264.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.264.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.264.dist-info/RECORD,,