ddi-fw 0.0.262__tar.gz → 0.0.264__tar.gz

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.262
+Version: 0.0.264
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.262"
+version = "0.0.264"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/src/ddi_fw/datasets/core.py

@@ -94,7 +94,7 @@ class BaseDataset(BaseModel, abc.ABC):
 
             train_data,test_data = np.stack(train_data.flatten().tolist()), np.stack(test_data.flatten().tolist())
             column = self.columns[0] if self.columns else 'default'
-            items.append([f'default', np.nan_to_num(train_data),
+            items.append([f'{column}', np.nan_to_num(train_data),
                           y_train_label, np.nan_to_num(test_data), y_test_label])
         else:
             for index, column in enumerate(self.columns):

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/src/ddi_fw/langchain/faiss_storage.py

@@ -414,8 +414,9 @@ def generate_embeddings(
         # Load embedding model
         try:
             model_kwargs = collection_config.get('model_kwargs')
+            kwargs = {"model_kwargs":model_kwargs}
             model = get_import(embedding_model_type)(
-                model_name=name, **model_kwargs)
+                model_name=name, **kwargs)
         except Exception as e:
             raise Exception(f"Unknown embedding model: {embedding_model_type}") from e
 
@@ -459,4 +460,48 @@ def generate_embeddings(
             vector_store_manager.generate_vector_store(docs)
 
             # Optionally persist/save
-            vector_store_manager.save(persist_dir)
+            vector_store_manager.save(persist_dir)
+            
+            
+import os
+import json
+
+def generate_embeddings_for_json_object(
+    obj_json: dict,
+    vector_store_manager_type: Type[BaseVectorStoreManager],
+    persist_root: str = "./embeddings",
+    new_model_names: Optional[List] = None,
+    docs=None
+):
+    """
+    Generate embeddings for all collections in the given JSON object, storing them in a container folder.
+
+    Args:
+        obj_json: JSON object with 'id', 'name', and 'collections' keys.
+        vector_store_manager_type: The vector store manager class to use.
+        persist_root: Root directory for all embeddings.
+        new_model_names: Optional list of model names to filter collections.
+        docs: Documents to embed (if needed).
+    """
+    obj_id = obj_json.get("id")
+    obj_name = obj_json.get("name")
+    collections = obj_json.get("collections", [])
+
+    if not obj_id:
+        raise ValueError("JSON object must have an 'id' field.")
+    if not collections:
+        raise ValueError("No collections found in the given JSON object.")
+
+    # Create container directory for this object
+    container_dir = os.path.join(persist_root, str(obj_id))
+    os.makedirs(container_dir, exist_ok=True)
+
+    # Call your existing function
+    generate_embeddings(
+        docs=docs,
+        vector_store_manager_type=vector_store_manager_type,
+        config_file=None,
+        new_model_names=new_model_names,
+        collections=collections,
+        persist_directory=container_dir
+    )

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/src/ddi_fw/pipeline/multi_pipeline.py

@@ -149,8 +149,9 @@ class MultiPipeline():
 
         # Default model configuration
         default_model = config.get("default_model", {})
-        default_model_type = get_import(default_model.get("model_type"))
-        default_model_params = default_model.get("params", {})
+        if default_model:
+            default_model_type = get_import(default_model.get("model_type"))
+            default_model_params = default_model.get("params", {})
         
         multi_modal = config.get("multi_modal")
         

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/src/ddi_fw/utils/kaggle.py

@@ -37,6 +37,7 @@ def create_kaggle_dataset(base_path: str, collections: list):
         
         # Ensure title is between 6 and 50 characters
         if not (6 <= len(title) <= 50):
+            raise ValueError(f"Title length for {title} must be between 6 and 50 characters.")
             continue  # Skip if title length is out of the expected range
 
         # Step 3: Define the metadata content

{ddi_fw-0.0.262 → ddi_fw-0.0.264}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.262
+Version: 0.0.264
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>