ddi-fw 0.0.263__tar.gz → 0.0.265__tar.gz

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.263
+Version: 0.0.265
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.263"
+version = "0.0.265"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw/datasets/core.py

@@ -75,12 +75,54 @@ class BaseDataset(BaseModel, abc.ABC):
     val_idx_arr: Optional[List[np.ndarray]] = None
     columns: List[str] = []
     additional_config: Optional[Dict[str, Any]] = None
+    input_processing: Optional[List[Dict[str, Any]]] = None
 
     class Config:
         arbitrary_types_allowed = True
+        
+    def process_input_data(self,data, processing_config=None):
+        
+        if not processing_config:
+            return data
+        
+        if processing_config.get("stack", False):
+            print("Stacking data...")
+            data = np.stack(data)
+            print(f"Data shape after stacking: {data.shape}")
+        if not isinstance(data, np.ndarray):
+            data = np.array(data)
+        # if processing_config.get("flatten", False):
+        #     data = np.stack(data.flatten().tolist())
+        # Ensure we start with a NumPy array
+       
+
+        # Normalize input
+        if processing_config.get("normalize", False):
+            data = data.astype(np.float32)
+            max_val = np.max(data)
+            if max_val > 1:
+                data /= max_val
+
+        # Reshape input (for images etc.)
+        if "reshape" in processing_config:
+            try:
+                target_shape = tuple(processing_config["reshape"])
+                data = data.reshape((-1, *target_shape))
+            except Exception as e:
+                raise ValueError(f"Reshape failed for data with shape {data.shape}: {e}")
+
+
+        return data
 
     # TODO columns yoksa tüm feature'lar alınıyor, bu pipeline'da nasıl yapılacak?
     def produce_inputs(self):
+        # Grouping the list by "column" key
+        grouped_data = defaultdict(dict)
+
+        if self.input_processing:
+            for item in self.input_processing:
+                grouped_data[item["column"]] = item
+                
         items = []
         if self.X_train is None or self.X_test is None:
             raise Exception("There is no data to produce inputs")
@@ -90,40 +132,30 @@ class BaseDataset(BaseModel, abc.ABC):
         if self.columns is None or len(self.columns) == 0 or len(self.columns) == 1:
             # If no columns or only one column are provided, do not change the data
             # and use the entire dataset as a single input.
-            train_data, test_data = self.X_train[:, :], self.X_test[:, :]
-
-            train_data,test_data = np.stack(train_data.flatten().tolist()), np.stack(test_data.flatten().tolist())
             column = self.columns[0] if self.columns else 'default'
-            items.append([f'default', np.nan_to_num(train_data),
+            train_data, test_data = self.X_train[:, :], self.X_test[:, :]
+            processing_config = grouped_data[column]
+            train_data = self.process_input_data(train_data, processing_config)
+            test_data = self.process_input_data(test_data, processing_config)
+            # train_data,test_data = np.stack(train_data.flatten().tolist()), np.stack(test_data.flatten().tolist())
+            items.append([f'{column}', np.nan_to_num(train_data),
                           y_train_label, np.nan_to_num(test_data), y_test_label])
         else:
             for index, column in enumerate(self.columns):
+                processing_config = grouped_data[column]
                 train_data, test_data = self.X_train[:,
                                                      index], self.X_test[:, index]
                 #TODO üstteki satır ile alttaki tek satır olsun, tolist() ile numpy array'e çevrilmesin, numpy array zaten ama uyarı verdiği için böyle
-                train_data,test_data = np.stack(train_data.tolist()), np.stack(test_data.tolist())
+                train_data = self.process_input_data(train_data, processing_config)
+                test_data = self.process_input_data(test_data, processing_config)
+                # train_data,test_data = np.stack(train_data.tolist()), np.stack(test_data.tolist())
                 items.append([f'{column}', np.nan_to_num(train_data),
                               y_train_label, np.nan_to_num(test_data), y_test_label])
 
                 # items.append([f'{column}_embedding', train_data,
                 #             y_train_label, test_data, y_test_label])
         return items
-
-    def produce_inputs_ex(self):
-        items = []
-        if self.X_train is None or self.X_test is None:
-            raise Exception("There is no data to produce inputs")
-        y_train_label, y_test_label = stack(self.y_train), stack(self.y_test)
-
-        for column in self.columns:
-            train_data, test_data = stack(
-                self.X_train[column]), stack(self.X_test[column])
-            items.append([f'{column}', np.nan_to_num(train_data),
-                          y_train_label, np.nan_to_num(test_data), y_test_label])
-
-            # items.append([f'{column}_embedding', train_data,
-            #             y_train_label, test_data, y_test_label])
-        return items
+ 
 
     @computed_field
     @property
@@ -294,72 +326,7 @@ class TextDatasetMixin(BaseModel):
 
     class Config:
         arbitrary_types_allowed = True
-
-    # def __create_or_update_embeddings__(self, embedding_dict, vector_db_persist_directory, vector_db_collection_name, column=None):
-    #     """
-    #     Fetch embeddings and metadata from a persistent Chroma vector database and update the provided embedding_dict.
-
-    #     Args:
-    #     - vector_db_persist_directory (str): The path to the directory where the Chroma vector database is stored.
-    #     - vector_db_collection_name (str): The name of the collection to query.
-    #     - embedding_dict (dict): The existing dictionary to update with embeddings.
-
-    #     """
-    #     if vector_db_persist_directory:
-    #         # Initialize the Chroma client and get the collection
-    #         vector_db = chromadb.PersistentClient(
-    #             path=vector_db_persist_directory)
-    #         collection = vector_db.get_collection(vector_db_collection_name)
-    #         # include = [IncludeEnum.embeddings, IncludeEnum.metadatas]
-    #         include: chromadb.Include = ["embeddings","metadatas"]
-    #         dictionary: chromadb.GetResult
-    #         # Fetch the embeddings and metadata
-    #         if column == None:
-    #             dictionary = collection.get(
-    #                 include=include
-    #                 # include=['embeddings', 'metadatas']
-    #             )
-    #             print(
-    #                 f"Embeddings are calculated from {vector_db_collection_name}")
-    #         else:
-    #             dictionary = collection.get(
-    #                 include=include,
-    #                 # include=['embeddings', 'metadatas'],
-    #                 where={
-    #                     "type": {"$eq": f"{column}"}})
-    #             print(
-    #                 f"Embeddings of {column} are calculated from {vector_db_collection_name}")
-
-    #         # Populate the embedding dictionary with embeddings from the vector database
-    #         metadatas = dictionary["metadatas"]
-    #         embeddings = dictionary["embeddings"]
-    #         if metadatas is None or embeddings is None:
-    #             raise ValueError(
-    #                 "The collection does not contain embeddings or metadatas.")
-    #         for metadata, embedding in zip(metadatas, embeddings):
-    #             embedding_dict[metadata["type"]
-    #                            ][metadata["id"]].append(embedding)
-
-    #     else:
-    #         raise ValueError(
-    #             "Persistent directory for the vector DB is not specified.")
-            
-    # def __initialize_embedding_dict(self):
-    #     embedding_dict = defaultdict(lambda: defaultdict(list))
-    #     if self.column_embedding_configs:
-    #         for item in self.column_embedding_configs:
-    #             col = item["column"]
-    #             col_db_dir = item["vector_db_persist_directory"]
-    #             col_db_collection = item["vector_db_collection_name"]
-    #             self.__create_or_update_embeddings__(embedding_dict, col_db_dir, col_db_collection, col)
-    #     elif self.vector_db_persist_directory:
-    #         self.__create_or_update_embeddings__(embedding_dict, self.vector_db_persist_directory, self.vector_db_collection_name)
-    #     else:
-    #         logging.warning("There is no configuration of Embeddings")
-    #         raise ValueError(
-    #             "There is no configuration of Embeddings. Please provide a vector database directory and collection name.")
-    #     return embedding_dict
-
+ 
     def __calculate_embedding_size(self):
         if not self.embedding_dict:
             raise ValueError("Embedding dictionary is not initialized, embedding size cannot be calculated.")
@@ -370,15 +337,12 @@ class TextDatasetMixin(BaseModel):
     def process_text(self):
         logging.info("Processing text data...")
  
-        # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
-        # kwargs = {"columns": self.columns}
-        # if self.ner_threshold:
-        #     for k, v in self.ner_threshold.items():
-        #         kwargs[k] = v
+      
         if not self.embedding_dict:
             if self.vector_store_manager is not None:
                 self.embedding_dict = self.vector_store_manager.initialize_embedding_dict()
-            # self.embedding_dict = self.__initialize_embedding_dict()    
+            else:
+                raise ValueError("Either embedding_dict or vector_store_manager must be provided for text processing.")
         self.__calculate_embedding_size()
          
 

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw/langchain/faiss_storage.py

@@ -460,4 +460,48 @@ def generate_embeddings(
             vector_store_manager.generate_vector_store(docs)
 
             # Optionally persist/save
-            vector_store_manager.save(persist_dir)
+            vector_store_manager.save(persist_dir)
+            
+            
+import os
+import json
+
+def generate_embeddings_for_json_object(
+    obj_json: dict,
+    vector_store_manager_type: Type[BaseVectorStoreManager],
+    persist_root: str = "./embeddings",
+    new_model_names: Optional[List] = None,
+    docs=None
+):
+    """
+    Generate embeddings for all collections in the given JSON object, storing them in a container folder.
+
+    Args:
+        obj_json: JSON object with 'id', 'name', and 'collections' keys.
+        vector_store_manager_type: The vector store manager class to use.
+        persist_root: Root directory for all embeddings.
+        new_model_names: Optional list of model names to filter collections.
+        docs: Documents to embed (if needed).
+    """
+    obj_id = obj_json.get("id")
+    obj_name = obj_json.get("name")
+    collections = obj_json.get("collections", [])
+
+    if not obj_id:
+        raise ValueError("JSON object must have an 'id' field.")
+    if not collections:
+        raise ValueError("No collections found in the given JSON object.")
+
+    # Create container directory for this object
+    container_dir = os.path.join(persist_root, str(obj_id))
+    os.makedirs(container_dir, exist_ok=True)
+
+    # Call your existing function
+    generate_embeddings(
+        docs=docs,
+        vector_store_manager_type=vector_store_manager_type,
+        config_file=None,
+        new_model_names=new_model_names,
+        collections=collections,
+        persist_directory=container_dir
+    )

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw/ml/ml_helper.py

@@ -3,7 +3,7 @@ from ddi_fw.ml.pytorch_wrapper import PTModelWrapper
 from ddi_fw.ml.tensorflow_wrapper import TFModelWrapper
 from ddi_fw.utils.package_helper import get_import
 import numpy as np
-from ddi_fw.ml.evaluation_helper import  evaluate
+from ddi_fw.ml.evaluation_helper import evaluate
 
 # import tf2onnx
 # import onnx
@@ -48,7 +48,8 @@ class MultiModalRunner:
     def __predict(self, single_results):
         item_dict = {t[0]: t for t in self.items}
         if self.default_model is None and not self.multi_modal:
-            raise Exception("Default model and multi modal cannot be None at the same time")
+            raise Exception(
+                "Default model and multi modal cannot be None at the same time")
 
         if self.multi_modal:
             for m in self.multi_modal:
@@ -56,49 +57,113 @@ class MultiModalRunner:
                 # input_type = m.get('input_type')
                 input = m.get('input')
                 inputs = m.get('inputs')
-                model_type = get_import(m.get("model_type"))
-                kwargs = m.get('params')
+                if m.get("model_type") is None:
+                    model_type = self.default_model.get("model_type")
+                    kwargs = self.default_model.get('params')
+                else:
+                    model_type = get_import(m.get("model_type"))
+                    kwargs = m.get('params')
+
+                if model_type is None:
+                    raise Exception(
+                        "model_type cannot be None, it should be defined in multi_modal or default_model")
+
                 T = self.__create_model(self.library)
                 single_modal = T(self.date, name, model_type,
-                                tracking_service=self.tracking_service,  **kwargs)
-                
-                if input is not None and inputs is not None:
-                    raise Exception("input and inputs should not be used together")
-                
+                                 tracking_service=self.tracking_service,  **kwargs)
+
+                if input and inputs:
+                    raise ValueError(
+                        "Only one of 'input' or 'inputs' should be defined.")
+                if not input and not inputs:
+                    raise ValueError(
+                        "At least one of 'input' or 'inputs' must be defined.")
+                    
+                if input and not isinstance(input, str):
+                    raise ValueError(
+                        "'input' should be a single string. For multiple inputs, use 'inputs'.")
+
+                # Get stacking and reshaping config
+                force_stack = m.get("force_stack", True)
+                reshape_dims = m.get("reshape")
+                train_data, train_label, test_data, test_label = None, None, None, None
+                # --- SINGLE INPUT CASE ---
                 if input:
                     item = item_dict[input]
-                    single_modal.set_data(
-                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+                    train_data = item[1]
+                    train_label = item[2]
+                    test_data = item[3]
+                    test_label = item[4]
+
+                    # Optional: force stack single input to simulate extra dimension
+                    if force_stack:
+                        train_data = np.expand_dims(train_data, axis=1)
+                        test_data = np.expand_dims(test_data, axis=1)
+
+                # --- MULTIPLE INPUTS CASE ---
                 elif inputs:
-                    # check keys
                     filtered_dict = {k: item_dict[k]
-                                    for k in inputs if k in item_dict}
-                    print(filtered_dict.keys())
+                                     for k in inputs if k in item_dict}
+                    if not filtered_dict:
+                        raise ValueError(
+                            f"No matching inputs found in item_dict for: {inputs}")
+
                     first_input = next(iter(filtered_dict.values()))
                     train_data_list = [f[1] for f in filtered_dict.values()]
                     test_data_list = [f[3] for f in filtered_dict.values()]
-                    train_data = np.stack(train_data_list, axis=1)
-                    test_data = np.stack(test_data_list, axis=1)
                     train_label = first_input[2]
                     test_label = first_input[4]
-                    single_modal.set_data(
-                        self.train_idx_arr, self.val_idx_arr, train_data, train_label, test_data, test_label)
+
+                    # Stack across inputs
+                    if len(train_data_list) == 1:
+                        train_data = train_data_list[0]
+                        test_data = test_data_list[0]
+                   
+                    if force_stack:
+                        train_data = np.stack(train_data_list, axis=1)
+                        test_data = np.stack(test_data_list, axis=1)
+                        
+                    else:
+                        # train_data = np.concatenate(train_data_list, axis=0)
+                        # test_data = np.concatenate(test_data_list, axis=0)
+                        train_data = np.array(train_data_list).T
+                        test_data = np.array(test_data_list).T
                 else:
                     raise Exception("check configurations")
+
+                # --- OPTIONAL: Reshape if needed ---
+                if reshape_dims:
+                    train_data = train_data.reshape((-1, *reshape_dims))
+                    test_data = test_data.reshape((-1, *reshape_dims))
+
+                # --- Finalize ---
+                single_modal.set_data(
+                    self.train_idx_arr, self.val_idx_arr,
+                    train_data, train_label,
+                    test_data, test_label
+                )
+
                 logs, metrics, prediction = single_modal.fit_and_evaluate()
                 self.result.add_metric(name, metrics)
                 single_results[name] = prediction
-        else: # TODO default model maybe?
+        else:  # TODO default model maybe?
             print("Default model will be used")
+            if self.default_model is None:
+                raise Exception(
+                    "Default model cannot be None if multi_modal is not defined")
+            if self.default_model.get("model_type") is None:
+                raise Exception(
+                    "model_type cannot be None, it should be defined in default_model")
+
             model_type = get_import(self.default_model.get("model_type"))
             kwargs = self.default_model.get('params')
             for item in self.items:
                 name = item[0]
                 T = self.__create_model(self.library)
                 single_modal = T(self.date, name, model_type,
-                                tracking_service=self.tracking_service,  **kwargs)
+                                 tracking_service=self.tracking_service,  **kwargs)
                 single_modal.set_data(
-                        self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
+                    self.train_idx_arr, self.val_idx_arr, item[1], item[2], item[3], item[4])
 
                 logs, metrics, prediction = single_modal.fit_and_evaluate()
                 self.result.add_metric(name, metrics)
@@ -116,14 +181,15 @@ class MultiModalRunner:
             combinations = []
             for i in range(2, len(l) + 1):
                 combinations.extend(list(itertools.combinations(l, i)))  # all
-                
+
         def _f():
             self.__predict(single_results)
             if combinations:
                 self.evaluate_combinations(single_results, combinations)
-            
+
         if self.tracking_service:
-            self.tracking_service.run(run_name=self.prefix, description="***", func = _f , nested_run=False)
+            self.tracking_service.run(
+                run_name=self.prefix, description="***", func=_f, nested_run=False)
         else:
             self.__predict(single_results)
             if combinations:
@@ -138,13 +204,13 @@ class MultiModalRunner:
                 def evaluate_combination(artifact_uri=None):
                     self.__evaluate_combinations(
                         single_results, combination, combination_descriptor, artifact_uri
-                )
-                 
-                self.tracking_service.run(run_name=combination_descriptor, 
-                                          description="***", 
-                                          nested_run=True, 
+                    )
+
+                self.tracking_service.run(run_name=combination_descriptor,
+                                          description="***",
+                                          nested_run=True,
                                           func=evaluate_combination)
-                
+
                 # with mlflow.start_run(run_name=combination_descriptor, description="***", nested=True) as combination_run:
                 #     self.__evaluate_combinations(
                 #         single_results, combination, combination_descriptor, combination_run.info.artifact_uri)

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw/pipeline/multi_pipeline.py

@@ -126,6 +126,7 @@ class MultiPipeline():
         dataset_splitter_type = get_import(dataset_config.get("dataset_splitter_type"))
         columns = dataset_config.get("columns", [])
         additional_config = dataset_config.get("additional_config", {})
+        input_processing = dataset_config.get("input_processing", [])
         
          # Vector database configuration
         vector_database = config.get("vector_databases", {})
@@ -149,18 +150,7 @@ class MultiPipeline():
 
         # Default model configuration
         default_model = config.get("default_model", {})
-        default_model_type = get_import(default_model.get("model_type"))
-        default_model_params = default_model.get("params", {})
-        
         multi_modal = config.get("multi_modal")
-        
-
-
-        #ner move it to related dataset
-        
-        # ner_data_file = config.get("ner_data_file")
-        # ner_threshold = config.get("ner_threshold")
-      
 
         combination_type = None
         kwargs_combination_params=None
@@ -184,6 +174,7 @@ class MultiPipeline():
                 dataset_additional_config=additional_config,
                 dataset_splitter_type=dataset_splitter_type,
                 columns=columns,
+                input_processing=input_processing,
                 vector_store_manager_type=vector_store_manager_type,
                 column_embedding_configs=column_embedding_configs,
                 vector_db_persist_directory=vector_db_persist_directory,

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw/pipeline/pipeline.py

@@ -22,6 +22,7 @@ class Pipeline(BaseModel):
     dataset_additional_config: Optional[Dict[str, Any]] = None
     dataset_splitter_type: Type[DatasetSplitter] = DatasetSplitter
     columns: Optional[List[str]] = None
+    input_processing: Optional[List[Dict[str, Any]]] = None
     embedding_dict: Optional[Dict[str, Any]] = None
     column_embedding_configs: Optional[List] = None
     vector_db_persist_directory: Optional[str] = None
@@ -76,13 +77,6 @@ class Pipeline(BaseModel):
         # 'enzyme','target','pathway','smile','all_text','indication', 'description','mechanism_of_action','pharmacodynamics', 'tui', 'cui', 'entities'
         kwargs = {"columns": self.columns,
                   "additional_config": self.dataset_additional_config}
-        # DDIMDL Dataset'e aktar
-        # if self.ner_threshold:
-        #     for k, v in self.ner_threshold.items():
-        #         kwargs[k] = v
-
-        # ner_df = CTakesNER(df=None).load(
-        #     filename=self.ner_data_file) if self.ner_data_file else None
 
         dataset_splitter = self.dataset_splitter_type()
         pooling_strategy = self.embedding_pooling_strategy_type(
@@ -97,12 +91,12 @@ class Pipeline(BaseModel):
         if self.vector_db_collection_name is not None:
             params["collection_name"] = self.vector_db_collection_name
 
-
-        vector_store_manager = self.vector_store_manager_type(**params) if self.vector_store_manager_type else None
+        vector_store_manager = self.vector_store_manager_type(
+            **params) if self.vector_store_manager_type else None
         if issubclass(self.dataset_type, TextDatasetMixin):
-            
+
             dataset = self.dataset_type(
-                vector_store_manager = vector_store_manager,
+                vector_store_manager=vector_store_manager,
                 embedding_dict=self.embedding_dict,
                 pooling_strategy=pooling_strategy,
                 column_embedding_configs=self.column_embedding_configs,
@@ -111,17 +105,20 @@ class Pipeline(BaseModel):
                 dataset_splitter_type=self.dataset_splitter_type,
                 **kwargs)
 
-        elif self.dataset_type == BaseDataset:
+        elif self.dataset_type == BaseDataset: ## !!! check it
             dataset = self.dataset_type(
                 dataset_splitter_type=self.dataset_splitter_type,
                 **kwargs)
         else:
-            dataset = self.dataset_type(**kwargs)
+            dataset = self.dataset_type(
+                dataset_splitter_type=self.dataset_splitter_type, **kwargs)
+            
+        dataset.input_processing = self.input_processing
 
         # X_train, X_test, y_train, y_test, train_indexes, test_indexes, train_idx_arr, val_idx_arr = dataset.load()
 
         dataset.load()
-        
+
         self._dataset = dataset
         self._train_idx_arr = dataset.train_idx_arr
         self._val_idx_arr = dataset.val_idx_arr

{ddi_fw-0.0.263 → ddi_fw-0.0.265}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.263
+Version: 0.0.265
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>