ddi-fw 0.0.266__py3-none-any.whl → 0.0.267__py3-none-any.whl

ddi_fw/datasets/__init__.py

@@ -1,5 +1,4 @@
-from .core import BaseDataset,TextDatasetMixin
+from .core import BaseDataset, TextDatasetMixin
 from .dataset_splitter import DatasetSplitter
-__all__  = ['BaseDataset', 'TextDatasetMixin', 'DatasetSplitter']
-
-
+from .processor import BaseInputProcessor, DefaultInputProcessor, ConcatInputProcessor
+__all__ = ['BaseDataset', 'TextDatasetMixin', 'DatasetSplitter']

ddi_fw/datasets/core.py

@@ -119,6 +119,7 @@ class BaseDataset(BaseModel, abc.ABC):
         return data
 
     # TODO columns yoksa tüm feature'lar alınıyor, bu pipeline'da nasıl yapılacak?
+    # TODO processor sınıfı kullanılsın
     def produce_inputs(self):
         # Grouping the list by "column" key
         grouped_data = defaultdict(dict)

ddi_fw/datasets/processor.py

@@ -0,0 +1,158 @@
+from typing import Optional
+import numpy as np
+
+
+class BaseInputProcessor:
+    def process1(self, data, processing_config=None):
+        raise NotImplementedError("Input processors must implement the process method.")
+    def process2(self, data, processing_config=None):
+        raise NotImplementedError("Input processors must implement the process method.")
+
+class DefaultInputProcessor(BaseInputProcessor):
+    def __init__(self):
+        pass
+
+    def process2(self, data, processing_config=None):
+        """
+        Processes input data according to the provided config.
+        Supports stacking, reshaping, and can use item_dict for advanced logic.
+        """
+        if processing_config is None:
+            raise ValueError("processing_config must be provided.")
+
+        force_stack = processing_config.get("force_stack", False)
+        reshape_dims = processing_config.get("reshape")
+        if type(data) is not list:
+        
+            # Optional: force stack single input to simulate extra dimension
+            if force_stack:
+                data = np.expand_dims(data, axis=1)
+        else:
+        # --- MULTIPLE INPUTS CASE ---
+            # Stack across inputs
+            if len(data) == 1:
+                data = data[0]
+            
+            if force_stack:
+                data = np.stack(data, axis=1)
+                
+            else:
+                data = np.array(data).T
+         
+
+        # --- OPTIONAL: Reshape if needed ---
+        if reshape_dims:
+            data = data.reshape((-1, *reshape_dims))
+        
+        return data
+
+
+    def process1(self, data, processing_config=None):
+        if not processing_config:
+            return data
+        if processing_config.get("flatten", False):
+            print("Flattening data...")
+            data = np.array(data).flatten()
+            print(f"Data shape after flattening: {data.shape}")
+        
+        if processing_config.get("stack", False):
+            print("Stacking data...")
+            data = np.stack(data)
+            print(f"Data shape after stacking: {data.shape}")
+        if not isinstance(data, np.ndarray):
+            data = np.array(data)
+        # if processing_config.get("flatten", False):
+        #     data = np.stack(data.flatten().tolist())
+        # Ensure we start with a NumPy array
+       
+
+        # Normalize input
+        if processing_config.get("normalize", False):
+            data = data.astype(np.float32)
+            max_val = np.max(data)
+            if max_val > 1:
+                data /= max_val
+
+        # Reshape input (for images etc.)
+        if "reshape" in processing_config:
+            try:
+                target_shape = tuple(processing_config["reshape"])
+                data = data.reshape((-1, *target_shape))
+            except Exception as e:
+                raise ValueError(f"Reshape failed for data with shape {data.shape}: {e}")
+
+
+        return data
+    
+    
+    
+import numpy as np
+import pandas as pd
+from sklearn.metrics.pairwise import cosine_similarity
+
+class ConcatInputProcessor(BaseInputProcessor):
+    def __init__(self, dataset, id_column, embedding_column, top_k=1):
+        self.ds = dataset  # Reference to the dataset instance
+        self.id_column = id_column
+        self.embedding_column = embedding_column
+        self.top_k = top_k
+        self.embeddings_array = None
+        self.id_list = None
+        self.id_to_idx = None
+        self.similarity_matrix = None
+        self.top_k_similar_df = None
+
+    def _prepare_embeddings(self, ids: Optional[list] = None):
+        if ids is None:
+            ids = self.ds.drugs_df[self.id_column].tolist()
+        df = pd.DataFrame.from_dict(self.ds.embedding_dict)
+        df = df[df.index.isin(ids)]
+        if self.embedding_column not in df.columns:
+            raise ValueError(f"Column '{self.embedding_column}' not found in embedding_dict.")
+        df['embeddings'] = df[self.embedding_column].apply(self.ds.pooling_strategy.apply)
+        df = df.dropna(subset=['embeddings'])
+        self.embeddings_array = np.stack(df['embeddings'].values).astype('float32')
+        self.id_list = list(df.index)
+        self.id_to_idx = {drug_id: idx for idx, drug_id in enumerate(self.id_list)}
+
+    def _compute_similarity_matrix(self):
+        self.similarity_matrix = cosine_similarity(self.embeddings_array)
+
+    def get_top_k_similar(self, top_k=None):
+        if top_k is None:
+            top_k = self.top_k
+        arr = self.similarity_matrix.copy()
+        np.fill_diagonal(arr, -np.inf)
+        top_k_idx = np.argpartition(arr, -top_k, axis=1)[:, -top_k:]
+        sorted_top_k_idx = np.argsort(arr[np.arange(arr.shape[0])[:, None], top_k_idx], axis=1)[:, ::-1]
+        final_top_k_idx = np.take_along_axis(top_k_idx, sorted_top_k_idx, axis=1)
+        top_k_ids_list = [[self.id_list[idx] for idx in row] for row in final_top_k_idx]
+        return pd.DataFrame({"drug_id": self.id_list, "top_similar_ids": top_k_ids_list}).set_index("drug_id")
+
+    def process(self, data, processing_config=None):
+        """
+        For each input vector, concatenate it with its top-k most similar vectors.
+        Assumes 'data' is a DataFrame with an id column and an embedding column.
+        """
+        # Prepare embeddings and similarity matrix if not already done
+        if self.embeddings_array is None or self.similarity_matrix is None:
+            self._prepare_embeddings()
+            self._compute_similarity_matrix()
+            self.top_k_similar_df = self.get_top_k_similar(self.top_k)
+
+        if self.top_k_similar_df is None:
+            raise ValueError("Top-k similar DataFrame not computed.")
+        # For each row in data, concatenate its embedding with its top-k similar embeddings
+        result = []
+        for idx, row in data.iterrows():
+            drug_id = row[self.id_column]
+            embedding = row[self.embedding_column]
+            similar_ids = self.top_k_similar_df.loc[drug_id, "top_similar_ids"]
+            similar_embeddings = []
+            for sim_id in similar_ids:
+                sim_idx = self.id_to_idx.get(sim_id)
+                if sim_idx is not None:
+                    similar_embeddings.append(self.embeddings_array[sim_idx])
+            concat_embedding = np.concatenate([embedding] + similar_embeddings)
+            result.append(concat_embedding)
+        return np.stack(result)

ddi_fw/ml/ml_helper.py

@@ -84,9 +84,18 @@ class MultiModalRunner:
                         "'input' should be a single string. For multiple inputs, use 'inputs'.")
 
                 # Get stacking and reshaping config
+                processor_type = m.get("processor", "ddi_fw.datasets.processor.DefaultInputProcessor")
+                processor = get_import(processor_type)  # Ensure the processor type is valid
                 force_stack = m.get("force_stack", True)
                 reshape_dims = m.get("reshape")
                 train_data, train_label, test_data, test_label = None, None, None, None
+                
+                # Prepare processing config with all context
+                processing_config = {
+                    "force_stack": force_stack,
+                    "reshape": reshape_dims
+                }
+                
                 # --- SINGLE INPUT CASE ---
                 if input:
                     item = item_dict[input]
@@ -94,11 +103,12 @@ class MultiModalRunner:
                     train_label = item[2]
                     test_data = item[3]
                     test_label = item[4]
+                   
 
-                    # Optional: force stack single input to simulate extra dimension
-                    if force_stack:
-                        train_data = np.expand_dims(train_data, axis=1)
-                        test_data = np.expand_dims(test_data, axis=1)
+                    # # Optional: force stack single input to simulate extra dimension
+                    # if force_stack:
+                    #     train_data = np.expand_dims(train_data, axis=1)
+                    #     test_data = np.expand_dims(test_data, axis=1)
 
                 # --- MULTIPLE INPUTS CASE ---
                 elif inputs:
@@ -109,33 +119,37 @@ class MultiModalRunner:
                             f"No matching inputs found in item_dict for: {inputs}")
 
                     first_input = next(iter(filtered_dict.values()))
-                    train_data_list = [f[1] for f in filtered_dict.values()]
-                    test_data_list = [f[3] for f in filtered_dict.values()]
+                    train_data = [f[1] for f in filtered_dict.values()]
+                    test_data = [f[3] for f in filtered_dict.values()]
                     train_label = first_input[2]
                     test_label = first_input[4]
 
-                    # Stack across inputs
-                    if len(train_data_list) == 1:
-                        train_data = train_data_list[0]
-                        test_data = test_data_list[0]
+                    # # Stack across inputs
+                    # if len(train_data_list) == 1:
+                    #     train_data = train_data_list[0]
+                    #     test_data = test_data_list[0]
                    
-                    if force_stack:
-                        train_data = np.stack(train_data_list, axis=1)
-                        test_data = np.stack(test_data_list, axis=1)
+                    # if force_stack:
+                    #     train_data = np.stack(train_data_list, axis=1)
+                    #     test_data = np.stack(test_data_list, axis=1)
                         
-                    else:
-                        # train_data = np.concatenate(train_data_list, axis=0)
-                        # test_data = np.concatenate(test_data_list, axis=0)
-                        train_data = np.array(train_data_list).T
-                        test_data = np.array(test_data_list).T
+                    # else:
+                    #     # train_data = np.concatenate(train_data_list, axis=0)
+                    #     # test_data = np.concatenate(test_data_list, axis=0)
+                    #     train_data = np.array(train_data_list).T
+                    #     test_data = np.array(test_data_list).T
                 else:
                     raise Exception("check configurations")
 
-                # --- OPTIONAL: Reshape if needed ---
-                if reshape_dims:
-                    train_data = train_data.reshape((-1, *reshape_dims))
-                    test_data = test_data.reshape((-1, *reshape_dims))
 
+                train_data = processor().process2(train_data, processing_config)
+                test_data = processor().process2(test_data, processing_config)
+                # # --- OPTIONAL: Reshape if needed ---
+                # if reshape_dims:
+                #     train_data = train_data.reshape((-1, *reshape_dims))
+                #     test_data = test_data.reshape((-1, *reshape_dims))
+
+           
                 # --- Finalize ---
                 single_modal.set_data(
                     self.train_idx_arr, self.val_idx_arr,

{ddi_fw-0.0.266.dist-info → ddi_fw-0.0.267.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.266
+Version: 0.0.267
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.266.dist-info → ddi_fw-0.0.267.dist-info}/RECORD

@@ -1,7 +1,8 @@
-ddi_fw/datasets/__init__.py,sha256=NozQvXPYIS01U0srZmcKhiqJgRDkD-C-VXHL6sKrFSw,166
-ddi_fw/datasets/core.py,sha256=tU31noFvOmJtCgLvfPKDYls0hNKD6T1ruk8vc5jeY_Y,15076
+ddi_fw/datasets/__init__.py,sha256=VBOLp6g2M86DXo1hgNfzz4CNpiJDHgtHqPns6cftbHc,250
+ddi_fw/datasets/core.py,sha256=g8p_lU7XOYGxjHajLPWqfWfw-NRHuludmdlvXs7d1cc,15122
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
+ddi_fw/datasets/processor.py,sha256=Cwuy7T8domLoNPphGdFgc9gL2qeDRnt_kub4i7LdXJ4,6524
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
 ddi_fw/langchain/__init__.py,sha256=97Y4lYuxShWqx5hfDbzf8VyV0HrM76fDlNp5xXusKQU,445
 ddi_fw/langchain/chroma_storage.py,sha256=fOxoJoaqqyOKqtfUtlq2zJd-XY03rARTDvrPE_9nY2I,15855
@@ -11,7 +12,7 @@ ddi_fw/langchain/sentence_splitter.py,sha256=NCcDdDWDnwZTZDqarg-5gSbcDFoAM_sxcgH
 ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
 ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
 ddi_fw/ml/evaluation_helper.py,sha256=2-7CLSgGTqLEk4HkgCVIOt-GxfLAn6SBozJghAtHb5M,11581
-ddi_fw/ml/ml_helper.py,sha256=CariGpbvLu5xK_9zXJtaI-BPQVVAHT1-JU3NyWSzjaQ,10701
+ddi_fw/ml/ml_helper.py,sha256=MO6bn0NW8sj8yc_HY5F-LZBU8XZJ57g8fOfcjfHNBkE,11377
 ddi_fw/ml/model_wrapper.py,sha256=38uBdHI4H_sjDKPWuhGXovUy_L1tpSNm5tEqCtwmlpY,973
 ddi_fw/ml/pytorch_wrapper.py,sha256=pe6UsjP2XeTgLxDnIUiodoyhJTGCxV27wD4Cjxysu2Q,8553
 ddi_fw/ml/tensorflow_wrapper.py,sha256=_mOXMpIkXx7lJySC2wtCDIDhSdtA8bQVEjKwJ5NQ7Io,16782
@@ -38,7 +39,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.266.dist-info/METADATA,sha256=2V947GjfIoq1n-26msOZT1pttLnEm7PKlFSBipjYL1I,2623
-ddi_fw-0.0.266.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ddi_fw-0.0.266.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.266.dist-info/RECORD,,
+ddi_fw-0.0.267.dist-info/METADATA,sha256=1IkpdIfCr5lRjvJ_KVWx_dnzd_2o1bAs7D74uVLR9cg,2623
+ddi_fw-0.0.267.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ddi_fw-0.0.267.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.267.dist-info/RECORD,,