ddi-fw 0.0.231__py3-none-any.whl → 0.0.232__py3-none-any.whl

ddi_fw/pipeline/ner_pipeline.py

@@ -6,6 +6,7 @@ import mlflow
 from pydantic import BaseModel, Field, model_validator, root_validator, validator
 from ddi_fw.datasets.core import BaseDataset
 from ddi_fw.datasets.dataset_splitter import DatasetSplitter
+from ddi_fw.ml.tracking_service import TrackingService
 from ddi_fw.vectorization.idf_helper import IDF
 from ddi_fw.ner.ner import CTakesNER
 from ddi_fw.ml.ml_helper import MultiModalRunner
@@ -33,6 +34,7 @@ class NerParameterSearch(BaseModel):
     increase_step: float = 0.5
 
     # Internal fields (not part of the input)
+    _tracking_service: TrackingService | None = None
     datasets: Dict[str, Any] = Field(default_factory=dict, exclude=True)
     items: List[Any] = Field(default_factory=list, exclude=True)
     # ner_df: Optional[Any] = Field(default=None, exclude=True)
@@ -42,6 +44,10 @@ class NerParameterSearch(BaseModel):
 
     class Config:
         arbitrary_types_allowed = True
+        
+    @property
+    def tracking_service(self) -> TrackingService | None:
+        return self._tracking_service
 
     # @root_validator(pre=True)
     @model_validator(mode="before")
@@ -61,6 +67,9 @@ class NerParameterSearch(BaseModel):
         return values
 
     def build(self):
+        self._tracking_service = TrackingService(self.experiment_name,
+                                                 backend=self.tracking_library, tracking_params=self.tracking_params)
+        
         """Build the datasets and items for the parameter search."""
         if not isinstance(self.dataset_type, type):
             raise TypeError("self.dataset_type must be a class, not an instance")
@@ -124,10 +133,20 @@ class NerParameterSearch(BaseModel):
                     self.datasets[item[0]] = dataset
 
                 self.items.extend(group_items)
-
-        self.y_test_label = self.items[0][4]
-        self.train_idx_arr = dataset.train_idx_arr
-        self.val_idx_arr = dataset.val_idx_arr
+                
+                # Set if y_test_label is None
+                # This ensures that y_test_label is set only once for the first dataset
+                if self.y_test_label is None:
+                    self.y_test_label = self.items[0][4]
+                    self.train_idx_arr = dataset.train_idx_arr
+                    self.val_idx_arr = dataset.val_idx_arr
+                
+                # Clear memory for the current dataset and items
+                del dataset
+                del group_items
+                import gc
+                gc.collect()
+      
 
     # def run(self):
     #     """Run the parameter search."""
@@ -149,18 +168,18 @@ class NerParameterSearch(BaseModel):
     #     result = multi_modal_runner.predict()
     #     return result
     
-        def run(self):
-            if self._tracking_service is None:
-                logging.warning("Tracking service is not initialized.")
-            else:
-                self._tracking_service.setup()
-
-            y_test_label = self.items[0][4]
-            multi_modal_runner = MultiModalRunner(
-                library=self.library, multi_modal=self.multi_modal, default_model=self.default_model, tracking_service=self._tracking_service)
-
-            multi_modal_runner.set_data(
-                self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
-            combinations = self.combinations if self.combinations is not None else []
-            result = multi_modal_runner.predict(combinations)
-            return result
+    def run(self):
+        if self._tracking_service is None:
+            logging.warning("Tracking service is not initialized.")
+        else:
+            self._tracking_service.setup()
+
+        y_test_label = self.items[0][4]
+        multi_modal_runner = MultiModalRunner(
+            library=self.library, multi_modal=self.multi_modal, default_model=self.default_model, tracking_service=self._tracking_service)
+
+        multi_modal_runner.set_data(
+            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+        # combinations = self.combinations if self.combinations is not None else []
+        result = multi_modal_runner.predict()
+        return result

{ddi_fw-0.0.231.dist-info → ddi_fw-0.0.232.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.231
+Version: 0.0.232
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.231.dist-info → ddi_fw-0.0.232.dist-info}/RECORD

@@ -86,7 +86,7 @@ ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,2
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=EjJnA3Vzd-WeEvUBaA2LDOy_iQ5-2eW2VhtxvvxDPfQ,9857
 ddi_fw/pipeline/multi_pipeline_org.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
-ddi_fw/pipeline/ner_pipeline.py,sha256=EhM9A4AarpLASDMf4TDUCYFOESB-V3jEj77KskTqjXw,7368
+ddi_fw/pipeline/ner_pipeline.py,sha256=BycxZvI7JRJ3s3HhYAgOxG2_lqrVnhv7ECOWSgVQhz4,8186
 ddi_fw/pipeline/pipeline.py,sha256=q1kMkW9-fOlrA4BOGUku40U_PuEYfcbtH2EvlRM4uTM,6243
 ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
@@ -101,7 +101,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
 ddi_fw/vectorization/feature_vector_generation.py,sha256=93G3QM28uoNlvlVz_BhV6ARxldpogiNJStxHdsgqTbU,6026
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.231.dist-info/METADATA,sha256=Cn7cy18IY6LxKcWxUf4Iad7ipvVZBQ0ZF5dwJnXwA08,2632
-ddi_fw-0.0.231.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-ddi_fw-0.0.231.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.231.dist-info/RECORD,,
+ddi_fw-0.0.232.dist-info/METADATA,sha256=CBSE9xsWEc0vxlCxw9NCLxTJWXmXoFTegN5wgrHVGvA,2632
+ddi_fw-0.0.232.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+ddi_fw-0.0.232.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.232.dist-info/RECORD,,