ddi-fw 0.0.230__tar.gz → 0.0.232__tar.gz

{ddi_fw-0.0.230 → ddi_fw-0.0.232}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.230
+Version: 0.0.232
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.230 → ddi_fw-0.0.232}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.230"
+version = "0.0.232"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.230 → ddi_fw-0.0.232}/src/ddi_fw/pipeline/ner_pipeline.py

@@ -6,6 +6,7 @@ import mlflow
 from pydantic import BaseModel, Field, model_validator, root_validator, validator
 from ddi_fw.datasets.core import BaseDataset
 from ddi_fw.datasets.dataset_splitter import DatasetSplitter
+from ddi_fw.ml.tracking_service import TrackingService
 from ddi_fw.vectorization.idf_helper import IDF
 from ddi_fw.ner.ner import CTakesNER
 from ddi_fw.ml.ml_helper import MultiModalRunner
@@ -33,6 +34,7 @@ class NerParameterSearch(BaseModel):
     increase_step: float = 0.5
 
     # Internal fields (not part of the input)
+    _tracking_service: TrackingService | None = None
     datasets: Dict[str, Any] = Field(default_factory=dict, exclude=True)
     items: List[Any] = Field(default_factory=list, exclude=True)
     # ner_df: Optional[Any] = Field(default=None, exclude=True)
@@ -42,6 +44,10 @@ class NerParameterSearch(BaseModel):
 
     class Config:
         arbitrary_types_allowed = True
+        
+    @property
+    def tracking_service(self) -> TrackingService | None:
+        return self._tracking_service
 
     # @root_validator(pre=True)
     @model_validator(mode="before")
@@ -61,6 +67,9 @@ class NerParameterSearch(BaseModel):
         return values
 
     def build(self):
+        self._tracking_service = TrackingService(self.experiment_name,
+                                                 backend=self.tracking_library, tracking_params=self.tracking_params)
+        
         """Build the datasets and items for the parameter search."""
         if not isinstance(self.dataset_type, type):
             raise TypeError("self.dataset_type must be a class, not an instance")
@@ -96,7 +105,9 @@ class NerParameterSearch(BaseModel):
                 "entities": 0,
             }
             if self.dataset_additional_config:
-                kwargs=  self.dataset_additional_config
+                additional_config=  self.dataset_additional_config
+            else:
+                additional_config={}
 
             for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
                 if column.startswith("tui"):
@@ -105,8 +116,8 @@ class NerParameterSearch(BaseModel):
                     thresholds["cui"] = threshold
                 if column.startswith("entities"):
                     thresholds["entities"] = threshold
-                kwargs['ner']['thresholds'] = thresholds
-                
+                additional_config['ner']['thresholds'] = thresholds
+                kwargs = {'additional_config': additional_config}
                 print(f"Loading dataset for column: {column} with threshold: {threshold}")
                 # Create a new dataset instance for each threshold
                 dataset = self.dataset_type(
@@ -122,10 +133,20 @@ class NerParameterSearch(BaseModel):
                     self.datasets[item[0]] = dataset
 
                 self.items.extend(group_items)
-
-        self.y_test_label = self.items[0][4]
-        self.train_idx_arr = dataset.train_idx_arr
-        self.val_idx_arr = dataset.val_idx_arr
+                
+                # Set if y_test_label is None
+                # This ensures that y_test_label is set only once for the first dataset
+                if self.y_test_label is None:
+                    self.y_test_label = self.items[0][4]
+                    self.train_idx_arr = dataset.train_idx_arr
+                    self.val_idx_arr = dataset.val_idx_arr
+                
+                # Clear memory for the current dataset and items
+                del dataset
+                del group_items
+                import gc
+                gc.collect()
+      
 
     # def run(self):
     #     """Run the parameter search."""
@@ -147,18 +168,18 @@ class NerParameterSearch(BaseModel):
     #     result = multi_modal_runner.predict()
     #     return result
     
-        def run(self):
-            if self._tracking_service is None:
-                logging.warning("Tracking service is not initialized.")
-            else:
-                self._tracking_service.setup()
-
-            y_test_label = self.items[0][4]
-            multi_modal_runner = MultiModalRunner(
-                library=self.library, multi_modal=self.multi_modal, default_model=self.default_model, tracking_service=self._tracking_service)
-
-            multi_modal_runner.set_data(
-                self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
-            combinations = self.combinations if self.combinations is not None else []
-            result = multi_modal_runner.predict(combinations)
-            return result
+    def run(self):
+        if self._tracking_service is None:
+            logging.warning("Tracking service is not initialized.")
+        else:
+            self._tracking_service.setup()
+
+        y_test_label = self.items[0][4]
+        multi_modal_runner = MultiModalRunner(
+            library=self.library, multi_modal=self.multi_modal, default_model=self.default_model, tracking_service=self._tracking_service)
+
+        multi_modal_runner.set_data(
+            self.items, self.train_idx_arr, self.val_idx_arr, y_test_label)
+        # combinations = self.combinations if self.combinations is not None else []
+        result = multi_modal_runner.predict()
+        return result

{ddi_fw-0.0.230 → ddi_fw-0.0.232}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.230
+Version: 0.0.232
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>