autogluon.multimodal 1.1.2b20241106__tar.gz → 1.1.2b20241108__tar.gz

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.multimodal
-Version: 1.1.2b20241106
+Version: 1.1.2b20241108
 Summary: Fast and Accurate ML in 3 Lines of Code
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/src/autogluon/multimodal/configs/pretrain/detection/dino/dino-4scale_r50_8xb2-12e_coco.py

@@ -84,7 +84,7 @@ model = dict(
 # train_pipeline, NOTE the img_scale and the Pad's size_divisor is different
 # from the default setting in mmdet.
 train_pipeline = [
-    dict(type="LoadImageFromFile", backend_args={{_base_.backend_args}}),
+    dict(type="LoadImageFromFile", backend_args={{_base_.backend_args}}),  # nosec
     dict(type="LoadAnnotations", with_bbox=True),
     dict(type="RandomFlip", prob=0.5),
     dict(

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/src/autogluon/multimodal/learners/object_detection.py

@@ -15,12 +15,13 @@ from ..utils import (
     check_if_packages_installed,
     cocoeval,
     convert_pred_to_xywh,
+    convert_result_df,
     create_fusion_model,
     extract_from_output,
     from_coco_or_voc,
     get_detection_classes,
     object_detection_data_to_df,
-    save_result_df,
+    save_result_coco_format,
     setup_save_path,
     split_train_tuning_data,
 )
@@ -39,8 +40,9 @@ class ObjectDetectionLearner(BaseLearner):
         hyperparameters: Optional[dict] = None,
         path: Optional[str] = None,
         verbosity: Optional[int] = 2,
-        num_classes: Optional[int] = None,  # TODO: can we infer this from train/predict data?
-        classes: Optional[list] = None,  # TODO: can we infer this from train/predict data?
+        num_classes: Optional[int] = None,
+        classes: Optional[list] = None,
+        category_ids: Optional[list] = None,
         warn_if_exist: Optional[bool] = True,
         enable_progress_bar: Optional[bool] = None,
         pretrained: Optional[bool] = True,
@@ -74,14 +76,17 @@ class ObjectDetectionLearner(BaseLearner):
         )
         check_if_packages_installed(problem_type=self._problem_type)
 
+        self._config = self.get_config_per_run(config=self._config, hyperparameters=hyperparameters)
+
         self._output_shape = num_classes
         self._classes = classes
+        self._category_ids = category_ids
         self._sample_data_path = sample_data_path
 
         # TODO: merge object detection and open vocabulary object detection
         self._label_column = "label"
         if self._sample_data_path is not None:
-            self._classes = get_detection_classes(self._sample_data_path)
+            self._classes, self._category_ids = get_detection_classes(self._sample_data_path)
             self._output_shape = len(self._classes)
 
         # TODO: merge _detection_anno_train and detection_anno_train?
@@ -97,15 +102,32 @@ class ObjectDetectionLearner(BaseLearner):
         """
         Return the classes of object detection.
         """
-        return self._model.model.CLASSES
+        if self._model.model.CLASSES is not None and self._classes is not None:
+            assert self._classes == self._model.model.CLASSES, f"{self._classes}\n{self._model.model.CLASSES}"
+        return self._classes if self._classes is not None else self._model.model.CLASSES
+
+    @property
+    def category_ids(self):
+        """
+        Return the classes of object detection.
+        """
+        return self._category_ids
 
     def setup_detection_train_tuning_data(self, max_num_tuning_data, seed, train_data, tuning_data):
         if isinstance(train_data, str):
             self._detection_anno_train = train_data
-            train_data = from_coco_or_voc(train_data, "train")  # TODO: Refactor to use convert_data_to_df
+            train_data = from_coco_or_voc(
+                train_data,
+                "train",
+                coco_root=self._config.model.mmdet_image.coco_root,
+            )  # TODO: Refactor to use convert_data_to_df
             if tuning_data is not None:
                 self.detection_anno_train = tuning_data
-                tuning_data = from_coco_or_voc(tuning_data, "val")  # TODO: Refactor to use convert_data_to_df
+                tuning_data = from_coco_or_voc(
+                    tuning_data,
+                    "val",
+                    coco_root=self._config.model.mmdet_image.coco_root,
+                )  # TODO: Refactor to use convert_data_to_df
                 if max_num_tuning_data is not None:
                     if len(tuning_data) > max_num_tuning_data:
                         tuning_data = tuning_data.sample(
@@ -114,10 +136,16 @@ class ObjectDetectionLearner(BaseLearner):
         elif isinstance(train_data, pd.DataFrame):
             self._detection_anno_train = None
             # sanity check dataframe columns
-            train_data = object_detection_data_to_df(train_data)
+            train_data = object_detection_data_to_df(
+                train_data,
+                coco_root=self._config.model.mmdet_image.coco_root,
+            )
             if tuning_data is not None:
                 self.detection_anno_train = tuning_data
-                tuning_data = object_detection_data_to_df(tuning_data)
+                tuning_data = object_detection_data_to_df(
+                    tuning_data,
+                    coco_root=self._config.model.mmdet_image.coco_root,
+                )
                 if max_num_tuning_data is not None:
                     if len(tuning_data) > max_num_tuning_data:
                         tuning_data = tuning_data.sample(
@@ -556,7 +584,9 @@ class ObjectDetectionLearner(BaseLearner):
         if isinstance(anno_file_or_df, str):
             anno_file = anno_file_or_df
             data = from_coco_or_voc(
-                anno_file, "test"
+                anno_file,
+                "test",
+                coco_root=self._config.model.mmdet_image.coco_root,
             )  # TODO: maybe remove default splits hardcoding (only used in VOC)
             if os.path.isdir(anno_file):
                 eval_tool = "torchmetrics"  # we can only use torchmetrics for VOC format evaluation.
@@ -636,7 +666,10 @@ class ObjectDetectionLearner(BaseLearner):
                 eval_tool=eval_tool,
             )
         else:
-            data = object_detection_data_to_df(data)
+            data = object_detection_data_to_df(
+                data,
+                coco_root=self._config.model.mmdet_image.coco_root,
+            )
             return self.evaluate_coco(
                 anno_file_or_df=data,
                 metrics=metrics,
@@ -648,6 +681,7 @@ class ObjectDetectionLearner(BaseLearner):
         self,
         data: Union[pd.DataFrame, dict, list, str],
         as_pandas: Optional[bool] = None,
+        as_coco: Optional[bool] = True,
         realtime: Optional[bool] = False,
         save_results: Optional[bool] = None,
         **kwargs,
@@ -662,6 +696,8 @@ class ObjectDetectionLearner(BaseLearner):
             follow same format (except for the `label` column).
         as_pandas
             Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).
+        as_coco
+            Whether to save the output as a COCO json file (True) or pandas DataFrame (False).
         realtime
             Whether to do realtime inference, which is efficient for small data (default False).
             If provided None, we would infer it on based on the data modalities
@@ -674,46 +710,65 @@ class ObjectDetectionLearner(BaseLearner):
         Array of predictions, one corresponding to each row in given dataset.
         """
         self.ensure_predict_ready()
+        if as_pandas is None and isinstance(data, pd.DataFrame):
+            as_pandas = True
+
         ret_type = BBOX
+
+        # only supports coco/voc format for OBJECT_DETECTION
         if self._problem_type == OBJECT_DETECTION:
-            data = object_detection_data_to_df(data)
-            if self._label_column not in data:
+            data_path = data
+            data_df = object_detection_data_to_df(
+                data_path,
+                coco_root=self._config.model.mmdet_image.coco_root,
+            )
+            if self._label_column not in data_df:
                 self._label_column = None
 
         outputs = self.predict_per_run(
-            data=data,
+            data=data_df,
             realtime=realtime,
             requires_label=False,
         )
         pred = extract_from_output(outputs=outputs, ret_type=ret_type)
-        if self._problem_type == OBJECT_DETECTION:
-            if self._model.output_bbox_format == XYWH:
-                pred = convert_pred_to_xywh(pred)
 
-        if save_results and self._problem_type == OBJECT_DETECTION:
+        self._save_path = setup_save_path(
+            old_save_path=self._save_path,
+            warn_if_exist=False,
+        )
+        result_path = os.path.join(self._save_path, "result.txt")
+
+        pred_df = convert_result_df(
+            pred=convert_pred_to_xywh(pred) if self._model.output_bbox_format == XYWH else pred,
+            data=data_df,
+            detection_classes=self.classes,
+            result_path=result_path,
+        )
+
+        if save_results:
             self._save_path = setup_save_path(
                 old_save_path=self._save_path,
                 warn_if_exist=False,
             )
-            save_result_df(
-                pred=pred,
-                data=data,
-                detection_classes=self._model.model.CLASSES,
-                result_path=os.path.join(self._save_path, "result.txt"),
-            )
-
-        if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:
-            # TODO: add prediction output in COCO format if as_pandas is False
-            # TODO: calling save_result_df to convert data to dataframe is not a good logic
-            # TODO: consider combining this with the above saving logic or using a different function.
-            pred = save_result_df(
-                pred=pred,
-                data=data,
-                detection_classes=self._model.model.CLASSES,
-                result_path=None,
-            )
+            if as_coco:
+                result_path = os.path.join(self._save_path, "result.json")
+                save_result_coco_format(
+                    data_path=data_path,
+                    pred=pred,
+                    category_ids=self.category_ids,
+                    result_path=result_path,
+                    coco_root=self._config.model.mmdet_image.coco_root,
+                )
+            else:
+                pred_df.to_csv(result_path, index=False)
+            logger.info(f"Saved detection results {'as coco' if as_coco else 'as dataframe'} to {result_path}")
 
-        return pred
+        if as_pandas:
+            return pred_df
+        else:
+            if self._model.output_bbox_format == XYWH:
+                pred = convert_pred_to_xywh(pred)
+            return pred
 
     def predict_proba(
         self,

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/src/autogluon/multimodal/predictor.py

@@ -605,6 +605,7 @@ class MultiModalPredictor:
         as_pandas: Optional[bool] = None,
         realtime: Optional[bool] = False,
         save_results: Optional[bool] = None,
+        **kwargs,
     ):
         """
         Predict the label column values for new data.
@@ -617,8 +618,8 @@ class MultiModalPredictor:
         candidate_data
             The candidate data from which to search the query data's matches.
         id_mappings
-             Id-to-content mappings. The contents can be text, image, etc.
-             This is used when data contain the query/response identifiers instead of their contents.
+            Id-to-content mappings. The contents can be text, image, etc.
+            This is used when data contain the query/response identifiers instead of their contents.
         as_pandas
             Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).
         realtime
@@ -627,10 +628,14 @@ class MultiModalPredictor:
             and sample number.
         save_results
             Whether to save the prediction results (only works for detection now)
+        **kwargs
+            Additional keyword arguments to pass to the underlying learner's predict method.
+            For example, `as_coco` for object detection tasks.
 
         Returns
         -------
         Array of predictions, one corresponding to each row in given dataset.
+        Format depends on the specific learner and provided arguments.
         """
         return self._learner.predict(
             data=data,
@@ -639,6 +644,7 @@ class MultiModalPredictor:
             realtime=realtime,
             save_results=save_results,
             id_mappings=id_mappings,
+            **kwargs,
         )
 
     def predict_proba(

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/src/autogluon/multimodal/utils/__init__.py

@@ -82,6 +82,7 @@ from .object_detection import (
     bbox_xyxy_to_xywh,
     cocoeval,
     convert_pred_to_xywh,
+    convert_result_df,
     from_coco,
     from_coco_or_voc,
     from_dict,
@@ -90,7 +91,6 @@ from .object_detection import (
     object_detection_data_to_df,
     object_detection_df_to_coco,
     save_result_coco_format,
-    save_result_df,
     save_result_voc_format,
     visualize_detection,
 )

{autogluon.multimodal-1.1.2b20241106 → autogluon.multimodal-1.1.2b20241108}/src/autogluon/multimodal/utils/hpo.py

@@ -50,9 +50,9 @@ def hpo_trial(sampled_hyperparameters, learner, checkpoint_dir=None, **_fit_args
     resources = context.get_trial_resources().required_resources
     num_cpus = int(resources.get("CPU"))
 
-    _fit_args["hyperparameters"] = (
-        sampled_hyperparameters  # The original hyperparameters is the search space, replace it with the hyperparameters sampled
-    )
+    # The original hyperparameters is the search space, replace it with the hyperparameters sampled
+    _fit_args["hyperparameters"] = sampled_hyperparameters
+
     _fit_args["save_path"] = context.get_trial_dir()  # We want to save each trial to a separate directory
     logger.debug(f"hpo trial save_path: {_fit_args['save_path']}")
     if checkpoint_dir is not None:
@@ -62,7 +62,15 @@ def hpo_trial(sampled_hyperparameters, learner, checkpoint_dir=None, **_fit_args
         learner.fit_per_run(**_fit_args)
 
 
-def build_final_learner(learner, best_trial_path, save_path, last_ckpt_path, is_matching, standalone, clean_ckpts):
+def build_final_learner(
+    learner,
+    best_trial_path,
+    save_path,
+    last_ckpt_path,
+    is_matching,
+    standalone,
+    clean_ckpts,
+):
     """
     Build the final learner after HPO is finished.
 
@@ -167,12 +175,13 @@ def hyperparameter_tune(hyperparameter_tune_kwargs, resources, is_matching=False
     mode = _fit_args.get("learner")._minmax_mode
     save_path = _fit_args.get("save_path")
     time_budget_s = _fit_args.get("max_time")
+    num_to_keep = hyperparameter_tune_kwargs.pop("num_to_keep", 3)
     if time_budget_s is not None:
         time_budget_s *= 0.95  # give some buffer time to ray
     try:
         run_config_kwargs = {
             "checkpoint_config": CheckpointConfig(
-                num_to_keep=3,
+                num_to_keep=num_to_keep,
                 checkpoint_score_attribute=metric,
             ),
         }