mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/feature_store/retrieval/spark_merger.py

@@ -13,45 +13,16 @@
 # limitations under the License.
 #
 
-import pandas as pd
-import semver
 
 import mlrun
+from mlrun.data_types.to_pandas import spark_df_to_pandas
 from mlrun.datastore.sources import ParquetSource
 from mlrun.datastore.targets import get_offline_target
+from mlrun.runtimes import RemoteSparkRuntime
+from mlrun.runtimes.sparkjob import Spark3Runtime
 from mlrun.utils.helpers import additional_filters_warning
 
-from ...runtimes import RemoteSparkRuntime
-from ...runtimes.sparkjob import Spark3Runtime
 from .base import BaseMerger
-from .conversion import PandasConversionMixin
-
-
-def spark_df_to_pandas(spark_df):
-    # as of pyspark 3.2.3, toPandas fails to convert timestamps unless we work around the issue
-    # when we upgrade pyspark, we should check whether this workaround is still necessary
-    # see https://stackoverflow.com/questions/76389694/transforming-pyspark-to-pandas-dataframe
-    if semver.parse(pd.__version__)["major"] >= 2:
-        import pyspark.sql.functions as pyspark_functions
-
-        type_conversion_dict = {}
-        for field in spark_df.schema.fields:
-            if str(field.dataType) == "TimestampType":
-                spark_df = spark_df.withColumn(
-                    field.name,
-                    pyspark_functions.date_format(
-                        pyspark_functions.to_timestamp(field.name),
-                        "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS",
-                    ),
-                )
-                type_conversion_dict[field.name] = "datetime64[ns]"
-
-        df = PandasConversionMixin.toPandas(spark_df)
-        if type_conversion_dict:
-            df = df.astype(type_conversion_dict)
-        return df
-    else:
-        return PandasConversionMixin.toPandas(spark_df)
 
 
 class SparkFeatureMerger(BaseMerger):
@@ -217,9 +188,13 @@ class SparkFeatureMerger(BaseMerger):
 
         if self.spark is None:
             # create spark context
-            self.spark = SparkSession.builder.appName(
-                f"vector-merger-{self.vector.metadata.name}"
-            ).getOrCreate()
+            self.spark = (
+                SparkSession.builder.appName(
+                    f"vector-merger-{self.vector.metadata.name}"
+                )
+                .config("spark.driver.memory", "2g")
+                .getOrCreate()
+            )
 
     def _get_engine_df(
         self,
@@ -231,10 +206,6 @@ class SparkFeatureMerger(BaseMerger):
         time_column=None,
         additional_filters=None,
     ):
-        mlrun.utils.helpers.additional_filters_warning(
-            additional_filters, self.__class__
-        )
-
         source_kwargs = {}
         if feature_set.spec.passthrough:
             if not feature_set.spec.source:

mlrun/feature_store/steps.py

@@ -743,3 +743,11 @@ class DropFeatures(StepToDict, MLRunStep):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"DropFeatures can only drop features, not entities: {dropped_entities}"
             )
+        if feature_set.spec.label_column in features:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"DropFeatures can not drop label_column: {feature_set.spec.label_column}"
+            )
+        if feature_set.spec.timestamp_key in features:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"DropFeatures can not drop timestamp_key: {feature_set.spec.timestamp_key}"
+            )

mlrun/frameworks/_common/plan.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
 from abc import ABC, abstractmethod
 
 import mlrun
 from mlrun.artifacts import Artifact
-from mlrun.utils.helpers import is_ipython
+from mlrun.utils.helpers import is_jupyter
 
 
 class Plan(ABC):
@@ -84,7 +84,7 @@ class Plan(ABC):
             return
 
         # Call the correct display method according to the kernel:
-        if is_ipython:
+        if is_jupyter:
             self._gui_display()
         else:
             self._cli_display()

mlrun/frameworks/_ml_common/plan.py

@@ -16,7 +16,7 @@ import json
 from abc import ABC, abstractmethod
 from enum import Enum
 
-from IPython.core.display import HTML, display
+from IPython.display import HTML, display
 
 import mlrun
 

mlrun/frameworks/parallel_coordinates.py

@@ -18,8 +18,7 @@ from typing import Union
 
 import numpy as np
 import pandas as pd
-from IPython.core.display import HTML
-from IPython.display import display
+from IPython.display import HTML, display
 from pandas.api.types import is_numeric_dtype, is_string_dtype
 
 import mlrun
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
                 fp.write("</body></html>")
             else:
                 fp.write(html)
-    if show or (show is None and mlrun.utils.is_ipython):
+    if show or (show is None and mlrun.utils.is_jupyter):
         display(HTML(html))
         if runs_list and len(runs_list) <= max_table_rows:
             display(HTML(html_table))

mlrun/frameworks/sklearn/mlrun_interface.py

@@ -97,7 +97,7 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
 
         def wrapper(
             self: SKLearnTypes.ModelType,
-            X: SKLearnTypes.DatasetType,
+            X: SKLearnTypes.DatasetType,  # noqa: N803 - should be lowercase "x", kept for BC
             y: SKLearnTypes.DatasetType = None,
             *args,
             **kwargs,
@@ -124,7 +124,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
 
         return wrapper
 
-    def mlrun_predict(self, X: SKLearnTypes.DatasetType, *args, **kwargs):
+    def mlrun_predict(
+        self,
+        X: SKLearnTypes.DatasetType,  # noqa: N803 - should be lowercase "x", kept for BC
+        *args,
+        **kwargs,
+    ):
         """
         MLRun's wrapper for the common ML API predict method.
         """
@@ -136,7 +141,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
 
         return y_pred
 
-    def mlrun_predict_proba(self, X: SKLearnTypes.DatasetType, *args, **kwargs):
+    def mlrun_predict_proba(
+        self,
+        X: SKLearnTypes.DatasetType,  # noqa: N803 - should be lowercase "x", kept for BC
+        *args,
+        **kwargs,
+    ):
         """
         MLRun's wrapper for the common ML API predict_proba method.
         """

mlrun/k8s_utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import re
+import warnings
 
 import kubernetes.client
 
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
     return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
 
 
-def verify_label_key(key: str):
+def verify_label_key(key: str, allow_k8s_prefix: bool = False):
     """
     Verify that the label key is valid for Kubernetes.
     Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
@@ -146,6 +147,10 @@ def verify_label_key(key: str):
         name = parts[0]
     elif len(parts) == 2:
         prefix, name = parts
+        if len(name) == 0:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Label key name cannot be empty when a prefix is set"
+            )
         if len(prefix) == 0:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Label key prefix cannot be empty"
@@ -173,7 +178,13 @@ def verify_label_key(key: str):
         mlrun.utils.regex.qualified_name,
     )
 
-    if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
+    # Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
+    # only when setting node selectors, not when adding new labels.
+    if (
+        key.startswith("k8s.io/")
+        or key.startswith("kubernetes.io/")
+        and not allow_k8s_prefix
+    ):
         raise mlrun.errors.MLRunInvalidArgumentError(
             "Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
         )
@@ -185,3 +196,38 @@ def verify_label_value(value, label_key):
         value,
         mlrun.utils.regex.label_value,
     )
+
+
+def validate_node_selectors(
+    node_selectors: dict[str, str], raise_on_error: bool = True
+) -> bool:
+    """
+    Ensures that user-defined node selectors adhere to Kubernetes label standards:
+    - Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
+    - Ensures values comply with Kubernetes label value rules.
+    - If raise_on_error is True, raises errors for invalid selectors.
+    - If raise_on_error is False, logs warnings for invalid selectors.
+    """
+
+    # Helper function for handling errors or warnings
+    def handle_invalid(message):
+        if raise_on_error:
+            raise
+        else:
+            warnings.warn(
+                f"{message}\n"
+                f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
+                f"Please note that invalid node selectors may cause issues with function scheduling."
+            )
+
+    node_selectors = node_selectors or {}
+    for key, value in node_selectors.items():
+        try:
+            verify_label_key(key, allow_k8s_prefix=True)
+            verify_label_value(value, label_key=key)
+        except mlrun.errors.MLRunInvalidArgumentError as err:
+            # An error or warning is raised by handle_invalid due to validation failure.
+            # Returning False indicates validation failed, allowing us to exit the function.
+            handle_invalid(str(err))
+            return False
+    return True

mlrun/launcher/client.py

@@ -14,7 +14,7 @@
 import abc
 from typing import Optional
 
-import IPython
+import IPython.display
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.errors
@@ -22,7 +22,7 @@ import mlrun.launcher.base as launcher
 import mlrun.lists
 import mlrun.model
 import mlrun.runtimes
-from mlrun.utils import logger
+import mlrun.utils
 
 
 class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
@@ -128,10 +128,10 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
         if result:
             results_tbl.append(result)
         else:
-            logger.info("no returned result (job may still be in progress)")
+            mlrun.utils.logger.info("no returned result (job may still be in progress)")
             results_tbl.append(run.to_dict())
 
-        if mlrun.utils.is_ipython and mlrun.mlconf.ipython_widget:
+        if mlrun.utils.is_jupyter and mlrun.mlconf.ipython_widget:
             results_tbl.show()
             print()
             ui_url = mlrun.utils.get_ui_url(project, uid)
@@ -147,9 +147,9 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
             project_flag = f"-p {project}" if project else ""
             info_cmd = f"mlrun get run {uid} {project_flag}"
             logs_cmd = f"mlrun logs {uid} {project_flag}"
-            logger.info(
+            mlrun.utils.logger.info(
                 "To track results use the CLI", info_cmd=info_cmd, logs_cmd=logs_cmd
             )
             ui_url = mlrun.utils.get_ui_url(project, uid)
             if ui_url:
-                logger.info("Or click for UI", ui_url=ui_url)
+                mlrun.utils.logger.info("Or click for UI", ui_url=ui_url)

mlrun/launcher/local.py

@@ -72,9 +72,9 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
         reset_on_run: Optional[bool] = None,
     ) -> "mlrun.run.RunObject":
         # do not allow local function to be scheduled
-        if self._is_run_local and schedule is not None:
+        if schedule is not None:
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "local and schedule cannot be used together"
+                f"Unexpected {schedule=} parameter for local function execution"
             )
 
         self.enrich_runtime(runtime, project)

mlrun/model.py

@@ -487,7 +487,7 @@ class ImageBuilder(ModelObj):
 
     def __init__(
         self,
-        functionSourceCode=None,
+        functionSourceCode=None,  # noqa: N803 - should be "snake_case", kept for BC
         source=None,
         image=None,
         base_image=None,
@@ -679,7 +679,25 @@ class ImageBuilder(ModelObj):
 
 
 class Notification(ModelObj):
-    """Notification specification"""
+    """Notification object
+
+    :param kind: notification implementation kind - slack, webhook, etc. See
+        :py:class:`mlrun.common.schemas.notification.NotificationKind`
+    :param name: for logging and identification
+    :param message: message content in the notification
+    :param severity: severity to display in the notification
+    :param when: list of statuses to trigger the notification: 'running', 'completed', 'error'
+    :param condition: optional condition to trigger the notification, a jinja2 expression that can use run data
+                      to evaluate if the notification should be sent in addition to the 'when' statuses.
+                      e.g.: '{{ run["status"]["results"]["accuracy"] < 0.9}}'
+    :param params: Implementation specific parameters for the notification implementation (e.g. slack webhook url,
+                   git repository details, etc.)
+    :param secret_params: secret parameters for the notification implementation, same as params but will be stored
+                          in a k8s secret and passed as a secret reference to the implementation.
+    :param status: notification status - pending, sent, error
+    :param sent_time: time the notification was sent
+    :param reason: failure reason if the notification failed to send
+    """
 
     def __init__(
         self,
@@ -737,19 +755,41 @@ class Notification(ModelObj):
             self.kind
         ).get_notification()
 
-        secret_params = self.secret_params
-        params = self.params
+        secret_params = self.secret_params or {}
+        params = self.params or {}
+
+        # if the secret_params are already masked - no need to validate
+        params_secret = secret_params.get("secret", "")
+        if params_secret:
+            if len(secret_params) > 1:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "When the 'secret' key is present, 'secret_params' should not contain any other keys."
+                )
+            return
 
         if not secret_params and not params:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Both 'secret_params' and 'params' are empty, at least one must be defined."
             )
-        if secret_params and params and secret_params != params:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "Both 'secret_params' and 'params' are defined but they contain different values"
-            )
 
-        notification_class.validate_params(secret_params or params)
+        notification_class.validate_params(secret_params | params)
+
+    def enrich_unmasked_secret_params_from_project_secret(self):
+        """
+        Fill the notification secret params from the project secret.
+        We are using this function instead of unmask_secret_params_from_project_secret when we run inside the
+        workflow runner pod that doesn't have access to the k8s secrets (but have access to the project secret)
+        """
+        secret = self.secret_params.get("secret")
+        if secret:
+            secret_value = mlrun.get_secret_or_env(secret)
+            if secret_value:
+                try:
+                    self.secret_params = json.loads(secret_value)
+                except ValueError as exc:
+                    raise mlrun.errors.MLRunValueError(
+                        "Failed to parse secret value"
+                    ) from exc
 
     @staticmethod
     def validate_notification_uniqueness(notifications: list["Notification"]):
@@ -1306,7 +1346,7 @@ class RunTemplate(ModelObj):
 
             task.with_input("data", "/file-dir/path/to/file")
             task.with_input("data", "s3://<bucket>/path/to/file")
-            task.with_input("data", "v3io://[<remote-host>]/<data-container>/path/to/file")
+            task.with_input("data", "v3io://<data-container>/path/to/file")
         """
         if not self.spec.inputs:
             self.spec.inputs = {}
@@ -1463,7 +1503,11 @@ class RunObject(RunTemplate):
     @property
     def error(self) -> str:
         """error string if failed"""
-        if self.status:
+        if (
+            self.status
+            and self.status.state
+            in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
+        ):
             unknown_error = ""
             if (
                 self.status.state
@@ -1479,20 +1523,43 @@ class RunObject(RunTemplate):
 
             return (
                 self.status.error
-                or self.status.reason
                 or self.status.status_text
+                or self.status.reason
                 or unknown_error
             )
         return ""
 
-    def output(self, key):
-        """return the value of a specific result or artifact by key"""
+    def output(self, key: str):
+        """
+        Return the value of a specific result or artifact by key.
+
+        This method waits for the outputs to complete and retrieves the value corresponding to the provided key.
+        If the key exists in the results, it returns the corresponding result value.
+        If not found in results, it attempts to fetch the artifact by key (cached in the run status).
+        If the artifact is not found, it tries to fetch the artifact URI by key.
+        If no artifact or result is found for the key, returns None.
+
+        :param key: The key of the result or artifact to retrieve.
+        :return: The value of the result or the artifact URI corresponding to the key, or None if not found.
+        """
         self._outputs_wait_for_completion()
+
+        # Check if the key exists in results and return the result value
         if self.status.results and key in self.status.results:
-            return self.status.results.get(key)
+            return self.status.results[key]
+
+        # Artifacts are usually cached in the run object under `status.artifacts`. However, the artifacts are not
+        # stored in the DB as part of the run. The server may enrich the run with the artifacts or provide
+        # `status.artifact_uris` instead. See mlrun.common.formatters.run.RunFormat.
+        # When running locally - `status.artifact_uri` does not exist in the run.
+        # When listing runs - `status.artifacts` does not exist in the run.
         artifact = self._artifact(key)
         if artifact:
             return get_artifact_target(artifact, self.metadata.project)
+
+        if self.status.artifact_uris and key in self.status.artifact_uris:
+            return self.status.artifact_uris[key]
+
         return None
 
     @property
@@ -1505,26 +1572,50 @@ class RunObject(RunTemplate):
 
     @property
     def outputs(self):
-        """return a dict of outputs, result values and artifact uris"""
-        outputs = {}
+        """
+        Return a dictionary of outputs, including result values and artifact URIs.
+
+        This method waits for the outputs to complete and combines result values
+        and artifact URIs into a single dictionary. If there are multiple artifacts
+        for the same key, only include the artifact that does not have the "latest" tag.
+        If there is no other tag, include the "latest" tag as a fallback.
+
+        :return: Dictionary containing result values and artifact URIs.
+        """
         self._outputs_wait_for_completion()
+        outputs = {}
+
+        # Add results if available
         if self.status.results:
-            outputs = {k: v for k, v in self.status.results.items()}
+            outputs.update(self.status.results)
+
+        # Artifacts are usually cached in the run object under `status.artifacts`. However, the artifacts are not
+        # stored in the DB as part of the run. The server may enrich the run with the artifacts or provide
+        # `status.artifact_uris` instead. See mlrun.common.formatters.run.RunFormat.
+        # When running locally - `status.artifact_uri` does not exist in the run.
+        # When listing runs - `status.artifacts` does not exist in the run.
         if self.status.artifacts:
-            for a in self.status.artifacts:
-                key = a["metadata"]["key"]
-                outputs[key] = get_artifact_target(a, self.metadata.project)
+            outputs.update(self._process_artifacts(self.status.artifacts))
+        elif self.status.artifact_uris:
+            outputs.update(self.status.artifact_uris)
+
         return outputs
 
-    def artifact(self, key) -> "mlrun.DataItem":
-        """return artifact DataItem by key"""
+    def artifact(self, key: str) -> "mlrun.DataItem":
+        """Return artifact DataItem by key.
+
+        This method waits for the outputs to complete, searches for the artifact matching the given key,
+        and returns a DataItem if the artifact is found.
+
+        :param key: The key of the artifact to find.
+        :return: A DataItem corresponding to the artifact with the given key, or None if no such artifact is found.
+        """
         self._outputs_wait_for_completion()
         artifact = self._artifact(key)
-        if artifact:
-            uri = get_artifact_target(artifact, self.metadata.project)
-            if uri:
-                return mlrun.get_dataitem(uri)
-        return None
+        if not artifact:
+            return None
+        uri = get_artifact_target(artifact, self.metadata.project)
+        return mlrun.get_dataitem(uri) if uri else None
 
     def _outputs_wait_for_completion(
         self,
@@ -1542,12 +1633,85 @@ class RunObject(RunTemplate):
             )
 
     def _artifact(self, key):
-        """return artifact DataItem by key"""
-        if self.status.artifacts:
-            for a in self.status.artifacts:
-                if a["metadata"]["key"] == key:
-                    return a
-        return None
+        """
+        Return the last artifact DataItem that matches the given key.
+
+        If multiple artifacts with the same key exist, return the last one in the list.
+        If there are artifacts with different tags, the method will return the one with a tag other than 'latest'
+        if available.
+        If no artifact with the given key is found, return None.
+
+        :param key: The key of the artifact to retrieve.
+        :return: The last artifact DataItem with the given key, or None if no such artifact is found.
+        """
+        if not self.status.artifacts:
+            return None
+
+        # Collect artifacts that match the key
+        matching_artifacts = [
+            artifact
+            for artifact in self.status.artifacts
+            if artifact["metadata"].get("key") == key
+        ]
+
+        if not matching_artifacts:
+            return None
+
+        # Sort matching artifacts by creation date in ascending order.
+        # The last element in the list will be the one created most recently.
+        # In case the `created` field does not exist in the artifact, that artifact will appear first in the sorted list
+        matching_artifacts.sort(
+            key=lambda artifact: artifact["metadata"].get("created", datetime.min)
+        )
+
+        # Filter out artifacts with 'latest' tag
+        non_latest_artifacts = [
+            artifact
+            for artifact in matching_artifacts
+            if artifact["metadata"].get("tag") != "latest"
+        ]
+
+        # Return the last non-'latest' artifact if available, otherwise return the last artifact
+        # In the case of only one tag, `status.artifacts` includes [v1, latest]. In that case, we want to return v1.
+        # In the case of multiple tags, `status.artifacts` includes [v1, latest, v2, v3].
+        # In that case, we need to return the last one (v3).
+        return (non_latest_artifacts or matching_artifacts)[-1]
+
+    def _process_artifacts(self, artifacts):
+        artifacts_by_key = {}
+
+        # Organize artifacts by key
+        for artifact in artifacts:
+            key = artifact["metadata"]["key"]
+            if key not in artifacts_by_key:
+                artifacts_by_key[key] = []
+            artifacts_by_key[key].append(artifact)
+
+        outputs = {}
+        for key, artifacts in artifacts_by_key.items():
+            # Sort matching artifacts by creation date in ascending order.
+            # The last element in the list will be the one created most recently.
+            # In case the `created` field does not exist in the artifactthat artifact will appear
+            # first in the sorted list
+            artifacts.sort(
+                key=lambda artifact: artifact["metadata"].get("created", datetime.min)
+            )
+
+            # Filter out artifacts with 'latest' tag
+            non_latest_artifacts = [
+                artifact
+                for artifact in artifacts
+                if artifact["metadata"].get("tag") != "latest"
+            ]
+
+            # Save the last non-'latest' artifact if available, otherwise save the last artifact
+            # In the case of only one tag, `artifacts` includes [v1, latest], in that case, we want to save v1.
+            # In the case of multiple tags, `artifacts` includes [v1, latest, v2, v3].
+            # In that case, we need to save the last one (v3).
+            artifact_to_save = (non_latest_artifacts or artifacts)[-1]
+            outputs[key] = get_artifact_target(artifact_to_save, self.metadata.project)
+
+        return outputs
 
     def uid(self):
         """run unique id"""
@@ -1664,6 +1828,11 @@ class RunObject(RunTemplate):
 
         return state
 
+    def abort(self):
+        """abort the run"""
+        db = mlrun.get_run_db()
+        db.abort_run(self.metadata.uid, self.metadata.project)
+
     @staticmethod
     def create_uri(project: str, uid: str, iteration: Union[int, str], tag: str = ""):
         if tag:
@@ -1892,6 +2061,8 @@ class DataSource(ModelObj):
     ]
     kind = None
 
+    _fields_to_serialize = ["start_time", "end_time"]
+
     def __init__(
         self,
         name: str = None,
@@ -1920,6 +2091,16 @@ class DataSource(ModelObj):
     def set_secrets(self, secrets):
         self._secrets = secrets
 
+    def _serialize_field(
+        self, struct: dict, field_name: str = None, strip: bool = False
+    ) -> typing.Any:
+        value = super()._serialize_field(struct, field_name, strip)
+        # We pull the field from self and not from struct because it was excluded from the struct when looping over
+        # the fields to save.
+        if field_name in ("start_time", "end_time") and isinstance(value, datetime):
+            return value.isoformat()
+        return value
+
 
 class DataTargetBase(ModelObj):
     """data target spec, specify a destination for the feature set data"""