mlrun 1.7.0rc16__py3-none-any.whl → 1.7.0rc18__py3-none-any.whl

mlrun/alerts/alert.py

@@ -15,8 +15,7 @@
 from typing import Union
 
 import mlrun
-import mlrun.common.schemas.alert as alert_constants
-from mlrun.common.schemas.notification import Notification
+import mlrun.common.schemas.alert as alert_objects
 from mlrun.model import ModelObj
 
 
@@ -36,17 +35,17 @@ class AlertConfig(ModelObj):
         self,
         project: str = None,
         name: str = None,
-        template: Union[alert_constants.AlertTemplate, str] = None,
+        template: Union[alert_objects.AlertTemplate, str] = None,
         description: str = None,
         summary: str = None,
-        severity: alert_constants.AlertSeverity = None,
-        trigger: alert_constants.AlertTrigger = None,
-        criteria: alert_constants.AlertCriteria = None,
-        reset_policy: alert_constants.ResetPolicy = None,
-        notifications: list[Notification] = None,
-        entities: alert_constants.EventEntities = None,
+        severity: alert_objects.AlertSeverity = None,
+        trigger: alert_objects.AlertTrigger = None,
+        criteria: alert_objects.AlertCriteria = None,
+        reset_policy: alert_objects.ResetPolicy = None,
+        notifications: list[alert_objects.AlertNotification] = None,
+        entities: alert_objects.EventEntities = None,
         id: int = None,
-        state: alert_constants.AlertActiveState = None,
+        state: alert_objects.AlertActiveState = None,
         created: str = None,
         count: int = None,
     ):
@@ -81,8 +80,10 @@ class AlertConfig(ModelObj):
             else self.entities
         )
         data["notifications"] = [
-            notification.dict() if not isinstance(notification, dict) else notification
-            for notification in self.notifications
+            notification_data.dict()
+            if not isinstance(notification_data, dict)
+            else notification_data
+            for notification_data in self.notifications
         ]
         data["trigger"] = (
             self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
@@ -95,35 +96,37 @@ class AlertConfig(ModelObj):
 
         entity_data = struct.get("entities")
         if entity_data:
-            entity_obj = alert_constants.EventEntities.parse_obj(entity_data)
+            entity_obj = alert_objects.EventEntities.parse_obj(entity_data)
             new_obj.entities = entity_obj
 
         notifications_data = struct.get("notifications")
         if notifications_data:
             notifications_objs = [
-                Notification.parse_obj(notification_data)
-                for notification_data in notifications_data
+                alert_objects.AlertNotification.parse_obj(notification)
+                for notification in notifications_data
             ]
             new_obj.notifications = notifications_objs
 
         trigger_data = struct.get("trigger")
         if trigger_data:
-            trigger_obj = alert_constants.AlertTrigger.parse_obj(trigger_data)
+            trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
             new_obj.trigger = trigger_obj
 
         return new_obj
 
-    def with_notifications(self, notifications: list[Notification]):
+    def with_notifications(self, notifications: list[alert_objects.AlertNotification]):
         if not isinstance(notifications, list) or not all(
-            isinstance(item, Notification) for item in notifications
+            isinstance(item, alert_objects.AlertNotification) for item in notifications
         ):
-            raise ValueError("Notifications parameter must be a list of notifications")
-        for notification in notifications:
-            self.notifications.append(notification)
+            raise ValueError(
+                "Notifications parameter must be a list of AlertNotification"
+            )
+        for notification_data in notifications:
+            self.notifications.append(notification_data)
         return self
 
-    def with_entities(self, entities: alert_constants.EventEntities):
-        if not isinstance(entities, alert_constants.EventEntities):
+    def with_entities(self, entities: alert_objects.EventEntities):
+        if not isinstance(entities, alert_objects.EventEntities):
             raise ValueError("Entities parameter must be of type: EventEntities")
         self.entities = entities
         return self
@@ -134,7 +137,7 @@ class AlertConfig(ModelObj):
             template = db.get_alert_template(template)
 
         # Extract parameters from the template and apply them to the AlertConfig object
-        self.description = template.description
+        self.summary = template.summary
         self.severity = template.severity
         self.criteria = template.criteria
         self.trigger = template.trigger

mlrun/artifacts/manager.py

@@ -72,6 +72,10 @@ class ArtifactProducer:
     def get_meta(self) -> dict:
         return {"kind": self.kind, "name": self.name, "tag": self.tag}
 
+    @property
+    def uid(self):
+        return None
+
 
 def dict_to_artifact(struct: dict) -> Artifact:
     kind = struct.get("kind", "")
@@ -262,7 +266,7 @@ class ArtifactManager:
         if target_path and item.is_dir and not target_path.endswith("/"):
             target_path += "/"
         target_path = template_artifact_path(
-            artifact_path=target_path, project=producer.project
+            artifact_path=target_path, project=producer.project, run_uid=producer.uid
         )
         item.target_path = target_path
 

mlrun/artifacts/model.py

@@ -509,7 +509,7 @@ def _get_extra(target, extra_data, is_dir=False):
 def _remove_tag_from_spec_yaml(model_spec):
     spec_dict = model_spec.to_dict()
     spec_dict["metadata"].pop("tag", None)
-    return yaml.dump(spec_dict)
+    return yaml.safe_dump(spec_dict)
 
 
 def update_model(

mlrun/common/runtimes/constants.py

@@ -136,6 +136,7 @@ class RunStates:
     unknown = "unknown"
     aborted = "aborted"
     aborting = "aborting"
+    skipped = "skipped"
 
     @staticmethod
     def all():
@@ -148,6 +149,7 @@ class RunStates:
             RunStates.unknown,
             RunStates.aborted,
             RunStates.aborting,
+            RunStates.skipped,
         ]
 
     @staticmethod
@@ -156,6 +158,7 @@ class RunStates:
             RunStates.completed,
             RunStates.error,
             RunStates.aborted,
+            RunStates.skipped,
         ]
 
     @staticmethod

mlrun/common/schemas/__init__.py

@@ -14,7 +14,13 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
-from .alert import AlertActiveState, AlertConfig, AlertTemplate, Event
+from .alert import (
+    AlertActiveState,
+    AlertConfig,
+    AlertNotification,
+    AlertTemplate,
+    Event,
+)
 from .api_gateway import (
     APIGateway,
     APIGatewayAuthenticationMode,
@@ -142,10 +148,10 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    MonitoringTSDBTables,
     PrometheusEndpoints,
     TimeSeriesConnector,
     TSDBTarget,
+    V3IOTSDBTables,
 )
 from .notification import (
     Notification,

mlrun/common/schemas/alert.py

@@ -22,7 +22,7 @@ from mlrun.common.types import StrEnum
 
 
 class EventEntityKind(StrEnum):
-    MODEL = "model"
+    MODEL_ENDPOINT_RESULT = "model-endpoint-result"
     JOB = "job"
 
 
@@ -33,14 +33,34 @@ class EventEntities(pydantic.BaseModel):
 
 
 class EventKind(StrEnum):
-    DRIFT_DETECTED = "drift_detected"
-    DRIFT_SUSPECTED = "drift_suspected"
+    DATA_DRIFT_DETECTED = "data_drift_detected"
+    DATA_DRIFT_SUSPECTED = "data_drift_suspected"
+    CONCEPT_DRIFT_DETECTED = "concept_drift_detected"
+    CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
+    MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
+    MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
+    MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
+    MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
+    MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
+    MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
     FAILED = "failed"
 
 
 _event_kind_entity_map = {
-    EventKind.DRIFT_SUSPECTED: [EventEntityKind.MODEL],
-    EventKind.DRIFT_DETECTED: [EventEntityKind.MODEL],
+    EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
+        EventEntityKind.MODEL_ENDPOINT_RESULT
+    ],
+    EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
+        EventEntityKind.MODEL_ENDPOINT_RESULT
+    ],
+    EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
+    EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
     EventKind.FAILED: [EventEntityKind.JOB],
 }
 
@@ -101,6 +121,18 @@ class ResetPolicy(StrEnum):
     AUTO = "auto"
 
 
+class AlertNotification(pydantic.BaseModel):
+    notification: Notification
+    cooldown_period: Annotated[
+        str,
+        pydantic.Field(
+            description="Period during which notifications "
+            "will not be sent after initial send. The format of this would be in time."
+            " e.g. 1d, 3h, 5m, 15s"
+        ),
+    ] = None
+
+
 class AlertConfig(pydantic.BaseModel):
     project: str
     id: int = None
@@ -111,7 +143,8 @@ class AlertConfig(pydantic.BaseModel):
         pydantic.Field(
             description=(
                 "String to be sent in the notifications generated."
-                "e.g. 'Model {{ $project }}/{{ $entity }} is drifting.'"
+                "e.g. 'Model {{project}}/{{entity}} is drifting.'"
+                "Supported variables: project, entity, name"
             )
         ),
     ]
@@ -121,10 +154,15 @@ class AlertConfig(pydantic.BaseModel):
     trigger: AlertTrigger
     criteria: Optional[AlertCriteria]
     reset_policy: ResetPolicy = ResetPolicy.MANUAL
-    notifications: pydantic.conlist(Notification, min_items=1)
+    notifications: pydantic.conlist(AlertNotification, min_items=1)
     state: AlertActiveState = AlertActiveState.INACTIVE
     count: Optional[int] = 0
 
+    def get_raw_notifications(self) -> list[Notification]:
+        return [
+            alert_notification.notification for alert_notification in self.notifications
+        ]
+
 
 class AlertsModes(StrEnum):
     enabled = "enabled"
@@ -144,8 +182,9 @@ class AlertTemplate(
     system_generated: bool = False
 
     # AlertConfig fields that are pre-defined
-    description: Optional[str] = (
-        "String to be sent in the generated notifications e.g. 'Model {{ $project }}/{{ $entity }} is drifting.'"
+    summary: Optional[str] = (
+        "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
+        "See AlertConfig.summary description"
     )
     severity: AlertSeverity
     trigger: AlertTrigger
@@ -156,7 +195,7 @@ class AlertTemplate(
     def templates_differ(self, other):
         return (
             self.template_description != other.template_description
-            or self.description != other.description
+            or self.summary != other.summary
             or self.severity != other.severity
             or self.trigger != other.trigger
             or self.reset_policy != other.reset_policy

mlrun/common/schemas/client_spec.py

@@ -59,6 +59,7 @@ class ClientSpec(pydantic.BaseModel):
     sql_url: typing.Optional[str]
     model_endpoint_monitoring_store_type: typing.Optional[str]
     model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
+    model_monitoring_tsdb_connection: typing.Optional[str]
     ce: typing.Optional[dict]
     # not passing them as one object as it possible client user would like to override only one of the params
     calculate_artifact_hash: typing.Optional[str]

mlrun/common/schemas/function.py

@@ -45,6 +45,9 @@ class FunctionState:
     # same goes for the build which is not coming from the pod, but is used and we can't just omit it for BC reasons
     build = "build"
 
+    # for pipeline steps
+    skipped = "skipped"
+
     @classmethod
     def get_function_state_from_pod_state(cls, pod_state: str):
         if pod_state == "succeeded":
@@ -60,6 +63,7 @@ class FunctionState:
         return [
             cls.ready,
             cls.error,
+            cls.skipped,
         ]
 
 

mlrun/common/schemas/model_monitoring/__init__.py

@@ -30,20 +30,22 @@ from .constants import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    MonitoringTSDBTables,
     ProjectSecretKeys,
     PrometheusEndpoints,
     PrometheusMetric,
     ResultData,
     SchedulingKeys,
+    TDEngineSuperTables,
     TimeSeriesConnector,
     TSDBTarget,
+    V3IOTSDBTables,
     VersionedModel,
     WriterEvent,
     WriterEventKind,
 )
 from .grafana import (
     GrafanaColumn,
+    GrafanaColumnType,
     GrafanaDataPoint,
     GrafanaNumberColumn,
     GrafanaStringColumn,

mlrun/common/schemas/model_monitoring/constants.py

@@ -81,6 +81,8 @@ class EventFieldType:
     DRIFT_DETECTED_THRESHOLD = "drift_detected_threshold"
     POSSIBLE_DRIFT_THRESHOLD = "possible_drift_threshold"
     SAMPLE_PARQUET_PATH = "sample_parquet_path"
+    TIME = "time"
+    TABLE_COLUMN = "table_column"
 
 
 class FeatureSetFeatures(MonitoringStrEnum):
@@ -171,6 +173,7 @@ class ProjectSecretKeys:
     PIPELINES_ACCESS_KEY = "MODEL_MONITORING_PIPELINES_ACCESS_KEY"
     KAFKA_BROKERS = "KAFKA_BROKERS"
     STREAM_PATH = "STREAM_PATH"
+    TSDB_CONNECTION = "TSDB_CONNECTION"
 
 
 class ModelMonitoringStoreKinds:
@@ -188,6 +191,7 @@ class SchedulingKeys:
 class FileTargetKind:
     ENDPOINTS = "endpoints"
     EVENTS = "events"
+    PREDICTIONS = "predictions"
     STREAM = "stream"
     PARQUET = "parquet"
     APPS_PARQUET = "apps_parquet"
@@ -229,12 +233,18 @@ class MonitoringFunctionNames(MonitoringStrEnum):
     WRITER = "model-monitoring-writer"
 
 
-class MonitoringTSDBTables(MonitoringStrEnum):
+class V3IOTSDBTables(MonitoringStrEnum):
     APP_RESULTS = "app-results"
     METRICS = "metrics"
     EVENTS = "events"
 
 
+class TDEngineSuperTables(MonitoringStrEnum):
+    APP_RESULTS = "app_results"
+    METRICS = "metrics"
+    PREDICTIONS = "predictions"
+
+
 @dataclass
 class FunctionURI:
     project: str
@@ -338,6 +348,7 @@ class ControllerPolicy:
 
 class TSDBTarget:
     V3IO_TSDB = "v3io-tsdb"
+    TDEngine = "tdengine"
     PROMETHEUS = "prometheus"
     APP_RESULTS_TABLE = "app-results"
     V3IO_BE = "tsdb"
@@ -347,3 +358,12 @@ class TSDBTarget:
 class HistogramDataDriftApplicationConstants:
     NAME = "histogram-data-drift"
     GENERAL_RESULT_NAME = "general_drift"
+
+
+class PredictionsQueryConstants:
+    DEFAULT_AGGREGATION_GRANULARITY = "10m"
+    INVOCATIONS = "invocations"
+
+
+class SpecialApps:
+    MLRUN_INFRA = "mlrun-infra"

mlrun/common/schemas/model_monitoring/grafana.py

@@ -11,12 +11,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 from typing import Optional, Union
 
 from pydantic import BaseModel
 
+import mlrun.common.types
+
+
+class GrafanaColumnType(mlrun.common.types.StrEnum):
+    NUMBER = "number"
+    STRING = "string"
+
 
 class GrafanaColumn(BaseModel):
     text: str
@@ -24,13 +30,11 @@ class GrafanaColumn(BaseModel):
 
 
 class GrafanaNumberColumn(GrafanaColumn):
-    text: str
-    type: str = "number"
+    type: str = GrafanaColumnType.NUMBER
 
 
 class GrafanaStringColumn(GrafanaColumn):
-    text: str
-    type: str = "string"
+    type: str = GrafanaColumnType.STRING
 
 
 class GrafanaTable(BaseModel):

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -298,6 +298,7 @@ class ModelEndpointList(BaseModel):
 
 class ModelEndpointMonitoringMetricType(mlrun.common.types.StrEnum):
     RESULT = "result"
+    METRIC = "metric"
 
 
 class ModelEndpointMonitoringMetric(BaseModel):
@@ -322,7 +323,7 @@ _FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
 _FQN_PATTERN = (
     rf"^(?P<project>{_FQN_PART_PATTERN})\."
     rf"(?P<app>{_FQN_PART_PATTERN})\."
-    rf"(?P<type>{_FQN_PART_PATTERN})\."
+    rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
     rf"(?P<name>{_FQN_PART_PATTERN})$"
 )
 _FQN_REGEX = re.compile(_FQN_PATTERN)
@@ -337,27 +338,37 @@ def _parse_metric_fqn_to_monitoring_metric(fqn: str) -> ModelEndpointMonitoringM
     )
 
 
+class _MetricPoint(NamedTuple):
+    timestamp: datetime
+    value: float
+
+
 class _ResultPoint(NamedTuple):
     timestamp: datetime
     value: float
     status: ResultStatusApp
 
 
-class _ModelEndpointMonitoringResultValuesBase(BaseModel):
+class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
     full_name: str
     type: ModelEndpointMonitoringMetricType
     data: bool
 
 
-class ModelEndpointMonitoringResultValues(_ModelEndpointMonitoringResultValuesBase):
-    full_name: str
-    type: ModelEndpointMonitoringMetricType
+class ModelEndpointMonitoringMetricValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.METRIC
+    values: list[_MetricPoint]
+    data: bool = True
+
+
+class ModelEndpointMonitoringResultValues(_ModelEndpointMonitoringMetricValuesBase):
+    type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT
     result_kind: ResultKindApp
     values: list[_ResultPoint]
     data: bool = True
 
 
-class ModelEndpointMonitoringResultNoData(_ModelEndpointMonitoringResultValuesBase):
+class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBase):
     full_name: str
     type: ModelEndpointMonitoringMetricType
     data: bool = False

mlrun/common/schemas/project.py

@@ -113,7 +113,9 @@ class ProjectSummary(pydantic.BaseModel):
     runs_completed_recent_count: int
     runs_failed_recent_count: int
     runs_running_count: int
-    schedules_count: int
+    distinct_schedules_count: int
+    distinct_scheduled_jobs_pending_count: int
+    distinct_scheduled_pipelines_pending_count: int
     pipelines_completed_recent_count: typing.Optional[int] = None
     pipelines_failed_recent_count: typing.Optional[int] = None
     pipelines_running_count: typing.Optional[int] = None

mlrun/config.py

@@ -232,6 +232,10 @@ default_config = {
         "databricks": {
             "artifact_directory_path": "/mlrun_databricks_runtime/artifacts_dictionaries"
         },
+        "application": {
+            "default_sidecar_internal_port": 8050,
+            "default_authentication_mode": "accessKey",
+        },
     },
     # TODO: function defaults should be moved to the function spec config above
     "function_defaults": {
@@ -517,7 +521,9 @@ default_config = {
         # See mlrun.model_monitoring.db.stores.ObjectStoreFactory for available options
         "store_type": "v3io-nosql",
         "endpoint_store_connection": "",
+        # See mlrun.model_monitoring.db.tsdb.ObjectTSDBFactory for available options
         "tsdb_connector_type": "v3io-tsdb",
+        "tsdb_connection": "",
     },
     "secret_stores": {
         # Use only in testing scenarios (such as integration tests) to avoid using k8s for secrets (will use in-memory
@@ -1402,14 +1408,14 @@ def read_env(env=None, prefix=env_prefix):
     if log_formatter_name := config.get("log_formatter"):
         import mlrun.utils.logger
 
-        log_formatter = mlrun.utils.create_formatter_instance(
+        log_formatter = mlrun.utils.resolve_formatter_by_kind(
             mlrun.utils.FormatterKinds(log_formatter_name)
         )
         current_handler = mlrun.utils.logger.get_handler("default")
         current_formatter_name = current_handler.formatter.__class__.__name__
-        desired_formatter_name = log_formatter.__class__.__name__
+        desired_formatter_name = log_formatter.__name__
         if current_formatter_name != desired_formatter_name:
-            current_handler.setFormatter(log_formatter)
+            current_handler.setFormatter(log_formatter())
 
     # The default function pod resource values are of type str; however, when reading from environment variable numbers,
     # it converts them to type int if contains only number, so we want to convert them to str.

mlrun/data_types/to_pandas.py

@@ -154,10 +154,10 @@ def toPandas(spark_df):
     column_counter = Counter(spark_df.columns)
 
     dtype = [None] * len(spark_df.schema)
-    for fieldIdx, field in enumerate(spark_df.schema):
+    for field_idx, field in enumerate(spark_df.schema):
         # For duplicate column name, we use `iloc` to access it.
         if column_counter[field.name] > 1:
-            pandas_col = pdf.iloc[:, fieldIdx]
+            pandas_col = pdf.iloc[:, field_idx]
         else:
             pandas_col = pdf[field.name]
 
@@ -171,12 +171,12 @@ def toPandas(spark_df):
             and field.nullable
             and pandas_col.isnull().any()
         ):
-            dtype[fieldIdx] = pandas_type
+            dtype[field_idx] = pandas_type
         # Ensure we fall back to nullable numpy types, even when whole column is null:
         if isinstance(field.dataType, IntegralType) and pandas_col.isnull().any():
-            dtype[fieldIdx] = np.float64
+            dtype[field_idx] = np.float64
         if isinstance(field.dataType, BooleanType) and pandas_col.isnull().any():
-            dtype[fieldIdx] = object
+            dtype[field_idx] = object
 
     df = pd.DataFrame()
     for index, t in enumerate(dtype):

mlrun/datastore/datastore.py

@@ -223,6 +223,11 @@ class StoreManager:
             subpath = url[len("memory://") :]
             return in_memory_store, subpath, url
 
+        elif schema in get_local_file_schema():
+            # parse_url() will drop the windows drive-letter from the path for url like "c:\a\b".
+            # As a workaround, we set subpath to the url.
+            subpath = url.replace("file://", "", 1)
+
         if not schema and endpoint:
             if endpoint in self._stores.keys():
                 return self._stores[endpoint], subpath, url
@@ -241,8 +246,7 @@ class StoreManager:
         )
         if not secrets and not mlrun.config.is_running_as_api():
             self._stores[store_key] = store
-        # in file stores in windows path like c:\a\b the drive letter is dropped from the path, so we return the url
-        return store, url if store.kind == "file" else subpath, url
+        return store, subpath, url
 
     def reset_secrets(self):
         self._secrets = {}

mlrun/datastore/redis.py

@@ -31,7 +31,7 @@ class RedisStore(DataStore):
     """
 
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
-        REDIS_DEFAULT_PORT = "6379"
+        redis_default_port = "6379"
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
         self.headers = None
 
@@ -49,7 +49,7 @@ class RedisStore(DataStore):
         user = self._get_secret_or_env("REDIS_USER", "", credentials_prefix)
         password = self._get_secret_or_env("REDIS_PASSWORD", "", credentials_prefix)
         host = parsed_endpoint.hostname
-        port = parsed_endpoint.port if parsed_endpoint.port else REDIS_DEFAULT_PORT
+        port = parsed_endpoint.port if parsed_endpoint.port else redis_default_port
         schema = parsed_endpoint.scheme
         if user or password:
             endpoint = f"{schema}://{user}:{password}@{host}:{port}"

mlrun/datastore/s3.py

@@ -198,6 +198,11 @@ class S3Store(DataStore):
         bucket = self.s3.Bucket(bucket)
         return [obj.key[key_length:] for obj in bucket.objects.filter(Prefix=key)]
 
+    def rm(self, path, recursive=False, maxdepth=None):
+        bucket, key = self.get_bucket_and_key(path)
+        path = f"{bucket}/{key}"
+        self.filesystem.rm(path=path, recursive=recursive, maxdepth=maxdepth)
+
 
 def parse_s3_bucket_and_key(s3_path):
     try: