mlrun 1.7.0rc9__py3-none-any.whl → 1.7.0rc11__py3-none-any.whl

mlrun/feature_store/steps.py

@@ -162,13 +162,19 @@ class MapValues(StepToDict, MLRunStep):
         example::
 
             # replace the value "U" with '0' in the age column
-            graph.to(MapValues(mapping={'age': {'U': '0'}}, with_original_features=True))
+            graph.to(MapValues(mapping={"age": {"U": "0"}}, with_original_features=True))
 
             # replace integers, example
-            graph.to(MapValues(mapping={'not': {0: 1, 1: 0}}))
+            graph.to(MapValues(mapping={"not": {0: 1, 1: 0}}))
 
             # replace by range, use -inf and inf for extended range
-            graph.to(MapValues(mapping={'numbers': {'ranges': {'negative': [-inf, 0], 'positive': [0, inf]}}}))
+            graph.to(
+                MapValues(
+                    mapping={
+                        "numbers": {"ranges": {"negative": [-inf, 0], "positive": [0, inf]}}
+                    }
+                )
+            )
 
         :param mapping: a dict with entry per column and the associated old/new values map
         :param with_original_features: set to True to keep the original features
@@ -424,8 +430,10 @@ class OneHotEncoder(StepToDict, MLRunStep):
 
         example::
 
-            mapping = {'category': ['food', 'health', 'transportation'],
-                       'gender': ['male', 'female']}
+            mapping = {
+                "category": ["food", "health", "transportation"],
+                "gender": ["male", "female"],
+            }
             graph.to(OneHotEncoder(mapping=one_hot_encoder_mapping))
 
         :param mapping: a dict of per column categories (to map to binary fields)
@@ -542,10 +550,12 @@ class DateExtractor(StepToDict, MLRunStep):
 
             # (taken from the fraud-detection end-to-end feature store demo)
             # Define the Transactions FeatureSet
-            transaction_set = fstore.FeatureSet("transactions",
-                                            entities=[fstore.Entity("source")],
-                                            timestamp_key='timestamp',
-                                            description="transactions feature set")
+            transaction_set = fstore.FeatureSet(
+                "transactions",
+                entities=[fstore.Entity("source")],
+                timestamp_key="timestamp",
+                description="transactions feature set",
+            )
 
             # Get FeatureSet computation graph
             transaction_graph = transaction_set.graph
@@ -553,11 +563,11 @@ class DateExtractor(StepToDict, MLRunStep):
             # Add the custom `DateExtractor` step
             # to the computation graph
             transaction_graph.to(
-                    class_name='DateExtractor',
-                    name='Extract Dates',
-                    parts = ['hour', 'day_of_week'],
-                    timestamp_col = 'timestamp',
-                )
+                class_name="DateExtractor",
+                name="Extract Dates",
+                parts=["hour", "day_of_week"],
+                timestamp_col="timestamp",
+            )
 
         :param parts: list of pandas style date-time parts you want to extract.
         :param timestamp_col: The name of the column containing the timestamps to extract from,
@@ -694,11 +704,12 @@ class DropFeatures(StepToDict, MLRunStep):
 
         example::
 
-            feature_set = fstore.FeatureSet("fs-new",
-                                        entities=[fstore.Entity("id")],
-                                        description="feature set",
-                                        engine="pandas",
-                                        )
+            feature_set = fstore.FeatureSet(
+                "fs-new",
+                entities=[fstore.Entity("id")],
+                description="feature set",
+                engine="pandas",
+            )
             # Pre-processing graph steps
             feature_set.graph.to(DropFeatures(features=["age"]))
             df_pandas = feature_set.ingest(data)

mlrun/features.py

@@ -238,10 +238,7 @@ class Validator(ModelObj):
             from mlrun.features import Validator
 
             # Add validator to the feature 'bid' with check type
-            quotes_set["bid"].validator = Validator(
-                check_type=True,
-                severity="info"
-            )
+            quotes_set["bid"].validator = Validator(check_type=True, severity="info")
 
         :param check_type:  check feature type e.g. True, False
         :param severity:    severity name e.g. info, warning, etc.
@@ -280,10 +277,7 @@ class MinMaxValidator(Validator):
 
             # Add validator to the feature 'bid', where valid
             # minimal value is 52
-            quotes_set["bid"].validator = MinMaxValidator(
-                min=52,
-                severity="info"
-            )
+            quotes_set["bid"].validator = MinMaxValidator(min=52, severity="info")
 
         :param check_type:  check feature type e.g. True, False
         :param severity:    severity name e.g. info, warning, etc.
@@ -344,9 +338,7 @@ class MinMaxLenValidator(Validator):
             # Add length validator to the feature 'ticker', where valid
             # minimal length is 1 and maximal length is 10
             quotes_set["ticker"].validator = MinMaxLenValidator(
-                min=1,
-                max=10,
-                severity="info"
+                min=1, max=10, severity="info"
             )
 
         :param check_type:  check feature type e.g. True, False
@@ -408,8 +400,7 @@ class RegexValidator(Validator):
             # expression '(\b[A-Za-z]{1}[0-9]{7}\b)' where valid values are
             # e.g. A1234567, z9874563, etc.
             quotes_set["name"].validator = RegexValidator(
-                regex=r"(\b[A-Za-z]{1}[0-9]{7}\b)",
-                severity="info"
+                regex=r"(\b[A-Za-z]{1}[0-9]{7}\b)", severity="info"
             )
 
         :param check_type:  check feature type e.g. True, False

mlrun/frameworks/auto_mlrun/auto_mlrun.py

@@ -363,7 +363,7 @@ class AutoMLRun:
 
                                              {
                                                  "/.../custom_model.py": "MyModel",
-                                                 "/.../custom_objects.py": ["object1", "object2"]
+                                                 "/.../custom_objects.py": ["object1", "object2"],
                                              }
 
                                          All the paths will be accessed from the given 'custom_objects_directory',
@@ -464,7 +464,7 @@ class AutoMLRun:
 
                                              {
                                                  "/.../custom_model.py": "MyModel",
-                                                 "/.../custom_objects.py": ["object1", "object2"]
+                                                 "/.../custom_objects.py": ["object1", "object2"],
                                              }
 
                                          All the paths will be accessed from the given 'custom_objects_directory',

mlrun/frameworks/lgbm/__init__.py

@@ -241,7 +241,7 @@ def apply_mlrun(
 
                                          {
                                              "/.../custom_model.py": "MyModel",
-                                             "/.../custom_objects.py": ["object1", "object2"]
+                                             "/.../custom_objects.py": ["object1", "object2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/frameworks/lgbm/callbacks/callback.py

@@ -63,11 +63,9 @@ class Callback(ABC):
             def on_train_end(self):
                 print("{self.name}: Done training!")
 
+
         apply_mlrun()
-        lgb.train(
-            ...,
-            callbacks=[ExampleCallback(name="Example")]
-        )
+        lgb.train(..., callbacks=[ExampleCallback(name="Example")])
     """
 
     def __init__(self, order: int = 10, before_iteration: bool = False):

mlrun/frameworks/lgbm/model_handler.py

@@ -103,7 +103,7 @@ class LGBMModelHandler(MLModelHandler):
 
                                              {
                                                  "/.../custom_model.py": "MyModel",
-                                                 "/.../custom_objects.py": ["object1", "object2"]
+                                                 "/.../custom_objects.py": ["object1", "object2"],
                                              }
 
                                          All the paths will be accessed from the given 'custom_objects_directory',

mlrun/frameworks/pytorch/__init__.py

@@ -112,7 +112,7 @@ def train(
 
                                             {
                                                 "/.../custom_optimizer.py": "optimizer",
-                                                "/.../custom_layers.py": ["layer1", "layer2"]
+                                                "/.../custom_layers.py": ["layer1", "layer2"],
                                             }
 
                                         All the paths will be accessed from the given 'custom_objects_directory',
@@ -264,7 +264,7 @@ def evaluate(
 
                                          {
                                              "/.../custom_optimizer.py": "optimizer",
-                                             "/.../custom_layers.py": ["layer1", "layer2"]
+                                             "/.../custom_layers.py": ["layer1", "layer2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/frameworks/sklearn/__init__.py

@@ -92,7 +92,7 @@ def apply_mlrun(
 
                                          {
                                              "/.../custom_model.py": "MyModel",
-                                             "/.../custom_objects.py": ["object1", "object2"]
+                                             "/.../custom_objects.py": ["object1", "object2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/frameworks/tf_keras/__init__.py

@@ -85,7 +85,7 @@ def apply_mlrun(
 
                                             {
                                                 "/.../custom_optimizer.py": "optimizer",
-                                                "/.../custom_layers.py": ["layer1", "layer2"]
+                                                "/.../custom_layers.py": ["layer1", "layer2"],
                                             }
 
                                         All the paths will be accessed from the given 'custom_objects_directory',

mlrun/frameworks/xgboost/__init__.py

@@ -90,7 +90,7 @@ def apply_mlrun(
 
                                          {
                                              "/.../custom_model.py": "MyModel",
-                                             "/.../custom_objects.py": ["object1", "object2"]
+                                             "/.../custom_objects.py": ["object1", "object2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning

mlrun/model.py

@@ -931,7 +931,7 @@ class RunSpec(ModelObj):
 
         >>> run_spec.inputs = {
         ...     "my_input": "...",
-        ...     "my_hinted_input : pandas.DataFrame": "..."
+        ...     "my_hinted_input : pandas.DataFrame": "...",
         ... }
 
         :param inputs: The inputs to set.
@@ -1275,7 +1275,7 @@ class RunTemplate(ModelObj):
 
         example::
 
-            grid_params = {"p1": [2,4,1], "p2": [10,20]}
+            grid_params = {"p1": [2, 4, 1], "p2": [10, 20]}
             task = mlrun.new_task("grid-search")
             task.with_hyper_params(grid_params, selector="max.accuracy")
         """

mlrun/model_monitoring/application.py

@@ -93,7 +93,11 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
                 endpoint_id: str,
                 output_stream_uri: str,
             ) -> ModelMonitoringApplicationResult:
-                self.context.log_artifact(TableArtifact("sample_df_stats", df=self.dict_to_histogram(sample_df_stats)))
+                self.context.log_artifact(
+                    TableArtifact(
+                        "sample_df_stats", df=self.dict_to_histogram(sample_df_stats)
+                    )
+                )
                 return ModelMonitoringApplicationResult(
                     name="data_drift_test",
                     value=0.5,
@@ -101,6 +105,7 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
                     status=mm_constant.ResultStatusApp.detected,
                 )
 
+
         # mlrun: end-code
     """
 
@@ -203,7 +208,11 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
             json.loads(event[mm_constant.ApplicationEvent.FEATURE_STATS]),
             ParquetTarget(
                 path=event[mm_constant.ApplicationEvent.SAMPLE_PARQUET_PATH]
-            ).as_df(start_time=start_time, end_time=end_time, time_column="timestamp"),
+            ).as_df(
+                start_time=start_time,
+                end_time=end_time,
+                time_column=mm_constant.FeatureSetFeatures.time_stamp(),
+            ),
             start_time,
             end_time,
             pd.Timestamp(event[mm_constant.ApplicationEvent.LAST_REQUEST]),

mlrun/model_monitoring/applications/histogram_data_drift.py

@@ -22,8 +22,8 @@ import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.model_monitoring.features_drift_table as mm_drift_table
 from mlrun.common.schemas.model_monitoring.constants import (
-    MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME,
     EventFieldType,
+    HistogramDataDriftApplicationConstants,
     ResultKindApp,
     ResultStatusApp,
 )
@@ -94,7 +94,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
     and the status is returned.
     """
 
-    NAME: Final[str] = MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+    NAME: Final[str] = HistogramDataDriftApplicationConstants.NAME
     METRIC_KIND: Final[ResultKindApp] = ResultKindApp.data_drift
 
     _REQUIRED_METRICS = {HellingerDistance, TotalVarianceDistance}
@@ -148,7 +148,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase):
         status = self._value_classifier.value_to_status(value)
         results.append(
             ModelMonitoringApplicationResult(
-                name="general_drift",
+                name=HistogramDataDriftApplicationConstants.GENERAL_RESULT_NAME,
                 value=value,
                 kind=self.METRIC_KIND,
                 status=status,

mlrun/model_monitoring/controller.py

@@ -354,7 +354,7 @@ class MonitoringApplicationController:
                             app.status.state == "ready"
                             # workaround for the default app, as its `status.state` is `None`
                             or app.metadata.name
-                            == mm_constants.MLRUN_HISTOGRAM_DATA_DRIFT_APP_NAME
+                            == mm_constants.HistogramDataDriftApplicationConstants.NAME
                         )
                     }
                 )
@@ -502,8 +502,7 @@ class MonitoringApplicationController:
 
                     # Get the current stats:
                     current_stats = calculate_inputs_statistics(
-                        sample_set_statistics=feature_stats,
-                        inputs=df,
+                        sample_set_statistics=feature_stats, inputs=df
                     )
 
                     cls._push_to_applications(

mlrun/model_monitoring/stream_processing.py

@@ -352,7 +352,6 @@ class EventStreamProcessor:
                     rate="10/m",
                     time_col=EventFieldType.TIMESTAMP,
                     container=self.tsdb_container,
-                    access_key=self.v3io_access_key,
                     v3io_frames=self.v3io_framesd,
                     infer_columns_from_data=True,
                     index_cols=[

mlrun/model_monitoring/writer.py

@@ -23,6 +23,7 @@ from v3io_frames.errors import Error as V3IOFramesError
 from v3io_frames.frames_pb2 import IGNORE
 
 import mlrun.common.model_monitoring
+import mlrun.common.schemas.alert as alert_constants
 import mlrun.model_monitoring
 import mlrun.model_monitoring.db.stores
 import mlrun.utils.v3io_clients
@@ -171,6 +172,29 @@ class ModelMonitoringWriter(StepToDict):
                 event=event,
             )
 
+    @staticmethod
+    def _generate_event_on_drift(
+        uid: str, drift_status: str, drift_value: float, project_name: str
+    ):
+        if (
+            drift_status == ResultStatusApp.detected
+            or drift_status == ResultStatusApp.potential_detection
+        ):
+            entity = {
+                "kind": alert_constants.EventEntityKind.MODEL,
+                "project": project_name,
+                "id": uid,
+            }
+            event_kind = (
+                alert_constants.EventKind.DRIFT_DETECTED
+                if drift_status == ResultStatusApp.detected
+                else alert_constants.EventKind.DRIFT_SUSPECTED
+            )
+            event_data = mlrun.common.schemas.Event(
+                kind=event_kind, entity=entity, value=drift_value
+            )
+            mlrun.get_run_db().generate_event(event_kind, event_data)
+
     @staticmethod
     def _reconstruct_event(event: _RawEvent) -> _AppResultEvent:
         """
@@ -201,4 +225,12 @@ class ModelMonitoringWriter(StepToDict):
         self._update_tsdb(event)
         self._update_kv_db(event)
         _Notifier(event=event, notification_pusher=self._custom_notifier).notify()
+
+        if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.enabled:
+            self._generate_event_on_drift(
+                event[WriterEvent.ENDPOINT_ID],
+                event[WriterEvent.RESULT_STATUS],
+                event[WriterEvent.RESULT_VALUE],
+                self.project,
+            )
         logger.info("Completed event DB writes")

mlrun/package/packagers_manager.py

@@ -92,6 +92,7 @@ class PackagersManager:
             from mlrun import Packager
             from x import XPackager
 
+
             class YPackager(Packager):
                 pass
 

mlrun/platforms/__init__.py

@@ -48,7 +48,7 @@ def watch_stream(
 
     example::
 
-        watch_stream('v3io:///users/admin/mystream')
+        watch_stream("v3io:///users/admin/mystream")
 
     :param url:        stream url
     :param shard_ids:  range or list of shard IDs

mlrun/platforms/other.py

@@ -33,7 +33,7 @@ def mount_pvc(pvc_name=None, volume_name="pipeline", volume_mount_path="/mnt/pip
     Usage::
 
         train = train_op(...)
-        train.apply(mount_pvc('claim-name', 'pipeline', '/mnt/pipeline'))
+        train.apply(mount_pvc("claim-name", "pipeline", "/mnt/pipeline"))
     """
     if "MLRUN_PVC_MOUNT" in os.environ:
         mount = os.environ.get("MLRUN_PVC_MOUNT")

mlrun/projects/operations.py

@@ -95,8 +95,11 @@ def run_function(
         MODEL_CLASS = "sklearn.ensemble.RandomForestClassifier"
         DATA_PATH = "s3://bigdata/data.parquet"
         function = mlrun.import_function("hub://auto-trainer")
-        run1 = run_function(function, params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                      inputs={"dataset": DATA_PATH})
+        run1 = run_function(
+            function,
+            params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+            inputs={"dataset": DATA_PATH},
+        )
 
     example (use with project)::
 
@@ -115,8 +118,12 @@ def run_function(
         @dsl.pipeline(name="test pipeline", description="test")
         def my_pipe(url=""):
             run1 = run_function("loaddata", params={"url": url}, outputs=["data"])
-            run2 = run_function("train", params={"label_columns": LABELS, "model_class": MODEL_CLASS},
-                                         inputs={"dataset": run1.outputs["data"]})
+            run2 = run_function(
+                "train",
+                params={"label_columns": LABELS, "model_class": MODEL_CLASS},
+                inputs={"dataset": run1.outputs["data"]},
+            )
+
 
         project.run(workflow_handler=my_pipe, arguments={"param1": 7})