mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (275) hide show
  1. mlrun/__init__.py +26 -22
  2. mlrun/__main__.py +15 -16
  3. mlrun/alerts/alert.py +150 -15
  4. mlrun/api/schemas/__init__.py +1 -9
  5. mlrun/artifacts/__init__.py +2 -3
  6. mlrun/artifacts/base.py +62 -19
  7. mlrun/artifacts/dataset.py +17 -17
  8. mlrun/artifacts/document.py +454 -0
  9. mlrun/artifacts/manager.py +28 -18
  10. mlrun/artifacts/model.py +91 -59
  11. mlrun/artifacts/plots.py +2 -2
  12. mlrun/common/constants.py +8 -0
  13. mlrun/common/formatters/__init__.py +1 -0
  14. mlrun/common/formatters/artifact.py +1 -1
  15. mlrun/common/formatters/feature_set.py +2 -0
  16. mlrun/common/formatters/function.py +1 -0
  17. mlrun/{model_monitoring/db/stores/v3io_kv/__init__.py → common/formatters/model_endpoint.py} +17 -0
  18. mlrun/common/formatters/pipeline.py +1 -2
  19. mlrun/common/formatters/project.py +9 -0
  20. mlrun/common/model_monitoring/__init__.py +0 -5
  21. mlrun/common/model_monitoring/helpers.py +12 -62
  22. mlrun/common/runtimes/constants.py +25 -4
  23. mlrun/common/schemas/__init__.py +9 -5
  24. mlrun/common/schemas/alert.py +114 -19
  25. mlrun/common/schemas/api_gateway.py +3 -3
  26. mlrun/common/schemas/artifact.py +22 -9
  27. mlrun/common/schemas/auth.py +8 -4
  28. mlrun/common/schemas/background_task.py +7 -7
  29. mlrun/common/schemas/client_spec.py +4 -4
  30. mlrun/common/schemas/clusterization_spec.py +2 -2
  31. mlrun/common/schemas/common.py +53 -3
  32. mlrun/common/schemas/constants.py +15 -0
  33. mlrun/common/schemas/datastore_profile.py +1 -1
  34. mlrun/common/schemas/feature_store.py +9 -9
  35. mlrun/common/schemas/frontend_spec.py +4 -4
  36. mlrun/common/schemas/function.py +10 -10
  37. mlrun/common/schemas/hub.py +1 -1
  38. mlrun/common/schemas/k8s.py +3 -3
  39. mlrun/common/schemas/memory_reports.py +3 -3
  40. mlrun/common/schemas/model_monitoring/__init__.py +4 -8
  41. mlrun/common/schemas/model_monitoring/constants.py +127 -46
  42. mlrun/common/schemas/model_monitoring/grafana.py +18 -12
  43. mlrun/common/schemas/model_monitoring/model_endpoints.py +154 -160
  44. mlrun/common/schemas/notification.py +24 -3
  45. mlrun/common/schemas/object.py +1 -1
  46. mlrun/common/schemas/pagination.py +4 -4
  47. mlrun/common/schemas/partition.py +142 -0
  48. mlrun/common/schemas/pipeline.py +3 -3
  49. mlrun/common/schemas/project.py +26 -18
  50. mlrun/common/schemas/runs.py +3 -3
  51. mlrun/common/schemas/runtime_resource.py +5 -5
  52. mlrun/common/schemas/schedule.py +1 -1
  53. mlrun/common/schemas/secret.py +1 -1
  54. mlrun/{model_monitoring/db/stores/sqldb/__init__.py → common/schemas/serving.py} +10 -1
  55. mlrun/common/schemas/tag.py +3 -3
  56. mlrun/common/schemas/workflow.py +6 -5
  57. mlrun/common/types.py +1 -0
  58. mlrun/config.py +157 -89
  59. mlrun/data_types/__init__.py +5 -3
  60. mlrun/data_types/infer.py +13 -3
  61. mlrun/data_types/spark.py +2 -1
  62. mlrun/datastore/__init__.py +59 -18
  63. mlrun/datastore/alibaba_oss.py +4 -1
  64. mlrun/datastore/azure_blob.py +4 -1
  65. mlrun/datastore/base.py +19 -24
  66. mlrun/datastore/datastore.py +10 -4
  67. mlrun/datastore/datastore_profile.py +178 -45
  68. mlrun/datastore/dbfs_store.py +4 -1
  69. mlrun/datastore/filestore.py +4 -1
  70. mlrun/datastore/google_cloud_storage.py +4 -1
  71. mlrun/datastore/hdfs.py +4 -1
  72. mlrun/datastore/inmem.py +4 -1
  73. mlrun/datastore/redis.py +4 -1
  74. mlrun/datastore/s3.py +14 -3
  75. mlrun/datastore/sources.py +89 -92
  76. mlrun/datastore/store_resources.py +7 -4
  77. mlrun/datastore/storeytargets.py +51 -16
  78. mlrun/datastore/targets.py +38 -31
  79. mlrun/datastore/utils.py +87 -4
  80. mlrun/datastore/v3io.py +4 -1
  81. mlrun/datastore/vectorstore.py +291 -0
  82. mlrun/datastore/wasbfs/fs.py +13 -12
  83. mlrun/db/base.py +286 -100
  84. mlrun/db/httpdb.py +1562 -490
  85. mlrun/db/nopdb.py +250 -83
  86. mlrun/errors.py +6 -2
  87. mlrun/execution.py +194 -50
  88. mlrun/feature_store/__init__.py +2 -10
  89. mlrun/feature_store/api.py +20 -458
  90. mlrun/feature_store/common.py +9 -9
  91. mlrun/feature_store/feature_set.py +20 -18
  92. mlrun/feature_store/feature_vector.py +105 -479
  93. mlrun/feature_store/feature_vector_utils.py +466 -0
  94. mlrun/feature_store/retrieval/base.py +15 -11
  95. mlrun/feature_store/retrieval/job.py +2 -1
  96. mlrun/feature_store/retrieval/storey_merger.py +1 -1
  97. mlrun/feature_store/steps.py +3 -3
  98. mlrun/features.py +30 -13
  99. mlrun/frameworks/__init__.py +1 -2
  100. mlrun/frameworks/_common/__init__.py +1 -2
  101. mlrun/frameworks/_common/artifacts_library.py +2 -2
  102. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  103. mlrun/frameworks/_common/model_handler.py +31 -31
  104. mlrun/frameworks/_common/producer.py +3 -1
  105. mlrun/frameworks/_dl_common/__init__.py +1 -2
  106. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  107. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  108. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  109. mlrun/frameworks/_ml_common/__init__.py +1 -2
  110. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  111. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  112. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  113. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  114. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  115. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  116. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  117. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  118. mlrun/frameworks/huggingface/__init__.py +1 -2
  119. mlrun/frameworks/huggingface/model_server.py +9 -9
  120. mlrun/frameworks/lgbm/__init__.py +47 -44
  121. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  122. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  123. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  124. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  125. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  126. mlrun/frameworks/lgbm/model_handler.py +15 -11
  127. mlrun/frameworks/lgbm/model_server.py +11 -7
  128. mlrun/frameworks/lgbm/utils.py +2 -2
  129. mlrun/frameworks/onnx/__init__.py +1 -2
  130. mlrun/frameworks/onnx/dataset.py +3 -3
  131. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  132. mlrun/frameworks/onnx/model_handler.py +7 -5
  133. mlrun/frameworks/onnx/model_server.py +8 -6
  134. mlrun/frameworks/parallel_coordinates.py +11 -11
  135. mlrun/frameworks/pytorch/__init__.py +22 -23
  136. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  137. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  138. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  139. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  140. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  141. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  142. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  143. mlrun/frameworks/pytorch/model_handler.py +21 -17
  144. mlrun/frameworks/pytorch/model_server.py +13 -9
  145. mlrun/frameworks/sklearn/__init__.py +19 -18
  146. mlrun/frameworks/sklearn/estimator.py +2 -2
  147. mlrun/frameworks/sklearn/metric.py +3 -3
  148. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  149. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  150. mlrun/frameworks/sklearn/model_handler.py +4 -3
  151. mlrun/frameworks/tf_keras/__init__.py +11 -12
  152. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  153. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  154. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  155. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  156. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  157. mlrun/frameworks/tf_keras/model_server.py +12 -8
  158. mlrun/frameworks/xgboost/__init__.py +19 -18
  159. mlrun/frameworks/xgboost/model_handler.py +13 -9
  160. mlrun/k8s_utils.py +2 -5
  161. mlrun/launcher/base.py +3 -4
  162. mlrun/launcher/client.py +2 -2
  163. mlrun/launcher/local.py +6 -2
  164. mlrun/launcher/remote.py +1 -1
  165. mlrun/lists.py +8 -4
  166. mlrun/model.py +132 -46
  167. mlrun/model_monitoring/__init__.py +3 -5
  168. mlrun/model_monitoring/api.py +113 -98
  169. mlrun/model_monitoring/applications/__init__.py +0 -5
  170. mlrun/model_monitoring/applications/_application_steps.py +81 -50
  171. mlrun/model_monitoring/applications/base.py +467 -14
  172. mlrun/model_monitoring/applications/context.py +212 -134
  173. mlrun/model_monitoring/{db/stores/base → applications/evidently}/__init__.py +6 -2
  174. mlrun/model_monitoring/applications/evidently/base.py +146 -0
  175. mlrun/model_monitoring/applications/histogram_data_drift.py +89 -56
  176. mlrun/model_monitoring/applications/results.py +67 -15
  177. mlrun/model_monitoring/controller.py +701 -315
  178. mlrun/model_monitoring/db/__init__.py +0 -2
  179. mlrun/model_monitoring/db/_schedules.py +242 -0
  180. mlrun/model_monitoring/db/_stats.py +189 -0
  181. mlrun/model_monitoring/db/tsdb/__init__.py +33 -22
  182. mlrun/model_monitoring/db/tsdb/base.py +243 -49
  183. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +76 -36
  184. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  185. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
  186. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +534 -88
  187. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  188. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +436 -106
  189. mlrun/model_monitoring/helpers.py +356 -114
  190. mlrun/model_monitoring/stream_processing.py +190 -345
  191. mlrun/model_monitoring/tracking_policy.py +11 -4
  192. mlrun/model_monitoring/writer.py +49 -90
  193. mlrun/package/__init__.py +3 -6
  194. mlrun/package/context_handler.py +2 -2
  195. mlrun/package/packager.py +12 -9
  196. mlrun/package/packagers/__init__.py +0 -2
  197. mlrun/package/packagers/default_packager.py +14 -11
  198. mlrun/package/packagers/numpy_packagers.py +16 -7
  199. mlrun/package/packagers/pandas_packagers.py +18 -18
  200. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  201. mlrun/package/packagers_manager.py +35 -32
  202. mlrun/package/utils/__init__.py +0 -3
  203. mlrun/package/utils/_pickler.py +6 -6
  204. mlrun/platforms/__init__.py +47 -16
  205. mlrun/platforms/iguazio.py +4 -1
  206. mlrun/projects/operations.py +30 -30
  207. mlrun/projects/pipelines.py +116 -47
  208. mlrun/projects/project.py +1292 -329
  209. mlrun/render.py +5 -9
  210. mlrun/run.py +57 -14
  211. mlrun/runtimes/__init__.py +1 -3
  212. mlrun/runtimes/base.py +30 -22
  213. mlrun/runtimes/daskjob.py +9 -9
  214. mlrun/runtimes/databricks_job/databricks_runtime.py +6 -5
  215. mlrun/runtimes/function_reference.py +5 -2
  216. mlrun/runtimes/generators.py +3 -2
  217. mlrun/runtimes/kubejob.py +6 -7
  218. mlrun/runtimes/mounts.py +574 -0
  219. mlrun/runtimes/mpijob/__init__.py +0 -2
  220. mlrun/runtimes/mpijob/abstract.py +7 -6
  221. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  222. mlrun/runtimes/nuclio/application/application.py +11 -13
  223. mlrun/runtimes/nuclio/application/reverse_proxy.go +66 -64
  224. mlrun/runtimes/nuclio/function.py +127 -70
  225. mlrun/runtimes/nuclio/serving.py +105 -37
  226. mlrun/runtimes/pod.py +159 -54
  227. mlrun/runtimes/remotesparkjob.py +3 -2
  228. mlrun/runtimes/sparkjob/__init__.py +0 -2
  229. mlrun/runtimes/sparkjob/spark3job.py +22 -12
  230. mlrun/runtimes/utils.py +7 -6
  231. mlrun/secrets.py +2 -2
  232. mlrun/serving/__init__.py +8 -0
  233. mlrun/serving/merger.py +7 -5
  234. mlrun/serving/remote.py +35 -22
  235. mlrun/serving/routers.py +186 -240
  236. mlrun/serving/server.py +41 -10
  237. mlrun/serving/states.py +432 -118
  238. mlrun/serving/utils.py +13 -2
  239. mlrun/serving/v1_serving.py +3 -2
  240. mlrun/serving/v2_serving.py +161 -203
  241. mlrun/track/__init__.py +1 -1
  242. mlrun/track/tracker.py +2 -2
  243. mlrun/track/trackers/mlflow_tracker.py +6 -5
  244. mlrun/utils/async_http.py +35 -22
  245. mlrun/utils/clones.py +7 -4
  246. mlrun/utils/helpers.py +511 -58
  247. mlrun/utils/logger.py +119 -13
  248. mlrun/utils/notifications/notification/__init__.py +22 -19
  249. mlrun/utils/notifications/notification/base.py +39 -15
  250. mlrun/utils/notifications/notification/console.py +6 -6
  251. mlrun/utils/notifications/notification/git.py +11 -11
  252. mlrun/utils/notifications/notification/ipython.py +10 -9
  253. mlrun/utils/notifications/notification/mail.py +176 -0
  254. mlrun/utils/notifications/notification/slack.py +16 -8
  255. mlrun/utils/notifications/notification/webhook.py +24 -8
  256. mlrun/utils/notifications/notification_pusher.py +191 -200
  257. mlrun/utils/regex.py +12 -2
  258. mlrun/utils/version/version.json +2 -2
  259. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/METADATA +81 -54
  260. mlrun-1.8.0.dist-info/RECORD +351 -0
  261. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/WHEEL +1 -1
  262. mlrun/model_monitoring/applications/evidently_base.py +0 -137
  263. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  264. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  265. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  266. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  267. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  268. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  269. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  270. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  271. mlrun/model_monitoring/model_endpoint.py +0 -118
  272. mlrun-1.7.2rc3.dist-info/RECORD +0 -351
  273. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/entry_points.txt +0 -0
  274. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info/licenses}/LICENSE +0 -0
  275. {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/top_level.txt +0 -0
mlrun/serving/routers.py CHANGED
@@ -18,6 +18,7 @@ import copy
18
18
  import json
19
19
  import traceback
20
20
  import typing
21
+ from datetime import timedelta
21
22
  from enum import Enum
22
23
  from io import BytesIO
23
24
  from typing import Union
@@ -28,11 +29,8 @@ import numpy as np
28
29
  import mlrun
29
30
  import mlrun.common.model_monitoring
30
31
  import mlrun.common.schemas.model_monitoring
31
- from mlrun.errors import err_to_str
32
32
  from mlrun.utils import logger, now_date
33
33
 
34
- from ..common.helpers import parse_versioned_object_uri
35
- from .server import GraphServer
36
34
  from .utils import RouterToDict, _extract_input_data, _update_result_body
37
35
  from .v2_serving import _ModelLogPusher
38
36
 
@@ -46,13 +44,13 @@ class BaseModelRouter(RouterToDict):
46
44
  def __init__(
47
45
  self,
48
46
  context=None,
49
- name: str = None,
47
+ name: typing.Optional[str] = None,
50
48
  routes=None,
51
- protocol: str = None,
52
- url_prefix: str = None,
53
- health_prefix: str = None,
54
- input_path: str = None,
55
- result_path: str = None,
49
+ protocol: typing.Optional[str] = None,
50
+ url_prefix: typing.Optional[str] = None,
51
+ health_prefix: typing.Optional[str] = None,
52
+ input_path: typing.Optional[str] = None,
53
+ result_path: typing.Optional[str] = None,
56
54
  **kwargs,
57
55
  ):
58
56
  """Model Serving Router, route between child models
@@ -81,6 +79,9 @@ class BaseModelRouter(RouterToDict):
81
79
  self.inputs_key = "instances" if self.protocol == "v1" else "inputs"
82
80
  self._input_path = input_path
83
81
  self._result_path = result_path
82
+ self._background_task_check_timestamp = None
83
+ self._background_task_terminate = False
84
+ self._background_task_current_state = None
84
85
  self.kwargs = kwargs
85
86
 
86
87
  def parse_event(self, event):
@@ -111,7 +112,7 @@ class BaseModelRouter(RouterToDict):
111
112
 
112
113
  return parsed_event
113
114
 
114
- def post_init(self, mode="sync"):
115
+ def post_init(self, mode="sync", **kwargs):
115
116
  self.context.logger.info(f"Loaded {list(self.routes.keys())}")
116
117
 
117
118
  def get_metadata(self):
@@ -138,6 +139,7 @@ class BaseModelRouter(RouterToDict):
138
139
  raise ValueError(
139
140
  f"illegal path prefix {urlpath}, must start with {self.url_prefix}"
140
141
  )
142
+ self._update_background_task_state(event)
141
143
  return event
142
144
 
143
145
  def do_event(self, event, *args, **kwargs):
@@ -163,6 +165,63 @@ class BaseModelRouter(RouterToDict):
163
165
  """run tasks after processing the event"""
164
166
  return event
165
167
 
168
+ def _get_background_task_status(
169
+ self,
170
+ ) -> mlrun.common.schemas.BackgroundTaskState:
171
+ self._background_task_check_timestamp = now_date()
172
+ server: mlrun.serving.GraphServer = getattr(
173
+ self.context, "_server", None
174
+ ) or getattr(self.context, "server", None)
175
+ if not self.context.is_mock:
176
+ if server.model_endpoint_creation_task_name:
177
+ background_task = mlrun.get_run_db().get_project_background_task(
178
+ server.project, server.model_endpoint_creation_task_name
179
+ )
180
+ logger.debug(
181
+ "Checking model endpoint creation task status",
182
+ task_name=server.model_endpoint_creation_task_name,
183
+ )
184
+ if (
185
+ background_task.status.state
186
+ in mlrun.common.schemas.BackgroundTaskState.terminal_states()
187
+ ):
188
+ logger.debug(
189
+ f"Model endpoint creation task completed with state {background_task.status.state}"
190
+ )
191
+ self._background_task_terminate = True
192
+ else: # in progress
193
+ logger.debug(
194
+ f"Model endpoint creation task is still in progress with the current state: "
195
+ f"{background_task.status.state}. Events will not be monitored for the next 15 seconds",
196
+ name=self.name,
197
+ background_task_check_timestamp=self._background_task_check_timestamp.isoformat(),
198
+ )
199
+ return background_task.status.state
200
+ else:
201
+ logger.debug(
202
+ "Model endpoint creation task name not provided",
203
+ )
204
+ elif self.context.monitoring_mock:
205
+ self._background_task_terminate = (
206
+ True # If mock monitoring we return success and terminate task check.
207
+ )
208
+ return mlrun.common.schemas.BackgroundTaskState.succeeded
209
+ self._background_task_terminate = True # If mock without monitoring we return failed and terminate task check.
210
+ return mlrun.common.schemas.BackgroundTaskState.failed
211
+
212
+ def _update_background_task_state(self, event):
213
+ if not self._background_task_terminate and (
214
+ self._background_task_check_timestamp is None
215
+ or now_date() - self._background_task_check_timestamp
216
+ >= timedelta(seconds=15)
217
+ ):
218
+ self._background_task_current_state = self._get_background_task_status()
219
+ if event.body:
220
+ event.body["background_task_state"] = (
221
+ self._background_task_current_state
222
+ or mlrun.common.schemas.BackgroundTaskState.running
223
+ )
224
+
166
225
 
167
226
  class ModelRouter(BaseModelRouter):
168
227
  def _resolve_route(self, body, urlpath):
@@ -249,11 +308,11 @@ class ParallelRun(BaseModelRouter):
249
308
  def __init__(
250
309
  self,
251
310
  context=None,
252
- name: str = None,
311
+ name: typing.Optional[str] = None,
253
312
  routes=None,
254
- protocol: str = None,
255
- url_prefix: str = None,
256
- health_prefix: str = None,
313
+ protocol: typing.Optional[str] = None,
314
+ url_prefix: typing.Optional[str] = None,
315
+ health_prefix: typing.Optional[str] = None,
257
316
  extend_event=None,
258
317
  executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
259
318
  **kwargs,
@@ -392,7 +451,7 @@ class ParallelRun(BaseModelRouter):
392
451
  self._pool = executor_class(
393
452
  max_workers=len(self.routes),
394
453
  initializer=ParallelRun.init_pool,
395
- initargs=(server, routes),
454
+ initargs=(server, routes, self.context.is_mock),
396
455
  )
397
456
  elif self.executor_type == ParallelRunnerModes.thread:
398
457
  executor_class = concurrent.futures.ThreadPoolExecutor
@@ -455,9 +514,9 @@ class ParallelRun(BaseModelRouter):
455
514
  return results
456
515
 
457
516
  @staticmethod
458
- def init_pool(server_spec, routes):
517
+ def init_pool(server_spec, routes, is_mock):
459
518
  server = mlrun.serving.GraphServer.from_dict(server_spec)
460
- server.init_states(None, None)
519
+ server.init_states(None, None, is_mock=is_mock)
461
520
  global local_routes
462
521
  for route in routes.values():
463
522
  route.context = server.context
@@ -481,13 +540,13 @@ class VotingEnsemble(ParallelRun):
481
540
  def __init__(
482
541
  self,
483
542
  context=None,
484
- name: str = None,
543
+ name: typing.Optional[str] = None,
485
544
  routes=None,
486
- protocol: str = None,
487
- url_prefix: str = None,
488
- health_prefix: str = None,
489
- vote_type: str = None,
490
- weights: dict[str, float] = None,
545
+ protocol: typing.Optional[str] = None,
546
+ url_prefix: typing.Optional[str] = None,
547
+ health_prefix: typing.Optional[str] = None,
548
+ vote_type: typing.Optional[str] = None,
549
+ weights: typing.Optional[dict[str, float]] = None,
491
550
  executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
492
551
  format_response_with_col_name_flag: bool = False,
493
552
  prediction_col_name: str = "prediction",
@@ -599,31 +658,33 @@ class VotingEnsemble(ParallelRun):
599
658
  self.vote_type = vote_type
600
659
  self.vote_flag = True if self.vote_type is not None else False
601
660
  self.weights = weights
602
- self._model_logger = (
603
- _ModelLogPusher(self, context)
604
- if context and context.stream.enabled
605
- else None
606
- )
607
- self.version = kwargs.get("version", "v1")
608
661
  self.log_router = True
609
662
  self.prediction_col_name = prediction_col_name or "prediction"
610
663
  self.format_response_with_col_name_flag = format_response_with_col_name_flag
611
- self.model_endpoint_uid = None
664
+ self.model_endpoint_uid = kwargs.get("model_endpoint_uid", None)
612
665
  self.shard_by_endpoint = shard_by_endpoint
666
+ self._model_logger = None
667
+ self.initialized = False
613
668
 
614
- def post_init(self, mode="sync"):
615
- server = getattr(self.context, "_server", None) or getattr(
616
- self.context, "server", None
617
- )
618
- if not server:
619
- logger.warn("GraphServer not initialized for VotingEnsemble instance")
620
- return
621
-
622
- if not self.context.is_mock or self.context.monitoring_mock:
623
- self.model_endpoint_uid = _init_endpoint_record(server, self)
624
-
669
+ def post_init(self, mode="sync", **kwargs):
625
670
  self._update_weights(self.weights)
626
671
 
672
+ def _lazy_init(self, event):
673
+ if event and isinstance(event, dict):
674
+ background_task_state = event.get("background_task_state", None)
675
+ if (
676
+ background_task_state
677
+ == mlrun.common.schemas.BackgroundTaskState.succeeded
678
+ ):
679
+ self._model_logger = (
680
+ _ModelLogPusher(self, self.context)
681
+ if self.context
682
+ and self.context.stream.enabled
683
+ and self.model_endpoint_uid
684
+ else None
685
+ )
686
+ self.initialized = True
687
+
627
688
  def _resolve_route(self, body, urlpath):
628
689
  """Resolves the appropriate model to send the event to.
629
690
  Supports:
@@ -814,7 +875,8 @@ class VotingEnsemble(ParallelRun):
814
875
  return self.logic(flattened_predictions, np.array(weights))
815
876
 
816
877
  def do_event(self, event, *args, **kwargs):
817
- """Handles incoming requests.
878
+ """
879
+ Handles incoming requests.
818
880
 
819
881
  Parameters
820
882
  ----------
@@ -827,12 +889,13 @@ class VotingEnsemble(ParallelRun):
827
889
  Event response after running the requested logic
828
890
  """
829
891
  start = now_date()
830
-
831
892
  # Handle and verify the request
832
893
  original_body = event.body
833
894
  event.body = _extract_input_data(self._input_path, event.body)
834
895
  event = self.preprocess(event)
835
896
  event = self._pre_handle_event(event)
897
+ if not self.initialized:
898
+ self._lazy_init(event.body)
836
899
 
837
900
  # Should we terminate the event?
838
901
  if hasattr(event, "terminated") and event.terminated:
@@ -879,14 +942,14 @@ class VotingEnsemble(ParallelRun):
879
942
  "model_name": self.name,
880
943
  "outputs": votes,
881
944
  }
882
- if self.version:
883
- response_body["model_version"] = self.version
945
+ if self.model_endpoint_uid:
946
+ response_body["model_endpoint_uid"] = self.model_endpoint_uid
884
947
  response.body = response_body
885
948
  elif name == self.name and event.method == "GET" and not subpath:
886
949
  response = copy.copy(event)
887
950
  response_body = {
888
951
  "name": self.name,
889
- "version": self.version or "",
952
+ "model_endpoint_uid": self.model_endpoint_uid or "",
890
953
  "inputs": [],
891
954
  "outputs": [],
892
955
  }
@@ -1000,146 +1063,25 @@ class VotingEnsemble(ParallelRun):
1000
1063
  self._weights[model] = 0
1001
1064
 
1002
1065
 
1003
- def _init_endpoint_record(
1004
- graph_server: GraphServer, voting_ensemble: VotingEnsemble
1005
- ) -> Union[str, None]:
1066
+ class EnrichmentModelRouter(ModelRouter):
1006
1067
  """
1007
- Initialize model endpoint record and write it into the DB. In general, this method retrieve the unique model
1008
- endpoint ID which is generated according to the function uri and the model version. If the model endpoint is
1009
- already exist in the DB, we skip the creation process. Otherwise, it writes the new model endpoint record to the DB.
1010
-
1011
- :param graph_server: A GraphServer object which will be used for getting the function uri.
1012
- :param voting_ensemble: Voting ensemble serving class. It contains important details for the model endpoint record
1013
- such as model name, model path, model version, and the ids of the children model endpoints.
1014
-
1015
- :return: Model endpoint unique ID.
1068
+ Model router with feature enrichment and imputing
1016
1069
  """
1017
1070
 
1018
- logger.info("Initializing endpoint records")
1019
-
1020
- # Generate required values for the model endpoint record
1021
- try:
1022
- # Getting project name from the function uri
1023
- project, uri, tag, hash_key = parse_versioned_object_uri(
1024
- graph_server.function_uri
1025
- )
1026
- except Exception as e:
1027
- logger.error("Failed to parse function URI", exc=err_to_str(e))
1028
- return None
1029
-
1030
- # Generating version model value based on the model name and model version
1031
- if voting_ensemble.version:
1032
- versioned_model_name = f"{voting_ensemble.name}:{voting_ensemble.version}"
1033
- else:
1034
- versioned_model_name = f"{voting_ensemble.name}:latest"
1035
-
1036
- # Generating model endpoint ID based on function uri and model version
1037
- endpoint_uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
1038
- function_uri=graph_server.function_uri, versioned_model=versioned_model_name
1039
- ).uid
1040
-
1041
- try:
1042
- model_ep = mlrun.get_run_db().get_model_endpoint(
1043
- project=project, endpoint_id=endpoint_uid
1044
- )
1045
- except mlrun.errors.MLRunNotFoundError:
1046
- model_ep = None
1047
- except mlrun.errors.MLRunBadRequestError as err:
1048
- logger.debug(
1049
- f"Cant reach to model endpoints store, due to : {err}",
1050
- )
1051
- return
1052
-
1053
- if voting_ensemble.context.server.track_models and not model_ep:
1054
- logger.info("Creating a new model endpoint record", endpoint_id=endpoint_uid)
1055
- # Get the children model endpoints ids
1056
- children_uids = []
1057
- for _, c in voting_ensemble.routes.items():
1058
- if hasattr(c, "endpoint_uid"):
1059
- children_uids.append(c.endpoint_uid)
1060
- model_endpoint = mlrun.common.schemas.ModelEndpoint(
1061
- metadata=mlrun.common.schemas.ModelEndpointMetadata(
1062
- project=project, uid=endpoint_uid
1063
- ),
1064
- spec=mlrun.common.schemas.ModelEndpointSpec(
1065
- function_uri=graph_server.function_uri,
1066
- model=versioned_model_name,
1067
- model_class=voting_ensemble.__class__.__name__,
1068
- stream_path=voting_ensemble.context.stream.stream_uri,
1069
- active=True,
1070
- monitoring_mode=mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled,
1071
- ),
1072
- status=mlrun.common.schemas.ModelEndpointStatus(
1073
- children=list(voting_ensemble.routes.keys()),
1074
- endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.ROUTER,
1075
- children_uids=children_uids,
1076
- ),
1077
- )
1078
-
1079
- db = mlrun.get_run_db()
1080
-
1081
- db.create_model_endpoint(
1082
- project=project,
1083
- endpoint_id=model_endpoint.metadata.uid,
1084
- model_endpoint=model_endpoint.dict(),
1085
- )
1086
-
1087
- # Update model endpoint children type
1088
- for model_endpoint in children_uids:
1089
- current_endpoint = db.get_model_endpoint(
1090
- project=project, endpoint_id=model_endpoint
1091
- )
1092
- current_endpoint.status.endpoint_type = (
1093
- mlrun.common.schemas.model_monitoring.EndpointType.LEAF_EP
1094
- )
1095
- db.create_model_endpoint(
1096
- project=project,
1097
- endpoint_id=model_endpoint,
1098
- model_endpoint=current_endpoint,
1099
- )
1100
- elif (
1101
- model_ep
1102
- and (
1103
- model_ep.spec.monitoring_mode
1104
- == mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
1105
- )
1106
- != voting_ensemble.context.server.track_models
1107
- ):
1108
- monitoring_mode = (
1109
- mlrun.common.schemas.model_monitoring.ModelMonitoringMode.enabled
1110
- if voting_ensemble.context.server.track_models
1111
- else mlrun.common.schemas.model_monitoring.ModelMonitoringMode.disabled
1112
- )
1113
- db = mlrun.get_run_db()
1114
- db.patch_model_endpoint(
1115
- project=project,
1116
- endpoint_id=endpoint_uid,
1117
- attributes={"monitoring_mode": monitoring_mode},
1118
- )
1119
- logger.debug(
1120
- f"Updating model endpoint monitoring_mode to {monitoring_mode}",
1121
- endpoint_id=endpoint_uid,
1122
- )
1123
-
1124
- return endpoint_uid
1125
-
1126
-
1127
- class EnrichmentModelRouter(ModelRouter):
1128
- """model router with feature enrichment and imputing"""
1129
-
1130
1071
  def __init__(
1131
1072
  self,
1132
1073
  context=None,
1133
- name: str = None,
1074
+ name: typing.Optional[str] = None,
1134
1075
  routes=None,
1135
- protocol: str = None,
1136
- url_prefix: str = None,
1137
- health_prefix: str = None,
1076
+ protocol: typing.Optional[str] = None,
1077
+ url_prefix: typing.Optional[str] = None,
1078
+ health_prefix: typing.Optional[str] = None,
1138
1079
  feature_vector_uri: str = "",
1139
- impute_policy: dict = None,
1080
+ impute_policy: typing.Optional[dict] = None,
1140
1081
  **kwargs,
1141
1082
  ):
1142
- """Model router with feature enrichment (from the feature store)
1083
+ """
1084
+ Model router with feature enrichment (from the feature store)
1143
1085
 
1144
1086
  The `EnrichmentModelRouter` class enrich the incoming event with real-time features
1145
1087
  read from a feature vector (in MLRun feature store) and forwards the enriched event to the child models
@@ -1147,27 +1089,25 @@ class EnrichmentModelRouter(ModelRouter):
1147
1089
  The feature vector is specified using the `feature_vector_uri`, in addition an imputing policy
1148
1090
  can be specified to substitute None/NaN values with pre defines constant or stats.
1149
1091
 
1150
- :param feature_vector_uri : feature vector uri in the form: [project/]name[:tag]
1151
- :param impute_policy : value imputing (substitute NaN/Inf values with statistical or constant value),
1152
- you can set the `impute_policy` parameter with the imputing policy, and specify which
1153
- constant or statistical value will be used instead of NaN/Inf value, this can be defined
1154
- per column or for all the columns ("*"). the replaced value can be fixed number for
1155
- constants or $mean, $max, $min, $std, $count for statistical values.
1156
- “*” is used to specify the default for all features, example:
1157
- impute_policy={"*": "$mean", "age": 33}
1092
+ :param feature_vector_uri: feature vector uri in the form: [project/]name[:tag]
1093
+ :param impute_policy: value imputing (substitute NaN/Inf values with statistical or constant value),
1094
+ you can set the `impute_policy` parameter with the imputing policy, and specify which constant or
1095
+ statistical value will be used instead of NaN/Inf value, this can be defined per column or
1096
+ for all the columns ("*"). The replaced value can be fixed number for constants or $mean, $max, $min, $std,
1097
+ $count for statistical values.
1098
+ “*” is used to specify the default for all features, example: impute_policy={"*": "$mean", "age": 33}
1158
1099
  :param context: for internal use (passed in init)
1159
1100
  :param name: step name
1160
1101
  :param routes: for internal use (routes passed in init)
1161
1102
  :param protocol: serving API protocol (default "v2")
1162
1103
  :param url_prefix: url prefix for the router (default /v2/models)
1163
1104
  :param health_prefix: health api url prefix (default /v2/health)
1164
- :param input_path: when specified selects the key/path in the event to use as body
1165
- this require that the event body will behave like a dict, example:
1166
- event: {"data": {"a": 5, "b": 7}}, input_path="data.b" means request body will be 7
1167
- :param result_path: selects the key/path in the event to write the results to
1168
- this require that the event body will behave like a dict, example:
1169
- event: {"x": 5} , result_path="resp" means the returned response will be written
1170
- to event["y"] resulting in {"x": 5, "resp": <result>}
1105
+ :param input_path: when specified selects the key/path in the event to use as body this require that the
1106
+ event body will behave like a dict, example: event: {"data": {"a": 5, "b": 7}}, input_path="data.b"
1107
+ means request body will be 7.
1108
+ :param result_path: selects the key/path in the event to write the results to this require that the event body
1109
+ will behave like a dict, example: event: {"x": 5} , result_path="resp" means the returned response will be
1110
+ written to event["y"] resulting in {"x": 5, "resp": <result>}
1171
1111
  :param kwargs: extra arguments
1172
1112
  """
1173
1113
  super().__init__(
@@ -1185,7 +1125,7 @@ class EnrichmentModelRouter(ModelRouter):
1185
1125
 
1186
1126
  self._feature_service = None
1187
1127
 
1188
- def post_init(self, mode="sync"):
1128
+ def post_init(self, mode="sync", **kwargs):
1189
1129
  from ..feature_store import get_feature_vector
1190
1130
 
1191
1131
  super().post_init(mode)
@@ -1206,33 +1146,37 @@ class EnrichmentModelRouter(ModelRouter):
1206
1146
 
1207
1147
 
1208
1148
  class EnrichmentVotingEnsemble(VotingEnsemble):
1209
- """Voting Ensemble with feature enrichment (from the feature store)"""
1149
+ """
1150
+ Voting Ensemble with feature enrichment (from the feature store)
1151
+ """
1210
1152
 
1211
1153
  def __init__(
1212
1154
  self,
1213
1155
  context=None,
1214
- name: str = None,
1156
+ name: typing.Optional[str] = None,
1215
1157
  routes=None,
1216
1158
  protocol=None,
1217
- url_prefix: str = None,
1218
- health_prefix: str = None,
1219
- vote_type: str = None,
1159
+ url_prefix: typing.Optional[str] = None,
1160
+ health_prefix: typing.Optional[str] = None,
1161
+ vote_type: typing.Optional[str] = None,
1220
1162
  executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
1221
- prediction_col_name: str = None,
1163
+ prediction_col_name: typing.Optional[str] = None,
1222
1164
  feature_vector_uri: str = "",
1223
- impute_policy: dict = None,
1165
+ impute_policy: typing.Optional[dict] = None,
1224
1166
  **kwargs,
1225
1167
  ):
1226
- """Voting Ensemble with feature enrichment (from the feature store)
1168
+ """
1169
+ Voting Ensemble with feature enrichment (from the feature store)
1227
1170
 
1228
1171
  The `EnrichmentVotingEnsemble` class enables to enrich the incoming event with real-time features
1229
1172
  read from a feature vector (in MLRun feature store) and apply prediction logic on top of
1230
1173
  the different added models.
1231
1174
 
1232
1175
  You can use it by calling:
1233
- - <prefix>/<model>[/versions/<ver>]/operation
1176
+
1177
+ - `<prefix>/<model>[/versions/<ver>]/operation`
1234
1178
  Sends the event to the specific <model>[/versions/<ver>]
1235
- - <prefix>/operation
1179
+ - `<prefix>/operation`
1236
1180
  Sends the event to all models and applies `vote(self, event)`
1237
1181
 
1238
1182
  The `VotingEnsemble` applies the following logic:
@@ -1243,7 +1187,7 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
1243
1187
  The feature vector is specified using the `feature_vector_uri`, in addition an imputing policy
1244
1188
  can be specified to substitute None/NaN values with pre defines constant or stats.
1245
1189
 
1246
- * When enabling model tracking via `set_tracking()` the ensemble logic
1190
+ When enabling model tracking via `set_tracking()` the ensemble logic
1247
1191
  predictions will appear with model name as the given VotingEnsemble name
1248
1192
  or "VotingEnsemble" by default.
1249
1193
 
@@ -1251,17 +1195,20 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
1251
1195
 
1252
1196
  # Define a serving function
1253
1197
  # Note: You can point the function to a file containing you own Router or Classifier Model class
1254
- # this basic class supports sklearn based models (with `<model>.predict()` api)
1255
- fn = mlrun.code_to_function(name='ensemble',
1256
- kind='serving',
1257
- filename='model-server.py'
1258
- image='mlrun/mlrun')
1198
+ # this basic class supports sklearn based models (with `<model>.predict()` api)
1199
+ fn = mlrun.code_to_function(
1200
+ name='ensemble',
1201
+ kind='serving',
1202
+ filename='model-server.py',
1203
+ image='mlrun/mlrun')
1204
+
1259
1205
 
1260
1206
  # Set the router class
1261
1207
  # You can set your own classes by simply changing the `class_name`
1262
- fn.set_topology(class_name='mlrun.serving.routers.EnrichmentVotingEnsemble',
1263
- feature_vector_uri="transactions-fraud",
1264
- impute_policy={"*": "$mean"})
1208
+ fn.set_topology(
1209
+ class_name='mlrun.serving.routers.EnrichmentVotingEnsemble',
1210
+ feature_vector_uri="transactions-fraud",
1211
+ impute_policy={"*": "$mean"})
1265
1212
 
1266
1213
  # Add models
1267
1214
  fn.add_model(<model_name>, <model_path>, <model_class_name>)
@@ -1283,35 +1230,32 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
1283
1230
  :param context: for internal use (passed in init)
1284
1231
  :param name: step name
1285
1232
  :param routes: for internal use (routes passed in init)
1286
- :param protocol: serving API protocol (default "v2")
1287
- :param url_prefix: url prefix for the router (default /v2/models)
1288
- :param health_prefix: health api url prefix (default /v2/health)
1289
- :param feature_vector_uri : feature vector uri in the form: [project/]name[:tag]
1290
- :param impute_policy : value imputing (substitute NaN/Inf values with statistical or constant value),
1291
- you can set the `impute_policy` parameter with the imputing policy, and specify which
1292
- constant or statistical value will be used instead of NaN/Inf value, this can be defined
1293
- per column or for all the columns ("*").
1294
- the replaced value can be fixed number for constants or $mean, $max, $min, $std, $count
1295
- for statistical values. “*” is used to specify the default for all features, example:
1296
- impute_policy={"*": "$mean", "age": 33}
1297
- :param input_path: when specified selects the key/path in the event to use as body
1298
- this require that the event body will behave like a dict, example:
1299
- event: {"data": {"a": 5, "b": 7}}, input_path="data.b" means request body will be 7
1300
- :param result_path: selects the key/path in the event to write the results to
1301
- this require that the event body will behave like a dict, example:
1302
- event: {"x": 5} , result_path="resp" means the returned response will be written
1303
- to event["y"] resulting in {"x": 5, "resp": <result>}
1304
- :param vote_type: Voting type to be used (from `VotingTypes`).
1305
- by default will try to self-deduct upon the first event:
1306
- - float prediction type: regression
1307
- - int prediction type: classification
1233
+ :param protocol: serving API protocol (default `v2`)
1234
+ :param url_prefix: url prefix for the router (default `/v2/models`)
1235
+ :param health_prefix: health api url prefix (default `/v2/health`)
1236
+ :param feature_vector_uri: feature vector uri in the form `[project/]name[:tag]`
1237
+ :param impute_policy: value imputing (substitute NaN/Inf values with statistical or constant value),
1238
+ you can set the `impute_policy` parameter with the imputing policy, and specify which constant or
1239
+ statistical value will be used instead of NaN/Inf value, this can be defined per column or for all
1240
+ the columns ("*"). The replaced value can be fixed number for constants or $mean, $max, $min, $std, $count
1241
+ for statistical values. “*” is used to specify the default for all features,
1242
+ example: impute_policy={"*": "$mean", "age": 33}
1243
+ :param input_path: when specified selects the key/path in the event to use as body this require that
1244
+ the event body will behave like a dict, example: event: {"data": {"a": 5, "b": 7}}, input_path="data.b"
1245
+ means request body will be 7.
1246
+ :param result_path: selects the key/path in the event to write the results to this require that the event body
1247
+ will behave like a dict, example: event: {"x": 5} , result_path="resp" means the returned response will be
1248
+ written to event["y"] resulting in {"x": 5, "resp": <result>}.
1249
+ :param vote_type: Voting type to be used (from `VotingTypes`). by default will try to self-deduct upon the
1250
+ first event:
1251
+ * float prediction type: regression
1252
+ * int prediction type: classification
1308
1253
  :param executor_type: Parallelism mechanism, out of `ParallelRunnerModes`, by default `threads`
1309
1254
  :param prediction_col_name: The dict key for the predictions column in the model's responses output.
1310
- Example: If the model returns
1311
- {id: <id>, model_name: <name>, outputs: {..., prediction: [<predictions>], ...}}
1312
- the prediction_col_name should be `prediction`.
1313
- by default, `prediction`
1314
- :param kwargs: extra arguments
1255
+ Example:
1256
+ If the model returns `{id: <id>, model_name: <name>, outputs: {..., prediction: [<predictions>], ...}}`,
1257
+ the prediction_col_name should be `prediction`. By default, `prediction`.
1258
+ :param kwargs: extra arguments
1315
1259
  """
1316
1260
  super().__init__(
1317
1261
  context=context,
@@ -1331,7 +1275,7 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
1331
1275
 
1332
1276
  self._feature_service = None
1333
1277
 
1334
- def post_init(self, mode="sync"):
1278
+ def post_init(self, mode="sync", **kwargs):
1335
1279
  from ..feature_store import get_feature_vector
1336
1280
 
1337
1281
  super().post_init(mode)
@@ -1342,7 +1286,9 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
1342
1286
  )
1343
1287
 
1344
1288
  def preprocess(self, event):
1345
- """Turn an entity identifier (source) to a Feature Vector"""
1289
+ """
1290
+ Turn an entity identifier (source) to a Feature Vector
1291
+ """
1346
1292
  if isinstance(event.body, (str, bytes)):
1347
1293
  event.body = json.loads(event.body)
1348
1294
  event.body["inputs"] = self._feature_service.get(