mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of mlrun might be problematic. Click here for more details.

Files changed (135) hide show
  1. mlrun/__main__.py +4 -2
  2. mlrun/alerts/alert.py +75 -8
  3. mlrun/artifacts/base.py +1 -0
  4. mlrun/artifacts/manager.py +9 -2
  5. mlrun/common/constants.py +4 -1
  6. mlrun/common/db/sql_session.py +3 -2
  7. mlrun/common/formatters/__init__.py +1 -0
  8. mlrun/common/formatters/artifact.py +1 -0
  9. mlrun/{model_monitoring/application.py → common/formatters/feature_set.py} +20 -6
  10. mlrun/common/formatters/run.py +3 -0
  11. mlrun/common/helpers.py +0 -1
  12. mlrun/common/schemas/__init__.py +3 -1
  13. mlrun/common/schemas/alert.py +15 -12
  14. mlrun/common/schemas/api_gateway.py +6 -6
  15. mlrun/common/schemas/auth.py +5 -0
  16. mlrun/common/schemas/client_spec.py +0 -1
  17. mlrun/common/schemas/common.py +7 -4
  18. mlrun/common/schemas/frontend_spec.py +7 -0
  19. mlrun/common/schemas/function.py +7 -0
  20. mlrun/common/schemas/model_monitoring/__init__.py +4 -3
  21. mlrun/common/schemas/model_monitoring/constants.py +41 -26
  22. mlrun/common/schemas/model_monitoring/model_endpoints.py +23 -47
  23. mlrun/common/schemas/notification.py +69 -12
  24. mlrun/common/schemas/project.py +45 -12
  25. mlrun/common/schemas/workflow.py +10 -2
  26. mlrun/common/types.py +1 -0
  27. mlrun/config.py +91 -35
  28. mlrun/data_types/data_types.py +6 -1
  29. mlrun/data_types/spark.py +2 -2
  30. mlrun/data_types/to_pandas.py +57 -25
  31. mlrun/datastore/__init__.py +1 -0
  32. mlrun/datastore/alibaba_oss.py +3 -2
  33. mlrun/datastore/azure_blob.py +125 -37
  34. mlrun/datastore/base.py +42 -21
  35. mlrun/datastore/datastore.py +4 -2
  36. mlrun/datastore/datastore_profile.py +1 -1
  37. mlrun/datastore/dbfs_store.py +3 -7
  38. mlrun/datastore/filestore.py +1 -3
  39. mlrun/datastore/google_cloud_storage.py +85 -29
  40. mlrun/datastore/inmem.py +4 -1
  41. mlrun/datastore/redis.py +1 -0
  42. mlrun/datastore/s3.py +25 -12
  43. mlrun/datastore/sources.py +76 -4
  44. mlrun/datastore/spark_utils.py +30 -0
  45. mlrun/datastore/storeytargets.py +151 -0
  46. mlrun/datastore/targets.py +102 -131
  47. mlrun/datastore/v3io.py +1 -0
  48. mlrun/db/base.py +15 -6
  49. mlrun/db/httpdb.py +57 -28
  50. mlrun/db/nopdb.py +29 -5
  51. mlrun/errors.py +20 -3
  52. mlrun/execution.py +46 -5
  53. mlrun/feature_store/api.py +25 -1
  54. mlrun/feature_store/common.py +6 -11
  55. mlrun/feature_store/feature_vector.py +3 -1
  56. mlrun/feature_store/retrieval/job.py +4 -1
  57. mlrun/feature_store/retrieval/spark_merger.py +10 -39
  58. mlrun/feature_store/steps.py +8 -0
  59. mlrun/frameworks/_common/plan.py +3 -3
  60. mlrun/frameworks/_ml_common/plan.py +1 -1
  61. mlrun/frameworks/parallel_coordinates.py +2 -3
  62. mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
  63. mlrun/k8s_utils.py +48 -2
  64. mlrun/launcher/client.py +6 -6
  65. mlrun/launcher/local.py +2 -2
  66. mlrun/model.py +215 -34
  67. mlrun/model_monitoring/api.py +38 -24
  68. mlrun/model_monitoring/applications/__init__.py +1 -2
  69. mlrun/model_monitoring/applications/_application_steps.py +60 -29
  70. mlrun/model_monitoring/applications/base.py +2 -174
  71. mlrun/model_monitoring/applications/context.py +197 -70
  72. mlrun/model_monitoring/applications/evidently_base.py +11 -85
  73. mlrun/model_monitoring/applications/histogram_data_drift.py +21 -16
  74. mlrun/model_monitoring/applications/results.py +4 -4
  75. mlrun/model_monitoring/controller.py +110 -282
  76. mlrun/model_monitoring/db/stores/__init__.py +8 -3
  77. mlrun/model_monitoring/db/stores/base/store.py +3 -0
  78. mlrun/model_monitoring/db/stores/sqldb/models/base.py +9 -7
  79. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +18 -3
  80. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +43 -23
  81. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +48 -35
  82. mlrun/model_monitoring/db/tsdb/__init__.py +7 -2
  83. mlrun/model_monitoring/db/tsdb/base.py +147 -15
  84. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +94 -55
  85. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +0 -3
  86. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +144 -38
  87. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +44 -3
  88. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +246 -57
  89. mlrun/model_monitoring/helpers.py +70 -50
  90. mlrun/model_monitoring/stream_processing.py +96 -195
  91. mlrun/model_monitoring/writer.py +13 -5
  92. mlrun/package/packagers/default_packager.py +2 -2
  93. mlrun/projects/operations.py +16 -8
  94. mlrun/projects/pipelines.py +126 -115
  95. mlrun/projects/project.py +286 -129
  96. mlrun/render.py +3 -3
  97. mlrun/run.py +38 -19
  98. mlrun/runtimes/__init__.py +19 -8
  99. mlrun/runtimes/base.py +4 -1
  100. mlrun/runtimes/daskjob.py +1 -1
  101. mlrun/runtimes/funcdoc.py +1 -1
  102. mlrun/runtimes/kubejob.py +6 -6
  103. mlrun/runtimes/local.py +12 -5
  104. mlrun/runtimes/nuclio/api_gateway.py +68 -8
  105. mlrun/runtimes/nuclio/application/application.py +307 -70
  106. mlrun/runtimes/nuclio/function.py +63 -14
  107. mlrun/runtimes/nuclio/serving.py +10 -10
  108. mlrun/runtimes/pod.py +25 -19
  109. mlrun/runtimes/remotesparkjob.py +2 -5
  110. mlrun/runtimes/sparkjob/spark3job.py +16 -17
  111. mlrun/runtimes/utils.py +34 -0
  112. mlrun/serving/routers.py +2 -5
  113. mlrun/serving/server.py +37 -19
  114. mlrun/serving/states.py +30 -3
  115. mlrun/serving/v2_serving.py +44 -35
  116. mlrun/track/trackers/mlflow_tracker.py +5 -0
  117. mlrun/utils/async_http.py +1 -1
  118. mlrun/utils/db.py +18 -0
  119. mlrun/utils/helpers.py +150 -36
  120. mlrun/utils/http.py +1 -1
  121. mlrun/utils/notifications/notification/__init__.py +0 -1
  122. mlrun/utils/notifications/notification/webhook.py +8 -1
  123. mlrun/utils/notifications/notification_pusher.py +1 -1
  124. mlrun/utils/v3io_clients.py +2 -2
  125. mlrun/utils/version/version.json +2 -2
  126. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/METADATA +153 -66
  127. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/RECORD +131 -134
  128. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/WHEEL +1 -1
  129. mlrun/feature_store/retrieval/conversion.py +0 -271
  130. mlrun/model_monitoring/controller_handler.py +0 -37
  131. mlrun/model_monitoring/evidently_application.py +0 -20
  132. mlrun/model_monitoring/prometheus.py +0 -216
  133. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/LICENSE +0 -0
  134. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/entry_points.txt +0 -0
  135. {mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/top_level.txt +0 -0
{mlrun-1.7.0rc28.dist-info → mlrun-1.7.0rc55.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mlrun
3
- Version: 1.7.0rc28
3
+ Version: 1.7.0rc55
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -28,46 +28,47 @@ Requires-Dist: aiohttp-retry ~=2.8
28
28
  Requires-Dist: click ~=8.1
29
29
  Requires-Dist: nest-asyncio ~=1.0
30
30
  Requires-Dist: ipython ~=8.10
31
- Requires-Dist: nuclio-jupyter ~=0.10.0
31
+ Requires-Dist: nuclio-jupyter ~=0.10.4
32
32
  Requires-Dist: numpy <1.27.0,>=1.16.5
33
33
  Requires-Dist: pandas <2.2,>=1.2
34
- Requires-Dist: pyarrow <15,>=10.0
34
+ Requires-Dist: pyarrow <18,>=10.0
35
35
  Requires-Dist: pyyaml <7,>=5.4.1
36
- Requires-Dist: requests ~=2.31
36
+ Requires-Dist: requests ~=2.32
37
37
  Requires-Dist: tabulate ~=0.8.6
38
- Requires-Dist: v3io ~=0.6.4
38
+ Requires-Dist: v3io ~=0.6.9
39
39
  Requires-Dist: pydantic <1.10.15,>=1.10.8
40
40
  Requires-Dist: mergedeep ~=1.3
41
41
  Requires-Dist: v3io-frames ~=0.10.14
42
42
  Requires-Dist: semver ~=3.0
43
43
  Requires-Dist: dependency-injector ~=4.41
44
- Requires-Dist: fsspec <2024.4,>=2023.9.2
44
+ Requires-Dist: fsspec <2024.7,>=2023.9.2
45
45
  Requires-Dist: v3iofs ~=0.1.17
46
- Requires-Dist: storey ~=1.7.20
46
+ Requires-Dist: storey ~=1.7.27
47
47
  Requires-Dist: inflection ~=0.5.0
48
48
  Requires-Dist: python-dotenv ~=0.17.0
49
- Requires-Dist: setuptools ~=69.1
49
+ Requires-Dist: setuptools ~=71.0
50
50
  Requires-Dist: deprecated ~=1.2
51
51
  Requires-Dist: jinja2 >=3.1.3,~=3.1
52
52
  Requires-Dist: orjson <4,>=3.9.15
53
- Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.2
54
- Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.2
53
+ Requires-Dist: mlrun-pipelines-kfp-common ~=0.1.9
54
+ Requires-Dist: mlrun-pipelines-kfp-v1-8 ~=0.1.6
55
55
  Provides-Extra: alibaba-oss
56
56
  Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
57
57
  Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
58
58
  Provides-Extra: all
59
59
  Requires-Dist: adlfs ==2023.9.0 ; extra == 'all'
60
- Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 'all'
60
+ Requires-Dist: aiobotocore <2.16,>=2.5.0 ; extra == 'all'
61
61
  Requires-Dist: avro ~=1.11 ; extra == 'all'
62
62
  Requires-Dist: azure-core ~=1.24 ; extra == 'all'
63
63
  Requires-Dist: azure-identity ~=1.5 ; extra == 'all'
64
64
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'all'
65
65
  Requires-Dist: bokeh >=2.4.2,~=2.4 ; extra == 'all'
66
- Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'all'
67
- Requires-Dist: dask ~=2023.9.0 ; extra == 'all'
66
+ Requires-Dist: boto3 <1.36,>=1.28.0 ; extra == 'all'
67
+ Requires-Dist: dask ~=2023.12.1 ; extra == 'all'
68
68
  Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'all'
69
- Requires-Dist: distributed ~=2023.9.0 ; extra == 'all'
70
- Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'all'
69
+ Requires-Dist: distributed ~=2023.12.1 ; extra == 'all'
70
+ Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'all'
71
+ Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'all'
71
72
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'all'
72
73
  Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'all'
73
74
  Requires-Dist: google-cloud ==0.34 ; extra == 'all'
@@ -77,13 +78,14 @@ Requires-Dist: mlflow ~=2.8 ; extra == 'all'
77
78
  Requires-Dist: msrest ~=0.6.21 ; extra == 'all'
78
79
  Requires-Dist: oss2 ==2.18.1 ; extra == 'all'
79
80
  Requires-Dist: ossfs ==2023.12.0 ; extra == 'all'
80
- Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
81
+ Requires-Dist: plotly ~=5.23 ; extra == 'all'
81
82
  Requires-Dist: pyopenssl >=23 ; extra == 'all'
82
83
  Requires-Dist: redis ~=4.3 ; extra == 'all'
83
- Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'all'
84
+ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'all'
84
85
  Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
85
86
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
86
- Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'all'
87
+ Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'all'
88
+ Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'all'
87
89
  Provides-Extra: api
88
90
  Requires-Dist: uvicorn ~=0.27.1 ; extra == 'api'
89
91
  Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'api'
@@ -96,7 +98,7 @@ Requires-Dist: sqlalchemy ~=1.4 ; extra == 'api'
96
98
  Requires-Dist: pymysql ~=1.0 ; extra == 'api'
97
99
  Requires-Dist: alembic ~=1.9 ; extra == 'api'
98
100
  Requires-Dist: timelength ~=1.1 ; extra == 'api'
99
- Requires-Dist: memray ~=1.12 ; extra == 'api'
101
+ Requires-Dist: memray ~=1.12 ; (sys_platform != "win32") and extra == 'api'
100
102
  Provides-Extra: azure-blob-storage
101
103
  Requires-Dist: msrest ~=0.6.21 ; extra == 'azure-blob-storage'
102
104
  Requires-Dist: azure-core ~=1.24 ; extra == 'azure-blob-storage'
@@ -110,80 +112,86 @@ Provides-Extra: bokeh
110
112
  Requires-Dist: bokeh >=2.4.2,~=2.4 ; extra == 'bokeh'
111
113
  Provides-Extra: complete
112
114
  Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete'
113
- Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 'complete'
115
+ Requires-Dist: aiobotocore <2.16,>=2.5.0 ; extra == 'complete'
114
116
  Requires-Dist: avro ~=1.11 ; extra == 'complete'
115
117
  Requires-Dist: azure-core ~=1.24 ; extra == 'complete'
116
118
  Requires-Dist: azure-identity ~=1.5 ; extra == 'complete'
117
119
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'complete'
118
- Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'complete'
119
- Requires-Dist: dask ~=2023.9.0 ; extra == 'complete'
120
+ Requires-Dist: boto3 <1.36,>=1.28.0 ; extra == 'complete'
121
+ Requires-Dist: dask ~=2023.12.1 ; extra == 'complete'
120
122
  Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete'
121
- Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete'
122
- Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'complete'
123
+ Requires-Dist: distributed ~=2023.12.1 ; extra == 'complete'
124
+ Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'complete'
125
+ Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'complete'
123
126
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete'
127
+ Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'complete'
128
+ Requires-Dist: google-cloud ==0.34 ; extra == 'complete'
124
129
  Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete'
125
130
  Requires-Dist: kafka-python ~=2.0 ; extra == 'complete'
126
131
  Requires-Dist: mlflow ~=2.8 ; extra == 'complete'
127
132
  Requires-Dist: msrest ~=0.6.21 ; extra == 'complete'
128
133
  Requires-Dist: oss2 ==2.18.1 ; extra == 'complete'
129
134
  Requires-Dist: ossfs ==2023.12.0 ; extra == 'complete'
130
- Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
135
+ Requires-Dist: plotly ~=5.23 ; extra == 'complete'
131
136
  Requires-Dist: pyopenssl >=23 ; extra == 'complete'
132
137
  Requires-Dist: redis ~=4.3 ; extra == 'complete'
133
- Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete'
138
+ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete'
134
139
  Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
135
140
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
136
- Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete'
141
+ Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'complete'
142
+ Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'complete'
137
143
  Provides-Extra: complete-api
138
144
  Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete-api'
139
- Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 'complete-api'
145
+ Requires-Dist: aiobotocore <2.16,>=2.5.0 ; extra == 'complete-api'
140
146
  Requires-Dist: alembic ~=1.9 ; extra == 'complete-api'
141
147
  Requires-Dist: apscheduler <4,>=3.10.3 ; extra == 'complete-api'
142
148
  Requires-Dist: avro ~=1.11 ; extra == 'complete-api'
143
149
  Requires-Dist: azure-core ~=1.24 ; extra == 'complete-api'
144
150
  Requires-Dist: azure-identity ~=1.5 ; extra == 'complete-api'
145
151
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'complete-api'
146
- Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'complete-api'
152
+ Requires-Dist: boto3 <1.36,>=1.28.0 ; extra == 'complete-api'
147
153
  Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'complete-api'
148
- Requires-Dist: dask ~=2023.9.0 ; extra == 'complete-api'
154
+ Requires-Dist: dask ~=2023.12.1 ; extra == 'complete-api'
149
155
  Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete-api'
150
- Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete-api'
156
+ Requires-Dist: distributed ~=2023.12.1 ; extra == 'complete-api'
151
157
  Requires-Dist: fastapi ~=0.110.0 ; extra == 'complete-api'
152
- Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'complete-api'
158
+ Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'complete-api'
159
+ Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'complete-api'
153
160
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete-api'
161
+ Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'complete-api'
162
+ Requires-Dist: google-cloud ==0.34 ; extra == 'complete-api'
154
163
  Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
155
164
  Requires-Dist: humanfriendly ~=10.0 ; extra == 'complete-api'
156
165
  Requires-Dist: igz-mgmt ~=0.2.0 ; extra == 'complete-api'
157
166
  Requires-Dist: kafka-python ~=2.0 ; extra == 'complete-api'
158
- Requires-Dist: memray ~=1.12 ; extra == 'complete-api'
159
167
  Requires-Dist: mlflow ~=2.8 ; extra == 'complete-api'
160
168
  Requires-Dist: msrest ~=0.6.21 ; extra == 'complete-api'
161
169
  Requires-Dist: objgraph ~=3.6 ; extra == 'complete-api'
162
170
  Requires-Dist: oss2 ==2.18.1 ; extra == 'complete-api'
163
171
  Requires-Dist: ossfs ==2023.12.0 ; extra == 'complete-api'
164
- Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete-api'
172
+ Requires-Dist: plotly ~=5.23 ; extra == 'complete-api'
165
173
  Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
166
174
  Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
167
175
  Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
168
- Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete-api'
176
+ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete-api'
169
177
  Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
170
178
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
171
- Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
179
+ Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'complete-api'
180
+ Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'complete-api'
172
181
  Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
173
182
  Requires-Dist: uvicorn ~=0.27.1 ; extra == 'complete-api'
183
+ Requires-Dist: memray ~=1.12 ; (sys_platform != "win32") and extra == 'complete-api'
174
184
  Provides-Extra: dask
175
- Requires-Dist: dask ~=2023.9.0 ; extra == 'dask'
176
- Requires-Dist: distributed ~=2023.9.0 ; extra == 'dask'
185
+ Requires-Dist: dask ~=2023.12.1 ; extra == 'dask'
186
+ Requires-Dist: distributed ~=2023.12.1 ; extra == 'dask'
177
187
  Provides-Extra: databricks-sdk
178
188
  Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'databricks-sdk'
179
189
  Provides-Extra: google-cloud
180
190
  Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'google-cloud'
181
191
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'google-cloud'
192
+ Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'google-cloud'
182
193
  Requires-Dist: google-cloud ==0.34 ; extra == 'google-cloud'
183
- Provides-Extra: google-cloud-bigquery
184
- Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'google-cloud-bigquery'
185
- Provides-Extra: google-cloud-storage
186
- Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'google-cloud-storage'
194
+ Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'google-cloud'
187
195
  Provides-Extra: graphviz
188
196
  Requires-Dist: graphviz ~=0.20.0 ; extra == 'graphviz'
189
197
  Provides-Extra: kafka
@@ -192,19 +200,20 @@ Requires-Dist: avro ~=1.11 ; extra == 'kafka'
192
200
  Provides-Extra: mlflow
193
201
  Requires-Dist: mlflow ~=2.8 ; extra == 'mlflow'
194
202
  Provides-Extra: plotly
195
- Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'plotly'
203
+ Requires-Dist: plotly ~=5.23 ; extra == 'plotly'
196
204
  Provides-Extra: redis
197
205
  Requires-Dist: redis ~=4.3 ; extra == 'redis'
198
206
  Provides-Extra: s3
199
- Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
200
- Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
201
- Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 's3'
207
+ Requires-Dist: boto3 <1.36,>=1.28.0 ; extra == 's3'
208
+ Requires-Dist: aiobotocore <2.16,>=2.5.0 ; extra == 's3'
209
+ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 's3'
202
210
  Provides-Extra: snowflake
203
211
  Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'snowflake'
204
212
  Provides-Extra: sqlalchemy
205
213
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
206
214
  Provides-Extra: tdengine
207
- Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'tdengine'
215
+ Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'tdengine'
216
+ Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'tdengine'
208
217
 
209
218
  <a id="top"></a>
210
219
  [![Build Status](https://github.com/mlrun/mlrun/actions/workflows/build.yaml/badge.svg?branch=development)](https://github.com/mlrun/mlrun/actions/workflows/build.yaml?query=branch%3Adevelopment)
@@ -220,19 +229,86 @@ Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'tdengine'
220
229
 
221
230
  # Using MLRun
222
231
 
223
- MLRun is an open MLOps platform for quickly building and managing continuous ML applications across their lifecycle. MLRun integrates into your development and CI/CD environment and automates the delivery of production data, ML pipelines, and online applications, significantly reducing engineering efforts, time to production, and computation resources.
232
+ MLRun is an open source AI orchestration platform for quickly building and managing continuous (gen) AI applications across their lifecycle. MLRun integrates into your development and CI/CD environment and automates the delivery of production data, ML pipelines, and online applications.
233
+ MLRun significantly reduces engineering efforts, time to production, and computation resources.
224
234
  With MLRun, you can choose any IDE on your local machine or on the cloud. MLRun breaks the silos between data, ML, software, and DevOps/MLOps teams, enabling collaboration and fast continuous improvements.
225
235
 
226
- Get started with MLRun [**Tutorials and Examples**](https://docs.mlrun.org/en/latest/tutorials/index.html), [**Installation and setup guide**](https://docs.mlrun.org/en/latest/install.html), or read about [**MLRun Architecture**](https://docs.mlrun.org/en/latest/architecture.html).
236
+ Get started with the MLRun [**Tutorials and Examples**](https://docs.mlrun.org/en/stable/tutorials/index.html) and the [**Installation and setup guide**](https://docs.mlrun.org/en/stable/install.html), or read about the [**MLRun Architecture**](https://docs.mlrun.org/en/stable/architecture.html).
237
+
238
+ This page explains how MLRun addresses the [**gen AI tasks**](#genai-tasks), [**MLOps tasks**](#mlops-tasks), and presents the [**MLRun core components**](#core-components).
239
+
240
+ See the supported data stores, development tools, services, platforms, etc., supported by MLRun's open architecture in **https://docs.mlrun.org/en/stable/ecosystem.html**.
241
+
242
+ ## Gen AI tasks
243
+
244
+ <p align="center"><img src="https://github.com/mlrun/mlrun/raw/development/docs/_static/images/ai-tasks.png" alt="ai-tasks" width="800"/></p><br>
245
+
246
+ Use MLRun to develop, scale, deploy, and monitor your AI model across your enterprise. The [**gen AI development workflow**](https://docs.mlrun.org/en/stable/genai/genai-flow.html)
247
+ section describes the different tasks and stages in detail.
248
+
249
+ ### Data management
250
+
251
+
252
+ MLRun supports batch or realtime data processing at scale, data lineage and versioning, structured and unstructured data, and more.
253
+ Removing inappropriate data at an early stage saves resources that would otherwise be required later on.
254
+
255
+
256
+ **Docs:**
257
+ [Using LLMs to process unstructured data](https://docs.mlrun.org/en/stable/genai/data-mgmt/unstructured-data.html)
258
+ [Vector databases](https://docs.mlrun.org/en/stable/genai/data-mgmt/vector-databases.html)
259
+ [Guardrails for data management](https://docs.mlrun.org/en/stable/genai/data-mgmt/guardrails-data.html)
260
+ **Demo:**
261
+ [Call center demo](https://github.com/mlrun/demo-call-center>`
262
+ **Video:**
263
+ [Call center](https://youtu.be/YycMbxRgLBA>`
264
+
265
+ ### Development
266
+ Use MLRun to build an automated ML pipeline to: collect data,
267
+ preprocess (prepare) the data, run the training pipeline, and evaluate the model.
268
+
269
+ **Docs:**
270
+ [Working with RAG](https://docs.mlrun.org/en/stable/genai/development/working-with-rag.html), [Evalating LLMs](https://docs.mlrun.org/en/stable/genai/development/evaluating-llms.html), [Fine tuning LLMS](https://docs.mlrun.org/en/stable/genai/development/fine-tuning-llms.html)
271
+ **Demos:**
272
+ [Call center demo](https://github.com/mlrun/demo-call-center), [Build & deploy custom (fine-tuned) LLM models and applications](https://github.com/mlrun/demo-llm-tuning/blob/main), [Interactive bot demo using LLMs](https://github.com/mlrun/demo-llm-bot/blob/main)
273
+ **Video:**
274
+ [Call center](https://youtu.be/YycMbxRgLBA)
275
+
276
+
277
+ ### Deployment
278
+ MLRun serving can productize the newly trained LLM as a serverless function using real-time auto-scaling Nuclio serverless functions.
279
+ The application pipeline includes all the steps from accepting events or data, contextualizing it with a state preparing the required model features,
280
+ inferring results using one or more models, and driving actions.
281
+
282
+
283
+ **Docs:**
284
+ [Serving gen AI models](https://docs.mlrun.org/en/stable/genai/deployment/genai_serving.html), GPU utilization](https://docs.mlrun.org/en/stable/genai/deployment/gpu_utilization.html), [Gen AI realtime serving graph](https://docs.mlrun.org/en/stable/genai/deployment/genai_serving_graph.html)
285
+ **Tutorial:**
286
+ [Deploy LLM using MLRun](https://docs.mlrun.org/en/stable/tutorials/genai_01_basic_tutorial.html)
287
+ **Demos:**
288
+ [Call center demo](https://github.com/mlrun/demo-call-center), [Build & deploy custom(fine-tuned)]LLM models and applications <https://github.com/mlrun/demo-llm-tuning/blob/main), [Interactive bot demo using LLMs]<https://github.com/mlrun/demo-llm-bot/blob/main)
289
+ **Video:**
290
+ [Call center]<https://youtu.be/YycMbxRgLBA)
291
+
292
+
293
+ ### Live Ops
294
+ Monitor all resources, data, model and application metrics to ensure performance. Then identify risks, control costs, and measure business KPIs.
295
+ Collect production data, metadata, and metrics to tune the model and application further, and to enable governance and explainability.
296
+
297
+
298
+ **Docs:**
299
+ [Model monitoring <monitoring](https://docs.mlrun.org/en/stable/concepts/monitoring.html), [Alerts and notifications](https://docs.mlrun.org/en/stable/concepts/alerts-notifications.html)
300
+ **Tutorials:**
301
+ [Deploy LLM using MLRun](https://docs.mlrun.org/en/stable/tutorials/genai_01_basic_tutorial.html), [Model monitoring using LLM](https://docs.mlrun.org/en/stable/tutorials/genai-02-monitoring-llm.html)
302
+ **Demo:**
303
+ [Build & deploy custom (fine-tuned) LLM models and applications](https://github.com/mlrun/demo-llm-tuning/blob/main)
227
304
 
228
- This page explains how MLRun addresses the [**MLOps Tasks**](#mlops-tasks) and the [**MLRun core components**](#core-components).
229
305
 
230
306
  <a id="mlops-tasks"></a>
231
307
  ## MLOps tasks
232
308
 
233
309
  <p align="center"><img src="https://github.com/mlrun/mlrun/raw/development/docs/_static/images/mlops-task.png" alt="mlrun-tasks" width="800"/></p><br>
234
310
 
235
- The [**MLOps development workflow**](https://docs.mlrun.org/en/latest/mlops-dev-flow.html) section describes the different tasks and stages in detail.
311
+ The [**MLOps development workflow**](https://docs.mlrun.org/en/stable/mlops-dev-flow.html) section describes the different tasks and stages in detail.
236
312
  MLRun can be used to automate and orchestrate all the different tasks or just specific tasks (and integrate them with what you have already deployed).
237
313
 
238
314
  ### Project management and CI/CD automation
@@ -241,32 +317,40 @@ In MLRun the assets, metadata, and services (data, functions, jobs, artifacts, m
241
317
  Projects can be imported/exported as a whole, mapped to git repositories or IDE projects (in PyCharm, VSCode, etc.), which enables versioning, collaboration, and CI/CD.
242
318
  Project access can be restricted to a set of users and roles.
243
319
 
244
- See: **Docs:** [Projects and Automation](https://docs.mlrun.org/en/latest/projects/project.html), [CI/CD Integration](https://docs.mlrun.org/en/latest/projects/ci-integration.html), **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorials/01-mlrun-basics.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorials/04-pipeline.html), **Video:** [Quick start](https://youtu.be/xI8KVGLlj7Q).
320
+ **Docs:** [Projects and Automation](https://docs.mlrun.org/en/stable/projects/project.html), [CI/CD Integration](https://docs.mlrun.org/en/stable/projects/ci-integration.html)
321
+ **Tutorials:** [Quick start](https://docs.mlrun.org/en/stable/tutorials/01-mlrun-basics.html), [Automated ML Pipeline](https://docs.mlrun.org/en/stable/tutorials/04-pipeline.html)
322
+ **Video:** [Quick start](https://youtu.be/xI8KVGLlj7Q).
245
323
 
246
324
  ### Ingest and process data
247
325
 
248
- MLRun provides abstract interfaces to various offline and online [**data sources**](https://docs.mlrun.org/en/latest/store/datastore.html), supports batch or realtime data processing at scale, data lineage and versioning, structured and unstructured data, and more.
249
- In addition, the MLRun [**Feature Store**](https://docs.mlrun.org/en/latest/feature-store/feature-store.html) automates the collection, transformation, storage, catalog, serving, and monitoring of data features across the ML lifecycle and enables feature reuse and sharing.
326
+ MLRun provides abstract interfaces to various offline and online [**data sources**](https://docs.mlrun.org/en/stable/store/datastore.html), supports batch or realtime data processing at scale, data lineage and versioning, structured and unstructured data, and more.
327
+ In addition, the MLRun [**Feature Store**](https://docs.mlrun.org/en/stable/feature-store/feature-store.html) automates the collection, transformation, storage, catalog, serving, and monitoring of data features across the ML lifecycle and enables feature reuse and sharing.
250
328
 
251
- See: **Docs:** [Ingest and process data](https://docs.mlrun.org/en/latest/data-prep/index.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/feature-store.html), [Data & Artifacts](https://docs.mlrun.org/en/latest/concepts/data.html); **Tutorials:** [Quick start](https://docs.mlrun.org/en/latest/tutorials/01-mlrun-basics.html), [Feature Store](https://docs.mlrun.org/en/latest/feature-store/basic-demo.html).
329
+ See: **Docs:** [Ingest and process data](https://docs.mlrun.org/en/stable/data-prep/index.html), [Feature Store](https://docs.mlrun.org/en/stable/feature-store/feature-store.html), [Data & Artifacts](https://docs.mlrun.org/en/stable/concepts/data.html)
330
+ **Tutorials:** [Quick start](https://docs.mlrun.org/en/stable/tutorials/01-mlrun-basics.html), [Feature Store](https://docs.mlrun.org/en/stable/feature-store/basic-demo.html).
252
331
 
253
332
  ### Develop and train models
254
333
 
255
334
  MLRun allows you to easily build ML pipelines that take data from various sources or the Feature Store and process it, train models at scale with multiple parameters, test models, tracks each experiments, register, version and deploy models, etc. MLRun provides scalable built-in or custom model training services, integrate with any framework and can work with 3rd party training/auto-ML services. You can also bring your own pre-trained model and use it in the pipeline.
256
335
 
257
- See: **Docs:** [Develop and train models](https://docs.mlrun.org/en/latest/development/index.html), [Model Training and Tracking](https://docs.mlrun.org/en/latest/development/model-training-tracking.html), [Batch Runs and Workflows](https://docs.mlrun.org/en/latest/concepts/runs-workflows.html); **Tutorials:** [Train, compare, and register models](https://docs.mlrun.org/en/latest/tutorials/02-model-training.html), [Automated ML Pipeline](https://docs.mlrun.org/en/latest/tutorials/04-pipeline.html); **Video:** [Train and compare models](https://youtu.be/bZgBsmLMdQo).
336
+ **Docs:** [Develop and train models](https://docs.mlrun.org/en/stable/development/index.html), [Model Training and Tracking](https://docs.mlrun.org/en/stable/development/model-training-tracking.html), [Batch Runs and Workflows](https://docs.mlrun.org/en/stable/concepts/runs-workflows.html)
337
+ **Tutorials:** [Train, compare, and register models](https://docs.mlrun.org/en/stable/tutorials/02-model-training.html), [Automated ML Pipeline](https://docs.mlrun.org/en/stable/tutorials/04-pipeline.html)
338
+ **Video:** [Train and compare models](https://youtu.be/bZgBsmLMdQo).
258
339
 
259
340
  ### Deploy models and applications
260
341
 
261
342
  MLRun rapidly deploys and manages production-grade real-time or batch application pipelines using elastic and resilient serverless functions. MLRun addresses the entire ML application: intercepting application/user requests, running data processing tasks, inferencing using one or more models, driving actions, and integrating with the application logic.
262
343
 
263
- See: **Docs:** [Deploy models and applications](https://docs.mlrun.org/en/latest/deployment/index.html), [Realtime Pipelines](https://docs.mlrun.org/en/latest/serving/serving-graph.html), [Batch Inference](https://docs.mlrun.org/en/latest/deployment/batch_inference.html), **Tutorials:** [Realtime Serving](https://docs.mlrun.org/en/latest/tutorials/03-model-serving.html), [Batch Inference](https://docs.mlrun.org/en/latest/tutorials/07-batch-infer.html), [Advanced Pipeline](https://docs.mlrun.org/en/latest/tutorials/07-batch-infer.html); **Video:** [Serving pre-trained models](https://youtu.be/OUjOus4dZfw).
344
+ **Docs:** [Deploy models and applications](https://docs.mlrun.org/en/stable/deployment/index.html), [Realtime Pipelines](https://docs.mlrun.org/en/stable/serving/serving-graph.html), [Batch Inference](https://docs.mlrun.org/en/stable/deployment/batch_inference.html)
345
+ **Tutorials:** [Realtime Serving](https://docs.mlrun.org/en/stable/tutorials/03-model-serving.html), [Batch Inference](https://docs.mlrun.org/en/stable/tutorials/07-batch-infer.html), [Advanced Pipeline](https://docs.mlrun.org/en/stable/tutorials/07-batch-infer.html)
346
+ **Video:** [Serving pre-trained models](https://youtu.be/OUjOus4dZfw).
264
347
 
265
- ### Monitor and alert
348
+ ### Model Monitoring
266
349
 
267
350
  Observability is built into the different MLRun objects (data, functions, jobs, models, pipelines, etc.), eliminating the need for complex integrations and code instrumentation. With MLRun, you can observe the application/model resource usage and model behavior (drift, performance, etc.), define custom app metrics, and trigger alerts or retraining jobs.
268
351
 
269
- See: **Docs:** [Monitor and alert](https://docs.mlrun.org/en/latest/monitoring/index.html), [Model Monitoring Overview](https://docs.mlrun.org/en/latest/monitoring/model-monitoring-deployment.html), **Tutorials:** [Model Monitoring & Drift Detection](https://docs.mlrun.org/en/latest/tutorials/05-model-monitoring.html).
352
+ **Docs:** [Model monitoring](https://docs.mlrun.org/en/stable/concepts/model-monitoring.html), [Model Monitoring Overview](https://docs.mlrun.org/en/stable/monitoring/model-monitoring-deployment.html)
353
+ **Tutorials:** [Model Monitoring & Drift Detection](https://docs.mlrun.org/en/stable/tutorials/05-model-monitoring.html).
270
354
 
271
355
 
272
356
  <a id="core-components"></a>
@@ -274,18 +358,21 @@ See: **Docs:** [Monitor and alert](https://docs.mlrun.org/en/latest/monitoring/i
274
358
 
275
359
  <p align="center"><img src="https://github.com/mlrun/mlrun/raw/development/docs/_static/images/mlops-core.png" alt="mlrun-core" width="800"/></p><br>
276
360
 
361
+
277
362
  MLRun includes the following major components:
278
363
 
279
- [**Project Management:**](https://docs.mlrun.org/en/latest/projects/project.html) A service (API, SDK, DB, UI) that manages the different project assets (data, functions, jobs, workflows, secrets, etc.) and provides central control and metadata layer.
364
+ [**Project Management:**](https://docs.mlrun.org/en/stable/projects/project.html) A service (API, SDK, DB, UI) that manages the different project assets (data, functions, jobs, workflows, secrets, etc.) and provides central control and metadata layer.
365
+
366
+ [**Functions:**](https://docs.mlrun.org/en/stable/runtimes/functions.html) automatically deployed software package with one or more methods and runtime-specific attributes (such as image, libraries, command, arguments, resources, etc.).
280
367
 
281
- [**Functions:**](https://docs.mlrun.org/en/latest/runtimes/functions.html) automatically deployed software package with one or more methods and runtime-specific attributes (such as image, libraries, command, arguments, resources, etc.).
368
+ [**Data & Artifacts:**](https://docs.mlrun.org/en/stable/concepts/data.html) Glueless connectivity to various data sources, metadata management, catalog, and versioning for structures/unstructured artifacts.
282
369
 
283
- [**Data & Artifacts:**](https://docs.mlrun.org/en/latest/concepts/data.html) Glueless connectivity to various data sources, metadata management, catalog, and versioning for structures/unstructured artifacts.
370
+ [**Batch Runs & Workflows:**](https://docs.mlrun.org/en/stable/concepts/runs-workflows.html) Execute one or more functions with specific parameters and collect, track, and compare all their results and artifacts.
284
371
 
285
- [**Feature Store:**](https://docs.mlrun.org/en/latest/feature-store/feature-store.html) automatically collects, prepares, catalogs, and serves production data features for development (offline) and real-time (online) deployment using minimal engineering effort.
372
+ [**Real-Time Serving Pipeline:**](https://docs.mlrun.org/en/stable/serving/serving-graph.html) Rapid deployment of scalable data and ML pipelines using real-time serverless technology, including API handling, data preparation/enrichment, model serving, ensembles, driving and measuring actions, etc.
286
373
 
287
- [**Batch Runs & Workflows:**](https://docs.mlrun.org/en/latest/concepts/runs-workflows.html) Execute one or more functions with specific parameters and collect, track, and compare all their results and artifacts.
374
+ [**Model monitoring:**](https://docs.mlrun.org/en/stable/monitoring/index.html) monitors data, models, resources, and production components and provides a feedback loop for exploring production data, identifying drift, alerting on anomalies or data quality issues, triggering retraining jobs, measuring business impact, etc.
288
375
 
289
- [**Real-Time Serving Pipeline:**](https://docs.mlrun.org/en/latest/serving/serving-graph.html) Rapid deployment of scalable data and ML pipelines using real-time serverless technology, including API handling, data preparation/enrichment, model serving, ensembles, driving and measuring actions, etc.
376
+ [**Alerts and notifications:**](https://docs.mlrun.org/en/stable/concepts/model-monitoring.html) Use alerts to identify and inform you of possible problem situations. Use notifications to report status on runs and pipelines.
290
377
 
291
- [**Real-Time monitoring:**](https://docs.mlrun.org/en/latest/monitoring/index.html) monitors data, models, resources, and production components and provides a feedback loop for exploring production data, identifying drift, alerting on anomalies or data quality issues, triggering retraining jobs, measuring business impact, etc.
378
+ [**Feature Store:**](https://docs.mlrun.org/en/stable/feature-store/feature-store.html) automatically collects, prepares, catalogs, and serves production data features for development (offline) and real-time (online) deployment using minimal engineering effort.