zenml-nightly 0.83.1.dev20250709__py3-none-any.whl → 0.83.1.dev20250710__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. zenml/VERSION +1 -1
  2. zenml/cli/login.py +141 -18
  3. zenml/cli/project.py +8 -6
  4. zenml/cli/utils.py +63 -16
  5. zenml/client.py +4 -1
  6. zenml/config/compiler.py +1 -0
  7. zenml/config/retry_config.py +5 -3
  8. zenml/config/step_configurations.py +7 -1
  9. zenml/console.py +4 -1
  10. zenml/constants.py +0 -1
  11. zenml/enums.py +13 -4
  12. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +58 -4
  13. zenml/integrations/kubernetes/orchestrators/kube_utils.py +172 -0
  14. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +37 -23
  15. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +92 -22
  16. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +59 -0
  17. zenml/logger.py +6 -4
  18. zenml/login/web_login.py +13 -6
  19. zenml/models/v2/core/model_version.py +9 -1
  20. zenml/models/v2/core/pipeline_run.py +1 -0
  21. zenml/models/v2/core/step_run.py +35 -1
  22. zenml/orchestrators/base_orchestrator.py +63 -8
  23. zenml/orchestrators/dag_runner.py +3 -1
  24. zenml/orchestrators/publish_utils.py +4 -1
  25. zenml/orchestrators/step_launcher.py +77 -139
  26. zenml/orchestrators/step_run_utils.py +16 -0
  27. zenml/orchestrators/step_runner.py +1 -4
  28. zenml/pipelines/pipeline_decorator.py +6 -1
  29. zenml/pipelines/pipeline_definition.py +7 -0
  30. zenml/zen_server/auth.py +0 -1
  31. zenml/zen_stores/migrations/versions/360fa84718bf_step_run_versioning.py +64 -0
  32. zenml/zen_stores/migrations/versions/85289fea86ff_adding_source_to_logs.py +1 -1
  33. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +21 -0
  34. zenml/zen_stores/schemas/pipeline_run_schemas.py +31 -2
  35. zenml/zen_stores/schemas/step_run_schemas.py +41 -17
  36. zenml/zen_stores/sql_zen_store.py +152 -32
  37. zenml/zen_stores/template_utils.py +29 -9
  38. zenml_nightly-0.83.1.dev20250710.dist-info/METADATA +499 -0
  39. {zenml_nightly-0.83.1.dev20250709.dist-info → zenml_nightly-0.83.1.dev20250710.dist-info}/RECORD +42 -41
  40. zenml_nightly-0.83.1.dev20250709.dist-info/METADATA +0 -538
  41. {zenml_nightly-0.83.1.dev20250709.dist-info → zenml_nightly-0.83.1.dev20250710.dist-info}/LICENSE +0 -0
  42. {zenml_nightly-0.83.1.dev20250709.dist-info → zenml_nightly-0.83.1.dev20250710.dist-info}/WHEEL +0 -0
  43. {zenml_nightly-0.83.1.dev20250709.dist-info → zenml_nightly-0.83.1.dev20250710.dist-info}/entry_points.txt +0 -0
@@ -1,538 +0,0 @@
1
- Metadata-Version: 2.3
2
- Name: zenml-nightly
3
- Version: 0.83.1.dev20250709
4
- Summary: ZenML: Write production-ready ML code.
5
- License: Apache-2.0
6
- Keywords: machine learning,production,pipeline,mlops,devops
7
- Author: ZenML GmbH
8
- Author-email: info@zenml.io
9
- Requires-Python: >=3.9,<3.13
10
- Classifier: Development Status :: 4 - Beta
11
- Classifier: Intended Audience :: Developers
12
- Classifier: Intended Audience :: Science/Research
13
- Classifier: Intended Audience :: System Administrators
14
- Classifier: License :: OSI Approved :: Apache Software License
15
- Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.9
17
- Classifier: Programming Language :: Python :: 3.10
18
- Classifier: Programming Language :: Python :: 3.11
19
- Classifier: Programming Language :: Python :: 3.12
20
- Classifier: Programming Language :: Python :: 3 :: Only
21
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
- Classifier: Topic :: System :: Distributed Computing
23
- Classifier: Typing :: Typed
24
- Provides-Extra: adlfs
25
- Provides-Extra: azureml
26
- Provides-Extra: connectors-aws
27
- Provides-Extra: connectors-azure
28
- Provides-Extra: connectors-gcp
29
- Provides-Extra: connectors-kubernetes
30
- Provides-Extra: dev
31
- Provides-Extra: gcsfs
32
- Provides-Extra: s3fs
33
- Provides-Extra: sagemaker
34
- Provides-Extra: secrets-aws
35
- Provides-Extra: secrets-azure
36
- Provides-Extra: secrets-gcp
37
- Provides-Extra: secrets-hashicorp
38
- Provides-Extra: server
39
- Provides-Extra: templates
40
- Provides-Extra: terraform
41
- Provides-Extra: vertex
42
- Requires-Dist: Jinja2 ; extra == "server"
43
- Requires-Dist: adlfs (>=2021.10.0) ; extra == "adlfs"
44
- Requires-Dist: alembic (>=1.8.1,<=1.15.2)
45
- Requires-Dist: aws-profile-manager (>=0.5.0) ; extra == "connectors-aws"
46
- Requires-Dist: azure-ai-ml (==1.23.1) ; extra == "azureml"
47
- Requires-Dist: azure-identity (>=1.4.0) ; extra == "secrets-azure" or extra == "connectors-azure"
48
- Requires-Dist: azure-keyvault-secrets (>=4.0.0) ; extra == "secrets-azure"
49
- Requires-Dist: azure-mgmt-containerregistry (>=10.0.0) ; extra == "connectors-azure"
50
- Requires-Dist: azure-mgmt-containerservice (>=20.0.0) ; extra == "connectors-azure"
51
- Requires-Dist: azure-mgmt-resource (>=21.0.0) ; extra == "connectors-azure"
52
- Requires-Dist: azure-mgmt-storage (>=20.0.0) ; extra == "connectors-azure"
53
- Requires-Dist: azure-storage-blob (>=12.0.0) ; extra == "connectors-azure"
54
- Requires-Dist: bandit (>=1.7.5,<2.0.0) ; extra == "dev"
55
- Requires-Dist: bcrypt (==4.0.1)
56
- Requires-Dist: boto3 (>=1.16.0) ; extra == "secrets-aws" or extra == "connectors-aws"
57
- Requires-Dist: click (>=8.0.1,<8.1.8)
58
- Requires-Dist: cloudpickle (>=2.0.0,<3)
59
- Requires-Dist: copier (>=8.1.0) ; extra == "templates"
60
- Requires-Dist: coverage[toml] (>=5.5,<6.0) ; extra == "dev"
61
- Requires-Dist: darglint (>=1.8.1,<2.0.0) ; extra == "dev"
62
- Requires-Dist: distro (>=1.6.0,<2.0.0)
63
- Requires-Dist: docker (>=7.1.0,<7.2.0)
64
- Requires-Dist: fastapi (>=0.100,<=0.115.8) ; extra == "server"
65
- Requires-Dist: gcsfs (>=2022.11.0) ; extra == "gcsfs"
66
- Requires-Dist: gitpython (>=3.1.18,<4.0.0)
67
- Requires-Dist: google-cloud-aiplatform (>=1.34.0) ; extra == "vertex"
68
- Requires-Dist: google-cloud-artifact-registry (>=1.11.3) ; extra == "connectors-gcp"
69
- Requires-Dist: google-cloud-container (>=2.21.0) ; extra == "connectors-gcp"
70
- Requires-Dist: google-cloud-pipeline-components (>=2.19.0) ; extra == "vertex"
71
- Requires-Dist: google-cloud-secret-manager (>=2.12.5) ; extra == "secrets-gcp"
72
- Requires-Dist: google-cloud-storage (>=2.9.0) ; extra == "connectors-gcp"
73
- Requires-Dist: hvac (>=0.11.2) ; extra == "secrets-hashicorp"
74
- Requires-Dist: hypothesis (>=6.43.1,<7.0.0) ; extra == "dev"
75
- Requires-Dist: importlib_metadata (<=7.0.0) ; python_version < "3.10"
76
- Requires-Dist: ipinfo (>=4.4.3) ; extra == "server"
77
- Requires-Dist: itsdangerous (>=2.2.0,<2.3.0) ; extra == "server"
78
- Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "templates"
79
- Requires-Dist: kfp (>=2.6.0) ; extra == "vertex"
80
- Requires-Dist: kubernetes (>=18.20.0) ; extra == "connectors-kubernetes" or extra == "connectors-aws" or extra == "connectors-gcp" or extra == "connectors-azure"
81
- Requires-Dist: maison (<2.0) ; extra == "dev"
82
- Requires-Dist: mike (>=1.1.2,<2.0.0) ; extra == "dev"
83
- Requires-Dist: mkdocs (>=1.6.1,<2.0.0) ; extra == "dev"
84
- Requires-Dist: mkdocs-autorefs (>=1.4.0,<2.0.0) ; extra == "dev"
85
- Requires-Dist: mkdocs-awesome-pages-plugin (>=2.10.1,<3.0.0) ; extra == "dev"
86
- Requires-Dist: mkdocs-material (==9.6.8) ; extra == "dev"
87
- Requires-Dist: mkdocstrings[python] (>=0.28.1,<0.29.0) ; extra == "dev"
88
- Requires-Dist: mypy (==1.7.1) ; extra == "dev"
89
- Requires-Dist: orjson (>=3.10.0,<3.11.0) ; extra == "server"
90
- Requires-Dist: packaging (>=24.1)
91
- Requires-Dist: passlib[bcrypt] (>=1.7.4,<1.8.0)
92
- Requires-Dist: psutil (>=5.0.0)
93
- Requires-Dist: pydantic (>=2.0,<2.11.2)
94
- Requires-Dist: pydantic-settings
95
- Requires-Dist: pyjwt[crypto] (==2.7.*) ; extra == "server"
96
- Requires-Dist: pyment (>=0.3.3,<0.4.0) ; extra == "dev"
97
- Requires-Dist: pymysql (>=1.1.1,<1.2.0)
98
- Requires-Dist: pytest (>=7.4.0,<8.0.0) ; extra == "dev"
99
- Requires-Dist: pytest-clarity (>=1.0.1,<2.0.0) ; extra == "dev"
100
- Requires-Dist: pytest-instafail (>=0.5.0) ; extra == "dev"
101
- Requires-Dist: pytest-mock (>=3.6.1,<4.0.0) ; extra == "dev"
102
- Requires-Dist: pytest-randomly (>=3.10.1,<4.0.0) ; extra == "dev"
103
- Requires-Dist: pytest-rerunfailures (>=13.0) ; extra == "dev"
104
- Requires-Dist: pytest-split (>=0.8.1,<0.9.0) ; extra == "dev"
105
- Requires-Dist: python-dateutil (>=2.8.1,<3.0.0)
106
- Requires-Dist: python-multipart (>=0.0.9,<0.1.0) ; extra == "server"
107
- Requires-Dist: pyyaml (>=6.0.1)
108
- Requires-Dist: pyyaml-include (<2.0) ; extra == "templates"
109
- Requires-Dist: requests (>=2.27.11,<3.0.0) ; extra == "connectors-azure"
110
- Requires-Dist: rich[jupyter] (>=12.0.0)
111
- Requires-Dist: ruff (>=0.1.7) ; extra == "templates" or extra == "dev"
112
- Requires-Dist: s3fs (>=2022.11.0,!=2025.3.1) ; extra == "s3fs"
113
- Requires-Dist: sagemaker (>=2.237.3) ; extra == "sagemaker"
114
- Requires-Dist: secure (>=0.3.0,<0.4.0) ; extra == "server"
115
- Requires-Dist: setuptools (>=70.0.0)
116
- Requires-Dist: sqlalchemy (>=2.0.0,<3.0.0)
117
- Requires-Dist: sqlalchemy_utils
118
- Requires-Dist: sqlmodel (==0.0.18)
119
- Requires-Dist: tldextract (>=5.1.0,<5.2.0) ; extra == "server"
120
- Requires-Dist: tox (>=3.24.3,<4.0.0) ; extra == "dev"
121
- Requires-Dist: types-Markdown (>=3.3.6,<4.0.0) ; extra == "dev"
122
- Requires-Dist: types-Pillow (>=9.2.1,<10.0.0) ; extra == "dev"
123
- Requires-Dist: types-PyMySQL (>=1.0.4,<2.0.0) ; extra == "dev"
124
- Requires-Dist: types-PyYAML (>=6.0.0,<7.0.0) ; extra == "dev"
125
- Requires-Dist: types-certifi (>=2021.10.8.0,<2022.0.0.0) ; extra == "dev"
126
- Requires-Dist: types-croniter (>=1.0.2,<2.0.0) ; extra == "dev"
127
- Requires-Dist: types-futures (>=3.3.1,<4.0.0) ; extra == "dev"
128
- Requires-Dist: types-paramiko (>=3.4.0) ; extra == "dev"
129
- Requires-Dist: types-passlib (>=1.7.7,<2.0.0) ; extra == "dev"
130
- Requires-Dist: types-protobuf (>=3.18.0,<4.0.0) ; extra == "dev"
131
- Requires-Dist: types-psutil (>=5.8.13,<6.0.0) ; extra == "dev"
132
- Requires-Dist: types-python-dateutil (>=2.8.2,<3.0.0) ; extra == "dev"
133
- Requires-Dist: types-python-slugify (>=5.0.2,<6.0.0) ; extra == "dev"
134
- Requires-Dist: types-redis (>=4.1.19,<5.0.0) ; extra == "dev"
135
- Requires-Dist: types-requests (>=2.27.11,<3.0.0) ; extra == "dev"
136
- Requires-Dist: types-setuptools (>=57.4.2,<58.0.0) ; extra == "dev"
137
- Requires-Dist: types-six (>=1.16.2,<2.0.0) ; extra == "dev"
138
- Requires-Dist: types-termcolor (>=1.1.2,<2.0.0) ; extra == "dev"
139
- Requires-Dist: typing-extensions (>=3.7.4) ; extra == "dev"
140
- Requires-Dist: uvicorn[standard] (>=0.17.5) ; extra == "server"
141
- Requires-Dist: yamlfix (>=1.16.0,<2.0.0) ; extra == "dev"
142
- Project-URL: Documentation, https://docs.zenml.io
143
- Project-URL: Homepage, https://zenml.io
144
- Project-URL: Repository, https://github.com/zenml-io/zenml
145
- Description-Content-Type: text/markdown
146
-
147
- <div align="center">
148
- <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=0fcbab94-8fbe-4a38-93e8-c2348450a42e" />
149
- <h1 align="center">Beyond The Demo: Production-Grade AI Systems</h1>
150
- <h3 align="center">ZenML brings battle-tested MLOps practices to your AI applications, handling evaluation, monitoring, and deployment at scale</h3>
151
- </div>
152
-
153
- <!-- PROJECT SHIELDS -->
154
- <!--
155
- *** I'm using markdown "reference style" links for readability.
156
- *** Reference links are enclosed in brackets [ ] instead of parentheses ( ).
157
- *** See the bottom of this document for the declaration of the reference variables
158
- *** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
159
- *** https://www.markdownguide.org/basic-syntax/#reference-style-links
160
- -->
161
-
162
- <div align="center">
163
-
164
- <!-- PROJECT LOGO -->
165
- <br />
166
- <a href="https://zenml.io">
167
- <img alt="ZenML Logo" src="docs/book/.gitbook/assets/header.png" alt="ZenML Logo">
168
- </a>
169
- <br />
170
-
171
- [![PyPi][pypi-shield]][pypi-url]
172
- [![PyPi][pypiversion-shield]][pypi-url]
173
- [![PyPi][downloads-shield]][downloads-url]
174
- [![Contributors][contributors-shield]][contributors-url]
175
- [![License][license-shield]][license-url]
176
- <!-- [![Build][build-shield]][build-url] -->
177
- <!-- [![CodeCov][codecov-shield]][codecov-url] -->
178
-
179
- </div>
180
-
181
- <!-- MARKDOWN LINKS & IMAGES -->
182
- <!-- https://www.markdownguide.org/basic-syntax/#reference-style-links -->
183
-
184
- [pypi-shield]: https://img.shields.io/pypi/pyversions/zenml?color=281158
185
-
186
- [pypi-url]: https://pypi.org/project/zenml/
187
-
188
- [pypiversion-shield]: https://img.shields.io/pypi/v/zenml?color=361776
189
-
190
- [downloads-shield]: https://img.shields.io/pypi/dm/zenml?color=431D93
191
-
192
- [downloads-url]: https://pypi.org/project/zenml/
193
-
194
- [codecov-shield]: https://img.shields.io/codecov/c/gh/zenml-io/zenml?color=7A3EF4
195
-
196
- [codecov-url]: https://codecov.io/gh/zenml-io/zenml
197
-
198
- [contributors-shield]: https://img.shields.io/github/contributors/zenml-io/zenml?color=7A3EF4
199
-
200
- [contributors-url]: https://github.com/zenml-io/zenml/graphs/contributors
201
-
202
- [license-shield]: https://img.shields.io/github/license/zenml-io/zenml?color=9565F6
203
-
204
- [license-url]: https://github.com/zenml-io/zenml/blob/main/LICENSE
205
-
206
- [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555
207
-
208
- [linkedin-url]: https://www.linkedin.com/company/zenml/
209
-
210
- [twitter-shield]: https://img.shields.io/twitter/follow/zenml_io?style=for-the-badge
211
-
212
- [twitter-url]: https://twitter.com/zenml_io
213
-
214
- [slack-shield]: https://img.shields.io/badge/-Slack-black.svg?style=for-the-badge&logo=linkedin&colorB=555
215
-
216
- [slack-url]: https://zenml.io/slack-invite
217
-
218
- [build-shield]: https://img.shields.io/github/workflow/status/zenml-io/zenml/Build,%20Lint,%20Unit%20&%20Integration%20Test/develop?logo=github&style=for-the-badge
219
-
220
- [build-url]: https://github.com/zenml-io/zenml/actions/workflows/ci.yml
221
-
222
- ---
223
-
224
- Need help with documentation? Visit our [docs site](https://docs.zenml.io) for comprehensive guides and tutorials, or browse the [SDK reference](https://sdkdocs.zenml.io/) to find specific functions and classes.
225
-
226
- ## ⭐️ Show Your Support
227
-
228
- If you find ZenML helpful or interesting, please consider giving us a star on GitHub. Your support helps promote the project and lets others know that it's worth checking out.
229
-
230
- Thank you for your support! 🌟
231
-
232
- [![Star this project](https://img.shields.io/github/stars/zenml-io/zenml?style=social)](https://github.com/zenml-io/zenml/stargazers)
233
-
234
- ## 🤸 Quickstart
235
- [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/quickstart.ipynb)
236
-
237
- [Install ZenML](https://docs.zenml.io/getting-started/installation) via [PyPI](https://pypi.org/project/zenml/). Python 3.9 - 3.12 is required:
238
-
239
- ```bash
240
- pip install "zenml[server]" notebook
241
- ```
242
-
243
- Take a tour with the guided quickstart by running:
244
-
245
- ```bash
246
- zenml go
247
- ```
248
-
249
- ## 🪄 From Prototype to Production: AI Made Simple
250
-
251
- ### Create AI pipelines with minimal code changes
252
-
253
- ZenML is an open-source framework that handles MLOps and LLMOps for engineers scaling AI beyond prototypes. Automate evaluation loops, track performance, and deploy updates across 100s of pipelines—all while your RAG apps run like clockwork.
254
-
255
- ```python
256
- from zenml import pipeline, step
257
-
258
- @step
259
- def load_rag_documents() -> dict:
260
- # Load and chunk documents for RAG pipeline
261
- documents = extract_web_content(url="https://www.zenml.io/")
262
- return {"chunks": chunk_documents(documents)}
263
-
264
- @step
265
- def generate_embeddings(data: dict) -> None:
266
- # Generate embeddings for RAG pipeline
267
- embeddings = embed_documents(data['chunks'])
268
- return {"embeddings": embeddings}
269
-
270
- @step
271
- def index_generator(
272
- embeddings: dict,
273
- ) -> str:
274
- # Generate index for RAG pipeline
275
- index = create_index(embeddings)
276
- return index.id
277
-
278
-
279
- @pipeline
280
- def rag_pipeline() -> str:
281
- documents = load_rag_documents()
282
- embeddings = generate_embeddings(documents)
283
- index = index_generator(embeddings)
284
- return index
285
- ```
286
- ![Running a ZenML pipeline](docs/book/.gitbook/assets/readme_simple_pipeline.gif)
287
-
288
- ### Easily provision an MLOps stack or reuse your existing infrastructure
289
-
290
- The framework is a gentle entry point for practitioners to build complex ML pipelines with little knowledge required of the underlying infrastructure complexity. ZenML pipelines can be run on AWS, GCP, Azure, Airflow, Kubeflow and even on Kubernetes without having to change any code or know underlying internals.
291
-
292
- ZenML provides different features to aid people to get started quickly on a remote setting as well. If you want to deploy a remote stack from scratch on your selected cloud provider, you can use the 1-click deployment feature either through the dashboard:
293
-
294
- ![Running a ZenML pipeline](docs/book/.gitbook/assets/one-click-deployment.gif)
295
-
296
- Or, through our CLI command:
297
-
298
- ```bash
299
- zenml stack deploy --provider aws
300
- ```
301
-
302
- Alternatively, if the necessary pieces of infrastructure are already deployed, you can register a cloud stack seamlessly through the stack wizard:
303
-
304
- ```bash
305
- zenml stack register <STACK_NAME> --provider aws
306
- ```
307
-
308
- Read more about [ZenML stacks](https://docs.zenml.io/user-guide/production-guide/understand-stacks).
309
-
310
- ### Run workloads easily on your production infrastructure
311
-
312
- Once you have your MLOps stack configured, you can easily run workloads on it:
313
-
314
- ```bash
315
- zenml stack set <STACK_NAME>
316
- python run.py
317
- ```
318
-
319
- ```python
320
- from zenml.config import ResourceSettings, DockerSettings
321
-
322
- @step(
323
- settings={
324
- "resources": ResourceSettings(memory="16GB", gpu_count="1", cpu_count="8"),
325
- "docker": DockerSettings(parent_image="pytorch/pytorch:1.12.1-cuda11.3-cudnn8-runtime")
326
- }
327
- )
328
- def training(...):
329
- ...
330
- ```
331
-
332
- ![Workloads with ZenML](docs/book/.gitbook/assets/readme_compute.gif)
333
-
334
- ### Track models, pipeline, and artifacts
335
-
336
- Create a complete lineage of who, where, and what data and models are produced.
337
-
338
- You'll be able to find out who produced which model, at what time, with which data, and on which version of the code. This guarantees full reproducibility and auditability.
339
-
340
- ```python
341
- from zenml import Model
342
-
343
- @step(model=Model(name="rag_llm", tags=["staging"]))
344
- def deploy_rag(index_id: str) -> str:
345
- deployment_id = deploy_to_endpoint(index_id)
346
- return deployment_id
347
- ```
348
-
349
- ![Exploring ZenML Models](docs/book/.gitbook/assets/readme_mcp.gif)
350
-
351
- ## 🚀 Key LLMOps Capabilities
352
-
353
- ### Continual RAG Improvement
354
- **Build production-ready retrieval systems**
355
-
356
- <div align="center">
357
- <img src="docs/book/.gitbook/assets/rag_zenml_home.png" width="800" alt="RAG Pipeline">
358
- </div>
359
-
360
- ZenML tracks document ingestion, embedding versions, and query patterns. Implement feedback loops and:
361
- - Fix your RAG logic based on production logs
362
- - Automatically re-ingest updated documents
363
- - A/B test different embedding models
364
- - Monitor retrieval quality metrics
365
-
366
- ### Reproducible Model Fine-Tuning
367
- **Confidence in model updates**
368
-
369
- <div align="center">
370
- <img src="docs/book/.gitbook/assets/finetune_zenml_home.png" width="800" alt="Finetuning Pipeline">
371
- </div>
372
-
373
- Maintain full lineage of SLM/LLM training runs:
374
- - Version training data and hyperparameters
375
- - Track performance across iterations
376
- - Automatically promote validated models
377
- - Roll back to previous versions if needed
378
-
379
- ### Purpose built for machine learning with integrations to your favorite tools
380
-
381
- While ZenML brings a lot of value out of the box, it also integrates into your existing tooling and infrastructure without you having to be locked in.
382
-
383
- ```python
384
- from bentoml._internal.bento import bento
385
-
386
- @step(on_failure=alert_slack, experiment_tracker="mlflow")
387
- def train_and_deploy(training_df: pd.DataFrame) -> bento.Bento
388
- mlflow.autolog()
389
- ...
390
- return bento
391
- ```
392
-
393
- ![Exploring ZenML Integrations](docs/book/.gitbook/assets/readme_integrations.gif)
394
-
395
- ## 🔄 Your LLM Framework Isn't Enough for Production
396
-
397
- While tools like LangChain and LlamaIndex help you **build** LLM workflows, ZenML helps you **productionize** them by adding:
398
-
399
- ✅ **Artifact Tracking** - Every vector store index, fine-tuned model, and evaluation result versioned automatically
400
- ✅ **Pipeline History** - See exactly what code/data produced each version of your RAG system
401
- ✅ **Stage Promotion** - Move validated pipelines from staging → production with one click
402
-
403
- ## 🖼️ Learning
404
-
405
- The best way to learn about ZenML is the [docs](https://docs.zenml.io/). We recommend beginning with the [Starter Guide](https://docs.zenml.io/user-guide/starter-guide) to get up and running quickly.
406
-
407
- If you are a visual learner, this 11-minute video tutorial is also a great start:
408
-
409
- [![Introductory Youtube Video](docs/book/.gitbook/assets/readme_youtube_thumbnail.png)](https://www.youtube.com/watch?v=wEVwIkDvUPs)
410
-
411
- And finally, here are some other examples and use cases for inspiration:
412
-
413
- 1. [E2E Batch Inference](examples/e2e/): Feature engineering, training, and inference pipelines for tabular machine learning.
414
- 2. [Basic NLP with BERT](examples/e2e_nlp/): Feature engineering, training, and inference focused on NLP.
415
- 3. [LLM RAG Pipeline with Langchain and OpenAI](https://github.com/zenml-io/zenml-projects/tree/main/zenml-support-agent): Using Langchain to create a simple RAG pipeline.
416
- 4. [Huggingface Model to Sagemaker Endpoint](https://github.com/zenml-io/zenml-projects/tree/main/huggingface-sagemaker): Automated MLOps on Amazon Sagemaker and HuggingFace
417
- 5. [LLMops](https://github.com/zenml-io/zenml-projects/tree/main/llm-complete-guide): Complete guide to do LLM with ZenML
418
-
419
-
420
- ## 📚 Learn from Books
421
-
422
- <div align="center">
423
- <a href="https://www.amazon.com/LLM-Engineers-Handbook-engineering-production/dp/1836200072">
424
- <img src="docs/book/.gitbook/assets/llm_engineering_handbook_cover.jpg" alt="LLM Engineer's Handbook Cover" width="200"/></img>
425
- </a>&nbsp;&nbsp;&nbsp;&nbsp;
426
- <a href="https://www.amazon.com/-/en/Andrew-McMahon/dp/1837631964">
427
- <img src="docs/book/.gitbook/assets/ml_engineering_with_python.jpg" alt="Machine Learning Engineering with Python Cover" width="200"/></img>
428
- </a>
429
- </br></br>
430
- </div>
431
-
432
- ZenML is featured in these comprehensive guides to modern MLOps and LLM engineering. Learn how to build production-ready machine learning systems with real-world examples and best practices.
433
-
434
- ## 🔋 Deploy ZenML
435
-
436
- For full functionality ZenML should be deployed on the cloud to
437
- enable collaborative features as the central MLOps interface for teams.
438
-
439
- Read more about various deployment options [here](https://docs.zenml.io/getting-started/deploying-zenml).
440
-
441
- Or, sign up for [ZenML Pro to get a fully managed server on a free trial](https://cloud.zenml.io/?utm_source=readme&utm_medium=referral_link&utm_campaign=cloud_promotion&utm_content=signup_link).
442
-
443
- ## Use ZenML with VS Code
444
-
445
- ZenML has a [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode) that allows you to inspect your stacks and pipeline runs directly from your editor. The extension also allows you to switch your stacks without needing to type any CLI commands.
446
-
447
- <details>
448
- <summary>🖥️ VS Code Extension in Action!</summary>
449
- <div align="center">
450
- <img width="60%" src="docs/book/.gitbook/assets/zenml-extension-shortened.gif" alt="ZenML Extension">
451
- </div>
452
- </details>
453
-
454
- ## 🗺 Roadmap
455
-
456
- ZenML is being built in public. The [roadmap](https://zenml.io/roadmap) is a regularly updated source of truth for the ZenML community to understand where the product is going in the short, medium, and long term.
457
-
458
- ZenML is managed by a [core team](https://zenml.io/company) of developers that are responsible for making key decisions and incorporating feedback from the community. The team oversees feedback via various channels,
459
- and you can directly influence the roadmap as follows:
460
-
461
- - Ask questions and share your thoughts on the [Discussions board](https://zenml.io/discussion).
462
- - Start a thread in our [Slack channel](https://zenml.io/slack).
463
- - [Create an issue](https://github.com/zenml-io/zenml/issues/new/choose) on our GitHub repo.
464
-
465
- ## 🙌 Contributing and Community
466
-
467
- We would love to develop ZenML together with our community! The best way to get started is to select any issue from the [`good-first-issue` label](https://github.com/issues?q=is%3Aopen+is%3Aissue+archived%3Afalse+user%3Azenml-io+label%3A%22good+first+issue%22) and open up a Pull Request!
468
-
469
- If you
470
- would like to contribute, please review our [Contributing
471
- Guide](CONTRIBUTING.md) for all relevant details.
472
-
473
- ## 🆘 Getting Help
474
-
475
- The first point of call should
476
- be [our Slack group](https://zenml.io/slack/).
477
- Ask your questions about bugs or specific use cases, and someone from
478
- the [core team](https://zenml.io/company) will respond.
479
- Or, if you
480
- prefer, [open an issue](https://github.com/zenml-io/zenml/issues/new/choose) on
481
- our GitHub repo.
482
-
483
- ## 📚 LLM-focused Learning Resources
484
-
485
- 1. [LL Complete Guide - Full RAG Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/llm-complete-guide) - Document ingestion, embedding management, and query serving
486
- 2. [LLM Fine-Tuning Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/zencoder) - From data prep to deployed model
487
- 3. [LLM Agents Example](https://github.com/zenml-io/zenml-projects/tree/main/zenml-support-agent) - Track conversation quality and tool usage
488
-
489
- ## 🤖 AI-Friendly Documentation with llms.txt
490
-
491
- ZenML implements the llms.txt standard to make our documentation more accessible to AI assistants and LLMs. Our implementation includes:
492
-
493
- - Base documentation at [zenml.io/llms.txt](https://zenml.io/llms.txt) with core user guides
494
- - Specialized files for different documentation aspects:
495
- - [Component guides](https://zenml.io/component-guide.txt) for integration details
496
- - [How-to guides](https://zenml.io/how-to-guides.txt) for practical implementations
497
- - [Complete documentation corpus](https://zenml.io/llms-full.txt) for comprehensive access
498
-
499
- This structured approach helps AI tools better understand and utilize ZenML's documentation, enabling more accurate code suggestions and improved documentation search.
500
-
501
- ## 📜 License
502
-
503
- ZenML is distributed under the terms of the Apache License Version 2.0.
504
- A complete version of the license is available in the [LICENSE](LICENSE) file in
505
- this repository. Any contribution made to this project will be licensed under
506
- the Apache License Version 2.0.
507
-
508
- <div>
509
- <p align="left">
510
- <div align="left">
511
- Join our <a href="https://zenml.io/slack" target="_blank">
512
- <img width="18" src="https://cdn3.iconfinder.com/data/icons/logos-and-brands-adobe/512/306_Slack-512.png" alt="Slack"/>
513
- <b>Slack Community</b> </a> and be part of the ZenML family.
514
- </div>
515
- <br />
516
- <a href="https://zenml.io/features">Features</a>
517
- ·
518
- <a href="https://zenml.io/roadmap">Roadmap</a>
519
- ·
520
- <a href="https://github.com/zenml-io/zenml/issues">Report Bug</a>
521
- ·
522
- <a href="https://zenml.io/pro">Sign up for ZenML Pro</a>
523
- ·
524
- <a href="https://www.zenml.io/blog">Read Blog</a>
525
- ·
526
- <a href="https://github.com/issues?q=is%3Aopen+is%3Aissue+archived%3Afalse+user%3Azenml-io+label%3A%22good+first+issue%22">Contribute to Open Source</a>
527
- ·
528
- <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
529
- <br />
530
- <br />
531
- 🎉 Version 0.83.1 is out. Check out the release notes
532
- <a href="https://github.com/zenml-io/zenml/releases">here</a>.
533
- <br />
534
- 🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.
535
- <br />
536
- </p>
537
- </div>
538
-