zenml-nightly 0.74.0.dev20250225__tar.gz → 0.74.0.dev20250226__tar.gz

{zenml_nightly-0.74.0.dev20250225 → zenml_nightly-0.74.0.dev20250226}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zenml-nightly
-Version: 0.74.0.dev20250225
+Version: 0.74.0.dev20250226
 Summary: ZenML: Write production-ready ML code.
 License: Apache-2.0
 Keywords: machine learning,production,pipeline,mlops,devops
@@ -79,10 +79,12 @@ Requires-Dist: kfp (>=2.6.0) ; extra == "vertex"
 Requires-Dist: kubernetes (>=18.20.0) ; extra == "connectors-kubernetes" or extra == "connectors-aws" or extra == "connectors-gcp" or extra == "connectors-azure"
 Requires-Dist: maison (<2.0) ; extra == "dev"
 Requires-Dist: mike (>=1.1.2,<2.0.0) ; extra == "dev"
-Requires-Dist: mkdocs (>=1.2.3,<2.0.0) ; extra == "dev"
+Requires-Dist: mkdocs (>=1.4.0,<2.0.0) ; extra == "dev"
+Requires-Dist: mkdocs-autorefs (>=0.4.1,<1.0.0) ; extra == "dev"
 Requires-Dist: mkdocs-awesome-pages-plugin (>=2.6.1,<3.0.0) ; extra == "dev"
-Requires-Dist: mkdocs-material (>=8.1.7,<9.0.0) ; extra == "dev"
-Requires-Dist: mkdocstrings (>=0.17.0,<0.18.0) ; extra == "dev"
+Requires-Dist: mkdocs-material (>=9.0.0,<10.0.0) ; extra == "dev"
+Requires-Dist: mkdocstrings (>=0.20.0,<1.0.0) ; extra == "dev"
+Requires-Dist: mkdocstrings-python (>=1.1.0,<2.0.0) ; extra == "dev"
 Requires-Dist: mypy (==1.7.1) ; extra == "dev"
 Requires-Dist: orjson (>=3.10.0,<3.11.0) ; extra == "server"
 Requires-Dist: packaging (>=24.1)
@@ -144,8 +146,8 @@ Description-Content-Type: text/markdown
 
 <div align="center">
   <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=0fcbab94-8fbe-4a38-93e8-c2348450a42e" />
-  <h1 align="center">Connecting data science teams seamlessly to cloud infrastructure.
-</h1>
+  <h1 align="center">Beyond The Demo: Production-Grade AI Systems</h1>
+  <h3 align="center">ZenML brings battle-tested MLOps practices to your AI applications, handling evaluation, monitoring, and deployment at scale</h3>
 </div>
 
 <!-- PROJECT SHIELDS -->
@@ -244,40 +246,44 @@ Take a tour with the guided quickstart by running:
 zenml go
 ```
 
-## 🪄 Simple, integrated, End-to-end MLOps 
+## 🪄 From Prototype to Production: AI Made Simple
 
-### Create machine learning pipelines with minimal code changes
+### Create AI pipelines with minimal code changes
 
-ZenML is a MLOps framework intended for data scientists or ML engineers looking to standardize machine learning practices. Just add `@step` and `@pipeline` to your existing Python functions to get going. Here is a toy example:
+ZenML is an open-source framework that handles MLOps and LLMOps for engineers scaling AI beyond prototypes. Automate evaluation loops, track performance, and deploy updates across 100s of pipelines—all while your RAG apps run like clockwork.
 
 ```python
 from zenml import pipeline, step
 
-@step  # Just add this decorator
-def load_data() -> dict:
-    training_data = [[1, 2], [3, 4], [5, 6]]
-    labels = [0, 1, 0]
-    return {'features': training_data, 'labels': labels}
+@step
+def load_rag_documents() -> dict:
+    # Load and chunk documents for RAG pipeline
+    documents = extract_web_content(url="https://www.zenml.io/")
+    return {"chunks": chunk_documents(documents)}
 
 @step
-def train_model(data: dict) -> None:
-    total_features = sum(map(sum, data['features']))
-    total_labels = sum(data['labels'])
-    
-    print(f"Trained model using {len(data['features'])} data points. "
-          f"Feature sum is {total_features}, label sum is {total_labels}")
+def generate_embeddings(data: dict) -> None:
+    # Generate embeddings for RAG pipeline
+    embeddings = embed_documents(data['chunks'])
+    return {"embeddings": embeddings}
 
-@pipeline  # This function combines steps together 
-def simple_ml_pipeline():
-    dataset = load_data()
-    train_model(dataset)
+@step
+def index_generator(
+    embeddings: dict,
+) -> str:
+    # Generate index for RAG pipeline
+    index = create_index(embeddings)
+    return index.id
+    
 
-if __name__ == "__main__":
-    run = simple_ml_pipeline()  # call this to run the pipeline
-   
+@pipeline
+def rag_pipeline() -> str:
+    documents = load_rag_documents()
+    embeddings = generate_embeddings(documents)
+    index = index_generator(embeddings)
+    return index
 ```
-
-![Running a ZenML pipeline](/docs/book/.gitbook/assets/readme_basic_pipeline.gif)
+![Running a ZenML pipeline](/docs/book/.gitbook/assets/readme_simple_pipeline.gif)
 
 ### Easily provision an MLOps stack or reuse your existing infrastructure
 
@@ -329,18 +335,47 @@ def training(...):
 
 Create a complete lineage of who, where, and what data and models are produced.
 
-You’ll be able to find out who produced which model, at what time, with which data, and on which version of the code. This guarantees full reproducibility and auditability.
+You'll be able to find out who produced which model, at what time, with which data, and on which version of the code. This guarantees full reproducibility and auditability.
 
 ```python
 from zenml import Model
 
-@step(model=Model(name="classification"))
-def trainer(training_df: pd.DataFrame) -> Annotated["model", torch.nn.Module]:
-	...
+@step(model=Model(name="rag_llm", tags=["staging"]))
+def deploy_rag(index_id: str) -> str:
+    deployment_id = deploy_to_endpoint(index_id)
+    return deployment_id
 ```
 
 ![Exploring ZenML Models](/docs/book/.gitbook/assets/readme_mcp.gif)
 
+## 🚀 Key LLMOps Capabilities
+
+### Continual RAG Improvement
+**Build production-ready retrieval systems**  
+
+<div align="center">
+  <img src="/docs/book/.gitbook/assets/rag_zenml_home.png" width="800" alt="RAG Pipeline">
+</div>
+
+ZenML tracks document ingestion, embedding versions, and query patterns. Implement feedback loops and:
+- Fix your RAG logic based on production logs
+- Automatically re-ingest updated documents
+- A/B test different embedding models
+- Monitor retrieval quality metrics
+
+### Reproducible Model Fine-Tuning
+**Confidence in model updates**
+
+<div align="center">
+  <img src="/docs/book/.gitbook/assets/finetune_zenml_home.png" width="800" alt="Finetuning Pipeline">
+</div>
+
+Maintain full lineage of SLM/LLM training runs:
+- Version training data and hyperparameters
+- Track performance across iterations
+- Automatically promote validated models
+- Roll back to previous versions if needed
+
 ### Purpose built for machine learning with integrations to your favorite tools
 
 While ZenML brings a lot of value out of the box, it also integrates into your existing tooling and infrastructure without you having to be locked in.
@@ -357,6 +392,14 @@ def train_and_deploy(training_df: pd.DataFrame) -> bento.Bento
 
 ![Exploring ZenML Integrations](/docs/book/.gitbook/assets/readme_integrations.gif)
 
+## 🔄 Your LLM Framework Isn't Enough for Production
+
+While tools like LangChain and LlamaIndex help you **build** LLM workflows, ZenML helps you **productionize** them by adding:
+
+✅ **Artifact Tracking** - Every vector store index, fine-tuned model, and evaluation result versioned automatically  
+✅ **Pipeline History** - See exactly what code/data produced each version of your RAG system  
+✅ **Stage Promotion** - Move validated pipelines from staging → production with one click  
+
 ## 🖼️ Learning
 
 The best way to learn about ZenML is the [docs](https://docs.zenml.io/). We recommend beginning with the [Starter Guide](https://docs.zenml.io/user-guide/starter-guide) to get up and running quickly.
@@ -441,13 +484,23 @@ Or, if you
 prefer, [open an issue](https://github.com/zenml-io/zenml/issues/new/choose) on
 our GitHub repo.
 
-## ⭐️ Show Your Support
+## 📚 LLM-focused Learning Resources
 
-If you find ZenML helpful or interesting, please consider giving us a star on GitHub. Your support helps promote the project and lets others know that it's worth checking out. 
+1. [LL Complete Guide - Full RAG Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/llm-complete-guide) - Document ingestion, embedding management, and query serving
+2. [LLM Fine-Tuning Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/llm-finetuning) - From data prep to deployed model
+3. [LLM Agents Example](https://github.com/zenml-io/zenml-projects/tree/main/llm-agents) - Track conversation quality and tool usage
 
-Thank you for your support! 🌟
+## 🤖 AI-Friendly Documentation with llms.txt
 
-[![Star this project](https://img.shields.io/github/stars/zenml-io/zenml?style=social)](https://github.com/zenml-io/zenml/stargazers)
+ZenML implements the llms.txt standard to make our documentation more accessible to AI assistants and LLMs. Our implementation includes:
+
+- Base documentation at [zenml.io/llms.txt](https://zenml.io/llms.txt) with core user guides
+- Specialized files for different documentation aspects:
+  - [Component guides](https://zenml.io/component-guide.txt) for integration details
+  - [How-to guides](https://zenml.io/how-to-guides.txt) for practical implementations
+  - [Complete documentation corpus](https://zenml.io/llms-full.txt) for comprehensive access
+
+This structured approach helps AI tools better understand and utilize ZenML's documentation, enabling more accurate code suggestions and improved documentation search.
 
 ## 📜 License
 

{zenml_nightly-0.74.0.dev20250225 → zenml_nightly-0.74.0.dev20250226}/README.md

@@ -1,7 +1,7 @@
 <div align="center">
   <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=0fcbab94-8fbe-4a38-93e8-c2348450a42e" />
-  <h1 align="center">Connecting data science teams seamlessly to cloud infrastructure.
-</h1>
+  <h1 align="center">Beyond The Demo: Production-Grade AI Systems</h1>
+  <h3 align="center">ZenML brings battle-tested MLOps practices to your AI applications, handling evaluation, monitoring, and deployment at scale</h3>
 </div>
 
 <!-- PROJECT SHIELDS -->
@@ -100,40 +100,44 @@ Take a tour with the guided quickstart by running:
 zenml go
 ```
 
-## 🪄 Simple, integrated, End-to-end MLOps 
+## 🪄 From Prototype to Production: AI Made Simple
 
-### Create machine learning pipelines with minimal code changes
+### Create AI pipelines with minimal code changes
 
-ZenML is a MLOps framework intended for data scientists or ML engineers looking to standardize machine learning practices. Just add `@step` and `@pipeline` to your existing Python functions to get going. Here is a toy example:
+ZenML is an open-source framework that handles MLOps and LLMOps for engineers scaling AI beyond prototypes. Automate evaluation loops, track performance, and deploy updates across 100s of pipelines—all while your RAG apps run like clockwork.
 
 ```python
 from zenml import pipeline, step
 
-@step  # Just add this decorator
-def load_data() -> dict:
-    training_data = [[1, 2], [3, 4], [5, 6]]
-    labels = [0, 1, 0]
-    return {'features': training_data, 'labels': labels}
+@step
+def load_rag_documents() -> dict:
+    # Load and chunk documents for RAG pipeline
+    documents = extract_web_content(url="https://www.zenml.io/")
+    return {"chunks": chunk_documents(documents)}
 
 @step
-def train_model(data: dict) -> None:
-    total_features = sum(map(sum, data['features']))
-    total_labels = sum(data['labels'])
-    
-    print(f"Trained model using {len(data['features'])} data points. "
-          f"Feature sum is {total_features}, label sum is {total_labels}")
+def generate_embeddings(data: dict) -> None:
+    # Generate embeddings for RAG pipeline
+    embeddings = embed_documents(data['chunks'])
+    return {"embeddings": embeddings}
 
-@pipeline  # This function combines steps together 
-def simple_ml_pipeline():
-    dataset = load_data()
-    train_model(dataset)
+@step
+def index_generator(
+    embeddings: dict,
+) -> str:
+    # Generate index for RAG pipeline
+    index = create_index(embeddings)
+    return index.id
+    
 
-if __name__ == "__main__":
-    run = simple_ml_pipeline()  # call this to run the pipeline
-   
+@pipeline
+def rag_pipeline() -> str:
+    documents = load_rag_documents()
+    embeddings = generate_embeddings(documents)
+    index = index_generator(embeddings)
+    return index
 ```
-
-![Running a ZenML pipeline](/docs/book/.gitbook/assets/readme_basic_pipeline.gif)
+![Running a ZenML pipeline](/docs/book/.gitbook/assets/readme_simple_pipeline.gif)
 
 ### Easily provision an MLOps stack or reuse your existing infrastructure
 
@@ -185,18 +189,47 @@ def training(...):
 
 Create a complete lineage of who, where, and what data and models are produced.
 
-You’ll be able to find out who produced which model, at what time, with which data, and on which version of the code. This guarantees full reproducibility and auditability.
+You'll be able to find out who produced which model, at what time, with which data, and on which version of the code. This guarantees full reproducibility and auditability.
 
 ```python
 from zenml import Model
 
-@step(model=Model(name="classification"))
-def trainer(training_df: pd.DataFrame) -> Annotated["model", torch.nn.Module]:
-	...
+@step(model=Model(name="rag_llm", tags=["staging"]))
+def deploy_rag(index_id: str) -> str:
+    deployment_id = deploy_to_endpoint(index_id)
+    return deployment_id
 ```
 
 ![Exploring ZenML Models](/docs/book/.gitbook/assets/readme_mcp.gif)
 
+## 🚀 Key LLMOps Capabilities
+
+### Continual RAG Improvement
+**Build production-ready retrieval systems**  
+
+<div align="center">
+  <img src="/docs/book/.gitbook/assets/rag_zenml_home.png" width="800" alt="RAG Pipeline">
+</div>
+
+ZenML tracks document ingestion, embedding versions, and query patterns. Implement feedback loops and:
+- Fix your RAG logic based on production logs
+- Automatically re-ingest updated documents
+- A/B test different embedding models
+- Monitor retrieval quality metrics
+
+### Reproducible Model Fine-Tuning
+**Confidence in model updates**
+
+<div align="center">
+  <img src="/docs/book/.gitbook/assets/finetune_zenml_home.png" width="800" alt="Finetuning Pipeline">
+</div>
+
+Maintain full lineage of SLM/LLM training runs:
+- Version training data and hyperparameters
+- Track performance across iterations
+- Automatically promote validated models
+- Roll back to previous versions if needed
+
 ### Purpose built for machine learning with integrations to your favorite tools
 
 While ZenML brings a lot of value out of the box, it also integrates into your existing tooling and infrastructure without you having to be locked in.
@@ -213,6 +246,14 @@ def train_and_deploy(training_df: pd.DataFrame) -> bento.Bento
 
 ![Exploring ZenML Integrations](/docs/book/.gitbook/assets/readme_integrations.gif)
 
+## 🔄 Your LLM Framework Isn't Enough for Production
+
+While tools like LangChain and LlamaIndex help you **build** LLM workflows, ZenML helps you **productionize** them by adding:
+
+✅ **Artifact Tracking** - Every vector store index, fine-tuned model, and evaluation result versioned automatically  
+✅ **Pipeline History** - See exactly what code/data produced each version of your RAG system  
+✅ **Stage Promotion** - Move validated pipelines from staging → production with one click  
+
 ## 🖼️ Learning
 
 The best way to learn about ZenML is the [docs](https://docs.zenml.io/). We recommend beginning with the [Starter Guide](https://docs.zenml.io/user-guide/starter-guide) to get up and running quickly.
@@ -297,13 +338,23 @@ Or, if you
 prefer, [open an issue](https://github.com/zenml-io/zenml/issues/new/choose) on
 our GitHub repo.
 
-## ⭐️ Show Your Support
+## 📚 LLM-focused Learning Resources
 
-If you find ZenML helpful or interesting, please consider giving us a star on GitHub. Your support helps promote the project and lets others know that it's worth checking out. 
+1. [LL Complete Guide - Full RAG Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/llm-complete-guide) - Document ingestion, embedding management, and query serving
+2. [LLM Fine-Tuning Pipeline](https://github.com/zenml-io/zenml-projects/tree/main/llm-finetuning) - From data prep to deployed model
+3. [LLM Agents Example](https://github.com/zenml-io/zenml-projects/tree/main/llm-agents) - Track conversation quality and tool usage
 
-Thank you for your support! 🌟
+## 🤖 AI-Friendly Documentation with llms.txt
 
-[![Star this project](https://img.shields.io/github/stars/zenml-io/zenml?style=social)](https://github.com/zenml-io/zenml/stargazers)
+ZenML implements the llms.txt standard to make our documentation more accessible to AI assistants and LLMs. Our implementation includes:
+
+- Base documentation at [zenml.io/llms.txt](https://zenml.io/llms.txt) with core user guides
+- Specialized files for different documentation aspects:
+  - [Component guides](https://zenml.io/component-guide.txt) for integration details
+  - [How-to guides](https://zenml.io/how-to-guides.txt) for practical implementations
+  - [Complete documentation corpus](https://zenml.io/llms-full.txt) for comprehensive access
+
+This structured approach helps AI tools better understand and utilize ZenML's documentation, enabling more accurate code suggestions and improved documentation search.
 
 ## 📜 License
 

{zenml_nightly-0.74.0.dev20250225 → zenml_nightly-0.74.0.dev20250226}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zenml-nightly"
-version = "0.74.0.dev20250225"
+version = "0.74.0.dev20250226"
 packages = [{ include = "zenml", from = "src" }]
 description = "ZenML: Write production-ready ML code."
 authors = ["ZenML GmbH <info@zenml.io>"]
@@ -155,11 +155,13 @@ pytest-rerunfailures = { version = ">=13.0", optional = true }
 pytest-split = { version = "^0.8.1", optional = true }
 
 # mkdocs including plugins
-mkdocs = { version = "^1.2.3", optional = true }
-mkdocs-material = { version = "^8.1.7", optional = true }
+mkdocs = { version = ">=1.4.0,<2.0.0", optional = true }
+mkdocs-material = { version = ">=9.0.0,<10.0.0", optional = true }
 mkdocs-awesome-pages-plugin = { version = "^2.6.1", optional = true }
-mkdocstrings = { version = "^0.17.0", optional = true }
-mike = { version = "^1.1.2", optional = true }
+mkdocstrings = { version = ">=0.20.0,<1.0.0", optional = true }
+mkdocstrings-python = { version = ">=1.1.0,<2.0.0", optional = true }
+mkdocs-autorefs = { version = ">=0.4.1,<1.0.0", optional = true }
+mike = { version = ">=1.1.2,<2.0.0", optional = true }
 
 # mypy type stubs
 types-certifi = { version = "^2021.10.8.0", optional = true }
@@ -255,6 +257,8 @@ dev = [
     "mkdocs-material",
     "mkdocs-awesome-pages-plugin",
     "mkdocstrings",
+    "mkdocstrings-python",
+    "mkdocs-autorefs",
     "mike",
     "maison",
     "types-certifi",
@@ -406,6 +410,7 @@ module = [
     "google.*",
     "transformers.*", # https://github.com/huggingface/transformers/issues/13390
     "datasets.*",
+    "langchain_community.*",
     "IPython.core.*",
 ]
 follow_imports = "skip"
@@ -481,7 +486,6 @@ module = [
     "prodigy.components.*",
     "prodigy.components.db.*",
     "transformers.*",
-    "langchain_community.*",
     "vllm.*",
     "numba.*",
     "uvloop.*",

zenml_nightly-0.74.0.dev20250226/src/zenml/VERSION

@@ -0,0 +1 @@
+0.74.0.dev20250226

{zenml_nightly-0.74.0.dev20250225 → zenml_nightly-0.74.0.dev20250226}/src/zenml/logging/step_logging.py

@@ -285,6 +285,28 @@ class StepLogsStorage:
         Args:
             force: whether to force a save even if the write conditions not met.
         """
+        import asyncio
+        import threading
+
+        # Most artifact stores are based on fsspec, which converts between
+        # sync and async operations by using a separate AIO thread.
+        # It may happen that the fsspec call itself will log something,
+        # which will trigger this method, which may then use fsspec again,
+        # causing a "Calling sync() from within a running loop" error, because
+        # the fsspec library does not expect sync calls being made as a result
+        # of a logging call made by itself.
+        # To avoid this, we simply check if we're running in the fsspec AIO
+        # thread and skip the save if that's the case.
+        try:
+            if (
+                asyncio.events.get_running_loop() is not None
+                and threading.current_thread().name == "fsspecIO"
+            ):
+                return
+        except RuntimeError:
+            # No running loop
+            pass
+
         if not self.disabled and (self._is_write_needed or force):
             # IMPORTANT: keep this as the first code line in this method! The
             # code that follows might still emit logging messages, which will

zenml_nightly-0.74.0.dev20250225/src/zenml/VERSION

@@ -1 +0,0 @@
-0.74.0.dev20250225