flowyml 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowyml-1.1.0/LICENSE +17 -0
- flowyml-1.1.0/PKG-INFO +372 -0
- flowyml-1.1.0/README.md +322 -0
- flowyml-1.1.0/flowyml/__init__.py +207 -0
- flowyml-1.1.0/flowyml/assets/__init__.py +22 -0
- flowyml-1.1.0/flowyml/assets/artifact.py +40 -0
- flowyml-1.1.0/flowyml/assets/base.py +209 -0
- flowyml-1.1.0/flowyml/assets/dataset.py +100 -0
- flowyml-1.1.0/flowyml/assets/featureset.py +301 -0
- flowyml-1.1.0/flowyml/assets/metrics.py +104 -0
- flowyml-1.1.0/flowyml/assets/model.py +82 -0
- flowyml-1.1.0/flowyml/assets/registry.py +157 -0
- flowyml-1.1.0/flowyml/assets/report.py +315 -0
- flowyml-1.1.0/flowyml/cli/__init__.py +5 -0
- flowyml-1.1.0/flowyml/cli/experiment.py +232 -0
- flowyml-1.1.0/flowyml/cli/init.py +256 -0
- flowyml-1.1.0/flowyml/cli/main.py +327 -0
- flowyml-1.1.0/flowyml/cli/run.py +75 -0
- flowyml-1.1.0/flowyml/cli/stack_cli.py +532 -0
- flowyml-1.1.0/flowyml/cli/ui.py +33 -0
- flowyml-1.1.0/flowyml/core/__init__.py +68 -0
- flowyml-1.1.0/flowyml/core/advanced_cache.py +274 -0
- flowyml-1.1.0/flowyml/core/approval.py +64 -0
- flowyml-1.1.0/flowyml/core/cache.py +203 -0
- flowyml-1.1.0/flowyml/core/checkpoint.py +148 -0
- flowyml-1.1.0/flowyml/core/conditional.py +373 -0
- flowyml-1.1.0/flowyml/core/context.py +155 -0
- flowyml-1.1.0/flowyml/core/error_handling.py +419 -0
- flowyml-1.1.0/flowyml/core/executor.py +354 -0
- flowyml-1.1.0/flowyml/core/graph.py +185 -0
- flowyml-1.1.0/flowyml/core/parallel.py +452 -0
- flowyml-1.1.0/flowyml/core/pipeline.py +764 -0
- flowyml-1.1.0/flowyml/core/project.py +253 -0
- flowyml-1.1.0/flowyml/core/resources.py +424 -0
- flowyml-1.1.0/flowyml/core/scheduler.py +630 -0
- flowyml-1.1.0/flowyml/core/scheduler_config.py +32 -0
- flowyml-1.1.0/flowyml/core/step.py +201 -0
- flowyml-1.1.0/flowyml/core/step_grouping.py +292 -0
- flowyml-1.1.0/flowyml/core/templates.py +226 -0
- flowyml-1.1.0/flowyml/core/versioning.py +217 -0
- flowyml-1.1.0/flowyml/integrations/__init__.py +1 -0
- flowyml-1.1.0/flowyml/integrations/keras.py +134 -0
- flowyml-1.1.0/flowyml/monitoring/__init__.py +1 -0
- flowyml-1.1.0/flowyml/monitoring/alerts.py +57 -0
- flowyml-1.1.0/flowyml/monitoring/data.py +102 -0
- flowyml-1.1.0/flowyml/monitoring/llm.py +160 -0
- flowyml-1.1.0/flowyml/monitoring/monitor.py +57 -0
- flowyml-1.1.0/flowyml/monitoring/notifications.py +246 -0
- flowyml-1.1.0/flowyml/registry/__init__.py +5 -0
- flowyml-1.1.0/flowyml/registry/model_registry.py +491 -0
- flowyml-1.1.0/flowyml/registry/pipeline_registry.py +55 -0
- flowyml-1.1.0/flowyml/stacks/__init__.py +27 -0
- flowyml-1.1.0/flowyml/stacks/base.py +77 -0
- flowyml-1.1.0/flowyml/stacks/bridge.py +288 -0
- flowyml-1.1.0/flowyml/stacks/components.py +155 -0
- flowyml-1.1.0/flowyml/stacks/gcp.py +499 -0
- flowyml-1.1.0/flowyml/stacks/local.py +112 -0
- flowyml-1.1.0/flowyml/stacks/migration.py +97 -0
- flowyml-1.1.0/flowyml/stacks/plugin_config.py +78 -0
- flowyml-1.1.0/flowyml/stacks/plugins.py +401 -0
- flowyml-1.1.0/flowyml/stacks/registry.py +226 -0
- flowyml-1.1.0/flowyml/storage/__init__.py +26 -0
- flowyml-1.1.0/flowyml/storage/artifacts.py +246 -0
- flowyml-1.1.0/flowyml/storage/materializers/__init__.py +20 -0
- flowyml-1.1.0/flowyml/storage/materializers/base.py +133 -0
- flowyml-1.1.0/flowyml/storage/materializers/keras.py +185 -0
- flowyml-1.1.0/flowyml/storage/materializers/numpy.py +94 -0
- flowyml-1.1.0/flowyml/storage/materializers/pandas.py +142 -0
- flowyml-1.1.0/flowyml/storage/materializers/pytorch.py +135 -0
- flowyml-1.1.0/flowyml/storage/materializers/sklearn.py +110 -0
- flowyml-1.1.0/flowyml/storage/materializers/tensorflow.py +152 -0
- flowyml-1.1.0/flowyml/storage/metadata.py +931 -0
- flowyml-1.1.0/flowyml/tracking/__init__.py +1 -0
- flowyml-1.1.0/flowyml/tracking/experiment.py +211 -0
- flowyml-1.1.0/flowyml/tracking/leaderboard.py +191 -0
- flowyml-1.1.0/flowyml/tracking/runs.py +145 -0
- flowyml-1.1.0/flowyml/ui/__init__.py +15 -0
- flowyml-1.1.0/flowyml/ui/backend/Dockerfile +31 -0
- flowyml-1.1.0/flowyml/ui/backend/__init__.py +0 -0
- flowyml-1.1.0/flowyml/ui/backend/auth.py +163 -0
- flowyml-1.1.0/flowyml/ui/backend/main.py +187 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/__init__.py +0 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/assets.py +45 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/execution.py +179 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/experiments.py +49 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/leaderboard.py +118 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/notifications.py +72 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/pipelines.py +110 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/plugins.py +192 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/projects.py +85 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/runs.py +66 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/schedules.py +222 -0
- flowyml-1.1.0/flowyml/ui/backend/routers/traces.py +84 -0
- flowyml-1.1.0/flowyml/ui/frontend/Dockerfile +20 -0
- flowyml-1.1.0/flowyml/ui/frontend/README.md +315 -0
- flowyml-1.1.0/flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +448 -0
- flowyml-1.1.0/flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +1 -0
- flowyml-1.1.0/flowyml/ui/frontend/dist/index.html +16 -0
- flowyml-1.1.0/flowyml/ui/frontend/index.html +15 -0
- flowyml-1.1.0/flowyml/ui/frontend/nginx.conf +26 -0
- flowyml-1.1.0/flowyml/ui/frontend/package-lock.json +3545 -0
- flowyml-1.1.0/flowyml/ui/frontend/package.json +33 -0
- flowyml-1.1.0/flowyml/ui/frontend/postcss.config.js +6 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/App.jsx +21 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/assets/page.jsx +397 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/dashboard/page.jsx +295 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/experiments/[experimentId]/page.jsx +255 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/experiments/page.jsx +360 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/leaderboard/page.jsx +133 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/pipelines/page.jsx +454 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/plugins/page.jsx +48 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/projects/page.jsx +292 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +682 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/runs/page.jsx +470 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/schedules/page.jsx +585 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/settings/page.jsx +314 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/tokens/page.jsx +456 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/app/traces/page.jsx +246 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/Layout.jsx +108 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/PipelineGraph.jsx +295 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/header/Header.jsx +72 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/AddPluginDialog.jsx +121 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/InstalledPlugins.jsx +124 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/PluginBrowser.jsx +167 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/PluginManager.jsx +60 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +145 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Badge.jsx +26 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Button.jsx +34 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Card.jsx +44 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/CodeSnippet.jsx +38 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/CollapsibleCard.jsx +53 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/DataView.jsx +175 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/EmptyState.jsx +49 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/ExecutionStatus.jsx +122 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/KeyValue.jsx +25 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/ProjectSelector.jsx +134 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/contexts/ProjectContext.jsx +79 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/contexts/ThemeContext.jsx +54 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/index.css +11 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/layouts/MainLayout.jsx +23 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/main.jsx +10 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/router/index.jsx +39 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/services/pluginService.js +90 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/utils/api.js +47 -0
- flowyml-1.1.0/flowyml/ui/frontend/src/utils/cn.js +6 -0
- flowyml-1.1.0/flowyml/ui/frontend/tailwind.config.js +31 -0
- flowyml-1.1.0/flowyml/ui/frontend/vite.config.js +21 -0
- flowyml-1.1.0/flowyml/ui/utils.py +77 -0
- flowyml-1.1.0/flowyml/utils/__init__.py +67 -0
- flowyml-1.1.0/flowyml/utils/config.py +308 -0
- flowyml-1.1.0/flowyml/utils/debug.py +240 -0
- flowyml-1.1.0/flowyml/utils/environment.py +346 -0
- flowyml-1.1.0/flowyml/utils/git.py +319 -0
- flowyml-1.1.0/flowyml/utils/logging.py +61 -0
- flowyml-1.1.0/flowyml/utils/performance.py +314 -0
- flowyml-1.1.0/flowyml/utils/stack_config.py +296 -0
- flowyml-1.1.0/flowyml/utils/validation.py +270 -0
- flowyml-1.1.0/pyproject.toml +135 -0
flowyml-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
Copyright 2024 Flowy Team
|
|
6
|
+
|
|
7
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
you may not use this file except in compliance with the License.
|
|
9
|
+
You may obtain a copy of the License at
|
|
10
|
+
|
|
11
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
|
|
13
|
+
Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
See the License for the specific language governing permissions and
|
|
17
|
+
limitations under the License.
|
flowyml-1.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flowyml
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Next-Generation ML Pipeline Framework
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Author: flowyml Team
|
|
8
|
+
Author-email: support@unicolab.ai
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Provides-Extra: all
|
|
22
|
+
Provides-Extra: gcp
|
|
23
|
+
Provides-Extra: pytorch
|
|
24
|
+
Provides-Extra: sklearn
|
|
25
|
+
Provides-Extra: tensorflow
|
|
26
|
+
Provides-Extra: ui
|
|
27
|
+
Requires-Dist: click (>=8.0.0)
|
|
28
|
+
Requires-Dist: croniter (>=2.0.1,<3.0.0)
|
|
29
|
+
Requires-Dist: fastapi (>=0.122.0,<0.123.0) ; extra == "ui"
|
|
30
|
+
Requires-Dist: google-cloud-aiplatform (>=1.35.0) ; extra == "gcp" or extra == "all"
|
|
31
|
+
Requires-Dist: google-cloud-storage (>=2.10.0) ; extra == "gcp" or extra == "all"
|
|
32
|
+
Requires-Dist: loguru (>=0.7.3,<0.8.0)
|
|
33
|
+
Requires-Dist: numpy (>=1.20.0)
|
|
34
|
+
Requires-Dist: pandas (>=1.3.0)
|
|
35
|
+
Requires-Dist: pydantic (>=2.0.0)
|
|
36
|
+
Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
|
|
37
|
+
Requires-Dist: pytz (>=2024.1,<2025.0)
|
|
38
|
+
Requires-Dist: pyyaml (>=6.0)
|
|
39
|
+
Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
|
|
40
|
+
Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
|
|
41
|
+
Requires-Dist: toml (>=0.10.2)
|
|
42
|
+
Requires-Dist: torch (>=2.0.0) ; extra == "pytorch" or extra == "all"
|
|
43
|
+
Requires-Dist: uvicorn[standard] (>=0.23.0) ; extra == "ui" or extra == "all"
|
|
44
|
+
Requires-Dist: websockets (>=11.0) ; extra == "ui" or extra == "all"
|
|
45
|
+
Project-URL: Documentation, https://unicolab.github.io/FlowyML/latest
|
|
46
|
+
Project-URL: Homepage, https://github.com/UnicoLab/FlowyML
|
|
47
|
+
Project-URL: Repository, https://github.com/UnicoLab/FlowyML
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# 🌊 flowyml
|
|
51
|
+
|
|
52
|
+
<p align="center">
|
|
53
|
+
<img src="docs/logo.png" width="350" alt="flowyml Logo"/>
|
|
54
|
+
<br>
|
|
55
|
+
<em>The Enterprise-Grade ML Pipeline Framework for Humans</em>
|
|
56
|
+
<br>
|
|
57
|
+
<br>
|
|
58
|
+
<p align="center">
|
|
59
|
+
<a href="https://github.com/UnicoLab/FlowyML/actions"><img src="https://img.shields.io/github/actions/workflow/status/UnicoLab/FlowyML/ci.yml?branch=main" alt="CI Status"></a>
|
|
60
|
+
<a href="https://pypi.org/project/flowyml/"><img src="https://img.shields.io/pypi/v/flowyml" alt="PyPI Version"></a>
|
|
61
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
|
|
62
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"></a>
|
|
63
|
+
<a href="https://unicolab.ai"><img src="https://img.shields.io/badge/UnicoLab-ai-red.svg" alt="UnicoLab"></a>
|
|
64
|
+
</p>
|
|
65
|
+
</p>
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
**flowyml** is the comprehensive ML pipeline framework that combines the **simplicity of a Python script** with the **power of an enterprise MLOps platform**.
|
|
70
|
+
|
|
71
|
+
## 🚀 Why flowyml?
|
|
72
|
+
|
|
73
|
+
| Feature | flowyml | Traditional Orchestrators |
|
|
74
|
+
|---------|---------|---------------------------|
|
|
75
|
+
| **Developer Experience** | 🐍 **Native Python** - No DSLs, no YAML hell. | 📜 Complex YAML or rigid DSLs. |
|
|
76
|
+
| **Context Awareness** | 🧠 **Auto-Injection** - Params are just function args. | 🔌 Manual wiring of every parameter. |
|
|
77
|
+
| **Caching** | ⚡ **Multi-Level** - Smart content-hashing & memoization. | 🐢 Basic file-timestamp checking. |
|
|
78
|
+
| **Asset Management** | 📦 **First-Class Assets** - Models & Datasets with lineage. | 📁 Generic file paths only. |
|
|
79
|
+
| **Architecture** | 🏗️ **Modular Stacks** - Local, Cloud, Hybrid. | 🔒 Vendor lock-in or complex setup. |
|
|
80
|
+
|
|
81
|
+
## 🚀 Feature Showcase
|
|
82
|
+
|
|
83
|
+
flowyml is a complete toolkit for building, debugging, and deploying ML applications.
|
|
84
|
+
|
|
85
|
+
### 1. Zero-Boilerplate Orchestration
|
|
86
|
+
Write pipelines as standard Python functions. No YAML, no DSLs.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
@step(outputs=["data"])
|
|
90
|
+
def load(): return [1, 2, 3]
|
|
91
|
+
|
|
92
|
+
@step(inputs=["data"], outputs=["model"])
|
|
93
|
+
def train(data): return Model.train(data)
|
|
94
|
+
|
|
95
|
+
# It's just Python!
|
|
96
|
+
pipeline = Pipeline("simple").add_step(load).add_step(train)
|
|
97
|
+
pipeline.run()
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### 2. 🧠 Intelligent Caching
|
|
101
|
+
Don't waste time re-running successful steps.
|
|
102
|
+
- **Code Hash**: Re-runs only when code changes.
|
|
103
|
+
- **Input Hash**: Re-runs only when data changes.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
# Only re-runs if 'data' changes, ignoring code changes
|
|
107
|
+
@step(cache="input_hash", outputs=["processed"])
|
|
108
|
+
def expensive_processing(data):
|
|
109
|
+
return process(data)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### 3. 🤖 LLM & GenAI Ready
|
|
113
|
+
Trace token usage, latency, and costs automatically.
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
@step
|
|
117
|
+
@trace_llm(model="gpt-4", tags=["production"])
|
|
118
|
+
def generate_summary(text: str):
|
|
119
|
+
# flowyml automatically tracks tokens, cost, and latency
|
|
120
|
+
return openai.ChatCompletion.create(...)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### 4. ⚡ Efficient Step Grouping
|
|
124
|
+
Group related steps to run in the same container - reduce overhead, maintain clarity.
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
# Run preprocessing steps in same container (shares resources)
|
|
128
|
+
@step(outputs=["raw"], execution_group="preprocessing")
|
|
129
|
+
def load(): return fetch_data()
|
|
130
|
+
|
|
131
|
+
@step(inputs=["raw"], outputs=["clean"], execution_group="preprocessing")
|
|
132
|
+
def clean(raw): return preprocess(raw)
|
|
133
|
+
|
|
134
|
+
# flowyml automatically aggregates resources (max CPU, memory, best GPU)
|
|
135
|
+
# and executes consecutively in same environment
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### 5. 🔀 Dynamic Workflows
|
|
139
|
+
Build adaptive workflows with conditional logic.
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
# Run 'deploy' only if model accuracy > 0.9
|
|
143
|
+
pipeline.add_step(
|
|
144
|
+
If(condition=lambda ctx: ctx["accuracy"] > 0.9)
|
|
145
|
+
.then(deploy_model)
|
|
146
|
+
.else_(notify_team)
|
|
147
|
+
)
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### 6. 🧩 Universal Plugin System
|
|
151
|
+
Extend flowyml with any tool - even wrap ZenML components!
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from flowyml.stacks.plugins import load_component
|
|
155
|
+
|
|
156
|
+
# Use any ZenML orchestrator, artifact store, or integration
|
|
157
|
+
k8s_orch = load_component("zenml:zenml.integrations.kubernetes.orchestrators.KubernetesOrchestrator")
|
|
158
|
+
mlflow = load_component("zenml:zenml.integrations.mlflow.MLflowExperimentTracker")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### 7. 👤 Human-in-the-Loop
|
|
162
|
+
Pause pipelines for manual approval or review.
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
from flowyml import approval
|
|
166
|
+
|
|
167
|
+
approval_step = approval(
|
|
168
|
+
name="approve_deployment",
|
|
169
|
+
approver="ml-team",
|
|
170
|
+
timeout_seconds=3600
|
|
171
|
+
)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### 8. 📊 Built-in Experiment Tracking
|
|
175
|
+
No external tools needed - tracking is built-in.
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from flowyml.tracking import Experiment
|
|
179
|
+
|
|
180
|
+
exp = Experiment("baseline_training")
|
|
181
|
+
exp.log_run(run_id="run_001", metrics={"accuracy": 0.95}, parameters={"lr": 0.01})
|
|
182
|
+
best = exp.get_best_run("accuracy", maximize=True)
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
### 9. 🏆 Model Leaderboard & Registry
|
|
186
|
+
Track, compare, and version your models.
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from flowyml import ModelLeaderboard
|
|
190
|
+
from flowyml.core import Model
|
|
191
|
+
|
|
192
|
+
# Track performance
|
|
193
|
+
leaderboard = ModelLeaderboard(metric="accuracy")
|
|
194
|
+
leaderboard.add_score(model_name="bert-base", score=0.92)
|
|
195
|
+
|
|
196
|
+
# Register models with stages
|
|
197
|
+
model = Model.create(artifact={...})
|
|
198
|
+
model.register(name="text_classifier", stage="production")
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### 10. 📅 Built-in Scheduling
|
|
202
|
+
Schedule recurring jobs without external orchestrators.
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from flowyml import PipelineScheduler
|
|
206
|
+
|
|
207
|
+
scheduler = PipelineScheduler()
|
|
208
|
+
scheduler.schedule_daily(
|
|
209
|
+
name="daily_training",
|
|
210
|
+
pipeline_func=lambda: pipeline.run(),
|
|
211
|
+
hour=2, minute=0
|
|
212
|
+
)
|
|
213
|
+
scheduler.start()
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
### 11. 🔔 Smart Notifications
|
|
217
|
+
Slack, Email, and custom alerts built-in.
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
from flowyml import configure_notifications
|
|
221
|
+
|
|
222
|
+
configure_notifications(
|
|
223
|
+
slack_webhook="https://hooks.slack.com/...",
|
|
224
|
+
email_config={...}
|
|
225
|
+
)
|
|
226
|
+
# Automatic notifications on pipeline success/failure
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### 12. 🎯 Step-Level Debugging
|
|
230
|
+
Set breakpoints and inspect state mid-pipeline.
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
from flowyml import StepDebugger
|
|
234
|
+
|
|
235
|
+
debugger = StepDebugger()
|
|
236
|
+
debugger.set_breakpoint("train_model")
|
|
237
|
+
pipeline.run(debug=True) # Pauses at breakpoint
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### 13. 📦 First-Class Assets
|
|
241
|
+
Specialized types for ML workflows.
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
from flowyml.core import Dataset, Model, Metrics, FeatureSet
|
|
245
|
+
|
|
246
|
+
dataset = Dataset.create(data=df, name="training_data")
|
|
247
|
+
model = Model.create(artifact=trained_model, score=0.95)
|
|
248
|
+
metrics = Metrics.create(values={"accuracy": 0.95})
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
### 14. 🔄 Smart Retries & Circuit Breakers
|
|
252
|
+
Handle failures gracefully.
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
@step(retry=3, timeout=300)
|
|
256
|
+
def flaky_api_call():
|
|
257
|
+
return external_api.fetch()
|
|
258
|
+
|
|
259
|
+
# Circuit breakers prevent cascading failures
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
### 15. 📈 Data Drift Detection
|
|
263
|
+
Monitor distribution shifts automatically.
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
from flowyml import detect_drift
|
|
267
|
+
|
|
268
|
+
drift = detect_drift(
|
|
269
|
+
reference_data=train_feature,
|
|
270
|
+
current_data=prod_feature,
|
|
271
|
+
threshold=0.1
|
|
272
|
+
)
|
|
273
|
+
if drift['drift_detected']:
|
|
274
|
+
trigger_retraining()
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
### 16. 🔍 Interactive Debugging
|
|
278
|
+
- **StepDebugger**: Set breakpoints in your pipeline.
|
|
279
|
+
- **Artifact Inspection**: View intermediate data in the UI.
|
|
280
|
+
- **Local Execution**: Run the exact same code locally as in production.
|
|
281
|
+
|
|
282
|
+
### 17. 🏭 Enterprise Production Features
|
|
283
|
+
- **🔄 Automatic Retries**: Handle transient failures.
|
|
284
|
+
- **⏰ Scheduling**: Built-in cron scheduler.
|
|
285
|
+
- **🔔 Notifications**: Slack/Email alerts.
|
|
286
|
+
- **🛡️ Circuit Breakers**: Stop cascading failures.
|
|
287
|
+
|
|
288
|
+
### 18. 🔌 Universal Integrations
|
|
289
|
+
- **ML Frameworks**: PyTorch, TensorFlow, Keras, Scikit-learn, HuggingFace.
|
|
290
|
+
- **Cloud Providers**: AWS, GCP, Azure (via plugins).
|
|
291
|
+
- **Tools**: MLflow, Weights & Biases, Great Expectations.
|
|
292
|
+
|
|
293
|
+
## 📦 Installation
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
# Install core
|
|
297
|
+
pip install flowyml
|
|
298
|
+
|
|
299
|
+
# Install with UI support
|
|
300
|
+
pip install "flowyml[ui]"
|
|
301
|
+
|
|
302
|
+
# Install with all features (recommended for dev)
|
|
303
|
+
pip install "flowyml[all]"
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
## ⚡ Quick Start
|
|
307
|
+
|
|
308
|
+
```python
|
|
309
|
+
from flowyml import Pipeline, step, context, Dataset, Model
|
|
310
|
+
|
|
311
|
+
# 1. Define your configuration (Auto-injected!)
|
|
312
|
+
ctx = context(
|
|
313
|
+
learning_rate=0.01,
|
|
314
|
+
batch_size=32,
|
|
315
|
+
epochs=10
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# 2. Define your steps (Pure Python)
|
|
319
|
+
@step(outputs=["dataset"])
|
|
320
|
+
def load_data(batch_size: int):
|
|
321
|
+
# 'batch_size' is automatically injected from context!
|
|
322
|
+
print(f"Loading data with batch size: {batch_size}")
|
|
323
|
+
return Dataset.create(data=[1, 2, 3], name="mnist")
|
|
324
|
+
|
|
325
|
+
@step(inputs=["dataset"], outputs=["model"])
|
|
326
|
+
def train(dataset: Dataset, learning_rate: float, epochs: int):
|
|
327
|
+
print(f"Training on {dataset.name} with lr={learning_rate}")
|
|
328
|
+
# Simulate training...
|
|
329
|
+
return Model.create(artifact={"weights": "..."}, score=0.98)
|
|
330
|
+
|
|
331
|
+
# 3. Run it!
|
|
332
|
+
pipeline = Pipeline("mnist_training", context=ctx)
|
|
333
|
+
pipeline.add_step(load_data)
|
|
334
|
+
pipeline.add_step(train)
|
|
335
|
+
|
|
336
|
+
result = pipeline.run()
|
|
337
|
+
|
|
338
|
+
print(f"Run ID: {result.run_id}")
|
|
339
|
+
print(f"Model Score: {result.outputs['model'].score}")
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
## 🖥️ The flowyml UI
|
|
343
|
+
|
|
344
|
+
Visualize your workflows, inspect artifacts, and monitor runs in real-time.
|
|
345
|
+
|
|
346
|
+
```bash
|
|
347
|
+
# Start the UI server
|
|
348
|
+
flowyml ui start --open-browser
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
Visit **http://localhost:8080** to access the dashboard.
|
|
352
|
+
|
|
353
|
+
## 📚 Documentation
|
|
354
|
+
|
|
355
|
+
- **[Getting Started](docs/getting-started.md)**: Your first 5 minutes with flowyml.
|
|
356
|
+
- **[Core Concepts](docs/core/pipelines.md)**: Deep dive into Pipelines, Steps, and Context.
|
|
357
|
+
- **[Advanced Features](docs/advanced/caching.md)**: Learn about Caching, Parallelism, and Conditional Execution.
|
|
358
|
+
- **[API Reference](docs/api/core.md)**: Detailed class and function documentation.
|
|
359
|
+
|
|
360
|
+
## 🤝 Contributing
|
|
361
|
+
|
|
362
|
+
We love contributions! Check out our [Contributing Guide](CONTRIBUTING.md) to get started.
|
|
363
|
+
|
|
364
|
+
## 📝 License
|
|
365
|
+
|
|
366
|
+
Apache 2.0 - See [LICENSE](LICENSE) for details.
|
|
367
|
+
|
|
368
|
+
---
|
|
369
|
+
<p align="center">
|
|
370
|
+
<strong>Built with ❤️ by <a href="https://unicolab.ai">UnicoLab</a></strong>
|
|
371
|
+
</p>
|
|
372
|
+
|
flowyml-1.1.0/README.md
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
# 🌊 flowyml
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="docs/logo.png" width="350" alt="flowyml Logo"/>
|
|
5
|
+
<br>
|
|
6
|
+
<em>The Enterprise-Grade ML Pipeline Framework for Humans</em>
|
|
7
|
+
<br>
|
|
8
|
+
<br>
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://github.com/UnicoLab/FlowyML/actions"><img src="https://img.shields.io/github/actions/workflow/status/UnicoLab/FlowyML/ci.yml?branch=main" alt="CI Status"></a>
|
|
11
|
+
<a href="https://pypi.org/project/flowyml/"><img src="https://img.shields.io/pypi/v/flowyml" alt="PyPI Version"></a>
|
|
12
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
|
|
13
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"></a>
|
|
14
|
+
<a href="https://unicolab.ai"><img src="https://img.shields.io/badge/UnicoLab-ai-red.svg" alt="UnicoLab"></a>
|
|
15
|
+
</p>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
**flowyml** is the comprehensive ML pipeline framework that combines the **simplicity of a Python script** with the **power of an enterprise MLOps platform**.
|
|
21
|
+
|
|
22
|
+
## 🚀 Why flowyml?
|
|
23
|
+
|
|
24
|
+
| Feature | flowyml | Traditional Orchestrators |
|
|
25
|
+
|---------|---------|---------------------------|
|
|
26
|
+
| **Developer Experience** | 🐍 **Native Python** - No DSLs, no YAML hell. | 📜 Complex YAML or rigid DSLs. |
|
|
27
|
+
| **Context Awareness** | 🧠 **Auto-Injection** - Params are just function args. | 🔌 Manual wiring of every parameter. |
|
|
28
|
+
| **Caching** | ⚡ **Multi-Level** - Smart content-hashing & memoization. | 🐢 Basic file-timestamp checking. |
|
|
29
|
+
| **Asset Management** | 📦 **First-Class Assets** - Models & Datasets with lineage. | 📁 Generic file paths only. |
|
|
30
|
+
| **Architecture** | 🏗️ **Modular Stacks** - Local, Cloud, Hybrid. | 🔒 Vendor lock-in or complex setup. |
|
|
31
|
+
|
|
32
|
+
## 🚀 Feature Showcase
|
|
33
|
+
|
|
34
|
+
flowyml is a complete toolkit for building, debugging, and deploying ML applications.
|
|
35
|
+
|
|
36
|
+
### 1. Zero-Boilerplate Orchestration
|
|
37
|
+
Write pipelines as standard Python functions. No YAML, no DSLs.
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
@step(outputs=["data"])
|
|
41
|
+
def load(): return [1, 2, 3]
|
|
42
|
+
|
|
43
|
+
@step(inputs=["data"], outputs=["model"])
|
|
44
|
+
def train(data): return Model.train(data)
|
|
45
|
+
|
|
46
|
+
# It's just Python!
|
|
47
|
+
pipeline = Pipeline("simple").add_step(load).add_step(train)
|
|
48
|
+
pipeline.run()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### 2. 🧠 Intelligent Caching
|
|
52
|
+
Don't waste time re-running successful steps.
|
|
53
|
+
- **Code Hash**: Re-runs only when code changes.
|
|
54
|
+
- **Input Hash**: Re-runs only when data changes.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
# Only re-runs if 'data' changes, ignoring code changes
|
|
58
|
+
@step(cache="input_hash", outputs=["processed"])
|
|
59
|
+
def expensive_processing(data):
|
|
60
|
+
return process(data)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### 3. 🤖 LLM & GenAI Ready
|
|
64
|
+
Trace token usage, latency, and costs automatically.
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
@step
|
|
68
|
+
@trace_llm(model="gpt-4", tags=["production"])
|
|
69
|
+
def generate_summary(text: str):
|
|
70
|
+
# flowyml automatically tracks tokens, cost, and latency
|
|
71
|
+
return openai.ChatCompletion.create(...)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### 4. ⚡ Efficient Step Grouping
|
|
75
|
+
Group related steps to run in the same container - reduce overhead, maintain clarity.
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
# Run preprocessing steps in same container (shares resources)
|
|
79
|
+
@step(outputs=["raw"], execution_group="preprocessing")
|
|
80
|
+
def load(): return fetch_data()
|
|
81
|
+
|
|
82
|
+
@step(inputs=["raw"], outputs=["clean"], execution_group="preprocessing")
|
|
83
|
+
def clean(raw): return preprocess(raw)
|
|
84
|
+
|
|
85
|
+
# flowyml automatically aggregates resources (max CPU, memory, best GPU)
|
|
86
|
+
# and executes consecutively in same environment
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 5. 🔀 Dynamic Workflows
|
|
90
|
+
Build adaptive workflows with conditional logic.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
# Run 'deploy' only if model accuracy > 0.9
|
|
94
|
+
pipeline.add_step(
|
|
95
|
+
If(condition=lambda ctx: ctx["accuracy"] > 0.9)
|
|
96
|
+
.then(deploy_model)
|
|
97
|
+
.else_(notify_team)
|
|
98
|
+
)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### 6. 🧩 Universal Plugin System
|
|
102
|
+
Extend flowyml with any tool - even wrap ZenML components!
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from flowyml.stacks.plugins import load_component
|
|
106
|
+
|
|
107
|
+
# Use any ZenML orchestrator, artifact store, or integration
|
|
108
|
+
k8s_orch = load_component("zenml:zenml.integrations.kubernetes.orchestrators.KubernetesOrchestrator")
|
|
109
|
+
mlflow = load_component("zenml:zenml.integrations.mlflow.MLflowExperimentTracker")
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### 7. 👤 Human-in-the-Loop
|
|
113
|
+
Pause pipelines for manual approval or review.
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from flowyml import approval
|
|
117
|
+
|
|
118
|
+
approval_step = approval(
|
|
119
|
+
name="approve_deployment",
|
|
120
|
+
approver="ml-team",
|
|
121
|
+
timeout_seconds=3600
|
|
122
|
+
)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### 8. 📊 Built-in Experiment Tracking
|
|
126
|
+
No external tools needed - tracking is built-in.
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from flowyml.tracking import Experiment
|
|
130
|
+
|
|
131
|
+
exp = Experiment("baseline_training")
|
|
132
|
+
exp.log_run(run_id="run_001", metrics={"accuracy": 0.95}, parameters={"lr": 0.01})
|
|
133
|
+
best = exp.get_best_run("accuracy", maximize=True)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### 9. 🏆 Model Leaderboard & Registry
|
|
137
|
+
Track, compare, and version your models.
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from flowyml import ModelLeaderboard
|
|
141
|
+
from flowyml.core import Model
|
|
142
|
+
|
|
143
|
+
# Track performance
|
|
144
|
+
leaderboard = ModelLeaderboard(metric="accuracy")
|
|
145
|
+
leaderboard.add_score(model_name="bert-base", score=0.92)
|
|
146
|
+
|
|
147
|
+
# Register models with stages
|
|
148
|
+
model = Model.create(artifact={...})
|
|
149
|
+
model.register(name="text_classifier", stage="production")
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### 10. 📅 Built-in Scheduling
|
|
153
|
+
Schedule recurring jobs without external orchestrators.
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from flowyml import PipelineScheduler
|
|
157
|
+
|
|
158
|
+
scheduler = PipelineScheduler()
|
|
159
|
+
scheduler.schedule_daily(
|
|
160
|
+
name="daily_training",
|
|
161
|
+
pipeline_func=lambda: pipeline.run(),
|
|
162
|
+
hour=2, minute=0
|
|
163
|
+
)
|
|
164
|
+
scheduler.start()
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### 11. 🔔 Smart Notifications
|
|
168
|
+
Slack, Email, and custom alerts built-in.
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
from flowyml import configure_notifications
|
|
172
|
+
|
|
173
|
+
configure_notifications(
|
|
174
|
+
slack_webhook="https://hooks.slack.com/...",
|
|
175
|
+
email_config={...}
|
|
176
|
+
)
|
|
177
|
+
# Automatic notifications on pipeline success/failure
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### 12. 🎯 Step-Level Debugging
|
|
181
|
+
Set breakpoints and inspect state mid-pipeline.
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from flowyml import StepDebugger
|
|
185
|
+
|
|
186
|
+
debugger = StepDebugger()
|
|
187
|
+
debugger.set_breakpoint("train_model")
|
|
188
|
+
pipeline.run(debug=True) # Pauses at breakpoint
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### 13. 📦 First-Class Assets
|
|
192
|
+
Specialized types for ML workflows.
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from flowyml.core import Dataset, Model, Metrics, FeatureSet
|
|
196
|
+
|
|
197
|
+
dataset = Dataset.create(data=df, name="training_data")
|
|
198
|
+
model = Model.create(artifact=trained_model, score=0.95)
|
|
199
|
+
metrics = Metrics.create(values={"accuracy": 0.95})
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### 14. 🔄 Smart Retries & Circuit Breakers
|
|
203
|
+
Handle failures gracefully.
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
@step(retry=3, timeout=300)
|
|
207
|
+
def flaky_api_call():
|
|
208
|
+
return external_api.fetch()
|
|
209
|
+
|
|
210
|
+
# Circuit breakers prevent cascading failures
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### 15. 📈 Data Drift Detection
|
|
214
|
+
Monitor distribution shifts automatically.
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from flowyml import detect_drift
|
|
218
|
+
|
|
219
|
+
drift = detect_drift(
|
|
220
|
+
reference_data=train_feature,
|
|
221
|
+
current_data=prod_feature,
|
|
222
|
+
threshold=0.1
|
|
223
|
+
)
|
|
224
|
+
if drift['drift_detected']:
|
|
225
|
+
trigger_retraining()
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
### 16. 🔍 Interactive Debugging
|
|
229
|
+
- **StepDebugger**: Set breakpoints in your pipeline.
|
|
230
|
+
- **Artifact Inspection**: View intermediate data in the UI.
|
|
231
|
+
- **Local Execution**: Run the exact same code locally as in production.
|
|
232
|
+
|
|
233
|
+
### 17. 🏭 Enterprise Production Features
|
|
234
|
+
- **🔄 Automatic Retries**: Handle transient failures.
|
|
235
|
+
- **⏰ Scheduling**: Built-in cron scheduler.
|
|
236
|
+
- **🔔 Notifications**: Slack/Email alerts.
|
|
237
|
+
- **🛡️ Circuit Breakers**: Stop cascading failures.
|
|
238
|
+
|
|
239
|
+
### 18. 🔌 Universal Integrations
|
|
240
|
+
- **ML Frameworks**: PyTorch, TensorFlow, Keras, Scikit-learn, HuggingFace.
|
|
241
|
+
- **Cloud Providers**: AWS, GCP, Azure (via plugins).
|
|
242
|
+
- **Tools**: MLflow, Weights & Biases, Great Expectations.
|
|
243
|
+
|
|
244
|
+
## 📦 Installation
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
# Install core
|
|
248
|
+
pip install flowyml
|
|
249
|
+
|
|
250
|
+
# Install with UI support
|
|
251
|
+
pip install "flowyml[ui]"
|
|
252
|
+
|
|
253
|
+
# Install with all features (recommended for dev)
|
|
254
|
+
pip install "flowyml[all]"
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## ⚡ Quick Start
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from flowyml import Pipeline, step, context, Dataset, Model
|
|
261
|
+
|
|
262
|
+
# 1. Define your configuration (Auto-injected!)
|
|
263
|
+
ctx = context(
|
|
264
|
+
learning_rate=0.01,
|
|
265
|
+
batch_size=32,
|
|
266
|
+
epochs=10
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
# 2. Define your steps (Pure Python)
|
|
270
|
+
@step(outputs=["dataset"])
|
|
271
|
+
def load_data(batch_size: int):
|
|
272
|
+
# 'batch_size' is automatically injected from context!
|
|
273
|
+
print(f"Loading data with batch size: {batch_size}")
|
|
274
|
+
return Dataset.create(data=[1, 2, 3], name="mnist")
|
|
275
|
+
|
|
276
|
+
@step(inputs=["dataset"], outputs=["model"])
|
|
277
|
+
def train(dataset: Dataset, learning_rate: float, epochs: int):
|
|
278
|
+
print(f"Training on {dataset.name} with lr={learning_rate}")
|
|
279
|
+
# Simulate training...
|
|
280
|
+
return Model.create(artifact={"weights": "..."}, score=0.98)
|
|
281
|
+
|
|
282
|
+
# 3. Run it!
|
|
283
|
+
pipeline = Pipeline("mnist_training", context=ctx)
|
|
284
|
+
pipeline.add_step(load_data)
|
|
285
|
+
pipeline.add_step(train)
|
|
286
|
+
|
|
287
|
+
result = pipeline.run()
|
|
288
|
+
|
|
289
|
+
print(f"Run ID: {result.run_id}")
|
|
290
|
+
print(f"Model Score: {result.outputs['model'].score}")
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
## 🖥️ The flowyml UI
|
|
294
|
+
|
|
295
|
+
Visualize your workflows, inspect artifacts, and monitor runs in real-time.
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
# Start the UI server
|
|
299
|
+
flowyml ui start --open-browser
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Visit **http://localhost:8080** to access the dashboard.
|
|
303
|
+
|
|
304
|
+
## 📚 Documentation
|
|
305
|
+
|
|
306
|
+
- **[Getting Started](docs/getting-started.md)**: Your first 5 minutes with flowyml.
|
|
307
|
+
- **[Core Concepts](docs/core/pipelines.md)**: Deep dive into Pipelines, Steps, and Context.
|
|
308
|
+
- **[Advanced Features](docs/advanced/caching.md)**: Learn about Caching, Parallelism, and Conditional Execution.
|
|
309
|
+
- **[API Reference](docs/api/core.md)**: Detailed class and function documentation.
|
|
310
|
+
|
|
311
|
+
## 🤝 Contributing
|
|
312
|
+
|
|
313
|
+
We love contributions! Check out our [Contributing Guide](CONTRIBUTING.md) to get started.
|
|
314
|
+
|
|
315
|
+
## 📝 License
|
|
316
|
+
|
|
317
|
+
Apache 2.0 - See [LICENSE](LICENSE) for details.
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
<p align="center">
|
|
321
|
+
<strong>Built with ❤️ by <a href="https://unicolab.ai">UnicoLab</a></strong>
|
|
322
|
+
</p>
|