flowyml 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. flowyml-1.1.0/LICENSE +17 -0
  2. flowyml-1.1.0/PKG-INFO +372 -0
  3. flowyml-1.1.0/README.md +322 -0
  4. flowyml-1.1.0/flowyml/__init__.py +207 -0
  5. flowyml-1.1.0/flowyml/assets/__init__.py +22 -0
  6. flowyml-1.1.0/flowyml/assets/artifact.py +40 -0
  7. flowyml-1.1.0/flowyml/assets/base.py +209 -0
  8. flowyml-1.1.0/flowyml/assets/dataset.py +100 -0
  9. flowyml-1.1.0/flowyml/assets/featureset.py +301 -0
  10. flowyml-1.1.0/flowyml/assets/metrics.py +104 -0
  11. flowyml-1.1.0/flowyml/assets/model.py +82 -0
  12. flowyml-1.1.0/flowyml/assets/registry.py +157 -0
  13. flowyml-1.1.0/flowyml/assets/report.py +315 -0
  14. flowyml-1.1.0/flowyml/cli/__init__.py +5 -0
  15. flowyml-1.1.0/flowyml/cli/experiment.py +232 -0
  16. flowyml-1.1.0/flowyml/cli/init.py +256 -0
  17. flowyml-1.1.0/flowyml/cli/main.py +327 -0
  18. flowyml-1.1.0/flowyml/cli/run.py +75 -0
  19. flowyml-1.1.0/flowyml/cli/stack_cli.py +532 -0
  20. flowyml-1.1.0/flowyml/cli/ui.py +33 -0
  21. flowyml-1.1.0/flowyml/core/__init__.py +68 -0
  22. flowyml-1.1.0/flowyml/core/advanced_cache.py +274 -0
  23. flowyml-1.1.0/flowyml/core/approval.py +64 -0
  24. flowyml-1.1.0/flowyml/core/cache.py +203 -0
  25. flowyml-1.1.0/flowyml/core/checkpoint.py +148 -0
  26. flowyml-1.1.0/flowyml/core/conditional.py +373 -0
  27. flowyml-1.1.0/flowyml/core/context.py +155 -0
  28. flowyml-1.1.0/flowyml/core/error_handling.py +419 -0
  29. flowyml-1.1.0/flowyml/core/executor.py +354 -0
  30. flowyml-1.1.0/flowyml/core/graph.py +185 -0
  31. flowyml-1.1.0/flowyml/core/parallel.py +452 -0
  32. flowyml-1.1.0/flowyml/core/pipeline.py +764 -0
  33. flowyml-1.1.0/flowyml/core/project.py +253 -0
  34. flowyml-1.1.0/flowyml/core/resources.py +424 -0
  35. flowyml-1.1.0/flowyml/core/scheduler.py +630 -0
  36. flowyml-1.1.0/flowyml/core/scheduler_config.py +32 -0
  37. flowyml-1.1.0/flowyml/core/step.py +201 -0
  38. flowyml-1.1.0/flowyml/core/step_grouping.py +292 -0
  39. flowyml-1.1.0/flowyml/core/templates.py +226 -0
  40. flowyml-1.1.0/flowyml/core/versioning.py +217 -0
  41. flowyml-1.1.0/flowyml/integrations/__init__.py +1 -0
  42. flowyml-1.1.0/flowyml/integrations/keras.py +134 -0
  43. flowyml-1.1.0/flowyml/monitoring/__init__.py +1 -0
  44. flowyml-1.1.0/flowyml/monitoring/alerts.py +57 -0
  45. flowyml-1.1.0/flowyml/monitoring/data.py +102 -0
  46. flowyml-1.1.0/flowyml/monitoring/llm.py +160 -0
  47. flowyml-1.1.0/flowyml/monitoring/monitor.py +57 -0
  48. flowyml-1.1.0/flowyml/monitoring/notifications.py +246 -0
  49. flowyml-1.1.0/flowyml/registry/__init__.py +5 -0
  50. flowyml-1.1.0/flowyml/registry/model_registry.py +491 -0
  51. flowyml-1.1.0/flowyml/registry/pipeline_registry.py +55 -0
  52. flowyml-1.1.0/flowyml/stacks/__init__.py +27 -0
  53. flowyml-1.1.0/flowyml/stacks/base.py +77 -0
  54. flowyml-1.1.0/flowyml/stacks/bridge.py +288 -0
  55. flowyml-1.1.0/flowyml/stacks/components.py +155 -0
  56. flowyml-1.1.0/flowyml/stacks/gcp.py +499 -0
  57. flowyml-1.1.0/flowyml/stacks/local.py +112 -0
  58. flowyml-1.1.0/flowyml/stacks/migration.py +97 -0
  59. flowyml-1.1.0/flowyml/stacks/plugin_config.py +78 -0
  60. flowyml-1.1.0/flowyml/stacks/plugins.py +401 -0
  61. flowyml-1.1.0/flowyml/stacks/registry.py +226 -0
  62. flowyml-1.1.0/flowyml/storage/__init__.py +26 -0
  63. flowyml-1.1.0/flowyml/storage/artifacts.py +246 -0
  64. flowyml-1.1.0/flowyml/storage/materializers/__init__.py +20 -0
  65. flowyml-1.1.0/flowyml/storage/materializers/base.py +133 -0
  66. flowyml-1.1.0/flowyml/storage/materializers/keras.py +185 -0
  67. flowyml-1.1.0/flowyml/storage/materializers/numpy.py +94 -0
  68. flowyml-1.1.0/flowyml/storage/materializers/pandas.py +142 -0
  69. flowyml-1.1.0/flowyml/storage/materializers/pytorch.py +135 -0
  70. flowyml-1.1.0/flowyml/storage/materializers/sklearn.py +110 -0
  71. flowyml-1.1.0/flowyml/storage/materializers/tensorflow.py +152 -0
  72. flowyml-1.1.0/flowyml/storage/metadata.py +931 -0
  73. flowyml-1.1.0/flowyml/tracking/__init__.py +1 -0
  74. flowyml-1.1.0/flowyml/tracking/experiment.py +211 -0
  75. flowyml-1.1.0/flowyml/tracking/leaderboard.py +191 -0
  76. flowyml-1.1.0/flowyml/tracking/runs.py +145 -0
  77. flowyml-1.1.0/flowyml/ui/__init__.py +15 -0
  78. flowyml-1.1.0/flowyml/ui/backend/Dockerfile +31 -0
  79. flowyml-1.1.0/flowyml/ui/backend/__init__.py +0 -0
  80. flowyml-1.1.0/flowyml/ui/backend/auth.py +163 -0
  81. flowyml-1.1.0/flowyml/ui/backend/main.py +187 -0
  82. flowyml-1.1.0/flowyml/ui/backend/routers/__init__.py +0 -0
  83. flowyml-1.1.0/flowyml/ui/backend/routers/assets.py +45 -0
  84. flowyml-1.1.0/flowyml/ui/backend/routers/execution.py +179 -0
  85. flowyml-1.1.0/flowyml/ui/backend/routers/experiments.py +49 -0
  86. flowyml-1.1.0/flowyml/ui/backend/routers/leaderboard.py +118 -0
  87. flowyml-1.1.0/flowyml/ui/backend/routers/notifications.py +72 -0
  88. flowyml-1.1.0/flowyml/ui/backend/routers/pipelines.py +110 -0
  89. flowyml-1.1.0/flowyml/ui/backend/routers/plugins.py +192 -0
  90. flowyml-1.1.0/flowyml/ui/backend/routers/projects.py +85 -0
  91. flowyml-1.1.0/flowyml/ui/backend/routers/runs.py +66 -0
  92. flowyml-1.1.0/flowyml/ui/backend/routers/schedules.py +222 -0
  93. flowyml-1.1.0/flowyml/ui/backend/routers/traces.py +84 -0
  94. flowyml-1.1.0/flowyml/ui/frontend/Dockerfile +20 -0
  95. flowyml-1.1.0/flowyml/ui/frontend/README.md +315 -0
  96. flowyml-1.1.0/flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +448 -0
  97. flowyml-1.1.0/flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +1 -0
  98. flowyml-1.1.0/flowyml/ui/frontend/dist/index.html +16 -0
  99. flowyml-1.1.0/flowyml/ui/frontend/index.html +15 -0
  100. flowyml-1.1.0/flowyml/ui/frontend/nginx.conf +26 -0
  101. flowyml-1.1.0/flowyml/ui/frontend/package-lock.json +3545 -0
  102. flowyml-1.1.0/flowyml/ui/frontend/package.json +33 -0
  103. flowyml-1.1.0/flowyml/ui/frontend/postcss.config.js +6 -0
  104. flowyml-1.1.0/flowyml/ui/frontend/src/App.jsx +21 -0
  105. flowyml-1.1.0/flowyml/ui/frontend/src/app/assets/page.jsx +397 -0
  106. flowyml-1.1.0/flowyml/ui/frontend/src/app/dashboard/page.jsx +295 -0
  107. flowyml-1.1.0/flowyml/ui/frontend/src/app/experiments/[experimentId]/page.jsx +255 -0
  108. flowyml-1.1.0/flowyml/ui/frontend/src/app/experiments/page.jsx +360 -0
  109. flowyml-1.1.0/flowyml/ui/frontend/src/app/leaderboard/page.jsx +133 -0
  110. flowyml-1.1.0/flowyml/ui/frontend/src/app/pipelines/page.jsx +454 -0
  111. flowyml-1.1.0/flowyml/ui/frontend/src/app/plugins/page.jsx +48 -0
  112. flowyml-1.1.0/flowyml/ui/frontend/src/app/projects/page.jsx +292 -0
  113. flowyml-1.1.0/flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +682 -0
  114. flowyml-1.1.0/flowyml/ui/frontend/src/app/runs/page.jsx +470 -0
  115. flowyml-1.1.0/flowyml/ui/frontend/src/app/schedules/page.jsx +585 -0
  116. flowyml-1.1.0/flowyml/ui/frontend/src/app/settings/page.jsx +314 -0
  117. flowyml-1.1.0/flowyml/ui/frontend/src/app/tokens/page.jsx +456 -0
  118. flowyml-1.1.0/flowyml/ui/frontend/src/app/traces/page.jsx +246 -0
  119. flowyml-1.1.0/flowyml/ui/frontend/src/components/Layout.jsx +108 -0
  120. flowyml-1.1.0/flowyml/ui/frontend/src/components/PipelineGraph.jsx +295 -0
  121. flowyml-1.1.0/flowyml/ui/frontend/src/components/header/Header.jsx +72 -0
  122. flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/AddPluginDialog.jsx +121 -0
  123. flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/InstalledPlugins.jsx +124 -0
  124. flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/PluginBrowser.jsx +167 -0
  125. flowyml-1.1.0/flowyml/ui/frontend/src/components/plugins/PluginManager.jsx +60 -0
  126. flowyml-1.1.0/flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +145 -0
  127. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Badge.jsx +26 -0
  128. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Button.jsx +34 -0
  129. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/Card.jsx +44 -0
  130. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/CodeSnippet.jsx +38 -0
  131. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/CollapsibleCard.jsx +53 -0
  132. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/DataView.jsx +175 -0
  133. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/EmptyState.jsx +49 -0
  134. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/ExecutionStatus.jsx +122 -0
  135. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/KeyValue.jsx +25 -0
  136. flowyml-1.1.0/flowyml/ui/frontend/src/components/ui/ProjectSelector.jsx +134 -0
  137. flowyml-1.1.0/flowyml/ui/frontend/src/contexts/ProjectContext.jsx +79 -0
  138. flowyml-1.1.0/flowyml/ui/frontend/src/contexts/ThemeContext.jsx +54 -0
  139. flowyml-1.1.0/flowyml/ui/frontend/src/index.css +11 -0
  140. flowyml-1.1.0/flowyml/ui/frontend/src/layouts/MainLayout.jsx +23 -0
  141. flowyml-1.1.0/flowyml/ui/frontend/src/main.jsx +10 -0
  142. flowyml-1.1.0/flowyml/ui/frontend/src/router/index.jsx +39 -0
  143. flowyml-1.1.0/flowyml/ui/frontend/src/services/pluginService.js +90 -0
  144. flowyml-1.1.0/flowyml/ui/frontend/src/utils/api.js +47 -0
  145. flowyml-1.1.0/flowyml/ui/frontend/src/utils/cn.js +6 -0
  146. flowyml-1.1.0/flowyml/ui/frontend/tailwind.config.js +31 -0
  147. flowyml-1.1.0/flowyml/ui/frontend/vite.config.js +21 -0
  148. flowyml-1.1.0/flowyml/ui/utils.py +77 -0
  149. flowyml-1.1.0/flowyml/utils/__init__.py +67 -0
  150. flowyml-1.1.0/flowyml/utils/config.py +308 -0
  151. flowyml-1.1.0/flowyml/utils/debug.py +240 -0
  152. flowyml-1.1.0/flowyml/utils/environment.py +346 -0
  153. flowyml-1.1.0/flowyml/utils/git.py +319 -0
  154. flowyml-1.1.0/flowyml/utils/logging.py +61 -0
  155. flowyml-1.1.0/flowyml/utils/performance.py +314 -0
  156. flowyml-1.1.0/flowyml/utils/stack_config.py +296 -0
  157. flowyml-1.1.0/flowyml/utils/validation.py +270 -0
  158. flowyml-1.1.0/pyproject.toml +135 -0
flowyml-1.1.0/LICENSE ADDED
@@ -0,0 +1,17 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ Copyright 2024 Flowy Team
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.
flowyml-1.1.0/PKG-INFO ADDED
@@ -0,0 +1,372 @@
1
+ Metadata-Version: 2.4
2
+ Name: flowyml
3
+ Version: 1.1.0
4
+ Summary: Next-Generation ML Pipeline Framework
5
+ License: Apache-2.0
6
+ License-File: LICENSE
7
+ Author: flowyml Team
8
+ Author-email: support@unicolab.ai
9
+ Requires-Python: >=3.10,<4.0
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Provides-Extra: all
22
+ Provides-Extra: gcp
23
+ Provides-Extra: pytorch
24
+ Provides-Extra: sklearn
25
+ Provides-Extra: tensorflow
26
+ Provides-Extra: ui
27
+ Requires-Dist: click (>=8.0.0)
28
+ Requires-Dist: croniter (>=2.0.1,<3.0.0)
29
+ Requires-Dist: fastapi (>=0.122.0,<0.123.0) ; extra == "ui"
30
+ Requires-Dist: google-cloud-aiplatform (>=1.35.0) ; extra == "gcp" or extra == "all"
31
+ Requires-Dist: google-cloud-storage (>=2.10.0) ; extra == "gcp" or extra == "all"
32
+ Requires-Dist: loguru (>=0.7.3,<0.8.0)
33
+ Requires-Dist: numpy (>=1.20.0)
34
+ Requires-Dist: pandas (>=1.3.0)
35
+ Requires-Dist: pydantic (>=2.0.0)
36
+ Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
37
+ Requires-Dist: pytz (>=2024.1,<2025.0)
38
+ Requires-Dist: pyyaml (>=6.0)
39
+ Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
40
+ Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
41
+ Requires-Dist: toml (>=0.10.2)
42
+ Requires-Dist: torch (>=2.0.0) ; extra == "pytorch" or extra == "all"
43
+ Requires-Dist: uvicorn[standard] (>=0.23.0) ; extra == "ui" or extra == "all"
44
+ Requires-Dist: websockets (>=11.0) ; extra == "ui" or extra == "all"
45
+ Project-URL: Documentation, https://unicolab.github.io/FlowyML/latest
46
+ Project-URL: Homepage, https://github.com/UnicoLab/FlowyML
47
+ Project-URL: Repository, https://github.com/UnicoLab/FlowyML
48
+ Description-Content-Type: text/markdown
49
+
50
+ # 🌊 flowyml
51
+
52
+ <p align="center">
53
+ <img src="docs/logo.png" width="350" alt="flowyml Logo"/>
54
+ <br>
55
+ <em>The Enterprise-Grade ML Pipeline Framework for Humans</em>
56
+ <br>
57
+ <br>
58
+ <p align="center">
59
+ <a href="https://github.com/UnicoLab/FlowyML/actions"><img src="https://img.shields.io/github/actions/workflow/status/UnicoLab/FlowyML/ci.yml?branch=main" alt="CI Status"></a>
60
+ <a href="https://pypi.org/project/flowyml/"><img src="https://img.shields.io/pypi/v/flowyml" alt="PyPI Version"></a>
61
+ <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
62
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"></a>
63
+ <a href="https://unicolab.ai"><img src="https://img.shields.io/badge/UnicoLab-ai-red.svg" alt="UnicoLab"></a>
64
+ </p>
65
+ </p>
66
+
67
+ ---
68
+
69
+ **flowyml** is the comprehensive ML pipeline framework that combines the **simplicity of a Python script** with the **power of an enterprise MLOps platform**.
70
+
71
+ ## 🚀 Why flowyml?
72
+
73
+ | Feature | flowyml | Traditional Orchestrators |
74
+ |---------|---------|---------------------------|
75
+ | **Developer Experience** | 🐍 **Native Python** - No DSLs, no YAML hell. | 📜 Complex YAML or rigid DSLs. |
76
+ | **Context Awareness** | 🧠 **Auto-Injection** - Params are just function args. | 🔌 Manual wiring of every parameter. |
77
+ | **Caching** | ⚡ **Multi-Level** - Smart content-hashing & memoization. | 🐢 Basic file-timestamp checking. |
78
+ | **Asset Management** | 📦 **First-Class Assets** - Models & Datasets with lineage. | 📁 Generic file paths only. |
79
+ | **Architecture** | 🏗️ **Modular Stacks** - Local, Cloud, Hybrid. | 🔒 Vendor lock-in or complex setup. |
80
+
81
+ ## 🚀 Feature Showcase
82
+
83
+ flowyml is a complete toolkit for building, debugging, and deploying ML applications.
84
+
85
+ ### 1. Zero-Boilerplate Orchestration
86
+ Write pipelines as standard Python functions. No YAML, no DSLs.
87
+
88
+ ```python
89
+ @step(outputs=["data"])
90
+ def load(): return [1, 2, 3]
91
+
92
+ @step(inputs=["data"], outputs=["model"])
93
+ def train(data): return Model.train(data)
94
+
95
+ # It's just Python!
96
+ pipeline = Pipeline("simple").add_step(load).add_step(train)
97
+ pipeline.run()
98
+ ```
99
+
100
+ ### 2. 🧠 Intelligent Caching
101
+ Don't waste time re-running successful steps.
102
+ - **Code Hash**: Re-runs only when code changes.
103
+ - **Input Hash**: Re-runs only when data changes.
104
+
105
+ ```python
106
+ # Only re-runs if 'data' changes, ignoring code changes
107
+ @step(cache="input_hash", outputs=["processed"])
108
+ def expensive_processing(data):
109
+ return process(data)
110
+ ```
111
+
112
+ ### 3. 🤖 LLM & GenAI Ready
113
+ Trace token usage, latency, and costs automatically.
114
+
115
+ ```python
116
+ @step
117
+ @trace_llm(model="gpt-4", tags=["production"])
118
+ def generate_summary(text: str):
119
+ # flowyml automatically tracks tokens, cost, and latency
120
+ return openai.ChatCompletion.create(...)
121
+ ```
122
+
123
+ ### 4. ⚡ Efficient Step Grouping
124
+ Group related steps to run in the same container - reduce overhead, maintain clarity.
125
+
126
+ ```python
127
+ # Run preprocessing steps in same container (shares resources)
128
+ @step(outputs=["raw"], execution_group="preprocessing")
129
+ def load(): return fetch_data()
130
+
131
+ @step(inputs=["raw"], outputs=["clean"], execution_group="preprocessing")
132
+ def clean(raw): return preprocess(raw)
133
+
134
+ # flowyml automatically aggregates resources (max CPU, memory, best GPU)
135
+ # and executes consecutively in same environment
136
+ ```
137
+
138
+ ### 5. 🔀 Dynamic Workflows
139
+ Build adaptive workflows with conditional logic.
140
+
141
+ ```python
142
+ # Run 'deploy' only if model accuracy > 0.9
143
+ pipeline.add_step(
144
+ If(condition=lambda ctx: ctx["accuracy"] > 0.9)
145
+ .then(deploy_model)
146
+ .else_(notify_team)
147
+ )
148
+ ```
149
+
150
+ ### 6. 🧩 Universal Plugin System
151
+ Extend flowyml with any tool - even wrap ZenML components!
152
+
153
+ ```python
154
+ from flowyml.stacks.plugins import load_component
155
+
156
+ # Use any ZenML orchestrator, artifact store, or integration
157
+ k8s_orch = load_component("zenml:zenml.integrations.kubernetes.orchestrators.KubernetesOrchestrator")
158
+ mlflow = load_component("zenml:zenml.integrations.mlflow.MLflowExperimentTracker")
159
+ ```
160
+
161
+ ### 7. 👤 Human-in-the-Loop
162
+ Pause pipelines for manual approval or review.
163
+
164
+ ```python
165
+ from flowyml import approval
166
+
167
+ approval_step = approval(
168
+ name="approve_deployment",
169
+ approver="ml-team",
170
+ timeout_seconds=3600
171
+ )
172
+ ```
173
+
174
+ ### 8. 📊 Built-in Experiment Tracking
175
+ No external tools needed - tracking is built-in.
176
+
177
+ ```python
178
+ from flowyml.tracking import Experiment
179
+
180
+ exp = Experiment("baseline_training")
181
+ exp.log_run(run_id="run_001", metrics={"accuracy": 0.95}, parameters={"lr": 0.01})
182
+ best = exp.get_best_run("accuracy", maximize=True)
183
+ ```
184
+
185
+ ### 9. 🏆 Model Leaderboard & Registry
186
+ Track, compare, and version your models.
187
+
188
+ ```python
189
+ from flowyml import ModelLeaderboard
190
+ from flowyml.core import Model
191
+
192
+ # Track performance
193
+ leaderboard = ModelLeaderboard(metric="accuracy")
194
+ leaderboard.add_score(model_name="bert-base", score=0.92)
195
+
196
+ # Register models with stages
197
+ model = Model.create(artifact={...})
198
+ model.register(name="text_classifier", stage="production")
199
+ ```
200
+
201
+ ### 10. 📅 Built-in Scheduling
202
+ Schedule recurring jobs without external orchestrators.
203
+
204
+ ```python
205
+ from flowyml import PipelineScheduler
206
+
207
+ scheduler = PipelineScheduler()
208
+ scheduler.schedule_daily(
209
+ name="daily_training",
210
+ pipeline_func=lambda: pipeline.run(),
211
+ hour=2, minute=0
212
+ )
213
+ scheduler.start()
214
+ ```
215
+
216
+ ### 11. 🔔 Smart Notifications
217
+ Slack, Email, and custom alerts built-in.
218
+
219
+ ```python
220
+ from flowyml import configure_notifications
221
+
222
+ configure_notifications(
223
+ slack_webhook="https://hooks.slack.com/...",
224
+ email_config={...}
225
+ )
226
+ # Automatic notifications on pipeline success/failure
227
+ ```
228
+
229
+ ### 12. 🎯 Step-Level Debugging
230
+ Set breakpoints and inspect state mid-pipeline.
231
+
232
+ ```python
233
+ from flowyml import StepDebugger
234
+
235
+ debugger = StepDebugger()
236
+ debugger.set_breakpoint("train_model")
237
+ pipeline.run(debug=True) # Pauses at breakpoint
238
+ ```
239
+
240
+ ### 13. 📦 First-Class Assets
241
+ Specialized types for ML workflows.
242
+
243
+ ```python
244
+ from flowyml.core import Dataset, Model, Metrics, FeatureSet
245
+
246
+ dataset = Dataset.create(data=df, name="training_data")
247
+ model = Model.create(artifact=trained_model, score=0.95)
248
+ metrics = Metrics.create(values={"accuracy": 0.95})
249
+ ```
250
+
251
+ ### 14. 🔄 Smart Retries & Circuit Breakers
252
+ Handle failures gracefully.
253
+
254
+ ```python
255
+ @step(retry=3, timeout=300)
256
+ def flaky_api_call():
257
+ return external_api.fetch()
258
+
259
+ # Circuit breakers prevent cascading failures
260
+ ```
261
+
262
+ ### 15. 📈 Data Drift Detection
263
+ Monitor distribution shifts automatically.
264
+
265
+ ```python
266
+ from flowyml import detect_drift
267
+
268
+ drift = detect_drift(
269
+ reference_data=train_feature,
270
+ current_data=prod_feature,
271
+ threshold=0.1
272
+ )
273
+ if drift['drift_detected']:
274
+ trigger_retraining()
275
+ ```
276
+
277
+ ### 16. 🔍 Interactive Debugging
278
+ - **StepDebugger**: Set breakpoints in your pipeline.
279
+ - **Artifact Inspection**: View intermediate data in the UI.
280
+ - **Local Execution**: Run the exact same code locally as in production.
281
+
282
+ ### 17. 🏭 Enterprise Production Features
283
+ - **🔄 Automatic Retries**: Handle transient failures.
284
+ - **⏰ Scheduling**: Built-in cron scheduler.
285
+ - **🔔 Notifications**: Slack/Email alerts.
286
+ - **🛡️ Circuit Breakers**: Stop cascading failures.
287
+
288
+ ### 18. 🔌 Universal Integrations
289
+ - **ML Frameworks**: PyTorch, TensorFlow, Keras, Scikit-learn, HuggingFace.
290
+ - **Cloud Providers**: AWS, GCP, Azure (via plugins).
291
+ - **Tools**: MLflow, Weights & Biases, Great Expectations.
292
+
293
+ ## 📦 Installation
294
+
295
+ ```bash
296
+ # Install core
297
+ pip install flowyml
298
+
299
+ # Install with UI support
300
+ pip install "flowyml[ui]"
301
+
302
+ # Install with all features (recommended for dev)
303
+ pip install "flowyml[all]"
304
+ ```
305
+
306
+ ## ⚡ Quick Start
307
+
308
+ ```python
309
+ from flowyml import Pipeline, step, context, Dataset, Model
310
+
311
+ # 1. Define your configuration (Auto-injected!)
312
+ ctx = context(
313
+ learning_rate=0.01,
314
+ batch_size=32,
315
+ epochs=10
316
+ )
317
+
318
+ # 2. Define your steps (Pure Python)
319
+ @step(outputs=["dataset"])
320
+ def load_data(batch_size: int):
321
+ # 'batch_size' is automatically injected from context!
322
+ print(f"Loading data with batch size: {batch_size}")
323
+ return Dataset.create(data=[1, 2, 3], name="mnist")
324
+
325
+ @step(inputs=["dataset"], outputs=["model"])
326
+ def train(dataset: Dataset, learning_rate: float, epochs: int):
327
+ print(f"Training on {dataset.name} with lr={learning_rate}")
328
+ # Simulate training...
329
+ return Model.create(artifact={"weights": "..."}, score=0.98)
330
+
331
+ # 3. Run it!
332
+ pipeline = Pipeline("mnist_training", context=ctx)
333
+ pipeline.add_step(load_data)
334
+ pipeline.add_step(train)
335
+
336
+ result = pipeline.run()
337
+
338
+ print(f"Run ID: {result.run_id}")
339
+ print(f"Model Score: {result.outputs['model'].score}")
340
+ ```
341
+
342
+ ## 🖥️ The flowyml UI
343
+
344
+ Visualize your workflows, inspect artifacts, and monitor runs in real-time.
345
+
346
+ ```bash
347
+ # Start the UI server
348
+ flowyml ui start --open-browser
349
+ ```
350
+
351
+ Visit **http://localhost:8080** to access the dashboard.
352
+
353
+ ## 📚 Documentation
354
+
355
+ - **[Getting Started](docs/getting-started.md)**: Your first 5 minutes with flowyml.
356
+ - **[Core Concepts](docs/core/pipelines.md)**: Deep dive into Pipelines, Steps, and Context.
357
+ - **[Advanced Features](docs/advanced/caching.md)**: Learn about Caching, Parallelism, and Conditional Execution.
358
+ - **[API Reference](docs/api/core.md)**: Detailed class and function documentation.
359
+
360
+ ## 🤝 Contributing
361
+
362
+ We love contributions! Check out our [Contributing Guide](CONTRIBUTING.md) to get started.
363
+
364
+ ## 📝 License
365
+
366
+ Apache 2.0 - See [LICENSE](LICENSE) for details.
367
+
368
+ ---
369
+ <p align="center">
370
+ <strong>Built with ❤️ by <a href="https://unicolab.ai">UnicoLab</a></strong>
371
+ </p>
372
+
@@ -0,0 +1,322 @@
1
+ # 🌊 flowyml
2
+
3
+ <p align="center">
4
+ <img src="docs/logo.png" width="350" alt="flowyml Logo"/>
5
+ <br>
6
+ <em>The Enterprise-Grade ML Pipeline Framework for Humans</em>
7
+ <br>
8
+ <br>
9
+ <p align="center">
10
+ <a href="https://github.com/UnicoLab/FlowyML/actions"><img src="https://img.shields.io/github/actions/workflow/status/UnicoLab/FlowyML/ci.yml?branch=main" alt="CI Status"></a>
11
+ <a href="https://pypi.org/project/flowyml/"><img src="https://img.shields.io/pypi/v/flowyml" alt="PyPI Version"></a>
12
+ <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
13
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"></a>
14
+ <a href="https://unicolab.ai"><img src="https://img.shields.io/badge/UnicoLab-ai-red.svg" alt="UnicoLab"></a>
15
+ </p>
16
+ </p>
17
+
18
+ ---
19
+
20
+ **flowyml** is the comprehensive ML pipeline framework that combines the **simplicity of a Python script** with the **power of an enterprise MLOps platform**.
21
+
22
+ ## 🚀 Why flowyml?
23
+
24
+ | Feature | flowyml | Traditional Orchestrators |
25
+ |---------|---------|---------------------------|
26
+ | **Developer Experience** | 🐍 **Native Python** - No DSLs, no YAML hell. | 📜 Complex YAML or rigid DSLs. |
27
+ | **Context Awareness** | 🧠 **Auto-Injection** - Params are just function args. | 🔌 Manual wiring of every parameter. |
28
+ | **Caching** | ⚡ **Multi-Level** - Smart content-hashing & memoization. | 🐢 Basic file-timestamp checking. |
29
+ | **Asset Management** | 📦 **First-Class Assets** - Models & Datasets with lineage. | 📁 Generic file paths only. |
30
+ | **Architecture** | 🏗️ **Modular Stacks** - Local, Cloud, Hybrid. | 🔒 Vendor lock-in or complex setup. |
31
+
32
+ ## 🚀 Feature Showcase
33
+
34
+ flowyml is a complete toolkit for building, debugging, and deploying ML applications.
35
+
36
+ ### 1. Zero-Boilerplate Orchestration
37
+ Write pipelines as standard Python functions. No YAML, no DSLs.
38
+
39
+ ```python
40
+ @step(outputs=["data"])
41
+ def load(): return [1, 2, 3]
42
+
43
+ @step(inputs=["data"], outputs=["model"])
44
+ def train(data): return Model.train(data)
45
+
46
+ # It's just Python!
47
+ pipeline = Pipeline("simple").add_step(load).add_step(train)
48
+ pipeline.run()
49
+ ```
50
+
51
+ ### 2. 🧠 Intelligent Caching
52
+ Don't waste time re-running successful steps.
53
+ - **Code Hash**: Re-runs only when code changes.
54
+ - **Input Hash**: Re-runs only when data changes.
55
+
56
+ ```python
57
+ # Only re-runs if 'data' changes, ignoring code changes
58
+ @step(cache="input_hash", outputs=["processed"])
59
+ def expensive_processing(data):
60
+ return process(data)
61
+ ```
62
+
63
+ ### 3. 🤖 LLM & GenAI Ready
64
+ Trace token usage, latency, and costs automatically.
65
+
66
+ ```python
67
+ @step
68
+ @trace_llm(model="gpt-4", tags=["production"])
69
+ def generate_summary(text: str):
70
+ # flowyml automatically tracks tokens, cost, and latency
71
+ return openai.ChatCompletion.create(...)
72
+ ```
73
+
74
+ ### 4. ⚡ Efficient Step Grouping
75
+ Group related steps to run in the same container - reduce overhead, maintain clarity.
76
+
77
+ ```python
78
+ # Run preprocessing steps in same container (shares resources)
79
+ @step(outputs=["raw"], execution_group="preprocessing")
80
+ def load(): return fetch_data()
81
+
82
+ @step(inputs=["raw"], outputs=["clean"], execution_group="preprocessing")
83
+ def clean(raw): return preprocess(raw)
84
+
85
+ # flowyml automatically aggregates resources (max CPU, memory, best GPU)
86
+ # and executes consecutively in same environment
87
+ ```
88
+
89
+ ### 5. 🔀 Dynamic Workflows
90
+ Build adaptive workflows with conditional logic.
91
+
92
+ ```python
93
+ # Run 'deploy' only if model accuracy > 0.9
94
+ pipeline.add_step(
95
+ If(condition=lambda ctx: ctx["accuracy"] > 0.9)
96
+ .then(deploy_model)
97
+ .else_(notify_team)
98
+ )
99
+ ```
100
+
101
+ ### 6. 🧩 Universal Plugin System
102
+ Extend flowyml with any tool - even wrap ZenML components!
103
+
104
+ ```python
105
+ from flowyml.stacks.plugins import load_component
106
+
107
+ # Use any ZenML orchestrator, artifact store, or integration
108
+ k8s_orch = load_component("zenml:zenml.integrations.kubernetes.orchestrators.KubernetesOrchestrator")
109
+ mlflow = load_component("zenml:zenml.integrations.mlflow.MLflowExperimentTracker")
110
+ ```
111
+
112
+ ### 7. 👤 Human-in-the-Loop
113
+ Pause pipelines for manual approval or review.
114
+
115
+ ```python
116
+ from flowyml import approval
117
+
118
+ approval_step = approval(
119
+ name="approve_deployment",
120
+ approver="ml-team",
121
+ timeout_seconds=3600
122
+ )
123
+ ```
124
+
125
+ ### 8. 📊 Built-in Experiment Tracking
126
+ No external tools needed - tracking is built-in.
127
+
128
+ ```python
129
+ from flowyml.tracking import Experiment
130
+
131
+ exp = Experiment("baseline_training")
132
+ exp.log_run(run_id="run_001", metrics={"accuracy": 0.95}, parameters={"lr": 0.01})
133
+ best = exp.get_best_run("accuracy", maximize=True)
134
+ ```
135
+
136
+ ### 9. 🏆 Model Leaderboard & Registry
137
+ Track, compare, and version your models.
138
+
139
+ ```python
140
+ from flowyml import ModelLeaderboard
141
+ from flowyml.core import Model
142
+
143
+ # Track performance
144
+ leaderboard = ModelLeaderboard(metric="accuracy")
145
+ leaderboard.add_score(model_name="bert-base", score=0.92)
146
+
147
+ # Register models with stages
148
+ model = Model.create(artifact={...})
149
+ model.register(name="text_classifier", stage="production")
150
+ ```
151
+
152
+ ### 10. 📅 Built-in Scheduling
153
+ Schedule recurring jobs without external orchestrators.
154
+
155
+ ```python
156
+ from flowyml import PipelineScheduler
157
+
158
+ scheduler = PipelineScheduler()
159
+ scheduler.schedule_daily(
160
+ name="daily_training",
161
+ pipeline_func=lambda: pipeline.run(),
162
+ hour=2, minute=0
163
+ )
164
+ scheduler.start()
165
+ ```
166
+
167
+ ### 11. 🔔 Smart Notifications
168
+ Slack, Email, and custom alerts built-in.
169
+
170
+ ```python
171
+ from flowyml import configure_notifications
172
+
173
+ configure_notifications(
174
+ slack_webhook="https://hooks.slack.com/...",
175
+ email_config={...}
176
+ )
177
+ # Automatic notifications on pipeline success/failure
178
+ ```
179
+
180
+ ### 12. 🎯 Step-Level Debugging
181
+ Set breakpoints and inspect state mid-pipeline.
182
+
183
+ ```python
184
+ from flowyml import StepDebugger
185
+
186
+ debugger = StepDebugger()
187
+ debugger.set_breakpoint("train_model")
188
+ pipeline.run(debug=True) # Pauses at breakpoint
189
+ ```
190
+
191
+ ### 13. 📦 First-Class Assets
192
+ Specialized types for ML workflows.
193
+
194
+ ```python
195
+ from flowyml.core import Dataset, Model, Metrics, FeatureSet
196
+
197
+ dataset = Dataset.create(data=df, name="training_data")
198
+ model = Model.create(artifact=trained_model, score=0.95)
199
+ metrics = Metrics.create(values={"accuracy": 0.95})
200
+ ```
201
+
202
+ ### 14. 🔄 Smart Retries & Circuit Breakers
203
+ Handle failures gracefully.
204
+
205
+ ```python
206
+ @step(retry=3, timeout=300)
207
+ def flaky_api_call():
208
+ return external_api.fetch()
209
+
210
+ # Circuit breakers prevent cascading failures
211
+ ```
212
+
213
+ ### 15. 📈 Data Drift Detection
214
+ Monitor distribution shifts automatically.
215
+
216
+ ```python
217
+ from flowyml import detect_drift
218
+
219
+ drift = detect_drift(
220
+ reference_data=train_feature,
221
+ current_data=prod_feature,
222
+ threshold=0.1
223
+ )
224
+ if drift['drift_detected']:
225
+ trigger_retraining()
226
+ ```
227
+
228
+ ### 16. 🔍 Interactive Debugging
229
+ - **StepDebugger**: Set breakpoints in your pipeline.
230
+ - **Artifact Inspection**: View intermediate data in the UI.
231
+ - **Local Execution**: Run the exact same code locally as in production.
232
+
233
+ ### 17. 🏭 Enterprise Production Features
234
+ - **🔄 Automatic Retries**: Handle transient failures.
235
+ - **⏰ Scheduling**: Built-in cron scheduler.
236
+ - **🔔 Notifications**: Slack/Email alerts.
237
+ - **🛡️ Circuit Breakers**: Stop cascading failures.
238
+
239
+ ### 18. 🔌 Universal Integrations
240
+ - **ML Frameworks**: PyTorch, TensorFlow, Keras, Scikit-learn, HuggingFace.
241
+ - **Cloud Providers**: AWS, GCP, Azure (via plugins).
242
+ - **Tools**: MLflow, Weights & Biases, Great Expectations.
243
+
244
+ ## 📦 Installation
245
+
246
+ ```bash
247
+ # Install core
248
+ pip install flowyml
249
+
250
+ # Install with UI support
251
+ pip install "flowyml[ui]"
252
+
253
+ # Install with all features (recommended for dev)
254
+ pip install "flowyml[all]"
255
+ ```
256
+
257
+ ## ⚡ Quick Start
258
+
259
+ ```python
260
+ from flowyml import Pipeline, step, context, Dataset, Model
261
+
262
+ # 1. Define your configuration (Auto-injected!)
263
+ ctx = context(
264
+ learning_rate=0.01,
265
+ batch_size=32,
266
+ epochs=10
267
+ )
268
+
269
+ # 2. Define your steps (Pure Python)
270
+ @step(outputs=["dataset"])
271
+ def load_data(batch_size: int):
272
+ # 'batch_size' is automatically injected from context!
273
+ print(f"Loading data with batch size: {batch_size}")
274
+ return Dataset.create(data=[1, 2, 3], name="mnist")
275
+
276
+ @step(inputs=["dataset"], outputs=["model"])
277
+ def train(dataset: Dataset, learning_rate: float, epochs: int):
278
+ print(f"Training on {dataset.name} with lr={learning_rate}")
279
+ # Simulate training...
280
+ return Model.create(artifact={"weights": "..."}, score=0.98)
281
+
282
+ # 3. Run it!
283
+ pipeline = Pipeline("mnist_training", context=ctx)
284
+ pipeline.add_step(load_data)
285
+ pipeline.add_step(train)
286
+
287
+ result = pipeline.run()
288
+
289
+ print(f"Run ID: {result.run_id}")
290
+ print(f"Model Score: {result.outputs['model'].score}")
291
+ ```
292
+
293
+ ## 🖥️ The flowyml UI
294
+
295
+ Visualize your workflows, inspect artifacts, and monitor runs in real-time.
296
+
297
+ ```bash
298
+ # Start the UI server
299
+ flowyml ui start --open-browser
300
+ ```
301
+
302
+ Visit **http://localhost:8080** to access the dashboard.
303
+
304
+ ## 📚 Documentation
305
+
306
+ - **[Getting Started](docs/getting-started.md)**: Your first 5 minutes with flowyml.
307
+ - **[Core Concepts](docs/core/pipelines.md)**: Deep dive into Pipelines, Steps, and Context.
308
+ - **[Advanced Features](docs/advanced/caching.md)**: Learn about Caching, Parallelism, and Conditional Execution.
309
+ - **[API Reference](docs/api/core.md)**: Detailed class and function documentation.
310
+
311
+ ## 🤝 Contributing
312
+
313
+ We love contributions! Check out our [Contributing Guide](CONTRIBUTING.md) to get started.
314
+
315
+ ## 📝 License
316
+
317
+ Apache 2.0 - See [LICENSE](LICENSE) for details.
318
+
319
+ ---
320
+ <p align="center">
321
+ <strong>Built with ❤️ by <a href="https://unicolab.ai">UnicoLab</a></strong>
322
+ </p>