flowyml 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. flowyml/__init__.py +3 -0
  2. flowyml/assets/base.py +10 -0
  3. flowyml/assets/metrics.py +6 -0
  4. flowyml/cli/main.py +108 -2
  5. flowyml/cli/run.py +9 -2
  6. flowyml/core/execution_status.py +52 -0
  7. flowyml/core/hooks.py +106 -0
  8. flowyml/core/observability.py +210 -0
  9. flowyml/core/orchestrator.py +274 -0
  10. flowyml/core/pipeline.py +193 -231
  11. flowyml/core/project.py +34 -2
  12. flowyml/core/remote_orchestrator.py +109 -0
  13. flowyml/core/resources.py +34 -17
  14. flowyml/core/retry_policy.py +80 -0
  15. flowyml/core/scheduler.py +9 -9
  16. flowyml/core/scheduler_config.py +2 -3
  17. flowyml/core/step.py +18 -1
  18. flowyml/core/submission_result.py +53 -0
  19. flowyml/integrations/keras.py +95 -22
  20. flowyml/monitoring/alerts.py +2 -2
  21. flowyml/stacks/__init__.py +15 -0
  22. flowyml/stacks/aws.py +599 -0
  23. flowyml/stacks/azure.py +295 -0
  24. flowyml/stacks/bridge.py +9 -9
  25. flowyml/stacks/components.py +24 -2
  26. flowyml/stacks/gcp.py +158 -11
  27. flowyml/stacks/local.py +5 -0
  28. flowyml/stacks/plugins.py +2 -2
  29. flowyml/stacks/registry.py +21 -0
  30. flowyml/storage/artifacts.py +15 -5
  31. flowyml/storage/materializers/__init__.py +2 -0
  32. flowyml/storage/materializers/base.py +33 -0
  33. flowyml/storage/materializers/cloudpickle.py +74 -0
  34. flowyml/storage/metadata.py +3 -881
  35. flowyml/storage/remote.py +590 -0
  36. flowyml/storage/sql.py +911 -0
  37. flowyml/ui/backend/dependencies.py +28 -0
  38. flowyml/ui/backend/main.py +43 -80
  39. flowyml/ui/backend/routers/assets.py +483 -17
  40. flowyml/ui/backend/routers/client.py +46 -0
  41. flowyml/ui/backend/routers/execution.py +13 -2
  42. flowyml/ui/backend/routers/experiments.py +97 -14
  43. flowyml/ui/backend/routers/metrics.py +168 -0
  44. flowyml/ui/backend/routers/pipelines.py +77 -12
  45. flowyml/ui/backend/routers/projects.py +33 -7
  46. flowyml/ui/backend/routers/runs.py +221 -12
  47. flowyml/ui/backend/routers/schedules.py +5 -21
  48. flowyml/ui/backend/routers/stats.py +14 -0
  49. flowyml/ui/backend/routers/traces.py +37 -53
  50. flowyml/ui/frontend/dist/assets/index-DcYwrn2j.css +1 -0
  51. flowyml/ui/frontend/dist/assets/index-Dlz_ygOL.js +592 -0
  52. flowyml/ui/frontend/dist/index.html +2 -2
  53. flowyml/ui/frontend/src/App.jsx +4 -1
  54. flowyml/ui/frontend/src/app/assets/page.jsx +260 -230
  55. flowyml/ui/frontend/src/app/dashboard/page.jsx +38 -7
  56. flowyml/ui/frontend/src/app/experiments/page.jsx +61 -314
  57. flowyml/ui/frontend/src/app/observability/page.jsx +277 -0
  58. flowyml/ui/frontend/src/app/pipelines/page.jsx +79 -402
  59. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectArtifactsList.jsx +151 -0
  60. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectExperimentsList.jsx +145 -0
  61. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHeader.jsx +45 -0
  62. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHierarchy.jsx +467 -0
  63. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectMetricsPanel.jsx +253 -0
  64. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectPipelinesList.jsx +105 -0
  65. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRelations.jsx +189 -0
  66. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRunsList.jsx +136 -0
  67. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectTabs.jsx +95 -0
  68. flowyml/ui/frontend/src/app/projects/[projectId]/page.jsx +326 -0
  69. flowyml/ui/frontend/src/app/projects/page.jsx +13 -3
  70. flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +79 -10
  71. flowyml/ui/frontend/src/app/runs/page.jsx +82 -424
  72. flowyml/ui/frontend/src/app/settings/page.jsx +1 -0
  73. flowyml/ui/frontend/src/app/tokens/page.jsx +62 -16
  74. flowyml/ui/frontend/src/components/AssetDetailsPanel.jsx +373 -0
  75. flowyml/ui/frontend/src/components/AssetLineageGraph.jsx +291 -0
  76. flowyml/ui/frontend/src/components/AssetStatsDashboard.jsx +302 -0
  77. flowyml/ui/frontend/src/components/AssetTreeHierarchy.jsx +477 -0
  78. flowyml/ui/frontend/src/components/ExperimentDetailsPanel.jsx +227 -0
  79. flowyml/ui/frontend/src/components/NavigationTree.jsx +401 -0
  80. flowyml/ui/frontend/src/components/PipelineDetailsPanel.jsx +239 -0
  81. flowyml/ui/frontend/src/components/PipelineGraph.jsx +67 -3
  82. flowyml/ui/frontend/src/components/ProjectSelector.jsx +115 -0
  83. flowyml/ui/frontend/src/components/RunDetailsPanel.jsx +298 -0
  84. flowyml/ui/frontend/src/components/header/Header.jsx +48 -1
  85. flowyml/ui/frontend/src/components/plugins/ZenMLIntegration.jsx +106 -0
  86. flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +52 -26
  87. flowyml/ui/frontend/src/components/ui/DataView.jsx +35 -17
  88. flowyml/ui/frontend/src/components/ui/ErrorBoundary.jsx +118 -0
  89. flowyml/ui/frontend/src/contexts/ProjectContext.jsx +2 -2
  90. flowyml/ui/frontend/src/contexts/ToastContext.jsx +116 -0
  91. flowyml/ui/frontend/src/layouts/MainLayout.jsx +5 -1
  92. flowyml/ui/frontend/src/router/index.jsx +4 -0
  93. flowyml/ui/frontend/src/utils/date.js +10 -0
  94. flowyml/ui/frontend/src/utils/downloads.js +11 -0
  95. flowyml/utils/config.py +6 -0
  96. flowyml/utils/stack_config.py +45 -3
  97. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/METADATA +44 -4
  98. flowyml-1.4.0.dist-info/RECORD +200 -0
  99. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/licenses/LICENSE +1 -1
  100. flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +0 -448
  101. flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +0 -1
  102. flowyml-1.2.0.dist-info/RECORD +0 -159
  103. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/WHEEL +0 -0
  104. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,277 @@
1
+ import React, { useEffect, useState } from 'react';
2
+ import { fetchApi } from '../../utils/api';
3
+ import { Activity, TrendingUp, Zap, Clock, BarChart2, CheckCircle, XCircle } from 'lucide-react';
4
+ import { Card } from '../../components/ui/Card';
5
+ import { Badge } from '../../components/ui/Badge';
6
+
7
+ export function Observability() {
8
+ const [orchestratorMetrics, setOrchestratorMetrics] = useState(null);
9
+ const [cacheMetrics, setCacheMetrics] = useState(null);
10
+ const [loading, setLoading] = useState(true);
11
+
12
+ useEffect(() => {
13
+ const fetchMetrics = async () => {
14
+ try {
15
+ const [orchRes, cacheRes] = await Promise.all([
16
+ fetchApi('/api/metrics/observability/orchestrator'),
17
+ fetchApi('/api/metrics/observability/cache')
18
+ ]);
19
+
20
+ const orchData = await orchRes.json();
21
+ const cacheData = await cacheRes.json();
22
+
23
+ setOrchestratorMetrics(orchData);
24
+ setCacheMetrics(cacheData);
25
+ } catch (error) {
26
+ console.error('Failed to fetch metrics:', error);
27
+ } finally {
28
+ setLoading(false);
29
+ }
30
+ };
31
+
32
+ fetchMetrics();
33
+ // Refresh every 30 seconds
34
+ const interval = setInterval(fetchMetrics, 30000);
35
+ return () => clearInterval(interval);
36
+ }, []);
37
+
38
+ if (loading) {
39
+ return (
40
+ <div className="flex items-center justify-center h-96">
41
+ <div className="animate-spin rounded-full h-12 w-12 border-b-2 border-primary-600"></div>
42
+ </div>
43
+ );
44
+ }
45
+
46
+ const successRate = orchestratorMetrics?.success_rate || 0;
47
+ const cacheHitRate = cacheMetrics?.cache_hit_rate || 0;
48
+
49
+ return (
50
+ <div className="p-6 space-y-6 max-w-7xl mx-auto">
51
+ {/* Header */}
52
+ <div className="flex items-center justify-between">
53
+ <div>
54
+ <h1 className="text-3xl font-bold text-slate-900 dark:text-white flex items-center gap-3">
55
+ <Activity className="text-primary-500" size={32} />
56
+ Observability Dashboard
57
+ </h1>
58
+ <p className="text-slate-500 mt-1">
59
+ System performance and metrics (Last 30 days)
60
+ </p>
61
+ </div>
62
+ <Badge variant="secondary" className="text-xs">
63
+ Auto-refresh: 30s
64
+ </Badge>
65
+ </div>
66
+
67
+ {/* Key Metrics Grid */}
68
+ <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
69
+ <MetricCard
70
+ icon={<BarChart2 size={24} />}
71
+ label="Total Runs"
72
+ value={orchestratorMetrics?.total_runs || 0}
73
+ color="blue"
74
+ />
75
+ <MetricCard
76
+ icon={<CheckCircle size={24} />}
77
+ label="Success Rate"
78
+ value={`${(successRate * 100).toFixed(1)}%`}
79
+ color="emerald"
80
+ trend={successRate >= 0.9 ? 'positive' : successRate >= 0.7 ? 'neutral' : 'negative'}
81
+ />
82
+ <MetricCard
83
+ icon={<Clock size={24} />}
84
+ label="Avg Duration"
85
+ value={`${(orchestratorMetrics?.avg_duration_seconds || 0).toFixed(2)}s`}
86
+ color="purple"
87
+ />
88
+ <MetricCard
89
+ icon={<Zap size={24} />}
90
+ label="Cache Hit Rate"
91
+ value={`${(cacheHitRate * 100).toFixed(1)}%`}
92
+ color="amber"
93
+ trend={cacheHitRate >= 0.5 ? 'positive' : cacheHitRate >= 0.2 ? 'neutral' : 'negative'}
94
+ />
95
+ </div>
96
+
97
+ {/* Detailed Sections */}
98
+ <div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
99
+ {/* Orchestrator Performance */}
100
+ <Card className="p-6">
101
+ <h3 className="text-lg font-bold text-slate-900 dark:text-white mb-4 flex items-center gap-2">
102
+ <TrendingUp className="text-primary-500" size={20} />
103
+ Orchestrator Performance
104
+ </h3>
105
+
106
+ <div className="space-y-4">
107
+ {/* Status Distribution */}
108
+ <div>
109
+ <h4 className="text-sm font-semibold text-slate-700 dark:text-slate-300 mb-3">
110
+ Status Distribution
111
+ </h4>
112
+ <div className="space-y-2">
113
+ {orchestratorMetrics?.status_distribution &&
114
+ Object.entries(orchestratorMetrics.status_distribution).map(([status, count]) => (
115
+ <div key={status} className="flex items-center justify-between">
116
+ <div className="flex items-center gap-2">
117
+ <div className={`w-3 h-3 rounded-full ${status === 'completed' ? 'bg-emerald-500' :
118
+ status === 'failed' ? 'bg-rose-500' :
119
+ 'bg-amber-500'
120
+ }`} />
121
+ <span className="text-sm text-slate-600 dark:text-slate-400 capitalize">
122
+ {status}
123
+ </span>
124
+ </div>
125
+ <div className="flex items-center gap-3">
126
+ <span className="text-sm font-mono font-semibold text-slate-900 dark:text-white">
127
+ {count}
128
+ </span>
129
+ <div className="w-24 bg-slate-200 dark:bg-slate-700 rounded-full h-2">
130
+ <div
131
+ className={`h-2 rounded-full ${status === 'completed' ? 'bg-emerald-500' :
132
+ status === 'failed' ? 'bg-rose-500' :
133
+ 'bg-amber-500'
134
+ }`}
135
+ style={{
136
+ width: `${(count / orchestratorMetrics.total_runs) * 100}%`
137
+ }}
138
+ />
139
+ </div>
140
+ </div>
141
+ </div>
142
+ ))
143
+ }
144
+ </div>
145
+ </div>
146
+
147
+ {/* Summary Stats */}
148
+ <div className="grid grid-cols-2 gap-3 pt-4 border-t border-slate-200 dark:border-slate-700">
149
+ <div className="p-3 bg-emerald-50 dark:bg-emerald-900/20 rounded-lg">
150
+ <div className="text-xs text-emerald-600 dark:text-emerald-400 mb-1">
151
+ Success Rate
152
+ </div>
153
+ <div className="text-2xl font-bold text-emerald-700 dark:text-emerald-300">
154
+ {(successRate * 100).toFixed(1)}%
155
+ </div>
156
+ </div>
157
+ <div className="p-3 bg-blue-50 dark:bg-blue-900/20 rounded-lg">
158
+ <div className="text-xs text-blue-600 dark:text-blue-400 mb-1">
159
+ Avg Duration
160
+ </div>
161
+ <div className="text-2xl font-bold text-blue-700 dark:text-blue-300">
162
+ {(orchestratorMetrics?.avg_duration_seconds || 0).toFixed(1)}s
163
+ </div>
164
+ </div>
165
+ </div>
166
+ </div>
167
+ </Card>
168
+
169
+ {/* Cache Performance */}
170
+ <Card className="p-6">
171
+ <h3 className="text-lg font-bold text-slate-900 dark:text-white mb-4 flex items-center gap-2">
172
+ <Zap className="text-amber-500" size={20} />
173
+ Cache Performance
174
+ </h3>
175
+
176
+ <div className="space-y-4">
177
+ {/* Cache Stats */}
178
+ <div className="grid grid-cols-2 gap-4">
179
+ <div className="p-4 bg-gradient-to-br from-amber-50 to-orange-50 dark:from-amber-900/20 dark:to-orange-900/20 rounded-xl border border-amber-100 dark:border-amber-800">
180
+ <div className="text-xs text-amber-600 dark:text-amber-400 mb-1">
181
+ Total Steps
182
+ </div>
183
+ <div className="text-3xl font-bold text-amber-700 dark:text-amber-300">
184
+ {cacheMetrics?.total_steps || 0}
185
+ </div>
186
+ </div>
187
+ <div className="p-4 bg-gradient-to-br from-emerald-50 to-teal-50 dark:from-emerald-900/20 dark:to-teal-900/20 rounded-xl border border-emerald-100 dark:border-emerald-800">
188
+ <div className="text-xs text-emerald-600 dark:text-emerald-400 mb-1">
189
+ Cached Steps
190
+ </div>
191
+ <div className="text-3xl font-bold text-emerald-700 dark:text-emerald-300">
192
+ {cacheMetrics?.cached_steps || 0}
193
+ </div>
194
+ </div>
195
+ </div>
196
+
197
+ {/* Cache Hit Rate Visualization */}
198
+ <div>
199
+ <div className="flex items-center justify-between mb-2">
200
+ <span className="text-sm font-semibold text-slate-700 dark:text-slate-300">
201
+ Cache Hit Rate
202
+ </span>
203
+ <span className="text-lg font-bold text-slate-900 dark:text-white">
204
+ {(cacheHitRate * 100).toFixed(1)}%
205
+ </span>
206
+ </div>
207
+ <div className="relative h-8 bg-slate-200 dark:bg-slate-700 rounded-full overflow-hidden">
208
+ <div
209
+ className="absolute inset-y-0 left-0 bg-gradient-to-r from-emerald-500 to-teal-500 flex items-center justify-end px-3 transition-all duration-500"
210
+ style={{ width: `${cacheHitRate * 100}%` }}
211
+ >
212
+ {cacheHitRate > 0.1 && (
213
+ <span className="text-xs font-bold text-white">
214
+ {(cacheHitRate * 100).toFixed(0)}%
215
+ </span>
216
+ )}
217
+ </div>
218
+ </div>
219
+ </div>
220
+
221
+ {/* Performance Impact */}
222
+ <div className="p-4 bg-blue-50 dark:bg-blue-900/20 rounded-xl border border-blue-100 dark:border-blue-800">
223
+ <div className="flex items-center gap-2 mb-2">
224
+ <TrendingUp size={16} className="text-blue-600" />
225
+ <span className="text-sm font-semibold text-blue-900 dark:text-blue-100">
226
+ Performance Impact
227
+ </span>
228
+ </div>
229
+ <p className="text-sm text-blue-700 dark:text-blue-300">
230
+ Cache saved approximately{' '}
231
+ <strong>{cacheMetrics?.cached_steps || 0}</strong> step executions,
232
+ improving pipeline efficiency.
233
+ </p>
234
+ </div>
235
+ </div>
236
+ </Card>
237
+ </div>
238
+ </div>
239
+ );
240
+ }
241
+
242
+ function MetricCard({ icon, label, value, color, trend }) {
243
+ const colorClasses = {
244
+ blue: "bg-blue-50 text-blue-600 dark:bg-blue-900/20 dark:text-blue-400",
245
+ emerald: "bg-emerald-50 text-emerald-600 dark:bg-emerald-900/20 dark:text-emerald-400",
246
+ purple: "bg-purple-50 text-purple-600 dark:bg-purple-900/20 dark:text-purple-400",
247
+ amber: "bg-amber-50 text-amber-600 dark:bg-amber-900/20 dark:text-amber-400",
248
+ };
249
+
250
+ const trendColors = {
251
+ positive: "text-emerald-600 dark:text-emerald-400",
252
+ neutral: "text-amber-600 dark:text-amber-400",
253
+ negative: "text-rose-600 dark:text-rose-400",
254
+ };
255
+
256
+ return (
257
+ <Card className="hover:shadow-lg transition-shadow duration-200">
258
+ <div className="flex items-center gap-4">
259
+ <div className={`p-3 rounded-xl ${colorClasses[color]}`}>
260
+ {icon}
261
+ </div>
262
+ <div className="flex-1">
263
+ <p className="text-sm text-slate-500 dark:text-slate-400 font-medium">{label}</p>
264
+ <div className="flex items-baseline gap-2">
265
+ <p className="text-2xl font-bold text-slate-900 dark:text-white">{value}</p>
266
+ {trend && (
267
+ <TrendingUp
268
+ size={16}
269
+ className={`${trendColors[trend]} ${trend === 'negative' ? 'rotate-180' : ''}`}
270
+ />
271
+ )}
272
+ </div>
273
+ </div>
274
+ </div>
275
+ </Card>
276
+ );
277
+ }