flowyml 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowyml/__init__.py +3 -0
- flowyml/assets/base.py +10 -0
- flowyml/assets/metrics.py +6 -0
- flowyml/cli/main.py +108 -2
- flowyml/cli/run.py +9 -2
- flowyml/core/execution_status.py +52 -0
- flowyml/core/hooks.py +106 -0
- flowyml/core/observability.py +210 -0
- flowyml/core/orchestrator.py +274 -0
- flowyml/core/pipeline.py +193 -231
- flowyml/core/project.py +34 -2
- flowyml/core/remote_orchestrator.py +109 -0
- flowyml/core/resources.py +34 -17
- flowyml/core/retry_policy.py +80 -0
- flowyml/core/scheduler.py +9 -9
- flowyml/core/scheduler_config.py +2 -3
- flowyml/core/step.py +18 -1
- flowyml/core/submission_result.py +53 -0
- flowyml/integrations/keras.py +95 -22
- flowyml/monitoring/alerts.py +2 -2
- flowyml/stacks/__init__.py +15 -0
- flowyml/stacks/aws.py +599 -0
- flowyml/stacks/azure.py +295 -0
- flowyml/stacks/bridge.py +9 -9
- flowyml/stacks/components.py +24 -2
- flowyml/stacks/gcp.py +158 -11
- flowyml/stacks/local.py +5 -0
- flowyml/stacks/plugins.py +2 -2
- flowyml/stacks/registry.py +21 -0
- flowyml/storage/artifacts.py +15 -5
- flowyml/storage/materializers/__init__.py +2 -0
- flowyml/storage/materializers/base.py +33 -0
- flowyml/storage/materializers/cloudpickle.py +74 -0
- flowyml/storage/metadata.py +3 -881
- flowyml/storage/remote.py +590 -0
- flowyml/storage/sql.py +911 -0
- flowyml/ui/backend/dependencies.py +28 -0
- flowyml/ui/backend/main.py +43 -80
- flowyml/ui/backend/routers/assets.py +483 -17
- flowyml/ui/backend/routers/client.py +46 -0
- flowyml/ui/backend/routers/execution.py +13 -2
- flowyml/ui/backend/routers/experiments.py +97 -14
- flowyml/ui/backend/routers/metrics.py +168 -0
- flowyml/ui/backend/routers/pipelines.py +77 -12
- flowyml/ui/backend/routers/projects.py +33 -7
- flowyml/ui/backend/routers/runs.py +221 -12
- flowyml/ui/backend/routers/schedules.py +5 -21
- flowyml/ui/backend/routers/stats.py +14 -0
- flowyml/ui/backend/routers/traces.py +37 -53
- flowyml/ui/frontend/dist/assets/index-DcYwrn2j.css +1 -0
- flowyml/ui/frontend/dist/assets/index-Dlz_ygOL.js +592 -0
- flowyml/ui/frontend/dist/index.html +2 -2
- flowyml/ui/frontend/src/App.jsx +4 -1
- flowyml/ui/frontend/src/app/assets/page.jsx +260 -230
- flowyml/ui/frontend/src/app/dashboard/page.jsx +38 -7
- flowyml/ui/frontend/src/app/experiments/page.jsx +61 -314
- flowyml/ui/frontend/src/app/observability/page.jsx +277 -0
- flowyml/ui/frontend/src/app/pipelines/page.jsx +79 -402
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectArtifactsList.jsx +151 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectExperimentsList.jsx +145 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHeader.jsx +45 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHierarchy.jsx +467 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectMetricsPanel.jsx +253 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectPipelinesList.jsx +105 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRelations.jsx +189 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRunsList.jsx +136 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectTabs.jsx +95 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/page.jsx +326 -0
- flowyml/ui/frontend/src/app/projects/page.jsx +13 -3
- flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +79 -10
- flowyml/ui/frontend/src/app/runs/page.jsx +82 -424
- flowyml/ui/frontend/src/app/settings/page.jsx +1 -0
- flowyml/ui/frontend/src/app/tokens/page.jsx +62 -16
- flowyml/ui/frontend/src/components/AssetDetailsPanel.jsx +373 -0
- flowyml/ui/frontend/src/components/AssetLineageGraph.jsx +291 -0
- flowyml/ui/frontend/src/components/AssetStatsDashboard.jsx +302 -0
- flowyml/ui/frontend/src/components/AssetTreeHierarchy.jsx +477 -0
- flowyml/ui/frontend/src/components/ExperimentDetailsPanel.jsx +227 -0
- flowyml/ui/frontend/src/components/NavigationTree.jsx +401 -0
- flowyml/ui/frontend/src/components/PipelineDetailsPanel.jsx +239 -0
- flowyml/ui/frontend/src/components/PipelineGraph.jsx +67 -3
- flowyml/ui/frontend/src/components/ProjectSelector.jsx +115 -0
- flowyml/ui/frontend/src/components/RunDetailsPanel.jsx +298 -0
- flowyml/ui/frontend/src/components/header/Header.jsx +48 -1
- flowyml/ui/frontend/src/components/plugins/ZenMLIntegration.jsx +106 -0
- flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +52 -26
- flowyml/ui/frontend/src/components/ui/DataView.jsx +35 -17
- flowyml/ui/frontend/src/components/ui/ErrorBoundary.jsx +118 -0
- flowyml/ui/frontend/src/contexts/ProjectContext.jsx +2 -2
- flowyml/ui/frontend/src/contexts/ToastContext.jsx +116 -0
- flowyml/ui/frontend/src/layouts/MainLayout.jsx +5 -1
- flowyml/ui/frontend/src/router/index.jsx +4 -0
- flowyml/ui/frontend/src/utils/date.js +10 -0
- flowyml/ui/frontend/src/utils/downloads.js +11 -0
- flowyml/utils/config.py +6 -0
- flowyml/utils/stack_config.py +45 -3
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/METADATA +44 -4
- flowyml-1.4.0.dist-info/RECORD +200 -0
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/licenses/LICENSE +1 -1
- flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +0 -448
- flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +0 -1
- flowyml-1.2.0.dist-info/RECORD +0 -159
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/WHEEL +0 -0
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import React, { useEffect, useState } from 'react';
|
|
2
|
+
import { fetchApi } from '../../utils/api';
|
|
3
|
+
import { Activity, TrendingUp, Zap, Clock, BarChart2, CheckCircle, XCircle } from 'lucide-react';
|
|
4
|
+
import { Card } from '../../components/ui/Card';
|
|
5
|
+
import { Badge } from '../../components/ui/Badge';
|
|
6
|
+
|
|
7
|
+
export function Observability() {
|
|
8
|
+
const [orchestratorMetrics, setOrchestratorMetrics] = useState(null);
|
|
9
|
+
const [cacheMetrics, setCacheMetrics] = useState(null);
|
|
10
|
+
const [loading, setLoading] = useState(true);
|
|
11
|
+
|
|
12
|
+
useEffect(() => {
|
|
13
|
+
const fetchMetrics = async () => {
|
|
14
|
+
try {
|
|
15
|
+
const [orchRes, cacheRes] = await Promise.all([
|
|
16
|
+
fetchApi('/api/metrics/observability/orchestrator'),
|
|
17
|
+
fetchApi('/api/metrics/observability/cache')
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
const orchData = await orchRes.json();
|
|
21
|
+
const cacheData = await cacheRes.json();
|
|
22
|
+
|
|
23
|
+
setOrchestratorMetrics(orchData);
|
|
24
|
+
setCacheMetrics(cacheData);
|
|
25
|
+
} catch (error) {
|
|
26
|
+
console.error('Failed to fetch metrics:', error);
|
|
27
|
+
} finally {
|
|
28
|
+
setLoading(false);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
fetchMetrics();
|
|
33
|
+
// Refresh every 30 seconds
|
|
34
|
+
const interval = setInterval(fetchMetrics, 30000);
|
|
35
|
+
return () => clearInterval(interval);
|
|
36
|
+
}, []);
|
|
37
|
+
|
|
38
|
+
if (loading) {
|
|
39
|
+
return (
|
|
40
|
+
<div className="flex items-center justify-center h-96">
|
|
41
|
+
<div className="animate-spin rounded-full h-12 w-12 border-b-2 border-primary-600"></div>
|
|
42
|
+
</div>
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const successRate = orchestratorMetrics?.success_rate || 0;
|
|
47
|
+
const cacheHitRate = cacheMetrics?.cache_hit_rate || 0;
|
|
48
|
+
|
|
49
|
+
return (
|
|
50
|
+
<div className="p-6 space-y-6 max-w-7xl mx-auto">
|
|
51
|
+
{/* Header */}
|
|
52
|
+
<div className="flex items-center justify-between">
|
|
53
|
+
<div>
|
|
54
|
+
<h1 className="text-3xl font-bold text-slate-900 dark:text-white flex items-center gap-3">
|
|
55
|
+
<Activity className="text-primary-500" size={32} />
|
|
56
|
+
Observability Dashboard
|
|
57
|
+
</h1>
|
|
58
|
+
<p className="text-slate-500 mt-1">
|
|
59
|
+
System performance and metrics (Last 30 days)
|
|
60
|
+
</p>
|
|
61
|
+
</div>
|
|
62
|
+
<Badge variant="secondary" className="text-xs">
|
|
63
|
+
Auto-refresh: 30s
|
|
64
|
+
</Badge>
|
|
65
|
+
</div>
|
|
66
|
+
|
|
67
|
+
{/* Key Metrics Grid */}
|
|
68
|
+
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
|
|
69
|
+
<MetricCard
|
|
70
|
+
icon={<BarChart2 size={24} />}
|
|
71
|
+
label="Total Runs"
|
|
72
|
+
value={orchestratorMetrics?.total_runs || 0}
|
|
73
|
+
color="blue"
|
|
74
|
+
/>
|
|
75
|
+
<MetricCard
|
|
76
|
+
icon={<CheckCircle size={24} />}
|
|
77
|
+
label="Success Rate"
|
|
78
|
+
value={`${(successRate * 100).toFixed(1)}%`}
|
|
79
|
+
color="emerald"
|
|
80
|
+
trend={successRate >= 0.9 ? 'positive' : successRate >= 0.7 ? 'neutral' : 'negative'}
|
|
81
|
+
/>
|
|
82
|
+
<MetricCard
|
|
83
|
+
icon={<Clock size={24} />}
|
|
84
|
+
label="Avg Duration"
|
|
85
|
+
value={`${(orchestratorMetrics?.avg_duration_seconds || 0).toFixed(2)}s`}
|
|
86
|
+
color="purple"
|
|
87
|
+
/>
|
|
88
|
+
<MetricCard
|
|
89
|
+
icon={<Zap size={24} />}
|
|
90
|
+
label="Cache Hit Rate"
|
|
91
|
+
value={`${(cacheHitRate * 100).toFixed(1)}%`}
|
|
92
|
+
color="amber"
|
|
93
|
+
trend={cacheHitRate >= 0.5 ? 'positive' : cacheHitRate >= 0.2 ? 'neutral' : 'negative'}
|
|
94
|
+
/>
|
|
95
|
+
</div>
|
|
96
|
+
|
|
97
|
+
{/* Detailed Sections */}
|
|
98
|
+
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
|
|
99
|
+
{/* Orchestrator Performance */}
|
|
100
|
+
<Card className="p-6">
|
|
101
|
+
<h3 className="text-lg font-bold text-slate-900 dark:text-white mb-4 flex items-center gap-2">
|
|
102
|
+
<TrendingUp className="text-primary-500" size={20} />
|
|
103
|
+
Orchestrator Performance
|
|
104
|
+
</h3>
|
|
105
|
+
|
|
106
|
+
<div className="space-y-4">
|
|
107
|
+
{/* Status Distribution */}
|
|
108
|
+
<div>
|
|
109
|
+
<h4 className="text-sm font-semibold text-slate-700 dark:text-slate-300 mb-3">
|
|
110
|
+
Status Distribution
|
|
111
|
+
</h4>
|
|
112
|
+
<div className="space-y-2">
|
|
113
|
+
{orchestratorMetrics?.status_distribution &&
|
|
114
|
+
Object.entries(orchestratorMetrics.status_distribution).map(([status, count]) => (
|
|
115
|
+
<div key={status} className="flex items-center justify-between">
|
|
116
|
+
<div className="flex items-center gap-2">
|
|
117
|
+
<div className={`w-3 h-3 rounded-full ${status === 'completed' ? 'bg-emerald-500' :
|
|
118
|
+
status === 'failed' ? 'bg-rose-500' :
|
|
119
|
+
'bg-amber-500'
|
|
120
|
+
}`} />
|
|
121
|
+
<span className="text-sm text-slate-600 dark:text-slate-400 capitalize">
|
|
122
|
+
{status}
|
|
123
|
+
</span>
|
|
124
|
+
</div>
|
|
125
|
+
<div className="flex items-center gap-3">
|
|
126
|
+
<span className="text-sm font-mono font-semibold text-slate-900 dark:text-white">
|
|
127
|
+
{count}
|
|
128
|
+
</span>
|
|
129
|
+
<div className="w-24 bg-slate-200 dark:bg-slate-700 rounded-full h-2">
|
|
130
|
+
<div
|
|
131
|
+
className={`h-2 rounded-full ${status === 'completed' ? 'bg-emerald-500' :
|
|
132
|
+
status === 'failed' ? 'bg-rose-500' :
|
|
133
|
+
'bg-amber-500'
|
|
134
|
+
}`}
|
|
135
|
+
style={{
|
|
136
|
+
width: `${(count / orchestratorMetrics.total_runs) * 100}%`
|
|
137
|
+
}}
|
|
138
|
+
/>
|
|
139
|
+
</div>
|
|
140
|
+
</div>
|
|
141
|
+
</div>
|
|
142
|
+
))
|
|
143
|
+
}
|
|
144
|
+
</div>
|
|
145
|
+
</div>
|
|
146
|
+
|
|
147
|
+
{/* Summary Stats */}
|
|
148
|
+
<div className="grid grid-cols-2 gap-3 pt-4 border-t border-slate-200 dark:border-slate-700">
|
|
149
|
+
<div className="p-3 bg-emerald-50 dark:bg-emerald-900/20 rounded-lg">
|
|
150
|
+
<div className="text-xs text-emerald-600 dark:text-emerald-400 mb-1">
|
|
151
|
+
Success Rate
|
|
152
|
+
</div>
|
|
153
|
+
<div className="text-2xl font-bold text-emerald-700 dark:text-emerald-300">
|
|
154
|
+
{(successRate * 100).toFixed(1)}%
|
|
155
|
+
</div>
|
|
156
|
+
</div>
|
|
157
|
+
<div className="p-3 bg-blue-50 dark:bg-blue-900/20 rounded-lg">
|
|
158
|
+
<div className="text-xs text-blue-600 dark:text-blue-400 mb-1">
|
|
159
|
+
Avg Duration
|
|
160
|
+
</div>
|
|
161
|
+
<div className="text-2xl font-bold text-blue-700 dark:text-blue-300">
|
|
162
|
+
{(orchestratorMetrics?.avg_duration_seconds || 0).toFixed(1)}s
|
|
163
|
+
</div>
|
|
164
|
+
</div>
|
|
165
|
+
</div>
|
|
166
|
+
</div>
|
|
167
|
+
</Card>
|
|
168
|
+
|
|
169
|
+
{/* Cache Performance */}
|
|
170
|
+
<Card className="p-6">
|
|
171
|
+
<h3 className="text-lg font-bold text-slate-900 dark:text-white mb-4 flex items-center gap-2">
|
|
172
|
+
<Zap className="text-amber-500" size={20} />
|
|
173
|
+
Cache Performance
|
|
174
|
+
</h3>
|
|
175
|
+
|
|
176
|
+
<div className="space-y-4">
|
|
177
|
+
{/* Cache Stats */}
|
|
178
|
+
<div className="grid grid-cols-2 gap-4">
|
|
179
|
+
<div className="p-4 bg-gradient-to-br from-amber-50 to-orange-50 dark:from-amber-900/20 dark:to-orange-900/20 rounded-xl border border-amber-100 dark:border-amber-800">
|
|
180
|
+
<div className="text-xs text-amber-600 dark:text-amber-400 mb-1">
|
|
181
|
+
Total Steps
|
|
182
|
+
</div>
|
|
183
|
+
<div className="text-3xl font-bold text-amber-700 dark:text-amber-300">
|
|
184
|
+
{cacheMetrics?.total_steps || 0}
|
|
185
|
+
</div>
|
|
186
|
+
</div>
|
|
187
|
+
<div className="p-4 bg-gradient-to-br from-emerald-50 to-teal-50 dark:from-emerald-900/20 dark:to-teal-900/20 rounded-xl border border-emerald-100 dark:border-emerald-800">
|
|
188
|
+
<div className="text-xs text-emerald-600 dark:text-emerald-400 mb-1">
|
|
189
|
+
Cached Steps
|
|
190
|
+
</div>
|
|
191
|
+
<div className="text-3xl font-bold text-emerald-700 dark:text-emerald-300">
|
|
192
|
+
{cacheMetrics?.cached_steps || 0}
|
|
193
|
+
</div>
|
|
194
|
+
</div>
|
|
195
|
+
</div>
|
|
196
|
+
|
|
197
|
+
{/* Cache Hit Rate Visualization */}
|
|
198
|
+
<div>
|
|
199
|
+
<div className="flex items-center justify-between mb-2">
|
|
200
|
+
<span className="text-sm font-semibold text-slate-700 dark:text-slate-300">
|
|
201
|
+
Cache Hit Rate
|
|
202
|
+
</span>
|
|
203
|
+
<span className="text-lg font-bold text-slate-900 dark:text-white">
|
|
204
|
+
{(cacheHitRate * 100).toFixed(1)}%
|
|
205
|
+
</span>
|
|
206
|
+
</div>
|
|
207
|
+
<div className="relative h-8 bg-slate-200 dark:bg-slate-700 rounded-full overflow-hidden">
|
|
208
|
+
<div
|
|
209
|
+
className="absolute inset-y-0 left-0 bg-gradient-to-r from-emerald-500 to-teal-500 flex items-center justify-end px-3 transition-all duration-500"
|
|
210
|
+
style={{ width: `${cacheHitRate * 100}%` }}
|
|
211
|
+
>
|
|
212
|
+
{cacheHitRate > 0.1 && (
|
|
213
|
+
<span className="text-xs font-bold text-white">
|
|
214
|
+
{(cacheHitRate * 100).toFixed(0)}%
|
|
215
|
+
</span>
|
|
216
|
+
)}
|
|
217
|
+
</div>
|
|
218
|
+
</div>
|
|
219
|
+
</div>
|
|
220
|
+
|
|
221
|
+
{/* Performance Impact */}
|
|
222
|
+
<div className="p-4 bg-blue-50 dark:bg-blue-900/20 rounded-xl border border-blue-100 dark:border-blue-800">
|
|
223
|
+
<div className="flex items-center gap-2 mb-2">
|
|
224
|
+
<TrendingUp size={16} className="text-blue-600" />
|
|
225
|
+
<span className="text-sm font-semibold text-blue-900 dark:text-blue-100">
|
|
226
|
+
Performance Impact
|
|
227
|
+
</span>
|
|
228
|
+
</div>
|
|
229
|
+
<p className="text-sm text-blue-700 dark:text-blue-300">
|
|
230
|
+
Cache saved approximately{' '}
|
|
231
|
+
<strong>{cacheMetrics?.cached_steps || 0}</strong> step executions,
|
|
232
|
+
improving pipeline efficiency.
|
|
233
|
+
</p>
|
|
234
|
+
</div>
|
|
235
|
+
</div>
|
|
236
|
+
</Card>
|
|
237
|
+
</div>
|
|
238
|
+
</div>
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function MetricCard({ icon, label, value, color, trend }) {
|
|
243
|
+
const colorClasses = {
|
|
244
|
+
blue: "bg-blue-50 text-blue-600 dark:bg-blue-900/20 dark:text-blue-400",
|
|
245
|
+
emerald: "bg-emerald-50 text-emerald-600 dark:bg-emerald-900/20 dark:text-emerald-400",
|
|
246
|
+
purple: "bg-purple-50 text-purple-600 dark:bg-purple-900/20 dark:text-purple-400",
|
|
247
|
+
amber: "bg-amber-50 text-amber-600 dark:bg-amber-900/20 dark:text-amber-400",
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
const trendColors = {
|
|
251
|
+
positive: "text-emerald-600 dark:text-emerald-400",
|
|
252
|
+
neutral: "text-amber-600 dark:text-amber-400",
|
|
253
|
+
negative: "text-rose-600 dark:text-rose-400",
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
return (
|
|
257
|
+
<Card className="hover:shadow-lg transition-shadow duration-200">
|
|
258
|
+
<div className="flex items-center gap-4">
|
|
259
|
+
<div className={`p-3 rounded-xl ${colorClasses[color]}`}>
|
|
260
|
+
{icon}
|
|
261
|
+
</div>
|
|
262
|
+
<div className="flex-1">
|
|
263
|
+
<p className="text-sm text-slate-500 dark:text-slate-400 font-medium">{label}</p>
|
|
264
|
+
<div className="flex items-baseline gap-2">
|
|
265
|
+
<p className="text-2xl font-bold text-slate-900 dark:text-white">{value}</p>
|
|
266
|
+
{trend && (
|
|
267
|
+
<TrendingUp
|
|
268
|
+
size={16}
|
|
269
|
+
className={`${trendColors[trend]} ${trend === 'negative' ? 'rotate-180' : ''}`}
|
|
270
|
+
/>
|
|
271
|
+
)}
|
|
272
|
+
</div>
|
|
273
|
+
</div>
|
|
274
|
+
</div>
|
|
275
|
+
</Card>
|
|
276
|
+
);
|
|
277
|
+
}
|