flowyml 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowyml/__init__.py +3 -0
- flowyml/assets/base.py +10 -0
- flowyml/assets/metrics.py +6 -0
- flowyml/cli/main.py +108 -2
- flowyml/cli/run.py +9 -2
- flowyml/core/execution_status.py +52 -0
- flowyml/core/hooks.py +106 -0
- flowyml/core/observability.py +210 -0
- flowyml/core/orchestrator.py +274 -0
- flowyml/core/pipeline.py +193 -231
- flowyml/core/project.py +34 -2
- flowyml/core/remote_orchestrator.py +109 -0
- flowyml/core/resources.py +34 -17
- flowyml/core/retry_policy.py +80 -0
- flowyml/core/scheduler.py +9 -9
- flowyml/core/scheduler_config.py +2 -3
- flowyml/core/step.py +18 -1
- flowyml/core/submission_result.py +53 -0
- flowyml/integrations/keras.py +95 -22
- flowyml/monitoring/alerts.py +2 -2
- flowyml/stacks/__init__.py +15 -0
- flowyml/stacks/aws.py +599 -0
- flowyml/stacks/azure.py +295 -0
- flowyml/stacks/bridge.py +9 -9
- flowyml/stacks/components.py +24 -2
- flowyml/stacks/gcp.py +158 -11
- flowyml/stacks/local.py +5 -0
- flowyml/stacks/plugins.py +2 -2
- flowyml/stacks/registry.py +21 -0
- flowyml/storage/artifacts.py +15 -5
- flowyml/storage/materializers/__init__.py +2 -0
- flowyml/storage/materializers/base.py +33 -0
- flowyml/storage/materializers/cloudpickle.py +74 -0
- flowyml/storage/metadata.py +3 -881
- flowyml/storage/remote.py +590 -0
- flowyml/storage/sql.py +911 -0
- flowyml/ui/backend/dependencies.py +28 -0
- flowyml/ui/backend/main.py +43 -80
- flowyml/ui/backend/routers/assets.py +483 -17
- flowyml/ui/backend/routers/client.py +46 -0
- flowyml/ui/backend/routers/execution.py +13 -2
- flowyml/ui/backend/routers/experiments.py +97 -14
- flowyml/ui/backend/routers/metrics.py +168 -0
- flowyml/ui/backend/routers/pipelines.py +77 -12
- flowyml/ui/backend/routers/projects.py +33 -7
- flowyml/ui/backend/routers/runs.py +221 -12
- flowyml/ui/backend/routers/schedules.py +5 -21
- flowyml/ui/backend/routers/stats.py +14 -0
- flowyml/ui/backend/routers/traces.py +37 -53
- flowyml/ui/frontend/dist/assets/index-DcYwrn2j.css +1 -0
- flowyml/ui/frontend/dist/assets/index-Dlz_ygOL.js +592 -0
- flowyml/ui/frontend/dist/index.html +2 -2
- flowyml/ui/frontend/src/App.jsx +4 -1
- flowyml/ui/frontend/src/app/assets/page.jsx +260 -230
- flowyml/ui/frontend/src/app/dashboard/page.jsx +38 -7
- flowyml/ui/frontend/src/app/experiments/page.jsx +61 -314
- flowyml/ui/frontend/src/app/observability/page.jsx +277 -0
- flowyml/ui/frontend/src/app/pipelines/page.jsx +79 -402
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectArtifactsList.jsx +151 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectExperimentsList.jsx +145 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHeader.jsx +45 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHierarchy.jsx +467 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectMetricsPanel.jsx +253 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectPipelinesList.jsx +105 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRelations.jsx +189 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRunsList.jsx +136 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectTabs.jsx +95 -0
- flowyml/ui/frontend/src/app/projects/[projectId]/page.jsx +326 -0
- flowyml/ui/frontend/src/app/projects/page.jsx +13 -3
- flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +79 -10
- flowyml/ui/frontend/src/app/runs/page.jsx +82 -424
- flowyml/ui/frontend/src/app/settings/page.jsx +1 -0
- flowyml/ui/frontend/src/app/tokens/page.jsx +62 -16
- flowyml/ui/frontend/src/components/AssetDetailsPanel.jsx +373 -0
- flowyml/ui/frontend/src/components/AssetLineageGraph.jsx +291 -0
- flowyml/ui/frontend/src/components/AssetStatsDashboard.jsx +302 -0
- flowyml/ui/frontend/src/components/AssetTreeHierarchy.jsx +477 -0
- flowyml/ui/frontend/src/components/ExperimentDetailsPanel.jsx +227 -0
- flowyml/ui/frontend/src/components/NavigationTree.jsx +401 -0
- flowyml/ui/frontend/src/components/PipelineDetailsPanel.jsx +239 -0
- flowyml/ui/frontend/src/components/PipelineGraph.jsx +67 -3
- flowyml/ui/frontend/src/components/ProjectSelector.jsx +115 -0
- flowyml/ui/frontend/src/components/RunDetailsPanel.jsx +298 -0
- flowyml/ui/frontend/src/components/header/Header.jsx +48 -1
- flowyml/ui/frontend/src/components/plugins/ZenMLIntegration.jsx +106 -0
- flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +52 -26
- flowyml/ui/frontend/src/components/ui/DataView.jsx +35 -17
- flowyml/ui/frontend/src/components/ui/ErrorBoundary.jsx +118 -0
- flowyml/ui/frontend/src/contexts/ProjectContext.jsx +2 -2
- flowyml/ui/frontend/src/contexts/ToastContext.jsx +116 -0
- flowyml/ui/frontend/src/layouts/MainLayout.jsx +5 -1
- flowyml/ui/frontend/src/router/index.jsx +4 -0
- flowyml/ui/frontend/src/utils/date.js +10 -0
- flowyml/ui/frontend/src/utils/downloads.js +11 -0
- flowyml/utils/config.py +6 -0
- flowyml/utils/stack_config.py +45 -3
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/METADATA +44 -4
- flowyml-1.4.0.dist-info/RECORD +200 -0
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/licenses/LICENSE +1 -1
- flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +0 -448
- flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +0 -1
- flowyml-1.2.0.dist-info/RECORD +0 -159
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/WHEEL +0 -0
- {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Backend dependencies."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from flowyml.storage.sql import SQLMetadataStore
|
|
5
|
+
from flowyml.utils.config import get_config
|
|
6
|
+
|
|
7
|
+
_store = None
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_store() -> SQLMetadataStore:
|
|
11
|
+
"""Get the metadata store instance.
|
|
12
|
+
|
|
13
|
+
Uses FLOWYML_DATABASE_URL if set, otherwise defaults to local SQLite.
|
|
14
|
+
"""
|
|
15
|
+
global _store
|
|
16
|
+
if _store is None:
|
|
17
|
+
config = get_config()
|
|
18
|
+
db_url = os.environ.get("FLOWYML_DATABASE_URL")
|
|
19
|
+
|
|
20
|
+
# If no explicit URL, use the config's metadata_db path
|
|
21
|
+
if not db_url:
|
|
22
|
+
db_path = config.metadata_db
|
|
23
|
+
# Ensure it's a string path for SQLMetadataStore
|
|
24
|
+
_store = SQLMetadataStore(db_path=str(db_path))
|
|
25
|
+
else:
|
|
26
|
+
_store = SQLMetadataStore(db_url=db_url)
|
|
27
|
+
|
|
28
|
+
return _store
|
flowyml/ui/backend/main.py
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
from fastapi import FastAPI
|
|
2
2
|
from fastapi.middleware.cors import CORSMiddleware
|
|
3
3
|
from fastapi.staticfiles import StaticFiles
|
|
4
|
-
from fastapi.responses import FileResponse
|
|
4
|
+
from fastapi.responses import FileResponse, JSONResponse
|
|
5
|
+
from fastapi.exceptions import RequestValidationError
|
|
5
6
|
import os
|
|
7
|
+
import traceback
|
|
8
|
+
|
|
9
|
+
from flowyml.monitoring.alerts import alert_manager, AlertLevel
|
|
6
10
|
|
|
7
11
|
# Include API routers
|
|
8
12
|
from flowyml.ui.backend.routers import (
|
|
@@ -17,6 +21,9 @@ from flowyml.ui.backend.routers import (
|
|
|
17
21
|
leaderboard,
|
|
18
22
|
execution,
|
|
19
23
|
plugins,
|
|
24
|
+
metrics,
|
|
25
|
+
client,
|
|
26
|
+
stats,
|
|
20
27
|
)
|
|
21
28
|
|
|
22
29
|
app = FastAPI(
|
|
@@ -52,6 +59,7 @@ async def get_public_config():
|
|
|
52
59
|
"remote_server_url": config.remote_server_url,
|
|
53
60
|
"remote_ui_url": config.remote_ui_url,
|
|
54
61
|
"enable_ui": config.enable_ui,
|
|
62
|
+
"remote_services": config.remote_services,
|
|
55
63
|
}
|
|
56
64
|
|
|
57
65
|
|
|
@@ -65,86 +73,10 @@ app.include_router(schedules.router, prefix="/api/schedules", tags=["schedules"]
|
|
|
65
73
|
app.include_router(notifications.router, prefix="/api/notifications", tags=["notifications"])
|
|
66
74
|
app.include_router(leaderboard.router, prefix="/api/leaderboard", tags=["leaderboard"])
|
|
67
75
|
app.include_router(execution.router, prefix="/api/execution", tags=["execution"])
|
|
76
|
+
app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
|
|
68
77
|
app.include_router(plugins.router, prefix="/api", tags=["plugins"])
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
# Stats endpoint for dashboard
|
|
72
|
-
@app.get("/api/stats")
|
|
73
|
-
async def get_stats(project: str = None):
|
|
74
|
-
"""Get overall statistics for the dashboard, optionally filtered by project."""
|
|
75
|
-
try:
|
|
76
|
-
from flowyml.storage.metadata import SQLiteMetadataStore
|
|
77
|
-
|
|
78
|
-
store = SQLiteMetadataStore()
|
|
79
|
-
|
|
80
|
-
# Get base stats
|
|
81
|
-
stats = store.get_statistics()
|
|
82
|
-
|
|
83
|
-
# Get run status counts (not in get_statistics yet)
|
|
84
|
-
# We can add this to get_statistics later, but for now let's query efficiently
|
|
85
|
-
import sqlite3
|
|
86
|
-
|
|
87
|
-
conn = sqlite3.connect(store.db_path)
|
|
88
|
-
cursor = conn.cursor()
|
|
89
|
-
|
|
90
|
-
if project:
|
|
91
|
-
cursor.execute(
|
|
92
|
-
"SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'completed'",
|
|
93
|
-
[project],
|
|
94
|
-
)
|
|
95
|
-
completed_runs = cursor.fetchone()[0]
|
|
96
|
-
|
|
97
|
-
cursor.execute(
|
|
98
|
-
"SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'failed'",
|
|
99
|
-
[project],
|
|
100
|
-
)
|
|
101
|
-
failed_runs = cursor.fetchone()[0]
|
|
102
|
-
|
|
103
|
-
cursor.execute(
|
|
104
|
-
"SELECT AVG(duration) FROM runs WHERE project = ? AND duration IS NOT NULL",
|
|
105
|
-
[project],
|
|
106
|
-
)
|
|
107
|
-
avg_duration = cursor.fetchone()[0] or 0
|
|
108
|
-
|
|
109
|
-
cursor.execute(
|
|
110
|
-
"SELECT COUNT(*) FROM runs WHERE project = ?",
|
|
111
|
-
[project],
|
|
112
|
-
)
|
|
113
|
-
total_runs = cursor.fetchone()[0]
|
|
114
|
-
else:
|
|
115
|
-
cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'completed'")
|
|
116
|
-
completed_runs = cursor.fetchone()[0]
|
|
117
|
-
|
|
118
|
-
cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'failed'")
|
|
119
|
-
failed_runs = cursor.fetchone()[0]
|
|
120
|
-
|
|
121
|
-
cursor.execute("SELECT AVG(duration) FROM runs WHERE duration IS NOT NULL")
|
|
122
|
-
avg_duration = cursor.fetchone()[0] or 0
|
|
123
|
-
|
|
124
|
-
cursor.execute("SELECT COUNT(*) FROM runs")
|
|
125
|
-
total_runs = cursor.fetchone()[0]
|
|
126
|
-
|
|
127
|
-
conn.close()
|
|
128
|
-
|
|
129
|
-
return {
|
|
130
|
-
"runs": total_runs if project else stats.get("total_runs", 0),
|
|
131
|
-
"completed_runs": completed_runs,
|
|
132
|
-
"failed_runs": failed_runs,
|
|
133
|
-
"pipelines": stats.get("total_pipelines", 0), # TODO: filter by project
|
|
134
|
-
"artifacts": stats.get("total_artifacts", 0), # TODO: filter by project
|
|
135
|
-
"avg_duration": avg_duration,
|
|
136
|
-
}
|
|
137
|
-
except Exception as e:
|
|
138
|
-
# Return default stats if there's an error
|
|
139
|
-
return {
|
|
140
|
-
"runs": 0,
|
|
141
|
-
"completed_runs": 0,
|
|
142
|
-
"failed_runs": 0,
|
|
143
|
-
"pipelines": 0,
|
|
144
|
-
"artifacts": 0,
|
|
145
|
-
"avg_duration": 0,
|
|
146
|
-
"error": str(e),
|
|
147
|
-
}
|
|
78
|
+
app.include_router(client.router, prefix="/api/client", tags=["client"])
|
|
79
|
+
app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
|
|
148
80
|
|
|
149
81
|
|
|
150
82
|
# Static file serving for frontend
|
|
@@ -185,3 +117,34 @@ else:
|
|
|
185
117
|
"message": "flowyml API is running.",
|
|
186
118
|
"detail": "Frontend not built. Run 'npm run build' in flowyml/ui/frontend to enable the UI.",
|
|
187
119
|
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@app.exception_handler(Exception)
|
|
123
|
+
async def global_exception_handler(request, exc):
|
|
124
|
+
error_msg = str(exc)
|
|
125
|
+
stack_trace = traceback.format_exc()
|
|
126
|
+
|
|
127
|
+
# Log and alert
|
|
128
|
+
alert_manager.send_alert(
|
|
129
|
+
title="Backend API Error",
|
|
130
|
+
message=f"Unhandled exception in {request.method} {request.url.path}: {error_msg}",
|
|
131
|
+
level=AlertLevel.ERROR,
|
|
132
|
+
metadata={"traceback": stack_trace, "path": request.url.path},
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
return JSONResponse(
|
|
136
|
+
status_code=500,
|
|
137
|
+
content={
|
|
138
|
+
"error": "Internal Server Error",
|
|
139
|
+
"message": "Something went wrong on our end. We've been notified.",
|
|
140
|
+
"detail": error_msg, # In prod maybe hide this, but for now it's useful
|
|
141
|
+
},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@app.exception_handler(RequestValidationError)
|
|
146
|
+
async def validation_exception_handler(request, exc):
|
|
147
|
+
return JSONResponse(
|
|
148
|
+
status_code=422,
|
|
149
|
+
content={"error": "Validation Error", "detail": exc.errors()},
|
|
150
|
+
)
|