flowyml 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. flowyml/__init__.py +3 -0
  2. flowyml/assets/base.py +10 -0
  3. flowyml/assets/metrics.py +6 -0
  4. flowyml/cli/main.py +108 -2
  5. flowyml/cli/run.py +9 -2
  6. flowyml/core/execution_status.py +52 -0
  7. flowyml/core/hooks.py +106 -0
  8. flowyml/core/observability.py +210 -0
  9. flowyml/core/orchestrator.py +274 -0
  10. flowyml/core/pipeline.py +193 -231
  11. flowyml/core/project.py +34 -2
  12. flowyml/core/remote_orchestrator.py +109 -0
  13. flowyml/core/resources.py +34 -17
  14. flowyml/core/retry_policy.py +80 -0
  15. flowyml/core/scheduler.py +9 -9
  16. flowyml/core/scheduler_config.py +2 -3
  17. flowyml/core/step.py +18 -1
  18. flowyml/core/submission_result.py +53 -0
  19. flowyml/integrations/keras.py +95 -22
  20. flowyml/monitoring/alerts.py +2 -2
  21. flowyml/stacks/__init__.py +15 -0
  22. flowyml/stacks/aws.py +599 -0
  23. flowyml/stacks/azure.py +295 -0
  24. flowyml/stacks/bridge.py +9 -9
  25. flowyml/stacks/components.py +24 -2
  26. flowyml/stacks/gcp.py +158 -11
  27. flowyml/stacks/local.py +5 -0
  28. flowyml/stacks/plugins.py +2 -2
  29. flowyml/stacks/registry.py +21 -0
  30. flowyml/storage/artifacts.py +15 -5
  31. flowyml/storage/materializers/__init__.py +2 -0
  32. flowyml/storage/materializers/base.py +33 -0
  33. flowyml/storage/materializers/cloudpickle.py +74 -0
  34. flowyml/storage/metadata.py +3 -881
  35. flowyml/storage/remote.py +590 -0
  36. flowyml/storage/sql.py +911 -0
  37. flowyml/ui/backend/dependencies.py +28 -0
  38. flowyml/ui/backend/main.py +43 -80
  39. flowyml/ui/backend/routers/assets.py +483 -17
  40. flowyml/ui/backend/routers/client.py +46 -0
  41. flowyml/ui/backend/routers/execution.py +13 -2
  42. flowyml/ui/backend/routers/experiments.py +97 -14
  43. flowyml/ui/backend/routers/metrics.py +168 -0
  44. flowyml/ui/backend/routers/pipelines.py +77 -12
  45. flowyml/ui/backend/routers/projects.py +33 -7
  46. flowyml/ui/backend/routers/runs.py +221 -12
  47. flowyml/ui/backend/routers/schedules.py +5 -21
  48. flowyml/ui/backend/routers/stats.py +14 -0
  49. flowyml/ui/backend/routers/traces.py +37 -53
  50. flowyml/ui/frontend/dist/assets/index-DcYwrn2j.css +1 -0
  51. flowyml/ui/frontend/dist/assets/index-Dlz_ygOL.js +592 -0
  52. flowyml/ui/frontend/dist/index.html +2 -2
  53. flowyml/ui/frontend/src/App.jsx +4 -1
  54. flowyml/ui/frontend/src/app/assets/page.jsx +260 -230
  55. flowyml/ui/frontend/src/app/dashboard/page.jsx +38 -7
  56. flowyml/ui/frontend/src/app/experiments/page.jsx +61 -314
  57. flowyml/ui/frontend/src/app/observability/page.jsx +277 -0
  58. flowyml/ui/frontend/src/app/pipelines/page.jsx +79 -402
  59. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectArtifactsList.jsx +151 -0
  60. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectExperimentsList.jsx +145 -0
  61. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHeader.jsx +45 -0
  62. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectHierarchy.jsx +467 -0
  63. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectMetricsPanel.jsx +253 -0
  64. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectPipelinesList.jsx +105 -0
  65. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRelations.jsx +189 -0
  66. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectRunsList.jsx +136 -0
  67. flowyml/ui/frontend/src/app/projects/[projectId]/_components/ProjectTabs.jsx +95 -0
  68. flowyml/ui/frontend/src/app/projects/[projectId]/page.jsx +326 -0
  69. flowyml/ui/frontend/src/app/projects/page.jsx +13 -3
  70. flowyml/ui/frontend/src/app/runs/[runId]/page.jsx +79 -10
  71. flowyml/ui/frontend/src/app/runs/page.jsx +82 -424
  72. flowyml/ui/frontend/src/app/settings/page.jsx +1 -0
  73. flowyml/ui/frontend/src/app/tokens/page.jsx +62 -16
  74. flowyml/ui/frontend/src/components/AssetDetailsPanel.jsx +373 -0
  75. flowyml/ui/frontend/src/components/AssetLineageGraph.jsx +291 -0
  76. flowyml/ui/frontend/src/components/AssetStatsDashboard.jsx +302 -0
  77. flowyml/ui/frontend/src/components/AssetTreeHierarchy.jsx +477 -0
  78. flowyml/ui/frontend/src/components/ExperimentDetailsPanel.jsx +227 -0
  79. flowyml/ui/frontend/src/components/NavigationTree.jsx +401 -0
  80. flowyml/ui/frontend/src/components/PipelineDetailsPanel.jsx +239 -0
  81. flowyml/ui/frontend/src/components/PipelineGraph.jsx +67 -3
  82. flowyml/ui/frontend/src/components/ProjectSelector.jsx +115 -0
  83. flowyml/ui/frontend/src/components/RunDetailsPanel.jsx +298 -0
  84. flowyml/ui/frontend/src/components/header/Header.jsx +48 -1
  85. flowyml/ui/frontend/src/components/plugins/ZenMLIntegration.jsx +106 -0
  86. flowyml/ui/frontend/src/components/sidebar/Sidebar.jsx +52 -26
  87. flowyml/ui/frontend/src/components/ui/DataView.jsx +35 -17
  88. flowyml/ui/frontend/src/components/ui/ErrorBoundary.jsx +118 -0
  89. flowyml/ui/frontend/src/contexts/ProjectContext.jsx +2 -2
  90. flowyml/ui/frontend/src/contexts/ToastContext.jsx +116 -0
  91. flowyml/ui/frontend/src/layouts/MainLayout.jsx +5 -1
  92. flowyml/ui/frontend/src/router/index.jsx +4 -0
  93. flowyml/ui/frontend/src/utils/date.js +10 -0
  94. flowyml/ui/frontend/src/utils/downloads.js +11 -0
  95. flowyml/utils/config.py +6 -0
  96. flowyml/utils/stack_config.py +45 -3
  97. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/METADATA +44 -4
  98. flowyml-1.4.0.dist-info/RECORD +200 -0
  99. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/licenses/LICENSE +1 -1
  100. flowyml/ui/frontend/dist/assets/index-DFNQnrUj.js +0 -448
  101. flowyml/ui/frontend/dist/assets/index-pWI271rZ.css +0 -1
  102. flowyml-1.2.0.dist-info/RECORD +0 -159
  103. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/WHEEL +0 -0
  104. {flowyml-1.2.0.dist-info → flowyml-1.4.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,28 @@
1
+ """Backend dependencies."""
2
+
3
+ import os
4
+ from flowyml.storage.sql import SQLMetadataStore
5
+ from flowyml.utils.config import get_config
6
+
7
+ _store = None
8
+
9
+
10
+ def get_store() -> SQLMetadataStore:
11
+ """Get the metadata store instance.
12
+
13
+ Uses FLOWYML_DATABASE_URL if set, otherwise defaults to local SQLite.
14
+ """
15
+ global _store
16
+ if _store is None:
17
+ config = get_config()
18
+ db_url = os.environ.get("FLOWYML_DATABASE_URL")
19
+
20
+ # If no explicit URL, use the config's metadata_db path
21
+ if not db_url:
22
+ db_path = config.metadata_db
23
+ # Ensure it's a string path for SQLMetadataStore
24
+ _store = SQLMetadataStore(db_path=str(db_path))
25
+ else:
26
+ _store = SQLMetadataStore(db_url=db_url)
27
+
28
+ return _store
@@ -1,8 +1,12 @@
1
1
  from fastapi import FastAPI
2
2
  from fastapi.middleware.cors import CORSMiddleware
3
3
  from fastapi.staticfiles import StaticFiles
4
- from fastapi.responses import FileResponse
4
+ from fastapi.responses import FileResponse, JSONResponse
5
+ from fastapi.exceptions import RequestValidationError
5
6
  import os
7
+ import traceback
8
+
9
+ from flowyml.monitoring.alerts import alert_manager, AlertLevel
6
10
 
7
11
  # Include API routers
8
12
  from flowyml.ui.backend.routers import (
@@ -17,6 +21,9 @@ from flowyml.ui.backend.routers import (
17
21
  leaderboard,
18
22
  execution,
19
23
  plugins,
24
+ metrics,
25
+ client,
26
+ stats,
20
27
  )
21
28
 
22
29
  app = FastAPI(
@@ -52,6 +59,7 @@ async def get_public_config():
52
59
  "remote_server_url": config.remote_server_url,
53
60
  "remote_ui_url": config.remote_ui_url,
54
61
  "enable_ui": config.enable_ui,
62
+ "remote_services": config.remote_services,
55
63
  }
56
64
 
57
65
 
@@ -65,86 +73,10 @@ app.include_router(schedules.router, prefix="/api/schedules", tags=["schedules"]
65
73
  app.include_router(notifications.router, prefix="/api/notifications", tags=["notifications"])
66
74
  app.include_router(leaderboard.router, prefix="/api/leaderboard", tags=["leaderboard"])
67
75
  app.include_router(execution.router, prefix="/api/execution", tags=["execution"])
76
+ app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
68
77
  app.include_router(plugins.router, prefix="/api", tags=["plugins"])
69
-
70
-
71
- # Stats endpoint for dashboard
72
- @app.get("/api/stats")
73
- async def get_stats(project: str = None):
74
- """Get overall statistics for the dashboard, optionally filtered by project."""
75
- try:
76
- from flowyml.storage.metadata import SQLiteMetadataStore
77
-
78
- store = SQLiteMetadataStore()
79
-
80
- # Get base stats
81
- stats = store.get_statistics()
82
-
83
- # Get run status counts (not in get_statistics yet)
84
- # We can add this to get_statistics later, but for now let's query efficiently
85
- import sqlite3
86
-
87
- conn = sqlite3.connect(store.db_path)
88
- cursor = conn.cursor()
89
-
90
- if project:
91
- cursor.execute(
92
- "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'completed'",
93
- [project],
94
- )
95
- completed_runs = cursor.fetchone()[0]
96
-
97
- cursor.execute(
98
- "SELECT COUNT(*) FROM runs WHERE project = ? AND status = 'failed'",
99
- [project],
100
- )
101
- failed_runs = cursor.fetchone()[0]
102
-
103
- cursor.execute(
104
- "SELECT AVG(duration) FROM runs WHERE project = ? AND duration IS NOT NULL",
105
- [project],
106
- )
107
- avg_duration = cursor.fetchone()[0] or 0
108
-
109
- cursor.execute(
110
- "SELECT COUNT(*) FROM runs WHERE project = ?",
111
- [project],
112
- )
113
- total_runs = cursor.fetchone()[0]
114
- else:
115
- cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'completed'")
116
- completed_runs = cursor.fetchone()[0]
117
-
118
- cursor.execute("SELECT COUNT(*) FROM runs WHERE status = 'failed'")
119
- failed_runs = cursor.fetchone()[0]
120
-
121
- cursor.execute("SELECT AVG(duration) FROM runs WHERE duration IS NOT NULL")
122
- avg_duration = cursor.fetchone()[0] or 0
123
-
124
- cursor.execute("SELECT COUNT(*) FROM runs")
125
- total_runs = cursor.fetchone()[0]
126
-
127
- conn.close()
128
-
129
- return {
130
- "runs": total_runs if project else stats.get("total_runs", 0),
131
- "completed_runs": completed_runs,
132
- "failed_runs": failed_runs,
133
- "pipelines": stats.get("total_pipelines", 0), # TODO: filter by project
134
- "artifacts": stats.get("total_artifacts", 0), # TODO: filter by project
135
- "avg_duration": avg_duration,
136
- }
137
- except Exception as e:
138
- # Return default stats if there's an error
139
- return {
140
- "runs": 0,
141
- "completed_runs": 0,
142
- "failed_runs": 0,
143
- "pipelines": 0,
144
- "artifacts": 0,
145
- "avg_duration": 0,
146
- "error": str(e),
147
- }
78
+ app.include_router(client.router, prefix="/api/client", tags=["client"])
79
+ app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
148
80
 
149
81
 
150
82
  # Static file serving for frontend
@@ -185,3 +117,34 @@ else:
185
117
  "message": "flowyml API is running.",
186
118
  "detail": "Frontend not built. Run 'npm run build' in flowyml/ui/frontend to enable the UI.",
187
119
  }
120
+
121
+
122
+ @app.exception_handler(Exception)
123
+ async def global_exception_handler(request, exc):
124
+ error_msg = str(exc)
125
+ stack_trace = traceback.format_exc()
126
+
127
+ # Log and alert
128
+ alert_manager.send_alert(
129
+ title="Backend API Error",
130
+ message=f"Unhandled exception in {request.method} {request.url.path}: {error_msg}",
131
+ level=AlertLevel.ERROR,
132
+ metadata={"traceback": stack_trace, "path": request.url.path},
133
+ )
134
+
135
+ return JSONResponse(
136
+ status_code=500,
137
+ content={
138
+ "error": "Internal Server Error",
139
+ "message": "Something went wrong on our end. We've been notified.",
140
+ "detail": error_msg, # In prod maybe hide this, but for now it's useful
141
+ },
142
+ )
143
+
144
+
145
+ @app.exception_handler(RequestValidationError)
146
+ async def validation_exception_handler(request, exc):
147
+ return JSONResponse(
148
+ status_code=422,
149
+ content={"error": "Validation Error", "detail": exc.errors()},
150
+ )