flowyml 1.5.0__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowyml/__init__.py +2 -1
- flowyml/assets/featureset.py +30 -5
- flowyml/assets/metrics.py +47 -4
- flowyml/cli/main.py +397 -0
- flowyml/cli/models.py +444 -0
- flowyml/cli/rich_utils.py +95 -0
- flowyml/core/checkpoint.py +6 -1
- flowyml/core/conditional.py +104 -0
- flowyml/core/display.py +595 -0
- flowyml/core/executor.py +27 -6
- flowyml/core/orchestrator.py +500 -7
- flowyml/core/pipeline.py +447 -11
- flowyml/core/project.py +4 -1
- flowyml/core/scheduler.py +225 -81
- flowyml/core/versioning.py +13 -4
- flowyml/registry/model_registry.py +1 -1
- flowyml/ui/frontend/dist/assets/index-By4trVyv.css +1 -0
- flowyml/ui/frontend/dist/assets/{index-DF8dJaFL.js → index-CX5RV2C9.js} +118 -117
- flowyml/ui/frontend/dist/index.html +2 -2
- flowyml/ui/frontend/src/components/PipelineGraph.jsx +43 -4
- flowyml/ui/server_manager.py +189 -0
- flowyml/ui/utils.py +66 -2
- flowyml/utils/config.py +7 -0
- {flowyml-1.5.0.dist-info → flowyml-1.7.0.dist-info}/METADATA +5 -3
- {flowyml-1.5.0.dist-info → flowyml-1.7.0.dist-info}/RECORD +28 -24
- flowyml/ui/frontend/dist/assets/index-CBUXOWze.css +0 -1
- {flowyml-1.5.0.dist-info → flowyml-1.7.0.dist-info}/WHEEL +0 -0
- {flowyml-1.5.0.dist-info → flowyml-1.7.0.dist-info}/entry_points.txt +0 -0
- {flowyml-1.5.0.dist-info → flowyml-1.7.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
<meta charset="UTF-8" />
|
|
6
6
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
7
7
|
<title>FlowyML</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
9
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-CX5RV2C9.js"></script>
|
|
9
|
+
<link rel="stylesheet" crossorigin href="/assets/index-By4trVyv.css">
|
|
10
10
|
</head>
|
|
11
11
|
|
|
12
12
|
<body>
|
|
@@ -83,10 +83,37 @@ export function PipelineGraph({ dag, steps, selectedStep, onStepSelect, onArtifa
|
|
|
83
83
|
return id;
|
|
84
84
|
};
|
|
85
85
|
|
|
86
|
+
// Map execution groups to colors
|
|
87
|
+
const groupColors = {};
|
|
88
|
+
const groupColorPalette = [
|
|
89
|
+
{ bg: 'bg-blue-50 dark:bg-blue-900/20', border: 'border-blue-400 dark:border-blue-500', text: 'text-blue-700 dark:text-blue-300', badge: 'bg-blue-100 dark:bg-blue-800 text-blue-700 dark:text-blue-300' },
|
|
90
|
+
{ bg: 'bg-purple-50 dark:bg-purple-900/20', border: 'border-purple-400 dark:border-purple-500', text: 'text-purple-700 dark:text-purple-300', badge: 'bg-purple-100 dark:bg-purple-800 text-purple-700 dark:text-purple-300' },
|
|
91
|
+
{ bg: 'bg-green-50 dark:bg-green-900/20', border: 'border-green-400 dark:border-green-500', text: 'text-green-700 dark:text-green-300', badge: 'bg-green-100 dark:bg-green-800 text-green-700 dark:text-green-300' },
|
|
92
|
+
{ bg: 'bg-orange-50 dark:bg-orange-900/20', border: 'border-orange-400 dark:border-orange-500', text: 'text-orange-700 dark:text-orange-300', badge: 'bg-orange-100 dark:bg-orange-800 text-orange-700 dark:text-orange-300' },
|
|
93
|
+
{ bg: 'bg-pink-50 dark:bg-pink-900/20', border: 'border-pink-400 dark:border-pink-500', text: 'text-pink-700 dark:text-pink-300', badge: 'bg-pink-100 dark:bg-pink-800 text-pink-700 dark:text-pink-300' },
|
|
94
|
+
{ bg: 'bg-cyan-50 dark:bg-cyan-900/20', border: 'border-cyan-400 dark:border-cyan-500', text: 'text-cyan-700 dark:text-cyan-300', badge: 'bg-cyan-100 dark:bg-cyan-800 text-cyan-700 dark:text-cyan-300' },
|
|
95
|
+
];
|
|
96
|
+
|
|
97
|
+
// First pass: collect all execution groups
|
|
98
|
+
const executionGroups = new Set();
|
|
99
|
+
dag.nodes.forEach(node => {
|
|
100
|
+
const stepData = steps?.[node.id] || {};
|
|
101
|
+
if (stepData.execution_group) {
|
|
102
|
+
executionGroups.add(stepData.execution_group);
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// Assign colors to groups
|
|
107
|
+
Array.from(executionGroups).forEach((group, idx) => {
|
|
108
|
+
groupColors[group] = groupColorPalette[idx % groupColorPalette.length];
|
|
109
|
+
});
|
|
110
|
+
|
|
86
111
|
// 1. Create Step Nodes and Connections
|
|
87
112
|
dag.nodes.forEach(node => {
|
|
88
113
|
const stepData = steps?.[node.id] || {};
|
|
89
114
|
const status = stepData.success ? 'success' : stepData.error ? 'failed' : stepData.running ? 'running' : 'pending';
|
|
115
|
+
const executionGroup = stepData.execution_group;
|
|
116
|
+
const groupColor = executionGroup ? groupColors[executionGroup] : null;
|
|
90
117
|
|
|
91
118
|
nodes.push({
|
|
92
119
|
id: node.id,
|
|
@@ -96,7 +123,9 @@ export function PipelineGraph({ dag, steps, selectedStep, onStepSelect, onArtifa
|
|
|
96
123
|
status,
|
|
97
124
|
duration: stepData.duration,
|
|
98
125
|
cached: stepData.cached,
|
|
99
|
-
selected: selectedStep === node.id
|
|
126
|
+
selected: selectedStep === node.id,
|
|
127
|
+
execution_group: executionGroup,
|
|
128
|
+
groupColor: groupColor
|
|
100
129
|
}
|
|
101
130
|
});
|
|
102
131
|
|
|
@@ -234,12 +263,15 @@ function CustomStepNode({ data }) {
|
|
|
234
263
|
};
|
|
235
264
|
|
|
236
265
|
const config = statusConfig[data.status] || statusConfig.pending;
|
|
266
|
+
const groupColor = data.groupColor;
|
|
267
|
+
const hasGroup = data.execution_group && groupColor;
|
|
237
268
|
|
|
238
269
|
return (
|
|
239
270
|
<div
|
|
240
271
|
className={`
|
|
241
272
|
relative px-4 py-3 rounded-lg border-2 transition-all duration-200
|
|
242
|
-
${
|
|
273
|
+
${hasGroup ? groupColor.bg : config.bg}
|
|
274
|
+
${hasGroup ? groupColor.border : config.border}
|
|
243
275
|
${data.selected ? `ring-4 ${config.ring} shadow-lg` : `hover:shadow-md ${config.shadow}`}
|
|
244
276
|
`}
|
|
245
277
|
style={{ width: stepNodeWidth, height: stepNodeHeight }}
|
|
@@ -252,10 +284,17 @@ function CustomStepNode({ data }) {
|
|
|
252
284
|
{config.icon}
|
|
253
285
|
</div>
|
|
254
286
|
<div className="min-w-0 flex-1">
|
|
255
|
-
<h3 className=
|
|
287
|
+
<h3 className={`font-bold text-sm truncate ${hasGroup ? groupColor.text : 'text-slate-900 dark:text-white'}`} title={data.label}>
|
|
256
288
|
{data.label}
|
|
257
289
|
</h3>
|
|
258
|
-
<
|
|
290
|
+
<div className="flex items-center gap-2 mt-0.5">
|
|
291
|
+
<p className="text-xs text-slate-500 capitalize">{data.status}</p>
|
|
292
|
+
{hasGroup && (
|
|
293
|
+
<span className={`text-[10px] font-semibold px-1.5 py-0.5 rounded ${groupColor.badge}`}>
|
|
294
|
+
{data.execution_group}
|
|
295
|
+
</span>
|
|
296
|
+
)}
|
|
297
|
+
</div>
|
|
259
298
|
</div>
|
|
260
299
|
</div>
|
|
261
300
|
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""UI Server Manager - Auto-start and manage the UI server in background."""
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
import time
|
|
5
|
+
import subprocess
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from flowyml.ui.utils import is_ui_running, get_ui_url, get_ui_host_port
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class UIServerManager:
|
|
12
|
+
"""Manages the UI server lifecycle in background threads."""
|
|
13
|
+
|
|
14
|
+
_instance: Optional["UIServerManager"] = None
|
|
15
|
+
_lock = threading.Lock()
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
self._server_thread: Optional[threading.Thread] = None
|
|
19
|
+
self._server_process: Optional[subprocess.Popen] = None
|
|
20
|
+
# Initialize from config/env vars
|
|
21
|
+
self._host, self._port = get_ui_host_port()
|
|
22
|
+
self._running = False
|
|
23
|
+
self._started = False
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def get_instance(cls) -> "UIServerManager":
|
|
27
|
+
"""Get singleton instance of UI server manager."""
|
|
28
|
+
if cls._instance is None:
|
|
29
|
+
with cls._lock:
|
|
30
|
+
if cls._instance is None:
|
|
31
|
+
cls._instance = cls()
|
|
32
|
+
return cls._instance
|
|
33
|
+
|
|
34
|
+
def ensure_running(self, host: str | None = None, port: int | None = None, auto_start: bool = True) -> bool:
|
|
35
|
+
"""Ensure UI server is running, start it if not and auto_start is True.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
host: Host to bind to (uses config/env if None)
|
|
39
|
+
port: Port to bind to (uses config/env if None)
|
|
40
|
+
auto_start: If True, automatically start server if not running
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
True if server is running, False otherwise
|
|
44
|
+
"""
|
|
45
|
+
# Use provided values or get from config
|
|
46
|
+
if host is None or port is None:
|
|
47
|
+
config_host, config_port = get_ui_host_port()
|
|
48
|
+
self._host = host if host is not None else config_host
|
|
49
|
+
self._port = port if port is not None else config_port
|
|
50
|
+
else:
|
|
51
|
+
self._host = host
|
|
52
|
+
self._port = port
|
|
53
|
+
|
|
54
|
+
# Check if already running
|
|
55
|
+
if is_ui_running(host, port):
|
|
56
|
+
return True
|
|
57
|
+
|
|
58
|
+
if not auto_start:
|
|
59
|
+
return False
|
|
60
|
+
|
|
61
|
+
# Try to start the server
|
|
62
|
+
return self.start(host, port)
|
|
63
|
+
|
|
64
|
+
def start(self, host: str | None = None, port: int | None = None) -> bool:
|
|
65
|
+
"""Start the UI server in a background thread.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
host: Host to bind to (uses config/env if None)
|
|
69
|
+
port: Port to bind to (uses config/env if None)
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
True if started successfully, False otherwise
|
|
73
|
+
"""
|
|
74
|
+
# Use provided values or get from config
|
|
75
|
+
if host is None or port is None:
|
|
76
|
+
config_host, config_port = get_ui_host_port()
|
|
77
|
+
self._host = host if host is not None else config_host
|
|
78
|
+
self._port = port if port is not None else config_port
|
|
79
|
+
else:
|
|
80
|
+
self._host = host
|
|
81
|
+
self._port = port
|
|
82
|
+
|
|
83
|
+
if self._running:
|
|
84
|
+
return is_ui_running(self._host, self._port)
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
# Check if UI dependencies are available
|
|
88
|
+
try:
|
|
89
|
+
import uvicorn # noqa: F401 - just check import
|
|
90
|
+
except ImportError:
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
# Capture host/port for closure
|
|
94
|
+
server_host = self._host
|
|
95
|
+
server_port = self._port
|
|
96
|
+
startup_error = {"error": None}
|
|
97
|
+
|
|
98
|
+
# Start server in a daemon thread
|
|
99
|
+
def run_server():
|
|
100
|
+
try:
|
|
101
|
+
import uvicorn
|
|
102
|
+
|
|
103
|
+
# Run uvicorn server (blocking call, but in daemon thread)
|
|
104
|
+
uvicorn.run(
|
|
105
|
+
"flowyml.ui.backend.main:app",
|
|
106
|
+
host=server_host,
|
|
107
|
+
port=server_port,
|
|
108
|
+
log_level="warning", # Show startup issues
|
|
109
|
+
access_log=False,
|
|
110
|
+
)
|
|
111
|
+
except Exception as e:
|
|
112
|
+
startup_error["error"] = str(e)
|
|
113
|
+
|
|
114
|
+
# Start in daemon thread
|
|
115
|
+
self._server_thread = threading.Thread(
|
|
116
|
+
target=run_server,
|
|
117
|
+
daemon=True,
|
|
118
|
+
name="flowyml-ui-server",
|
|
119
|
+
)
|
|
120
|
+
self._server_thread.start()
|
|
121
|
+
self._running = True
|
|
122
|
+
self._started = True
|
|
123
|
+
|
|
124
|
+
# Wait a bit for server to start (up to 8 seconds)
|
|
125
|
+
max_wait = 8
|
|
126
|
+
for _ in range(max_wait * 10): # Check every 100ms
|
|
127
|
+
time.sleep(0.1)
|
|
128
|
+
# Check if server started successfully
|
|
129
|
+
if is_ui_running(server_host, server_port):
|
|
130
|
+
return True
|
|
131
|
+
# Check if we have an error
|
|
132
|
+
if startup_error["error"]:
|
|
133
|
+
self._running = False
|
|
134
|
+
return False
|
|
135
|
+
|
|
136
|
+
# If we get here, server didn't start in time
|
|
137
|
+
self._running = False
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
except Exception:
|
|
141
|
+
self._running = False
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
def stop(self) -> None:
|
|
145
|
+
"""Stop the UI server."""
|
|
146
|
+
# Since we're using a daemon thread, it will be killed when main process exits
|
|
147
|
+
# For now, we just mark it as stopped
|
|
148
|
+
self._running = False
|
|
149
|
+
self._server_thread = None
|
|
150
|
+
|
|
151
|
+
def get_url(self) -> Optional[str]:
|
|
152
|
+
"""Get the URL of the running UI server.
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
URL string if server is running, None otherwise
|
|
156
|
+
"""
|
|
157
|
+
return get_ui_url(self._host, self._port)
|
|
158
|
+
|
|
159
|
+
def is_running(self) -> bool:
|
|
160
|
+
"""Check if UI server is running."""
|
|
161
|
+
return is_ui_running(self._host, self._port)
|
|
162
|
+
|
|
163
|
+
def get_run_url(self, run_id: str) -> Optional[str]:
|
|
164
|
+
"""Get URL to view a specific pipeline run.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
run_id: ID of the pipeline run
|
|
168
|
+
|
|
169
|
+
Returns:
|
|
170
|
+
URL string if server is running, None otherwise
|
|
171
|
+
"""
|
|
172
|
+
base_url = self.get_url()
|
|
173
|
+
if base_url:
|
|
174
|
+
return f"{base_url}/runs/{run_id}"
|
|
175
|
+
return None
|
|
176
|
+
|
|
177
|
+
def get_pipeline_url(self, pipeline_name: str) -> Optional[str]:
|
|
178
|
+
"""Get URL to view a specific pipeline.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
pipeline_name: Name of the pipeline
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
URL string if server is running, None otherwise
|
|
185
|
+
"""
|
|
186
|
+
base_url = self.get_url()
|
|
187
|
+
if base_url:
|
|
188
|
+
return f"{base_url}/pipelines/{pipeline_name}"
|
|
189
|
+
return None
|
flowyml/ui/utils.py
CHANGED
|
@@ -1,8 +1,69 @@
|
|
|
1
1
|
"""UI utility functions for checking UI server status and getting URLs."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
3
4
|
import http.client
|
|
4
5
|
|
|
5
6
|
|
|
7
|
+
def get_ui_server_url() -> str:
|
|
8
|
+
"""Get the UI server URL from configuration or environment variables.
|
|
9
|
+
|
|
10
|
+
Priority order:
|
|
11
|
+
1. FLOWYML_SERVER_URL environment variable (explicit override)
|
|
12
|
+
2. FLOWYML_REMOTE_UI_URL from config (for centralized deployments)
|
|
13
|
+
3. FLOWYML_UI_HOST and FLOWYML_UI_PORT from config/env
|
|
14
|
+
4. Default: http://localhost:8080
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
Base URL of the UI server (e.g., "http://localhost:8080" or "https://flowyml.example.com")
|
|
18
|
+
"""
|
|
19
|
+
# Check for explicit server URL override
|
|
20
|
+
server_url = os.getenv("FLOWYML_SERVER_URL")
|
|
21
|
+
if server_url:
|
|
22
|
+
return server_url.rstrip("/")
|
|
23
|
+
|
|
24
|
+
# Check for remote UI URL (centralized deployment)
|
|
25
|
+
try:
|
|
26
|
+
from flowyml.utils.config import get_config
|
|
27
|
+
|
|
28
|
+
config = get_config()
|
|
29
|
+
if config.remote_ui_url:
|
|
30
|
+
return config.remote_ui_url.rstrip("/")
|
|
31
|
+
|
|
32
|
+
# Use config values for host/port
|
|
33
|
+
host = os.getenv("FLOWYML_UI_HOST", config.ui_host)
|
|
34
|
+
port = int(os.getenv("FLOWYML_UI_PORT", str(config.ui_port)))
|
|
35
|
+
|
|
36
|
+
# Determine protocol based on port (443 = https, else http)
|
|
37
|
+
protocol = "https" if port == 443 else "http"
|
|
38
|
+
|
|
39
|
+
return f"{protocol}://{host}:{port}"
|
|
40
|
+
except Exception:
|
|
41
|
+
# Fallback to defaults
|
|
42
|
+
host = os.getenv("FLOWYML_UI_HOST", "localhost")
|
|
43
|
+
port = int(os.getenv("FLOWYML_UI_PORT", "8080"))
|
|
44
|
+
protocol = "https" if port == 443 else "http"
|
|
45
|
+
return f"{protocol}://{host}:{port}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_ui_host_port() -> tuple[str, int]:
|
|
49
|
+
"""Get UI host and port from configuration.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
Tuple of (host, port)
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
from flowyml.utils.config import get_config
|
|
56
|
+
|
|
57
|
+
config = get_config()
|
|
58
|
+
host = os.getenv("FLOWYML_UI_HOST", config.ui_host)
|
|
59
|
+
port = int(os.getenv("FLOWYML_UI_PORT", str(config.ui_port)))
|
|
60
|
+
return (host, port)
|
|
61
|
+
except Exception:
|
|
62
|
+
host = os.getenv("FLOWYML_UI_HOST", "localhost")
|
|
63
|
+
port = int(os.getenv("FLOWYML_UI_PORT", "8080"))
|
|
64
|
+
return (host, port)
|
|
65
|
+
|
|
66
|
+
|
|
6
67
|
def is_ui_running(host: str = "localhost", port: int = 8080) -> bool:
|
|
7
68
|
"""Check if the flowyml UI server is running.
|
|
8
69
|
|
|
@@ -17,12 +78,14 @@ def is_ui_running(host: str = "localhost", port: int = 8080) -> bool:
|
|
|
17
78
|
conn = http.client.HTTPConnection(host, port, timeout=2)
|
|
18
79
|
conn.request("GET", "/api/health")
|
|
19
80
|
response = conn.getresponse()
|
|
20
|
-
conn.close()
|
|
21
81
|
|
|
22
82
|
# Check if response is successful and from flowyml
|
|
83
|
+
# Note: must read data BEFORE closing connection
|
|
23
84
|
if response.status == 200:
|
|
24
85
|
data = response.read().decode("utf-8")
|
|
86
|
+
conn.close()
|
|
25
87
|
return "flowyml" in data.lower() or "ok" in data.lower()
|
|
88
|
+
conn.close()
|
|
26
89
|
return False
|
|
27
90
|
except Exception:
|
|
28
91
|
return False
|
|
@@ -39,7 +102,8 @@ def get_ui_url(host: str = "localhost", port: int = 8080) -> str | None:
|
|
|
39
102
|
URL string if server is running, None otherwise
|
|
40
103
|
"""
|
|
41
104
|
if is_ui_running(host, port):
|
|
42
|
-
|
|
105
|
+
protocol = "https" if port == 443 else "http"
|
|
106
|
+
return f"{protocol}://{host}:{port}"
|
|
43
107
|
return None
|
|
44
108
|
|
|
45
109
|
|
flowyml/utils/config.py
CHANGED
|
@@ -27,9 +27,11 @@ class FlowymlConfig:
|
|
|
27
27
|
remote_ui_url: str = ""
|
|
28
28
|
remote_services: list[dict[str, str]] = field(default_factory=list)
|
|
29
29
|
enable_caching: bool = True
|
|
30
|
+
enable_checkpointing: bool = True # Enable checkpointing by default
|
|
30
31
|
enable_logging: bool = True
|
|
31
32
|
log_level: str = "INFO"
|
|
32
33
|
max_cache_size_mb: int = 10000 # 10GB default
|
|
34
|
+
checkpoint_dir: Path = field(default_factory=lambda: Path(".flowyml/checkpoints"))
|
|
33
35
|
|
|
34
36
|
# UI settings
|
|
35
37
|
ui_host: str = "localhost"
|
|
@@ -63,6 +65,7 @@ class FlowymlConfig:
|
|
|
63
65
|
"runs_dir",
|
|
64
66
|
"experiments_dir",
|
|
65
67
|
"projects_dir",
|
|
68
|
+
"checkpoint_dir",
|
|
66
69
|
]:
|
|
67
70
|
value = getattr(self, field_name)
|
|
68
71
|
if not isinstance(value, Path):
|
|
@@ -76,6 +79,7 @@ class FlowymlConfig:
|
|
|
76
79
|
self.runs_dir.mkdir(parents=True, exist_ok=True)
|
|
77
80
|
self.experiments_dir.mkdir(parents=True, exist_ok=True)
|
|
78
81
|
self.projects_dir.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
|
|
79
83
|
|
|
80
84
|
# Create metadata db parent dir
|
|
81
85
|
self.metadata_db.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -90,12 +94,14 @@ class FlowymlConfig:
|
|
|
90
94
|
"runs_dir": str(self.runs_dir),
|
|
91
95
|
"experiments_dir": str(self.experiments_dir),
|
|
92
96
|
"projects_dir": str(self.projects_dir),
|
|
97
|
+
"checkpoint_dir": str(self.checkpoint_dir),
|
|
93
98
|
"default_stack": self.default_stack,
|
|
94
99
|
"execution_mode": self.execution_mode,
|
|
95
100
|
"remote_server_url": self.remote_server_url,
|
|
96
101
|
"remote_ui_url": self.remote_ui_url,
|
|
97
102
|
"remote_services": self.remote_services,
|
|
98
103
|
"enable_caching": self.enable_caching,
|
|
104
|
+
"enable_checkpointing": self.enable_checkpointing,
|
|
99
105
|
"enable_logging": self.enable_logging,
|
|
100
106
|
"log_level": self.log_level,
|
|
101
107
|
"max_cache_size_mb": self.max_cache_size_mb,
|
|
@@ -274,6 +280,7 @@ def get_env_config() -> dict[str, Any]:
|
|
|
274
280
|
"flowyml_EXECUTION_MODE": "execution_mode",
|
|
275
281
|
"flowyml_REMOTE_SERVER_URL": "remote_server_url",
|
|
276
282
|
"flowyml_REMOTE_UI_URL": "remote_ui_url",
|
|
283
|
+
"flowyml_SERVER_URL": "remote_ui_url", # Alias for FLOWYML_SERVER_URL -> remote_ui_url
|
|
277
284
|
"flowyml_ENABLE_CACHING": "enable_caching",
|
|
278
285
|
"flowyml_LOG_LEVEL": "log_level",
|
|
279
286
|
"flowyml_UI_HOST": "ui_host",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flowyml
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.7.0
|
|
4
4
|
Summary: Next-Generation ML Pipeline Framework
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -23,13 +23,14 @@ Provides-Extra: aws
|
|
|
23
23
|
Provides-Extra: azure
|
|
24
24
|
Provides-Extra: gcp
|
|
25
25
|
Provides-Extra: pytorch
|
|
26
|
+
Provides-Extra: rich
|
|
26
27
|
Provides-Extra: sklearn
|
|
27
28
|
Provides-Extra: tensorflow
|
|
28
29
|
Provides-Extra: ui
|
|
29
30
|
Requires-Dist: click (>=8.0.0)
|
|
30
31
|
Requires-Dist: cloudpickle (>=2.0.0)
|
|
31
32
|
Requires-Dist: croniter (>=2.0.1,<3.0.0)
|
|
32
|
-
Requires-Dist: fastapi (>=0.122.0,<0.123.0) ; extra == "ui"
|
|
33
|
+
Requires-Dist: fastapi (>=0.122.0,<0.123.0) ; extra == "ui" or extra == "all"
|
|
33
34
|
Requires-Dist: google-cloud-aiplatform (>=1.35.0) ; extra == "gcp" or extra == "all"
|
|
34
35
|
Requires-Dist: google-cloud-storage (>=2.10.0) ; extra == "gcp" or extra == "all"
|
|
35
36
|
Requires-Dist: httpx (>=0.24,<0.28)
|
|
@@ -41,6 +42,7 @@ Requires-Dist: pydantic (>=2.0.0)
|
|
|
41
42
|
Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
|
|
42
43
|
Requires-Dist: pytz (>=2024.1,<2025.0)
|
|
43
44
|
Requires-Dist: pyyaml (>=6.0)
|
|
45
|
+
Requires-Dist: rich (>=13.0.0) ; extra == "rich"
|
|
44
46
|
Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
|
|
45
47
|
Requires-Dist: sqlalchemy (>=2.0.0)
|
|
46
48
|
Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
|
|
@@ -254,7 +256,7 @@ pipeline.run(debug=True) # Pauses at breakpoint
|
|
|
254
256
|
Assets are not just files; they are first-class citizens with lineage, metadata, and versioning.
|
|
255
257
|
|
|
256
258
|
```python
|
|
257
|
-
from flowyml
|
|
259
|
+
from flowyml import Dataset, Model, Metrics, FeatureSet
|
|
258
260
|
|
|
259
261
|
# Assets track their producer, lineage, and metadata automatically
|
|
260
262
|
dataset = Dataset.create(data=df, name="training_data", metadata={"source": "s3"})
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
-
flowyml/__init__.py,sha256=
|
|
1
|
+
flowyml/__init__.py,sha256=QqY5Rx6LdF1buwbLqMispN7xBV0DIiNqqpYjucxgr7w,5361
|
|
2
2
|
flowyml/assets/__init__.py,sha256=bbuzbW_PckrdekZEhcbJAwxhx0VWt8V4UeLLIute244,570
|
|
3
3
|
flowyml/assets/artifact.py,sha256=UTcNhjCvA7AmOlIVeMAo7kkKDZUbn5meUXz7CzWRy0c,1171
|
|
4
4
|
flowyml/assets/base.py,sha256=MGZiwbnwx551r_mxZXusrjAo_BvhhdNJkLlPgs7BE4M,6435
|
|
5
5
|
flowyml/assets/dataset.py,sha256=tjxcT0FGmC_0vajOWAKJCIRm1O77AeALIELHHuiyTjA,3003
|
|
6
|
-
flowyml/assets/featureset.py,sha256=
|
|
7
|
-
flowyml/assets/metrics.py,sha256=
|
|
6
|
+
flowyml/assets/featureset.py,sha256=JRlGXEQQBrh5wgamOBWNd76lbRio0K2O75nMsoM1ZoQ,11211
|
|
7
|
+
flowyml/assets/metrics.py,sha256=_f9qBRkl6ByvfVd4iCr64OAqiEEWqnRsbSC-aNjPQes,5097
|
|
8
8
|
flowyml/assets/model.py,sha256=Fa7Lw5AfrkLM_L16tNPCy7vSButMWE_W3wixC-pVANk,2649
|
|
9
9
|
flowyml/assets/registry.py,sha256=aYZOeRFGERgLCtUyVAtW7MbFlSofAN1sBSfO2FNj4ls,4830
|
|
10
10
|
flowyml/assets/report.py,sha256=CR1aI_08GereO-qsVwvy4JdG5_Du5rMcTfoqJ_1jmu8,9207
|
|
11
11
|
flowyml/cli/__init__.py,sha256=bMA7grr-wiy3LeAjGFSeSG2WQwlXDnQKIeFP7X4-HhM,83
|
|
12
12
|
flowyml/cli/experiment.py,sha256=ryPzfOlPKtCksjW9E7umJEVmE_5lNjAZqacbYX-azos,6284
|
|
13
13
|
flowyml/cli/init.py,sha256=FAlk_xhiCYYI_HoiVBqPfiZVC4mGJmDWxqY7R77E7UY,6204
|
|
14
|
-
flowyml/cli/main.py,sha256=
|
|
14
|
+
flowyml/cli/main.py,sha256=mltcYGsH-xdvGE5itDoORDyCjyuiNvnS9Ui9-J_qETU,29613
|
|
15
|
+
flowyml/cli/models.py,sha256=fg5Ry3J_FaPpoHtnm4Ou-wNGMTrNS9ufYZhdonIZhKU,17772
|
|
16
|
+
flowyml/cli/rich_utils.py,sha256=-jUZ71oNYVc9O3doCuw0Tx6EBuiNb-erIVXhOFJTEEc,2645
|
|
15
17
|
flowyml/cli/run.py,sha256=bkpEhE9LOqP3baNE0_inDGw29eYfToyb-8pCUS8084A,2293
|
|
16
18
|
flowyml/cli/stack_cli.py,sha256=XrJnOVq9_p6gjurxIV1a-20kAFZutEN58Vdzi2hpm74,16822
|
|
17
19
|
flowyml/cli/ui.py,sha256=0_R6-YdmtSTWPVZJWjdYV-4FEjSzuOqtlHwz4RmirkA,874
|
|
@@ -19,29 +21,30 @@ flowyml/core/__init__.py,sha256=zvRAQQ8ySPrDXfjLb2wmg9kibbCfGPs69Fi86xkuv2A,1494
|
|
|
19
21
|
flowyml/core/advanced_cache.py,sha256=Rs9z81nNA-j8ShodKHR52veSaEPLVU0Jsg-L-fM4Z7Y,8342
|
|
20
22
|
flowyml/core/approval.py,sha256=kHyHcKtrPp9xxa1gFOGpd5LXAKtBoHjLwMCqNz_uUGs,2288
|
|
21
23
|
flowyml/core/cache.py,sha256=rIzIQ-GX2sVO7qAckpZMfmxDLKQU34XJKJp38zywIpk,6197
|
|
22
|
-
flowyml/core/checkpoint.py,sha256=
|
|
23
|
-
flowyml/core/conditional.py,sha256=
|
|
24
|
+
flowyml/core/checkpoint.py,sha256=EBKAi0UqWkCOOiz5wyXY_SBDBWJH4POwH_E6-AI_h4s,4794
|
|
25
|
+
flowyml/core/conditional.py,sha256=Huwb_dt6ZRr_OGjmA826IaXxwGQkUchGX8pnT5pHJ0g,11991
|
|
24
26
|
flowyml/core/context.py,sha256=M0_K_REzIByJlF-2INCgHqDWFDKXmNQb-2HSmPt_WIY,4816
|
|
27
|
+
flowyml/core/display.py,sha256=_zVGZAlqG_0n2NxVZPavbtMpFGsV2SKFhzY5s07YIms,22138
|
|
25
28
|
flowyml/core/error_handling.py,sha256=TovzbOFzQYBHMMM8NjsR8WcbmbdtVjXuW8_fsdbgsPA,11952
|
|
26
29
|
flowyml/core/execution_status.py,sha256=vlQOOzglpSea4z9_csqnPkFP27jJuXjAke6EQbPMq3g,1347
|
|
27
|
-
flowyml/core/executor.py,sha256=
|
|
30
|
+
flowyml/core/executor.py,sha256=ljBV472nJyp_06meeeHmD_aomm_JdJD6y03OiPvMtEA,19168
|
|
28
31
|
flowyml/core/graph.py,sha256=drVezsgYwva-b8X5FsjExtaW-7NsjYMkchTIPYyb5HQ,6065
|
|
29
32
|
flowyml/core/hooks.py,sha256=UOqrNY1m3lmVql1FRls6AXDNAV3dxxZl-zO6ijEeRW8,4022
|
|
30
33
|
flowyml/core/observability.py,sha256=NW05m8zki1TwKeRBBCtAP1UwguMcJasGz8HHgMVbjAU,7011
|
|
31
|
-
flowyml/core/orchestrator.py,sha256=
|
|
34
|
+
flowyml/core/orchestrator.py,sha256=8duXRaVNJD6V7s7LBTRESJC7a8dBfQau-UQIk7UFo2Y,37733
|
|
32
35
|
flowyml/core/parallel.py,sha256=KGstDu32i9FFKZV0bBRrm1kl3bKjcHmL2js48dVjWlk,12492
|
|
33
|
-
flowyml/core/pipeline.py,sha256=
|
|
34
|
-
flowyml/core/project.py,sha256=
|
|
36
|
+
flowyml/core/pipeline.py,sha256=4Up-3Iz5_jy578l3-B8jvKkHYxLo6bmieut5jS6JNQ4,46076
|
|
37
|
+
flowyml/core/project.py,sha256=7Cv_MUnXrD2e69Pzwo4ThXhGyjvP45De6CqL3Z2BtiA,8997
|
|
35
38
|
flowyml/core/remote_orchestrator.py,sha256=LpHlNQslq14OliNZogBGf0fpu6c5n9T7f3BpftahM8Q,3527
|
|
36
39
|
flowyml/core/resources.py,sha256=5TimjjSOEedALgNVVjRrvU-cHujKebyFjU4YxImvqZE,14402
|
|
37
40
|
flowyml/core/retry_policy.py,sha256=OKiazbSfGm01BMAhVrjYrGbGZdceTgckGZjbcDeKK1c,2358
|
|
38
|
-
flowyml/core/scheduler.py,sha256=
|
|
41
|
+
flowyml/core/scheduler.py,sha256=2xXOsQ7klvkiA6wIAzrsgN15_GJB08daTCajrxkK4vU,28779
|
|
39
42
|
flowyml/core/scheduler_config.py,sha256=bMlt9a7ap_-kKImiRbEAhuGkzvvpJiQPu8NXIfdwhEA,1203
|
|
40
43
|
flowyml/core/step.py,sha256=h_y4Yx_uJSL6D8jaL1vhAjtYFUjt-64RGsJHapJxCss,7871
|
|
41
44
|
flowyml/core/step_grouping.py,sha256=wBMZbOLZW5ghN8n8J3CmzKFKTaytRzrLkHJ-AwCuP5E,9941
|
|
42
45
|
flowyml/core/submission_result.py,sha256=bPPCC9pxqCZTTV0CdK1znJi4Rhw8QRCqQVwuVFr0b20,1692
|
|
43
46
|
flowyml/core/templates.py,sha256=r7WbxZv8-BV3_3oX27ACGSitIsmNbNd4VeitpI2uJeg,6359
|
|
44
|
-
flowyml/core/versioning.py,sha256=
|
|
47
|
+
flowyml/core/versioning.py,sha256=4UeX9IBTsPF0M4BjK2McYMMjI9kjp2u263XlSYjk4QY,7691
|
|
45
48
|
flowyml/integrations/__init__.py,sha256=SQWPFwFs_1gLGlJ6TDo2bcFKwBrv1PkwhZvEiELV1ok,28
|
|
46
49
|
flowyml/integrations/keras.py,sha256=L-qX5Luhah_XfLMsfx7BMXTYs2b_hg2fMJWf78ydAJg,8216
|
|
47
50
|
flowyml/monitoring/__init__.py,sha256=nkqmfdvLGMd44NfrIvH2vwVZBJGDVUn-qpei192WJp4,33
|
|
@@ -51,7 +54,7 @@ flowyml/monitoring/llm.py,sha256=ULq4rmsQ_BY_QoekztG-NwqMuj4UkQnP1AuzhRfjmq4,480
|
|
|
51
54
|
flowyml/monitoring/monitor.py,sha256=rdhUUVYUSyGNm3fQPRc3qS8Z7fr3S_d-lOxJQdwNA8o,1448
|
|
52
55
|
flowyml/monitoring/notifications.py,sha256=Z0pzuUL5cI_9oKy3mEcyYFsv5GgK7HHXtYy_dow-vEE,7484
|
|
53
56
|
flowyml/registry/__init__.py,sha256=fClcK1W1hxBvjaHvXZYK9-VIW7R7Oc1hWf_rqnpnzPw,196
|
|
54
|
-
flowyml/registry/model_registry.py,sha256
|
|
57
|
+
flowyml/registry/model_registry.py,sha256=bC-hepSyJLEz8pSPIr5bp5y67KXnPgf6Ylts5IPJQ-g,14235
|
|
55
58
|
flowyml/registry/pipeline_registry.py,sha256=fEJvN5sQW-Nt-lZg5LfjMyRSKNskiOBlpPxfFq30--s,1552
|
|
56
59
|
flowyml/stacks/__init__.py,sha256=5FOapdb-qoXtZz0wa1_I1OZT5SBITarJjNy4WT_OyPU,1246
|
|
57
60
|
flowyml/stacks/aws.py,sha256=8u2TcKT-Z-PFfgaswjW87UrFi_VpA7JmK4Ev3NEYtHY,20958
|
|
@@ -107,9 +110,9 @@ flowyml/ui/backend/routers/traces.py,sha256=xWxEuOe6TKDrZwWkZG5D3LuHEj_pmGnhJZ-5
|
|
|
107
110
|
flowyml/ui/backend/routers/websocket.py,sha256=PdGJKdG78yD5nVq7KIF6MwGyzlWHbbQDOJz2eFIb9X8,4908
|
|
108
111
|
flowyml/ui/frontend/Dockerfile,sha256=BJWOdU5mnk4e2H6rARiGOO6TF-YCRxN7mOIWPv0EIAM,314
|
|
109
112
|
flowyml/ui/frontend/README.md,sha256=M5il2PEdNVCSm1zo1rZAXqQ0nTj5frCNXmi5-d1ZKIM,7099
|
|
110
|
-
flowyml/ui/frontend/dist/assets/index-
|
|
111
|
-
flowyml/ui/frontend/dist/assets/index-
|
|
112
|
-
flowyml/ui/frontend/dist/index.html,sha256=
|
|
113
|
+
flowyml/ui/frontend/dist/assets/index-By4trVyv.css,sha256=rDcWIcfFdSPd9bVKIspoO0S4fXu9QYE6j-MQtSlC6rI,76734
|
|
114
|
+
flowyml/ui/frontend/dist/assets/index-CX5RV2C9.js,sha256=zqoVw8lcKEXOCHVaJTbMTIW-F4ihR3ujzAWEg-uvxoI,1295556
|
|
115
|
+
flowyml/ui/frontend/dist/index.html,sha256=vAbCc_3jA5yuMTq5vElxZ1oEjf2-qUpRTUfQ6g-F13E,375
|
|
113
116
|
flowyml/ui/frontend/index.html,sha256=Y_Xm2LhORPIkzwz4VADHCmfDSuTkuT_gGVLRWpDOtis,279
|
|
114
117
|
flowyml/ui/frontend/nginx.conf,sha256=MdOpD7y2SJzpxwlHgjMVadTvlXTtZaRXfWO-WHmtuls,637
|
|
115
118
|
flowyml/ui/frontend/package-lock.json,sha256=qYEiw3iu7uXCjZcnog5tActXcv98B6shm23VW-xi_Qg,135309
|
|
@@ -152,7 +155,7 @@ flowyml/ui/frontend/src/components/ExperimentDetailsPanel.jsx,sha256=mV5PuWzNRaO
|
|
|
152
155
|
flowyml/ui/frontend/src/components/Layout.jsx,sha256=CsoaFrWe-5iRyAsVyu2rmJqhoC2ltckEzhVscsXlmsI,5423
|
|
153
156
|
flowyml/ui/frontend/src/components/NavigationTree.jsx,sha256=9os95nqyBNtL1INR-zOZD3OhIFKXqFI1q3zDlASfHgU,15710
|
|
154
157
|
flowyml/ui/frontend/src/components/PipelineDetailsPanel.jsx,sha256=pNM8oy-hWXQmCZJtyr_kEzgIcdJMdqdPN6Vjxwc3YQ4,10932
|
|
155
|
-
flowyml/ui/frontend/src/components/PipelineGraph.jsx,sha256=
|
|
158
|
+
flowyml/ui/frontend/src/components/PipelineGraph.jsx,sha256=cJE5n4_fOXL879DrBEZLcHX_eIdkOzqfh6VqVe4y4l8,17050
|
|
156
159
|
flowyml/ui/frontend/src/components/ProjectSelector.jsx,sha256=L-nmTUMwy-5wD-mFizztybqnw-wd97y6mCD2Q9kh4B8,5580
|
|
157
160
|
flowyml/ui/frontend/src/components/RunDetailsPanel.jsx,sha256=OCB1vn-xH04FrNtYHkK8yU03b7kQTkZ2OVgJ_k8-bxY,16169
|
|
158
161
|
flowyml/ui/frontend/src/components/header/Header.jsx,sha256=9KrdWTglIg0lVu68a1fm_wnDGW7lurEhbcq3Nby6VBk,6278
|
|
@@ -187,9 +190,10 @@ flowyml/ui/frontend/src/utils/date.js,sha256=8tYLT-TspihDCbziiYCNaBjvMa5ez155kBx
|
|
|
187
190
|
flowyml/ui/frontend/src/utils/downloads.js,sha256=2w3uSOiAktiCWAj56bTSZUx3eNA9QZt1qkVCzX3YrdY,305
|
|
188
191
|
flowyml/ui/frontend/tailwind.config.js,sha256=__nxDJC93bzcg8Ro9uxt4c2DiErpUCJfi4B-zNRooYg,813
|
|
189
192
|
flowyml/ui/frontend/vite.config.js,sha256=b4JAsNo2yU4wRdTKf7ppBKsaw6WW447LrS0V4AbXkbk,401
|
|
190
|
-
flowyml/ui/
|
|
193
|
+
flowyml/ui/server_manager.py,sha256=Se37U4aXnzZsG4oh5RmKnGmmisXQ-8lOhaod0QbzSzc,6274
|
|
194
|
+
flowyml/ui/utils.py,sha256=FeM5zga3Dbbn_HRKRRdjLrt0HbnCrxXk0NN7NqPeJII,4516
|
|
191
195
|
flowyml/utils/__init__.py,sha256=eA_YZEZCnCvWRdcqH8IzwWIO-LSUI5-8sbA9mU6xBto,1490
|
|
192
|
-
flowyml/utils/config.py,sha256=
|
|
196
|
+
flowyml/utils/config.py,sha256=Oeywfo2vptI0-yF28AwmtIbf3fb-2SME-zbkfKfQgqs,10280
|
|
193
197
|
flowyml/utils/debug.py,sha256=zcHZxGLbuSImLdcfn1V7CwfaDzc3SunXdV-pWR_UW90,6536
|
|
194
198
|
flowyml/utils/environment.py,sha256=3vqyHBldrCdENrMYtYH0rsE7bPJ2IkLulzWWwWzMCmE,9166
|
|
195
199
|
flowyml/utils/git.py,sha256=TFbHPgt8xpwHE5qg8bhq369IM4yQFU53MfvsZs0auNw,7745
|
|
@@ -197,8 +201,8 @@ flowyml/utils/logging.py,sha256=PBJDFlGdp1mePS6A3g08dnGAB-v8jTlcNxEsYs9WSBo,1371
|
|
|
197
201
|
flowyml/utils/performance.py,sha256=-ne9v9ddEltiKRPk-AerM1R3Gwwd_oCRKtNyHARWd4k,8655
|
|
198
202
|
flowyml/utils/stack_config.py,sha256=STX1niArJzvu0YsqUQmrNJ0WTeMVW_setYNH36BlbVI,10826
|
|
199
203
|
flowyml/utils/validation.py,sha256=mClumVro0bl_XXxT1zWPlRI6M_iZa3z2SZ0QUdmTOqs,10199
|
|
200
|
-
flowyml-1.
|
|
201
|
-
flowyml-1.
|
|
202
|
-
flowyml-1.
|
|
203
|
-
flowyml-1.
|
|
204
|
-
flowyml-1.
|
|
204
|
+
flowyml-1.7.0.dist-info/METADATA,sha256=ocS0ENtzA4OxxIyBctONIvO8VLiJnOaFjCww9WGuKcY,15536
|
|
205
|
+
flowyml-1.7.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
206
|
+
flowyml-1.7.0.dist-info/entry_points.txt,sha256=yuF-dOC4rbyJ2Aqi4CMRBxFhqIRoKO6Mhh6jfiQEVjI,48
|
|
207
|
+
flowyml-1.7.0.dist-info/licenses/LICENSE,sha256=DRBRWOEjKZQBvy1WZwxyvp2NmnC1whW9Ef7v0Oo-p_g,626
|
|
208
|
+
flowyml-1.7.0.dist-info/RECORD,,
|