flashstudio 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flashstudio might be problematic. Click here for more details.
- flashstudio/__init__.py +5 -0
- flashstudio/app.py +194 -0
- flashstudio/cli.py +18 -0
- flashstudio/components/__init__.py +0 -0
- flashstudio/components/project_manager.py +490 -0
- flashstudio/components/sidebar.py +87 -0
- flashstudio/components/styles.py +346 -0
- flashstudio/components/wizard.py +45 -0
- flashstudio/components/zone_drawer/__init__.py +177 -0
- flashstudio/constants.py +245 -0
- flashstudio/core/__init__.py +3 -0
- flashstudio/core/constants.py +3 -0
- flashstudio/launcher.py +288 -0
- flashstudio/pages/__init__.py +8 -0
- flashstudio/pages/dashboard/__init__.py +3 -0
- flashstudio/pages/dashboard/_common.py +10 -0
- flashstudio/pages/dashboard/overview/__init__.py +3 -0
- flashstudio/pages/dashboard/overview/metrics.py +33 -0
- flashstudio/pages/dashboard/overview/pipeline_status.py +35 -0
- flashstudio/pages/dashboard/overview/project_banner.py +23 -0
- flashstudio/pages/dashboard/overview/quick_actions.py +28 -0
- flashstudio/pages/dashboard/overview/recent_runs.py +99 -0
- flashstudio/pages/dashboard/overview/tab.py +35 -0
- flashstudio/pages/dashboard/page.py +17 -0
- flashstudio/pages/data/__init__.py +3 -0
- flashstudio/pages/data/_common.py +107 -0
- flashstudio/pages/data/download/__init__.py +3 -0
- flashstudio/pages/data/download/tab.py +75 -0
- flashstudio/pages/data/helpers.py +338 -0
- flashstudio/pages/data/page.py +50 -0
- flashstudio/pages/data/preview/__init__.py +3 -0
- flashstudio/pages/data/preview/tab.py +139 -0
- flashstudio/pages/data/upload/__init__.py +3 -0
- flashstudio/pages/data/upload/tab.py +156 -0
- flashstudio/pages/data/verify/__init__.py +3 -0
- flashstudio/pages/data/verify/tab.py +108 -0
- flashstudio/pages/export/__init__.py +3 -0
- flashstudio/pages/export/page.py +54 -0
- flashstudio/pages/export/weights/__init__.py +4 -0
- flashstudio/pages/export/weights/exporter.py +60 -0
- flashstudio/pages/export/weights/settings.py +86 -0
- flashstudio/pages/inference/__init__.py +3 -0
- flashstudio/pages/inference/_common.py +76 -0
- flashstudio/pages/inference/data/__init__.py +3 -0
- flashstudio/pages/inference/data/tab.py +121 -0
- flashstudio/pages/inference/model/__init__.py +3 -0
- flashstudio/pages/inference/model/tab.py +62 -0
- flashstudio/pages/inference/page.py +42 -0
- flashstudio/pages/inference/run/__init__.py +3 -0
- flashstudio/pages/inference/run/detection.py +133 -0
- flashstudio/pages/inference/run/tab.py +161 -0
- flashstudio/pages/inference/run/video.py +195 -0
- flashstudio/pages/inference/solution/__init__.py +3 -0
- flashstudio/pages/inference/solution/tab.py +121 -0
- flashstudio/pages/model/__init__.py +3 -0
- flashstudio/pages/model/_common.py +3 -0
- flashstudio/pages/model/advanced/__init__.py +3 -0
- flashstudio/pages/model/advanced/tab.py +54 -0
- flashstudio/pages/model/architecture/__init__.py +3 -0
- flashstudio/pages/model/architecture/tab.py +90 -0
- flashstudio/pages/model/augment/__init__.py +3 -0
- flashstudio/pages/model/augment/tab.py +18 -0
- flashstudio/pages/model/hyperparams/__init__.py +4 -0
- flashstudio/pages/model/hyperparams/tab.py +89 -0
- flashstudio/pages/model/page.py +30 -0
- flashstudio/pages/model/utils/__init__.py +3 -0
- flashstudio/pages/model/utils/config.py +27 -0
- flashstudio/pages/training/__init__.py +3 -0
- flashstudio/pages/training/_common.py +24 -0
- flashstudio/pages/training/launch/__init__.py +3 -0
- flashstudio/pages/training/launch/controls.py +84 -0
- flashstudio/pages/training/launch/dialogs.py +247 -0
- flashstudio/pages/training/launch/preflight.py +77 -0
- flashstudio/pages/training/launch/runner.py +440 -0
- flashstudio/pages/training/launch/tab.py +134 -0
- flashstudio/pages/training/monitor/__init__.py +3 -0
- flashstudio/pages/training/monitor/checkpoints.py +96 -0
- flashstudio/pages/training/monitor/curves.py +113 -0
- flashstudio/pages/training/monitor/dashboard.py +129 -0
- flashstudio/pages/training/monitor/gt_verification.py +122 -0
- flashstudio/pages/training/monitor/log_viewer.py +43 -0
- flashstudio/pages/training/monitor/parsers.py +253 -0
- flashstudio/pages/training/monitor/run_meta.py +165 -0
- flashstudio/pages/training/monitor/tab.py +142 -0
- flashstudio/pages/training/monitor/visualizations.py +52 -0
- flashstudio/pages/training/page.py +134 -0
- flashstudio/utils/__init__.py +152 -0
- flashstudio/utils/config.py +104 -0
- flashstudio/utils/device.py +67 -0
- flashstudio/utils/filesystem.py +67 -0
- flashstudio/utils/jobs.py +95 -0
- flashstudio/utils/training_hooks.py +266 -0
- flashstudio-0.0.1.dist-info/METADATA +107 -0
- flashstudio-0.0.1.dist-info/RECORD +97 -0
- flashstudio-0.0.1.dist-info/WHEEL +5 -0
- flashstudio-0.0.1.dist-info/entry_points.txt +2 -0
- flashstudio-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Run metadata extraction for the monitor tab."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import glob as glob_module
|
|
6
|
+
|
|
7
|
+
from flashstudio.constants import (
|
|
8
|
+
CKPT_BEST_INFERENCE, CKPT_BEST, CKPT_LAST_INFERENCE,
|
|
9
|
+
CKPT_LAST, CKPT_FINAL_INFERENCE, CKPT_FINAL_FP16,
|
|
10
|
+
TRAINING_LOG_CSV, format_bytes,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_run_meta(run_dir: str) -> dict:
|
|
15
|
+
"""Extract metadata from a training run for display."""
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
name = os.path.basename(run_dir)
|
|
19
|
+
meta = {
|
|
20
|
+
"display_name": name,
|
|
21
|
+
"status": "unknown",
|
|
22
|
+
"date": "",
|
|
23
|
+
"epochs": "?",
|
|
24
|
+
"mAP": None,
|
|
25
|
+
"model": "",
|
|
26
|
+
"dataset": "",
|
|
27
|
+
"size": "",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Date from folder modification time
|
|
31
|
+
try:
|
|
32
|
+
mtime = os.path.getmtime(run_dir)
|
|
33
|
+
meta["date"] = time.strftime("%b %d %H:%M", time.localtime(mtime))
|
|
34
|
+
except OSError:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
# Folder size
|
|
38
|
+
try:
|
|
39
|
+
total = 0
|
|
40
|
+
for dirpath, _dirs, files in os.walk(run_dir):
|
|
41
|
+
for f in files:
|
|
42
|
+
total += os.path.getsize(os.path.join(dirpath, f))
|
|
43
|
+
meta["size"] = format_bytes(total)
|
|
44
|
+
except OSError:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
# Status: check for checkpoints
|
|
48
|
+
has_final = (os.path.isfile(os.path.join(run_dir, CKPT_FINAL_INFERENCE))
|
|
49
|
+
or os.path.isfile(os.path.join(run_dir, CKPT_FINAL_FP16)))
|
|
50
|
+
has_best = (os.path.isfile(os.path.join(run_dir, CKPT_BEST_INFERENCE))
|
|
51
|
+
or os.path.isfile(os.path.join(run_dir, CKPT_BEST)))
|
|
52
|
+
has_last = (os.path.isfile(os.path.join(run_dir, CKPT_LAST))
|
|
53
|
+
or os.path.isfile(os.path.join(run_dir, CKPT_LAST_INFERENCE)))
|
|
54
|
+
from flashstudio.constants import TRAINING_LOG_GLOB
|
|
55
|
+
log_files = glob_module.glob(os.path.join(run_dir, TRAINING_LOG_GLOB))
|
|
56
|
+
has_log = bool(log_files)
|
|
57
|
+
has_csv = os.path.isfile(os.path.join(run_dir, TRAINING_LOG_CSV))
|
|
58
|
+
|
|
59
|
+
if has_final:
|
|
60
|
+
meta["status"] = "Complete"
|
|
61
|
+
elif has_best and has_last:
|
|
62
|
+
# Check log for "Training Complete!" to distinguish complete vs in-progress
|
|
63
|
+
completed_in_log = False
|
|
64
|
+
if log_files:
|
|
65
|
+
try:
|
|
66
|
+
with open(max(log_files, key=lambda p: os.path.getsize(p)), "r", encoding="utf-8", errors="replace") as _f:
|
|
67
|
+
_tail = _f.readlines()[-5:]
|
|
68
|
+
completed_in_log = any("Training Complete!" in l for l in _tail)
|
|
69
|
+
except OSError:
|
|
70
|
+
pass
|
|
71
|
+
meta["status"] = "Complete" if completed_in_log else "In Progress"
|
|
72
|
+
elif has_last and (has_log or has_csv):
|
|
73
|
+
meta["status"] = "In Progress"
|
|
74
|
+
elif has_log or has_csv:
|
|
75
|
+
meta["status"] = "Started"
|
|
76
|
+
else:
|
|
77
|
+
meta["status"] = "Empty"
|
|
78
|
+
|
|
79
|
+
# Extract model/epochs from log header (use largest log file)
|
|
80
|
+
if log_files:
|
|
81
|
+
try:
|
|
82
|
+
best_log = max(log_files, key=lambda p: os.path.getsize(p))
|
|
83
|
+
with open(best_log, "r", encoding="utf-8", errors="replace") as f:
|
|
84
|
+
header_lines = f.readlines()[:30]
|
|
85
|
+
for line in header_lines:
|
|
86
|
+
lc = line.strip()
|
|
87
|
+
# FlashDet format: "Model Size: FlashDet-N"
|
|
88
|
+
if "Model Size:" in lc and not meta["model"]:
|
|
89
|
+
meta["model"] = lc.split("Model Size:")[-1].strip()
|
|
90
|
+
# Older/pico format: "Model: p, Input: (320, 320)"
|
|
91
|
+
if not meta["model"]:
|
|
92
|
+
mm = re.search(r"Model:\s*(\w+),\s*Input:\s*\((\d+),\s*(\d+)\)", lc)
|
|
93
|
+
if mm:
|
|
94
|
+
meta["model"] = f"{mm.group(1)} ({mm.group(2)}x{mm.group(3)})"
|
|
95
|
+
# Input size on separate line: "Input Size: (320, 320)"
|
|
96
|
+
if "Input Size:" in lc and not meta.get("input_size"):
|
|
97
|
+
ism = re.search(r"Input Size:\s*\((\d+),\s*(\d+)\)", lc)
|
|
98
|
+
if ism:
|
|
99
|
+
meta["input_size"] = f"{ism.group(1)}x{ism.group(2)}"
|
|
100
|
+
# Epochs on own line: "Epochs: 1000"
|
|
101
|
+
if "Epochs:" in lc and meta["epochs"] == "?":
|
|
102
|
+
em = re.search(r"Epochs:\s*(\d+)", lc)
|
|
103
|
+
if em:
|
|
104
|
+
meta["epochs"] = em.group(1)
|
|
105
|
+
# Also: "Epochs: 100, Batch: 16, LR: 0.001" (compact format)
|
|
106
|
+
if "Batch Size:" in lc and not meta.get("batch_size"):
|
|
107
|
+
bm = re.search(r"Batch Size:\s*(\d+)", lc)
|
|
108
|
+
if bm:
|
|
109
|
+
meta["batch_size"] = bm.group(1)
|
|
110
|
+
# Device
|
|
111
|
+
if "Device:" in lc and not meta.get("device"):
|
|
112
|
+
meta["device"] = lc.split("Device:")[-1].strip()
|
|
113
|
+
except OSError:
|
|
114
|
+
pass
|
|
115
|
+
|
|
116
|
+
# Fallback: parse training_log.csv for epoch count and mAP
|
|
117
|
+
csv_path = os.path.join(run_dir, TRAINING_LOG_CSV)
|
|
118
|
+
if os.path.isfile(csv_path) and meta["epochs"] == "?":
|
|
119
|
+
try:
|
|
120
|
+
import csv
|
|
121
|
+
with open(csv_path, "r") as f:
|
|
122
|
+
reader = csv.DictReader(f)
|
|
123
|
+
rows = list(reader)
|
|
124
|
+
if rows:
|
|
125
|
+
meta["epochs"] = len(rows)
|
|
126
|
+
mAP_vals = []
|
|
127
|
+
for r in rows:
|
|
128
|
+
v = r.get("mAP@0.5") or r.get("val_mAP") or ""
|
|
129
|
+
if isinstance(v, str) and v.strip():
|
|
130
|
+
try:
|
|
131
|
+
mAP_vals.append(float(v))
|
|
132
|
+
except ValueError:
|
|
133
|
+
pass
|
|
134
|
+
if mAP_vals and max(mAP_vals) > 0 and not meta["mAP"]:
|
|
135
|
+
meta["mAP"] = max(mAP_vals)
|
|
136
|
+
except Exception:
|
|
137
|
+
pass
|
|
138
|
+
|
|
139
|
+
# Fallback: parse log tail for "Best mAP@0.5:" and epoch count
|
|
140
|
+
if (meta["epochs"] == "?" or not meta["mAP"]) and log_files:
|
|
141
|
+
try:
|
|
142
|
+
best_log = max(log_files, key=lambda p: os.path.getsize(p))
|
|
143
|
+
with open(best_log, "r", encoding="utf-8", errors="replace") as f:
|
|
144
|
+
tail = f.readlines()[-20:]
|
|
145
|
+
for line in tail:
|
|
146
|
+
bm = re.search(r"Best mAP@0\.5:\s*([\d.]+)", line)
|
|
147
|
+
if bm and not meta["mAP"]:
|
|
148
|
+
meta["mAP"] = float(bm.group(1))
|
|
149
|
+
if meta["epochs"] == "?":
|
|
150
|
+
for line in tail:
|
|
151
|
+
em = re.search(r"Epoch (\d+)/(\d+)", line)
|
|
152
|
+
if em:
|
|
153
|
+
meta["epochs"] = em.group(1)
|
|
154
|
+
except OSError:
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
# Build enriched display name
|
|
158
|
+
parts = [name]
|
|
159
|
+
if meta["model"]:
|
|
160
|
+
parts.append(meta["model"])
|
|
161
|
+
if meta["mAP"]:
|
|
162
|
+
parts.append(f"mAP={meta['mAP']:.3f}")
|
|
163
|
+
meta["display_name"] = " | ".join(parts) if len(parts) > 1 else name
|
|
164
|
+
|
|
165
|
+
return meta
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Monitor tab — browse and inspect training runs."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
from flashstudio.constants import AUTOREFRESH_INTERVAL_MS, TRAINING_LOG_CSV
|
|
7
|
+
from flashstudio.pages.training._common import _get_save_dir
|
|
8
|
+
from flashstudio.pages.training.monitor.run_meta import _get_run_meta
|
|
9
|
+
from flashstudio.pages.training.monitor.dashboard import _render_run_dashboard
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _render_monitor_tab():
|
|
13
|
+
"""Tab: monitor an existing training run — choose workspace, browse runs."""
|
|
14
|
+
# Auto-refresh toggle
|
|
15
|
+
auto_refresh = st.session_state.get("monitor_auto_refresh", False)
|
|
16
|
+
is_training = st.session_state.get("training_active", False) and not st.session_state.get("training_paused", False)
|
|
17
|
+
if is_training or auto_refresh:
|
|
18
|
+
try:
|
|
19
|
+
from streamlit_autorefresh import st_autorefresh
|
|
20
|
+
st_autorefresh(interval=AUTOREFRESH_INTERVAL_MS, limit=None, key="monitor_autorefresh")
|
|
21
|
+
except ImportError:
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
# Monitor workspace — defaults to Save Dir from Model > Advanced
|
|
25
|
+
save_dir = _get_save_dir()
|
|
26
|
+
if "monitor_workspace" not in st.session_state:
|
|
27
|
+
st.session_state["monitor_workspace"] = save_dir
|
|
28
|
+
|
|
29
|
+
wc1, wc2, wc3, wc4 = st.columns([5, 1, 1, 1])
|
|
30
|
+
with wc1:
|
|
31
|
+
workspace = st.text_input("Workspace Path", key="monitor_workspace",
|
|
32
|
+
label_visibility="collapsed")
|
|
33
|
+
with wc2:
|
|
34
|
+
if st.button("Sync", key="mon_sync_btn", use_container_width=True,
|
|
35
|
+
help="Sync with Save Dir from Model > Advanced"):
|
|
36
|
+
st.session_state["monitor_workspace"] = save_dir
|
|
37
|
+
st.rerun()
|
|
38
|
+
with wc3:
|
|
39
|
+
if st.button("Refresh", key="mon_refresh_btn", use_container_width=True):
|
|
40
|
+
st.rerun()
|
|
41
|
+
with wc4:
|
|
42
|
+
if "monitor_auto_refresh" not in st.session_state:
|
|
43
|
+
st.session_state["monitor_auto_refresh"] = is_training
|
|
44
|
+
st.toggle("Auto", key="monitor_auto_refresh")
|
|
45
|
+
|
|
46
|
+
if not os.path.isdir(workspace):
|
|
47
|
+
st.warning("Workspace not found")
|
|
48
|
+
_render_run_dashboard(None)
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
def _is_training_run(folder_path):
|
|
52
|
+
"""Check if a folder looks like a FlashDet training run."""
|
|
53
|
+
try:
|
|
54
|
+
entries = os.listdir(folder_path)
|
|
55
|
+
except OSError:
|
|
56
|
+
return False
|
|
57
|
+
has_log = any(f.startswith("train_") and f.endswith(".log") for f in entries)
|
|
58
|
+
has_csv = TRAINING_LOG_CSV in entries
|
|
59
|
+
has_ckpt = any(f.endswith(".pth") for f in entries)
|
|
60
|
+
return has_log or has_csv or has_ckpt
|
|
61
|
+
|
|
62
|
+
# Collect training run folders (max 2 levels deep)
|
|
63
|
+
all_folders = []
|
|
64
|
+
for d in sorted(os.listdir(workspace)):
|
|
65
|
+
full = os.path.join(workspace, d)
|
|
66
|
+
if os.path.isdir(full):
|
|
67
|
+
if _is_training_run(full):
|
|
68
|
+
all_folders.append(d)
|
|
69
|
+
else:
|
|
70
|
+
try:
|
|
71
|
+
for sd in sorted(os.listdir(full)):
|
|
72
|
+
sf = os.path.join(full, sd)
|
|
73
|
+
if os.path.isdir(sf) and _is_training_run(sf):
|
|
74
|
+
all_folders.append(os.path.join(d, sd))
|
|
75
|
+
except OSError:
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
if not all_folders:
|
|
79
|
+
st.info("No runs found. Start training first.")
|
|
80
|
+
_render_run_dashboard(None)
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
# Sort by modification time (most recent first)
|
|
84
|
+
all_folders.sort(
|
|
85
|
+
key=lambda d: os.path.getmtime(os.path.join(workspace, d)),
|
|
86
|
+
reverse=True,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
labels = []
|
|
90
|
+
for r in all_folders:
|
|
91
|
+
meta = _get_run_meta(os.path.join(workspace, r))
|
|
92
|
+
parts = [r]
|
|
93
|
+
if meta["mAP"]:
|
|
94
|
+
parts.append(f"mAP={meta['mAP']:.3f}")
|
|
95
|
+
if meta["size"]:
|
|
96
|
+
parts.append(meta["size"])
|
|
97
|
+
labels.append(" | ".join(parts))
|
|
98
|
+
|
|
99
|
+
sc1, sc2 = st.columns([7, 1])
|
|
100
|
+
with sc1:
|
|
101
|
+
selected_idx = st.selectbox("Select Run", range(len(all_folders)),
|
|
102
|
+
format_func=lambda i: labels[i],
|
|
103
|
+
key="selected_run_idx", label_visibility="collapsed")
|
|
104
|
+
selected_run = all_folders[selected_idx]
|
|
105
|
+
run_dir = os.path.join(workspace, selected_run)
|
|
106
|
+
with sc2:
|
|
107
|
+
if st.button("Del", key="delete_run_btn", help="Delete this run"):
|
|
108
|
+
st.session_state["show_delete_run"] = selected_run
|
|
109
|
+
|
|
110
|
+
if st.session_state.get("show_delete_run"):
|
|
111
|
+
dc1, dc2 = st.columns([1, 1])
|
|
112
|
+
with dc1:
|
|
113
|
+
if st.button(f"Confirm delete {st.session_state['show_delete_run']}", type="primary",
|
|
114
|
+
key="confirm_delete_run"):
|
|
115
|
+
import shutil
|
|
116
|
+
from flashstudio.utils import flash
|
|
117
|
+
deleted_name = st.session_state["show_delete_run"]
|
|
118
|
+
shutil.rmtree(os.path.join(workspace, deleted_name), ignore_errors=True)
|
|
119
|
+
st.session_state.pop("show_delete_run", None)
|
|
120
|
+
flash(f"Deleted run `{deleted_name}`", "success")
|
|
121
|
+
st.rerun()
|
|
122
|
+
with dc2:
|
|
123
|
+
if st.button("Cancel", key="cancel_delete_run"):
|
|
124
|
+
st.session_state.pop("show_delete_run", None)
|
|
125
|
+
st.rerun()
|
|
126
|
+
|
|
127
|
+
meta = _get_run_meta(run_dir)
|
|
128
|
+
m1, m2, m3, m4, m5, m6 = st.columns(6)
|
|
129
|
+
with m1:
|
|
130
|
+
st.metric("Status", meta["status"][:10])
|
|
131
|
+
with m2:
|
|
132
|
+
st.metric("Model", (meta["model"] or "—")[:12])
|
|
133
|
+
with m3:
|
|
134
|
+
st.metric("Epochs", meta["epochs"])
|
|
135
|
+
with m4:
|
|
136
|
+
st.metric("mAP@50", f"{meta['mAP']:.3f}" if meta["mAP"] else "—")
|
|
137
|
+
with m5:
|
|
138
|
+
st.metric("Size", meta["size"] or "—")
|
|
139
|
+
with m6:
|
|
140
|
+
st.metric("Date", meta["date"][:8])
|
|
141
|
+
|
|
142
|
+
_render_run_dashboard(run_dir)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Training visualizations rendering."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
from flashstudio.constants import IMG_EXTENSIONS, VIS_SKIP_FILE, VIS_QUEUE_SIZE
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _render_visualizations(run_dir):
|
|
10
|
+
"""Show GT-vs-Prediction images in a FIFO queue — latest N shown, oldest dropped."""
|
|
11
|
+
vis_dir = os.path.join(run_dir, "visualizations")
|
|
12
|
+
all_images = []
|
|
13
|
+
if os.path.isdir(vis_dir):
|
|
14
|
+
for f in sorted(os.listdir(vis_dir)):
|
|
15
|
+
if f.lower().endswith(IMG_EXTENSIONS) and f != VIS_SKIP_FILE:
|
|
16
|
+
all_images.append(os.path.join(vis_dir, f))
|
|
17
|
+
|
|
18
|
+
if not all_images:
|
|
19
|
+
st.info("No visualization images yet. GT vs Prediction images appear here during training.")
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
queue = all_images[-VIS_QUEUE_SIZE:]
|
|
23
|
+
|
|
24
|
+
st.caption(f"Showing latest {len(queue)} of {len(all_images)} total | Queue drops oldest when new arrives")
|
|
25
|
+
|
|
26
|
+
cols = st.columns(VIS_QUEUE_SIZE)
|
|
27
|
+
for i, img_path in enumerate(queue):
|
|
28
|
+
fname = os.path.basename(img_path)
|
|
29
|
+
label = fname.replace(".jpg", "").replace(".png", "").replace("_", " ").title()
|
|
30
|
+
with cols[i]:
|
|
31
|
+
st.image(img_path, caption=label, use_container_width=True)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _render_image_grid(img_dir, description, max_images=3):
|
|
35
|
+
"""Render images from a directory in a grid (max 3 by default)."""
|
|
36
|
+
if not os.path.isdir(img_dir):
|
|
37
|
+
st.info(f"Directory not found: {img_dir}")
|
|
38
|
+
return
|
|
39
|
+
|
|
40
|
+
images = sorted([f for f in os.listdir(img_dir) if f.endswith((".jpg", ".png"))])
|
|
41
|
+
if not images:
|
|
42
|
+
st.info("No images found.")
|
|
43
|
+
return
|
|
44
|
+
|
|
45
|
+
shown = images[:max_images]
|
|
46
|
+
st.caption(f"{description} — {len(shown)} images" + (f" (of {len(images)} total)" if len(images) > max_images else ""))
|
|
47
|
+
|
|
48
|
+
cols = st.columns(len(shown))
|
|
49
|
+
for i, col in enumerate(cols):
|
|
50
|
+
img_path = os.path.join(img_dir, shown[i])
|
|
51
|
+
with col:
|
|
52
|
+
st.image(img_path, caption=shown[i][:30], use_container_width=True)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""FlashStudio — Training Dashboard Page (reads real FlashDet workspace output)."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
from flashstudio.constants import COMPLETE_MARKERS
|
|
7
|
+
from flashstudio.pages.training.launch.tab import _render_start_tab
|
|
8
|
+
from flashstudio.pages.training.monitor.tab import _render_monitor_tab
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _check_training_process():
|
|
12
|
+
"""Check if the training subprocess is still alive; extract errors if it crashed."""
|
|
13
|
+
pid = st.session_state.get("training_pid")
|
|
14
|
+
if not pid or not st.session_state.get("training_active"):
|
|
15
|
+
return
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
os.kill(pid, 0)
|
|
19
|
+
return
|
|
20
|
+
except (ProcessLookupError, PermissionError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
log_path = st.session_state.get("training_log_file", "")
|
|
24
|
+
error_lines = []
|
|
25
|
+
last_lines = []
|
|
26
|
+
if log_path and os.path.isfile(log_path):
|
|
27
|
+
try:
|
|
28
|
+
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
|
|
29
|
+
all_lines = f.readlines()
|
|
30
|
+
last_lines = all_lines[-30:] if len(all_lines) > 30 else all_lines
|
|
31
|
+
|
|
32
|
+
in_traceback = False
|
|
33
|
+
for line in all_lines:
|
|
34
|
+
stripped = line.strip()
|
|
35
|
+
if "Traceback" in stripped:
|
|
36
|
+
in_traceback = True
|
|
37
|
+
error_lines = [stripped]
|
|
38
|
+
elif in_traceback:
|
|
39
|
+
error_lines.append(stripped)
|
|
40
|
+
elif stripped.startswith("[") and "[ERROR]" in stripped:
|
|
41
|
+
error_lines.append(stripped)
|
|
42
|
+
elif "Error:" in stripped or "Exception:" in stripped:
|
|
43
|
+
error_lines.append(stripped)
|
|
44
|
+
except OSError:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
st.session_state["training_active"] = False
|
|
48
|
+
st.session_state["training_pid"] = None
|
|
49
|
+
fh = st.session_state.pop("_log_file_handle", None)
|
|
50
|
+
if fh and not fh.closed:
|
|
51
|
+
try:
|
|
52
|
+
fh.close()
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
if error_lines:
|
|
57
|
+
st.session_state["training_status"] = "Failed"
|
|
58
|
+
st.session_state["training_error"] = "\n".join(error_lines[-15:])
|
|
59
|
+
else:
|
|
60
|
+
complete_markers = COMPLETE_MARKERS
|
|
61
|
+
run_path = st.session_state.get("active_run_path", "")
|
|
62
|
+
completed = any(
|
|
63
|
+
os.path.isfile(os.path.join(run_path, m)) for m in complete_markers
|
|
64
|
+
) if run_path else False
|
|
65
|
+
if not completed and last_lines:
|
|
66
|
+
completed = any("Training Complete!" in l for l in last_lines)
|
|
67
|
+
if completed:
|
|
68
|
+
st.session_state["training_status"] = "Complete"
|
|
69
|
+
else:
|
|
70
|
+
tail = "".join(last_lines[-5:]).strip() if last_lines else "No log output"
|
|
71
|
+
st.session_state["training_status"] = "Stopped"
|
|
72
|
+
st.session_state["training_error"] = f"Training process exited.\n\nLast log output:\n{tail}"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def render_training_page():
|
|
76
|
+
"""Render training dashboard — ultra-compact."""
|
|
77
|
+
from flashstudio.components.styles import render_page_header
|
|
78
|
+
from flashstudio.utils import show_flashes
|
|
79
|
+
render_page_header("", "Training")
|
|
80
|
+
|
|
81
|
+
show_flashes()
|
|
82
|
+
|
|
83
|
+
if "training_active" not in st.session_state:
|
|
84
|
+
st.session_state["training_active"] = False
|
|
85
|
+
st.session_state["training_status"] = "Not started"
|
|
86
|
+
if "training_paused" not in st.session_state:
|
|
87
|
+
st.session_state["training_paused"] = False
|
|
88
|
+
|
|
89
|
+
_check_training_process()
|
|
90
|
+
|
|
91
|
+
training_err = st.session_state.pop("training_error", None)
|
|
92
|
+
if training_err:
|
|
93
|
+
lines = training_err.strip().split("\n")
|
|
94
|
+
error_summary = ""
|
|
95
|
+
for line in reversed(lines):
|
|
96
|
+
stripped = line.strip()
|
|
97
|
+
if "Error:" in stripped or "Exception:" in stripped:
|
|
98
|
+
error_summary = stripped
|
|
99
|
+
break
|
|
100
|
+
if stripped.startswith("[") and "[ERROR]" in stripped:
|
|
101
|
+
error_summary = stripped
|
|
102
|
+
break
|
|
103
|
+
if error_summary:
|
|
104
|
+
st.error(f"**Training Error:** {error_summary}")
|
|
105
|
+
else:
|
|
106
|
+
st.error("**Training failed** — see log details below")
|
|
107
|
+
with st.expander("Full error log", expanded=True):
|
|
108
|
+
st.code(training_err, language="text")
|
|
109
|
+
|
|
110
|
+
is_active = st.session_state.get("training_active", False)
|
|
111
|
+
pid = st.session_state.get("training_pid")
|
|
112
|
+
status = st.session_state.get("training_status", "")
|
|
113
|
+
if is_active:
|
|
114
|
+
if st.session_state.get("training_paused"):
|
|
115
|
+
st.warning(f"Paused (PID {pid})")
|
|
116
|
+
else:
|
|
117
|
+
st.success(f"Running (PID {pid})")
|
|
118
|
+
elif status == "Failed":
|
|
119
|
+
st.error("Training failed")
|
|
120
|
+
elif status == "Complete":
|
|
121
|
+
st.success("Training complete")
|
|
122
|
+
|
|
123
|
+
goto_monitor = st.session_state.pop("_goto_monitor", False)
|
|
124
|
+
is_active = st.session_state.get("training_active", False)
|
|
125
|
+
|
|
126
|
+
if goto_monitor or is_active:
|
|
127
|
+
tab_monitor, tab_start = st.tabs(["Monitor", "Launch"])
|
|
128
|
+
else:
|
|
129
|
+
tab_start, tab_monitor = st.tabs(["Launch", "Monitor"])
|
|
130
|
+
|
|
131
|
+
with tab_start:
|
|
132
|
+
_render_start_tab()
|
|
133
|
+
with tab_monitor:
|
|
134
|
+
_render_monitor_tab()
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""FlashStudio shared utilities."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from flashstudio.constants import (
|
|
6
|
+
TRAIN_EPOCHS, TRAIN_BATCH_SIZE, TRAIN_LR, TRAIN_IMG_SIZE,
|
|
7
|
+
TRAIN_WEIGHT_DECAY, TRAIN_WARMUP_EPOCHS, TRAIN_PATIENCE,
|
|
8
|
+
TRAIN_NUM_WORKERS, TRAIN_GRAD_ACCUM, DEFAULT_MODEL_ARCH,
|
|
9
|
+
DEFAULT_ARCH_FAMILY, DEFAULT_SAVE_DIR,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
# Central defaults — single source of truth, no hardcoding elsewhere
|
|
13
|
+
DEFAULTS = {
|
|
14
|
+
"epochs": TRAIN_EPOCHS,
|
|
15
|
+
"batch_size": TRAIN_BATCH_SIZE,
|
|
16
|
+
"lr": TRAIN_LR,
|
|
17
|
+
"img_size": TRAIN_IMG_SIZE,
|
|
18
|
+
"weight_decay": TRAIN_WEIGHT_DECAY,
|
|
19
|
+
"warmup_epochs": TRAIN_WARMUP_EPOCHS,
|
|
20
|
+
"patience": TRAIN_PATIENCE,
|
|
21
|
+
"num_workers": TRAIN_NUM_WORKERS,
|
|
22
|
+
"grad_accum": TRAIN_GRAD_ACCUM,
|
|
23
|
+
"model_arch": DEFAULT_MODEL_ARCH,
|
|
24
|
+
"arch_family": DEFAULT_ARCH_FAMILY,
|
|
25
|
+
"save_dir": DEFAULT_SAVE_DIR,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def get_default(key: str):
|
|
30
|
+
"""Get a default value for a session state key."""
|
|
31
|
+
return DEFAULTS.get(key)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_state(key: str):
|
|
35
|
+
"""Get a session state value with config mirror fallback.
|
|
36
|
+
|
|
37
|
+
Priority: _config_mirror > session_state > DEFAULTS.
|
|
38
|
+
Mirror is kept fresh by update_config_mirror() on every Model page render,
|
|
39
|
+
so it always holds the latest user choices. Mirror wins over session_state
|
|
40
|
+
because Streamlit drops widget keys when navigating away, and
|
|
41
|
+
_ensure_config_in_session may re-seed stale values from the project file.
|
|
42
|
+
"""
|
|
43
|
+
import streamlit as st
|
|
44
|
+
mirror = st.session_state.get("_config_mirror", {})
|
|
45
|
+
if key in mirror:
|
|
46
|
+
return mirror[key]
|
|
47
|
+
return st.session_state.get(key, DEFAULTS.get(key))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def update_config_mirror():
|
|
51
|
+
"""Sync _config_mirror with current session_state values.
|
|
52
|
+
|
|
53
|
+
Call this after Model page renders so the mirror always holds the
|
|
54
|
+
latest user choices, surviving page navigations that clear widget keys.
|
|
55
|
+
"""
|
|
56
|
+
import streamlit as st
|
|
57
|
+
mirror_keys = [
|
|
58
|
+
"epochs", "batch_size", "lr", "img_size", "weight_decay",
|
|
59
|
+
"warmup_epochs", "patience", "num_workers", "grad_accum",
|
|
60
|
+
"val_interval", "lr_final_ratio", "optimizer",
|
|
61
|
+
"model_arch", "arch_family", "pico_backbone",
|
|
62
|
+
"amp", "grad_clip", "multiscale",
|
|
63
|
+
"aug_mosaic", "aug_mixup", "aug_copypaste",
|
|
64
|
+
"finetune_strategy", "pretrain_option", "custom_weights",
|
|
65
|
+
"lora_variant", "lora_rank", "lora_alpha", "lora_dropout", "lora_targets",
|
|
66
|
+
"activation_checkpointing", "activation_offloading", "optimizer_in_bwd",
|
|
67
|
+
"compile_model", "use_8bit_optimizer", "ddp", "chunked_loss", "chunk_size",
|
|
68
|
+
"save_dir", "save_best", "resume_training",
|
|
69
|
+
]
|
|
70
|
+
mirror = st.session_state.get("_config_mirror", {})
|
|
71
|
+
for k in mirror_keys:
|
|
72
|
+
if k in st.session_state:
|
|
73
|
+
mirror[k] = st.session_state[k]
|
|
74
|
+
st.session_state["_config_mirror"] = mirror
|
|
75
|
+
|
|
76
|
+
from flashstudio.utils.device import (
|
|
77
|
+
is_colab,
|
|
78
|
+
get_device,
|
|
79
|
+
get_gpu_info,
|
|
80
|
+
get_colab_runtime_type,
|
|
81
|
+
has_cuda,
|
|
82
|
+
)
|
|
83
|
+
from flashstudio.utils.filesystem import (
|
|
84
|
+
dir_size_bytes,
|
|
85
|
+
dir_size_str,
|
|
86
|
+
ensure_dir,
|
|
87
|
+
safe_rmtree,
|
|
88
|
+
)
|
|
89
|
+
from flashstudio.utils.config import (
|
|
90
|
+
build_training_config,
|
|
91
|
+
apply_training_config,
|
|
92
|
+
save_config_yaml,
|
|
93
|
+
load_config_yaml,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def flash(msg: str, kind: str = "success"):
|
|
98
|
+
"""Queue a flash message to display on the next render.
|
|
99
|
+
|
|
100
|
+
kind: 'success', 'error', 'warning', 'info'
|
|
101
|
+
"""
|
|
102
|
+
import streamlit as st
|
|
103
|
+
msgs = st.session_state.setdefault("_flash_messages", [])
|
|
104
|
+
msgs.append((kind, msg))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def show_flashes():
|
|
108
|
+
"""Display and clear all queued flash messages at the top of a page."""
|
|
109
|
+
import streamlit as st
|
|
110
|
+
msgs = st.session_state.pop("_flash_messages", [])
|
|
111
|
+
for kind, msg in msgs:
|
|
112
|
+
getattr(st, kind, st.info)(msg)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def get_class_names_str() -> str:
|
|
116
|
+
"""Return class names as a newline-separated string, regardless of stored type."""
|
|
117
|
+
import streamlit as st
|
|
118
|
+
raw = st.session_state.get("class_names", "")
|
|
119
|
+
if isinstance(raw, list):
|
|
120
|
+
return "\n".join(raw)
|
|
121
|
+
return str(raw)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def get_class_names_list() -> list:
|
|
125
|
+
"""Return class names as a list of strings."""
|
|
126
|
+
raw = get_class_names_str()
|
|
127
|
+
return [c.strip() for c in raw.strip().split("\n") if c.strip()]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
__all__ = [
|
|
131
|
+
"DEFAULTS",
|
|
132
|
+
"get_default",
|
|
133
|
+
"get_state",
|
|
134
|
+
"update_config_mirror",
|
|
135
|
+
"is_colab",
|
|
136
|
+
"get_device",
|
|
137
|
+
"get_gpu_info",
|
|
138
|
+
"get_colab_runtime_type",
|
|
139
|
+
"has_cuda",
|
|
140
|
+
"dir_size_bytes",
|
|
141
|
+
"dir_size_str",
|
|
142
|
+
"ensure_dir",
|
|
143
|
+
"safe_rmtree",
|
|
144
|
+
"build_training_config",
|
|
145
|
+
"apply_training_config",
|
|
146
|
+
"save_config_yaml",
|
|
147
|
+
"load_config_yaml",
|
|
148
|
+
"flash",
|
|
149
|
+
"show_flashes",
|
|
150
|
+
"get_class_names_str",
|
|
151
|
+
"get_class_names_list",
|
|
152
|
+
]
|