flashstudio 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flashstudio might be problematic. Click here for more details.
- flashstudio/__init__.py +5 -0
- flashstudio/app.py +194 -0
- flashstudio/cli.py +18 -0
- flashstudio/components/__init__.py +0 -0
- flashstudio/components/project_manager.py +490 -0
- flashstudio/components/sidebar.py +87 -0
- flashstudio/components/styles.py +346 -0
- flashstudio/components/wizard.py +45 -0
- flashstudio/components/zone_drawer/__init__.py +177 -0
- flashstudio/constants.py +245 -0
- flashstudio/core/__init__.py +3 -0
- flashstudio/core/constants.py +3 -0
- flashstudio/launcher.py +288 -0
- flashstudio/pages/__init__.py +8 -0
- flashstudio/pages/dashboard/__init__.py +3 -0
- flashstudio/pages/dashboard/_common.py +10 -0
- flashstudio/pages/dashboard/overview/__init__.py +3 -0
- flashstudio/pages/dashboard/overview/metrics.py +33 -0
- flashstudio/pages/dashboard/overview/pipeline_status.py +35 -0
- flashstudio/pages/dashboard/overview/project_banner.py +23 -0
- flashstudio/pages/dashboard/overview/quick_actions.py +28 -0
- flashstudio/pages/dashboard/overview/recent_runs.py +99 -0
- flashstudio/pages/dashboard/overview/tab.py +35 -0
- flashstudio/pages/dashboard/page.py +17 -0
- flashstudio/pages/data/__init__.py +3 -0
- flashstudio/pages/data/_common.py +107 -0
- flashstudio/pages/data/download/__init__.py +3 -0
- flashstudio/pages/data/download/tab.py +75 -0
- flashstudio/pages/data/helpers.py +338 -0
- flashstudio/pages/data/page.py +50 -0
- flashstudio/pages/data/preview/__init__.py +3 -0
- flashstudio/pages/data/preview/tab.py +139 -0
- flashstudio/pages/data/upload/__init__.py +3 -0
- flashstudio/pages/data/upload/tab.py +156 -0
- flashstudio/pages/data/verify/__init__.py +3 -0
- flashstudio/pages/data/verify/tab.py +108 -0
- flashstudio/pages/export/__init__.py +3 -0
- flashstudio/pages/export/page.py +54 -0
- flashstudio/pages/export/weights/__init__.py +4 -0
- flashstudio/pages/export/weights/exporter.py +60 -0
- flashstudio/pages/export/weights/settings.py +86 -0
- flashstudio/pages/inference/__init__.py +3 -0
- flashstudio/pages/inference/_common.py +76 -0
- flashstudio/pages/inference/data/__init__.py +3 -0
- flashstudio/pages/inference/data/tab.py +121 -0
- flashstudio/pages/inference/model/__init__.py +3 -0
- flashstudio/pages/inference/model/tab.py +62 -0
- flashstudio/pages/inference/page.py +42 -0
- flashstudio/pages/inference/run/__init__.py +3 -0
- flashstudio/pages/inference/run/detection.py +133 -0
- flashstudio/pages/inference/run/tab.py +161 -0
- flashstudio/pages/inference/run/video.py +195 -0
- flashstudio/pages/inference/solution/__init__.py +3 -0
- flashstudio/pages/inference/solution/tab.py +121 -0
- flashstudio/pages/model/__init__.py +3 -0
- flashstudio/pages/model/_common.py +3 -0
- flashstudio/pages/model/advanced/__init__.py +3 -0
- flashstudio/pages/model/advanced/tab.py +54 -0
- flashstudio/pages/model/architecture/__init__.py +3 -0
- flashstudio/pages/model/architecture/tab.py +90 -0
- flashstudio/pages/model/augment/__init__.py +3 -0
- flashstudio/pages/model/augment/tab.py +18 -0
- flashstudio/pages/model/hyperparams/__init__.py +4 -0
- flashstudio/pages/model/hyperparams/tab.py +89 -0
- flashstudio/pages/model/page.py +30 -0
- flashstudio/pages/model/utils/__init__.py +3 -0
- flashstudio/pages/model/utils/config.py +27 -0
- flashstudio/pages/training/__init__.py +3 -0
- flashstudio/pages/training/_common.py +24 -0
- flashstudio/pages/training/launch/__init__.py +3 -0
- flashstudio/pages/training/launch/controls.py +84 -0
- flashstudio/pages/training/launch/dialogs.py +247 -0
- flashstudio/pages/training/launch/preflight.py +77 -0
- flashstudio/pages/training/launch/runner.py +440 -0
- flashstudio/pages/training/launch/tab.py +134 -0
- flashstudio/pages/training/monitor/__init__.py +3 -0
- flashstudio/pages/training/monitor/checkpoints.py +96 -0
- flashstudio/pages/training/monitor/curves.py +113 -0
- flashstudio/pages/training/monitor/dashboard.py +129 -0
- flashstudio/pages/training/monitor/gt_verification.py +122 -0
- flashstudio/pages/training/monitor/log_viewer.py +43 -0
- flashstudio/pages/training/monitor/parsers.py +253 -0
- flashstudio/pages/training/monitor/run_meta.py +165 -0
- flashstudio/pages/training/monitor/tab.py +142 -0
- flashstudio/pages/training/monitor/visualizations.py +52 -0
- flashstudio/pages/training/page.py +134 -0
- flashstudio/utils/__init__.py +152 -0
- flashstudio/utils/config.py +104 -0
- flashstudio/utils/device.py +67 -0
- flashstudio/utils/filesystem.py +67 -0
- flashstudio/utils/jobs.py +95 -0
- flashstudio/utils/training_hooks.py +266 -0
- flashstudio-0.0.1.dist-info/METADATA +107 -0
- flashstudio-0.0.1.dist-info/RECORD +97 -0
- flashstudio-0.0.1.dist-info/WHEEL +5 -0
- flashstudio-0.0.1.dist-info/entry_points.txt +2 -0
- flashstudio-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Training curves rendering."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
import plotly.graph_objects as go
|
|
6
|
+
from plotly.subplots import make_subplots
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _render_curves(history, run_dir):
|
|
10
|
+
"""Render training curves — from FlashDet's plots/ images and CSV data."""
|
|
11
|
+
from flashstudio.constants import VIS_DIR_NAMES
|
|
12
|
+
plots_dir = os.path.join(run_dir, VIS_DIR_NAMES[1])
|
|
13
|
+
training_curves_img = os.path.join(plots_dir, "training_curves.png")
|
|
14
|
+
map_curve_img = os.path.join(plots_dir, "mAP_curve.png")
|
|
15
|
+
|
|
16
|
+
if os.path.isfile(training_curves_img) or os.path.isfile(map_curve_img):
|
|
17
|
+
st.caption("FlashDet Generated Plots")
|
|
18
|
+
cc1, cc2 = st.columns(2)
|
|
19
|
+
if os.path.isfile(training_curves_img):
|
|
20
|
+
with cc1:
|
|
21
|
+
st.image(training_curves_img, caption="Training Curves", use_container_width=True)
|
|
22
|
+
if os.path.isfile(map_curve_img):
|
|
23
|
+
with cc2:
|
|
24
|
+
st.image(map_curve_img, caption="mAP Curve", use_container_width=True)
|
|
25
|
+
|
|
26
|
+
if not history or not history.get("train_loss"):
|
|
27
|
+
if not os.path.isfile(training_curves_img):
|
|
28
|
+
st.info("No training data yet.")
|
|
29
|
+
return
|
|
30
|
+
|
|
31
|
+
st.caption("Interactive Charts")
|
|
32
|
+
epochs = history.get("epochs", [])
|
|
33
|
+
losses = [x for x in history["train_loss"] if x is not None]
|
|
34
|
+
epochs_for_loss = [epochs[i] if i < len(epochs) else i + 1
|
|
35
|
+
for i, x in enumerate(history["train_loss"]) if x is not None]
|
|
36
|
+
|
|
37
|
+
# Determine sub-loss keys (CSV uses train_box/train_cls, log uses o2m_cls/o2m_box)
|
|
38
|
+
has_csv_subloss = any(x for x in history.get("train_box", []) if x is not None)
|
|
39
|
+
has_log_subloss = any(x for x in history.get("o2m_cls", []) if x is not None)
|
|
40
|
+
|
|
41
|
+
fig = make_subplots(
|
|
42
|
+
rows=2, cols=2,
|
|
43
|
+
subplot_titles=("Total Loss", "mAP@0.5",
|
|
44
|
+
"Sub-Losses (box / cls / l1)" if has_csv_subloss else "Sub-Losses (o2m/o2o)",
|
|
45
|
+
"Learning Rate"),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
fig.add_trace(
|
|
49
|
+
go.Scatter(x=epochs_for_loss, y=losses, mode="lines+markers",
|
|
50
|
+
name="Train Loss", line=dict(color="#7C3AED", width=2), marker=dict(size=4)),
|
|
51
|
+
row=1, col=1,
|
|
52
|
+
)
|
|
53
|
+
if history.get("val_loss"):
|
|
54
|
+
ve = history.get("val_epochs", list(range(1, len(history["val_loss"]) + 1)))
|
|
55
|
+
fig.add_trace(
|
|
56
|
+
go.Scatter(x=ve, y=history["val_loss"], mode="lines+markers",
|
|
57
|
+
name="Val Loss", line=dict(color="#F59E0B", width=2), marker=dict(size=4)),
|
|
58
|
+
row=1, col=1,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
if history.get("mAP50"):
|
|
62
|
+
ve = history.get("val_epochs", list(range(1, len(history["mAP50"]) + 1)))
|
|
63
|
+
fig.add_trace(
|
|
64
|
+
go.Scatter(x=ve, y=history["mAP50"], mode="lines+markers",
|
|
65
|
+
name="mAP@0.5", line=dict(color="#10B981", width=2), marker=dict(size=5)),
|
|
66
|
+
row=1, col=2,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
# Sub-losses from CSV (train_box, train_cls, train_l1)
|
|
70
|
+
if has_csv_subloss:
|
|
71
|
+
for key, color, dash in [
|
|
72
|
+
("train_cls", "#EF4444", None), ("train_box", "#3B82F6", None),
|
|
73
|
+
("train_l1", "#F97316", "dash"),
|
|
74
|
+
("val_cls", "#EF4444", "dot"), ("val_box", "#3B82F6", "dot"),
|
|
75
|
+
]:
|
|
76
|
+
vals = [x for x in history.get(key, []) if x is not None]
|
|
77
|
+
if vals:
|
|
78
|
+
ep = list(range(1, len(vals) + 1))
|
|
79
|
+
fig.add_trace(
|
|
80
|
+
go.Scatter(x=ep, y=vals, mode="lines", name=key,
|
|
81
|
+
line=dict(color=color, width=1.5, dash=dash)),
|
|
82
|
+
row=2, col=1,
|
|
83
|
+
)
|
|
84
|
+
elif has_log_subloss:
|
|
85
|
+
for key, color, dash in [
|
|
86
|
+
("o2m_cls", "#EF4444", None), ("o2m_box", "#F97316", "dash"),
|
|
87
|
+
("o2o_cls", "#3B82F6", None), ("o2o_box", "#10B981", "dash"),
|
|
88
|
+
]:
|
|
89
|
+
vals = [x for x in history.get(key, []) if x is not None]
|
|
90
|
+
if vals:
|
|
91
|
+
ep = list(range(1, len(vals) + 1))
|
|
92
|
+
fig.add_trace(
|
|
93
|
+
go.Scatter(x=ep, y=vals, mode="lines", name=key,
|
|
94
|
+
line=dict(color=color, width=1.5, dash=dash)),
|
|
95
|
+
row=2, col=1,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
if history.get("lr"):
|
|
99
|
+
lr_epochs = list(range(1, len(history["lr"]) + 1))
|
|
100
|
+
fig.add_trace(
|
|
101
|
+
go.Scatter(x=lr_epochs, y=history["lr"], mode="lines",
|
|
102
|
+
name="LR", line=dict(color="#6366F1", width=2)),
|
|
103
|
+
row=2, col=2,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
fig.update_layout(
|
|
107
|
+
template="plotly_white",
|
|
108
|
+
height=400,
|
|
109
|
+
margin=dict(l=30, r=10, t=30, b=25),
|
|
110
|
+
showlegend=True,
|
|
111
|
+
legend=dict(orientation="h", yanchor="bottom", y=1.02, font=dict(size=10)),
|
|
112
|
+
)
|
|
113
|
+
st.plotly_chart(fig, use_container_width=True)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Run dashboard — main display panel for a single training run."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
from flashstudio.pages.training.monitor.parsers import _find_log_file, _parse_training_csv, _parse_training_log
|
|
7
|
+
from flashstudio.pages.training.monitor.curves import _render_curves
|
|
8
|
+
from flashstudio.pages.training.monitor.visualizations import _render_visualizations
|
|
9
|
+
from flashstudio.pages.training.monitor.gt_verification import _render_gt_verification
|
|
10
|
+
from flashstudio.pages.training.monitor.log_viewer import _render_full_log
|
|
11
|
+
from flashstudio.pages.training.monitor.checkpoints import _render_checkpoints
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _render_run_dashboard(run_dir):
|
|
15
|
+
"""Render full dashboard for a single training run — always shows all 5 sub-tabs."""
|
|
16
|
+
log_file = None
|
|
17
|
+
history = None
|
|
18
|
+
|
|
19
|
+
if run_dir and os.path.isdir(run_dir):
|
|
20
|
+
log_file = _find_log_file(run_dir)
|
|
21
|
+
# Prefer CSV (clean numeric data), fall back to log parsing
|
|
22
|
+
history = _parse_training_csv(run_dir)
|
|
23
|
+
if not history:
|
|
24
|
+
history = _parse_training_log(log_file) if log_file else None
|
|
25
|
+
_render_metrics_from_history(history, run_dir)
|
|
26
|
+
|
|
27
|
+
# Check for errors in log file — show error banner at top
|
|
28
|
+
if log_file and os.path.isfile(log_file):
|
|
29
|
+
try:
|
|
30
|
+
with open(log_file) as f:
|
|
31
|
+
log_content = f.read()
|
|
32
|
+
if "Traceback" in log_content:
|
|
33
|
+
all_lines = log_content.strip().split("\n")
|
|
34
|
+
tb_idx = -1
|
|
35
|
+
for idx_l, ln in enumerate(all_lines):
|
|
36
|
+
if "Traceback" in ln:
|
|
37
|
+
tb_idx = idx_l
|
|
38
|
+
if tb_idx >= 0:
|
|
39
|
+
error_summary = "\n".join(all_lines[tb_idx:][-10:])
|
|
40
|
+
st.error(f"Training error detected. Check the **Log** tab for details.\n```\n{error_summary}\n```")
|
|
41
|
+
except OSError:
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
tab_curves, tab_viz, tab_gt, tab_log, tab_files = st.tabs([
|
|
45
|
+
"Curves", "Visualizations", "Ground Truth", "Log", "Files"
|
|
46
|
+
])
|
|
47
|
+
|
|
48
|
+
with tab_curves:
|
|
49
|
+
if run_dir and os.path.isdir(run_dir):
|
|
50
|
+
_render_curves(history, run_dir)
|
|
51
|
+
else:
|
|
52
|
+
st.info("No training data yet. Start a training run to see curves.")
|
|
53
|
+
|
|
54
|
+
with tab_viz:
|
|
55
|
+
if run_dir and os.path.isdir(run_dir):
|
|
56
|
+
_render_visualizations(run_dir)
|
|
57
|
+
else:
|
|
58
|
+
st.info("No visualizations yet.")
|
|
59
|
+
|
|
60
|
+
with tab_gt:
|
|
61
|
+
if run_dir and os.path.isdir(run_dir):
|
|
62
|
+
_render_gt_verification(run_dir)
|
|
63
|
+
else:
|
|
64
|
+
st.info("No GT verification data yet.")
|
|
65
|
+
|
|
66
|
+
with tab_log:
|
|
67
|
+
if log_file:
|
|
68
|
+
_render_full_log(log_file)
|
|
69
|
+
else:
|
|
70
|
+
st.info("No training log yet.")
|
|
71
|
+
|
|
72
|
+
with tab_files:
|
|
73
|
+
if run_dir and os.path.isdir(run_dir):
|
|
74
|
+
_render_checkpoints(run_dir)
|
|
75
|
+
else:
|
|
76
|
+
st.info("No checkpoint files yet.")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _render_metrics_from_history(history, run_dir):
|
|
80
|
+
"""Render top metric cards from parsed history (CSV or log-based)."""
|
|
81
|
+
cols = st.columns(7)
|
|
82
|
+
|
|
83
|
+
if history:
|
|
84
|
+
n_epochs = len(history["epochs"])
|
|
85
|
+
total = history.get("total_epochs") or n_epochs
|
|
86
|
+
|
|
87
|
+
with cols[0]:
|
|
88
|
+
st.metric("Epoch", f"{n_epochs}/{total}")
|
|
89
|
+
with cols[1]:
|
|
90
|
+
losses = [x for x in history["train_loss"] if x is not None]
|
|
91
|
+
val = f"{losses[-1]:.1f}" if losses else "—"
|
|
92
|
+
st.metric("Train Loss", val)
|
|
93
|
+
with cols[2]:
|
|
94
|
+
vl = [x for x in history.get("val_loss", []) if x is not None]
|
|
95
|
+
val = f"{vl[-1]:.2f}" if vl else "—"
|
|
96
|
+
st.metric("Val Loss", val)
|
|
97
|
+
with cols[3]:
|
|
98
|
+
mp = [x for x in history.get("mAP50", []) if x is not None]
|
|
99
|
+
val = f"{mp[-1]:.4f}" if mp else "—"
|
|
100
|
+
st.metric("mAP@0.5", val)
|
|
101
|
+
with cols[4]:
|
|
102
|
+
best = f"{max(mp):.4f}" if mp else "—"
|
|
103
|
+
st.metric("Best mAP", best)
|
|
104
|
+
with cols[5]:
|
|
105
|
+
lr_val = f"{history['lr'][-1]:.2e}" if history.get("lr") else "—"
|
|
106
|
+
st.metric("LR", lr_val)
|
|
107
|
+
with cols[6]:
|
|
108
|
+
model = history.get("model_info", "")
|
|
109
|
+
device = history.get("device", "")
|
|
110
|
+
label = model or device or "—"
|
|
111
|
+
st.metric("Model", label[:12])
|
|
112
|
+
|
|
113
|
+
if n_epochs < total:
|
|
114
|
+
st.progress(n_epochs / max(total, 1))
|
|
115
|
+
epoch_times = history.get("epoch_time", [])
|
|
116
|
+
if epoch_times:
|
|
117
|
+
avg_time = sum(epoch_times) / len(epoch_times)
|
|
118
|
+
remaining = (total - n_epochs) * avg_time
|
|
119
|
+
if remaining > 3600:
|
|
120
|
+
eta = f"~{remaining / 3600:.1f}h"
|
|
121
|
+
elif remaining > 60:
|
|
122
|
+
eta = f"~{remaining / 60:.0f}m"
|
|
123
|
+
else:
|
|
124
|
+
eta = f"~{remaining:.0f}s"
|
|
125
|
+
st.caption(f"ETA: {eta} ({avg_time:.1f}s/epoch)")
|
|
126
|
+
else:
|
|
127
|
+
for col in cols:
|
|
128
|
+
with col:
|
|
129
|
+
st.metric("—", "No data")
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Ground truth verification display."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import json
|
|
6
|
+
import streamlit as st
|
|
7
|
+
|
|
8
|
+
from flashstudio.pages.training.monitor.parsers import _find_log_file
|
|
9
|
+
from flashstudio.pages.training.monitor.visualizations import _render_image_grid
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _render_gt_verification(run_dir):
|
|
13
|
+
"""GT verification — from dedicated directory, or extracted from log file."""
|
|
14
|
+
from flashstudio.constants import GT_VERIFICATION_DIR, GT_REPORT_FILE, GT_SUMMARY_FILE
|
|
15
|
+
gt_dir = os.path.join(run_dir, GT_VERIFICATION_DIR)
|
|
16
|
+
|
|
17
|
+
if os.path.isdir(gt_dir):
|
|
18
|
+
report_path = os.path.join(gt_dir, GT_REPORT_FILE)
|
|
19
|
+
summary_path = os.path.join(gt_dir, GT_SUMMARY_FILE)
|
|
20
|
+
|
|
21
|
+
if os.path.isfile(report_path):
|
|
22
|
+
with open(report_path) as f:
|
|
23
|
+
report = json.load(f)
|
|
24
|
+
st.success("Verification: PASSED") if report.get("passed") else st.error("Verification: FAILED")
|
|
25
|
+
|
|
26
|
+
tc = report.get("splits", {}).get("train", {}).get("coco", {})
|
|
27
|
+
vc = report.get("splits", {}).get("val", {}).get("coco", {})
|
|
28
|
+
td = report.get("splits", {}).get("train", {}).get("dataloader", {})
|
|
29
|
+
|
|
30
|
+
m1, m2, m3, m4, m5, m6 = st.columns(6)
|
|
31
|
+
with m1:
|
|
32
|
+
st.metric("Train Imgs", tc.get("num_images", 0))
|
|
33
|
+
with m2:
|
|
34
|
+
st.metric("Train Ann", tc.get("num_annotations", 0))
|
|
35
|
+
with m3:
|
|
36
|
+
st.metric("Val Imgs", vc.get("num_images", 0))
|
|
37
|
+
with m4:
|
|
38
|
+
st.metric("Val Ann", vc.get("num_annotations", 0))
|
|
39
|
+
with m5:
|
|
40
|
+
st.metric("Classes", report.get("num_classes", "?"))
|
|
41
|
+
with m6:
|
|
42
|
+
st.metric("Avg Boxes/Img", f"{td.get('avg_boxes_per_sample', 0):.1f}")
|
|
43
|
+
|
|
44
|
+
if os.path.isfile(summary_path):
|
|
45
|
+
with st.expander("Verification Summary", expanded=False):
|
|
46
|
+
with open(summary_path) as f:
|
|
47
|
+
st.code(f.read(), language="text")
|
|
48
|
+
|
|
49
|
+
raw_dir = os.path.join(gt_dir, "images", "raw")
|
|
50
|
+
dl_dir = os.path.join(gt_dir, "images", "dataloader")
|
|
51
|
+
gt_tab_raw, gt_tab_dl = st.tabs(["Raw GT Images", "Dataloader GT Images"])
|
|
52
|
+
with gt_tab_raw:
|
|
53
|
+
_render_image_grid(raw_dir, "Raw ground truth (before augmentations)")
|
|
54
|
+
with gt_tab_dl:
|
|
55
|
+
_render_image_grid(dl_dir, "After dataloader transforms & augmentations")
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
# No dedicated directory — extract verification info from the log file
|
|
59
|
+
log_file = _find_log_file(run_dir)
|
|
60
|
+
if not log_file:
|
|
61
|
+
st.info("No GT verification data yet. Start training to generate.")
|
|
62
|
+
return
|
|
63
|
+
|
|
64
|
+
with open(log_file, "r", encoding="utf-8", errors="replace") as f:
|
|
65
|
+
log_content = f.read()
|
|
66
|
+
|
|
67
|
+
# Extract dataset verification block from log
|
|
68
|
+
ver_start = log_content.find("Dataset Verification")
|
|
69
|
+
if ver_start == -1:
|
|
70
|
+
st.info("No dataset verification found in log.")
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
ver_end = log_content.find("Starting training", ver_start)
|
|
74
|
+
if ver_end == -1:
|
|
75
|
+
ver_end = ver_start + 800
|
|
76
|
+
ver_block = log_content[ver_start:ver_end].strip()
|
|
77
|
+
|
|
78
|
+
# Parse verification info
|
|
79
|
+
train_imgs = re.search(r"train.*?Images:\s*(\d+)", ver_block, re.DOTALL)
|
|
80
|
+
train_ann = re.search(r"train.*?Annotations:\s*(\d+)", ver_block, re.DOTALL)
|
|
81
|
+
train_found = re.search(r"train.*?Files found:\s*(\d+)/(\d+)", ver_block, re.DOTALL)
|
|
82
|
+
val_imgs = re.search(r"(?:valid|val).*?Images:\s*(\d+)", ver_block, re.DOTALL)
|
|
83
|
+
val_ann = re.search(r"(?:valid|val).*?Annotations:\s*(\d+)", ver_block, re.DOTALL)
|
|
84
|
+
|
|
85
|
+
# Extract classes from header
|
|
86
|
+
cls_m = re.search(r"Classes \((\d+)\): \[(.+?)\]", log_content)
|
|
87
|
+
num_cls = int(cls_m.group(1)) if cls_m else "?"
|
|
88
|
+
class_names = [c.strip().strip("'") for c in cls_m.group(2).split(",")] if cls_m else []
|
|
89
|
+
|
|
90
|
+
# Check if all passed
|
|
91
|
+
train_ok = "✓ train" in ver_block
|
|
92
|
+
val_ok = "✓ valid" in ver_block or "✓ val" in ver_block
|
|
93
|
+
if train_ok and val_ok:
|
|
94
|
+
st.success("Dataset Verification: PASSED")
|
|
95
|
+
elif train_ok:
|
|
96
|
+
st.warning("Dataset Verification: Train OK, Val missing")
|
|
97
|
+
else:
|
|
98
|
+
st.error("Dataset Verification: FAILED")
|
|
99
|
+
|
|
100
|
+
m1, m2, m3, m4, m5, m6 = st.columns(6)
|
|
101
|
+
with m1:
|
|
102
|
+
st.metric("Train Imgs", train_imgs.group(1) if train_imgs else "—")
|
|
103
|
+
with m2:
|
|
104
|
+
st.metric("Train Ann", train_ann.group(1) if train_ann else "—")
|
|
105
|
+
with m3:
|
|
106
|
+
st.metric("Val Imgs", val_imgs.group(1) if val_imgs else "—")
|
|
107
|
+
with m4:
|
|
108
|
+
st.metric("Val Ann", val_ann.group(1) if val_ann else "—")
|
|
109
|
+
with m5:
|
|
110
|
+
st.metric("Classes", num_cls)
|
|
111
|
+
with m6:
|
|
112
|
+
if train_found:
|
|
113
|
+
st.metric("Files OK", f"{train_found.group(1)}/{train_found.group(2)}")
|
|
114
|
+
else:
|
|
115
|
+
st.metric("Files OK", "—")
|
|
116
|
+
|
|
117
|
+
if class_names:
|
|
118
|
+
with st.expander(f"Class Names ({len(class_names)})", expanded=False):
|
|
119
|
+
st.code("\n".join(class_names), language="text")
|
|
120
|
+
|
|
121
|
+
with st.expander("Verification Log", expanded=False):
|
|
122
|
+
st.code(ver_block, language="text")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Training log viewer."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _render_full_log(log_path):
|
|
8
|
+
"""Log viewer with refresh and error detection."""
|
|
9
|
+
if not log_path or not os.path.isfile(log_path):
|
|
10
|
+
st.info("No log found.")
|
|
11
|
+
return
|
|
12
|
+
|
|
13
|
+
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
|
|
14
|
+
content = f.read()
|
|
15
|
+
lines = content.strip().split("\n") if content.strip() else []
|
|
16
|
+
|
|
17
|
+
# Detect the LAST error block (Traceback + exception)
|
|
18
|
+
error_lines = []
|
|
19
|
+
last_tb_start = -1
|
|
20
|
+
for i, line in enumerate(lines):
|
|
21
|
+
if "Traceback" in line:
|
|
22
|
+
last_tb_start = i
|
|
23
|
+
if last_tb_start >= 0:
|
|
24
|
+
error_lines = lines[last_tb_start:]
|
|
25
|
+
has_error = bool(error_lines)
|
|
26
|
+
|
|
27
|
+
lc1, lc2, lc3 = st.columns([4, 1, 1])
|
|
28
|
+
with lc1:
|
|
29
|
+
st.caption(f"`{os.path.basename(log_path)}` — {len(lines)} lines")
|
|
30
|
+
with lc2:
|
|
31
|
+
show_all = st.checkbox("Full", value=False, key="show_full_log")
|
|
32
|
+
with lc3:
|
|
33
|
+
if st.button("Refresh", key="refresh_log_btn", use_container_width=True):
|
|
34
|
+
st.rerun()
|
|
35
|
+
|
|
36
|
+
if has_error and error_lines:
|
|
37
|
+
st.error("Training failed with error:")
|
|
38
|
+
st.code("\n".join(error_lines[-20:]), language="python")
|
|
39
|
+
|
|
40
|
+
if lines:
|
|
41
|
+
st.code("\n".join(lines if show_all else lines[-30:]), language="bash")
|
|
42
|
+
else:
|
|
43
|
+
st.warning("Log file is empty. Training may still be starting...")
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Training log and CSV parsers."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import glob as glob_module
|
|
6
|
+
|
|
7
|
+
from flashstudio.constants import TRAINING_LOG_CSV, TRAINING_LOG_GLOB
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _find_log_file(run_dir: str):
|
|
11
|
+
"""Find the best training log file — prefer the largest (stdout-redirected has more data)."""
|
|
12
|
+
logs = glob_module.glob(os.path.join(run_dir, TRAINING_LOG_GLOB))
|
|
13
|
+
if not logs:
|
|
14
|
+
return None
|
|
15
|
+
return max(logs, key=lambda p: os.path.getsize(p))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parse_training_csv(run_dir: str):
|
|
19
|
+
"""Parse training_log.csv — the primary metrics source from FlashDet."""
|
|
20
|
+
csv_path = os.path.join(run_dir, TRAINING_LOG_CSV)
|
|
21
|
+
if not os.path.isfile(csv_path):
|
|
22
|
+
return None
|
|
23
|
+
|
|
24
|
+
import csv
|
|
25
|
+
history = {
|
|
26
|
+
"epochs": [], "lr": [], "train_loss": [], "val_loss": [], "mAP50": [],
|
|
27
|
+
"train_box": [], "train_cls": [], "train_l1": [],
|
|
28
|
+
"val_box": [], "val_cls": [], "val_l1": [],
|
|
29
|
+
"model_info": "", "device": "", "classes": [],
|
|
30
|
+
"total_epochs": 0, "batch_size": 0,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
with open(csv_path, "r") as f:
|
|
35
|
+
reader = csv.DictReader(f)
|
|
36
|
+
for row in reader:
|
|
37
|
+
try:
|
|
38
|
+
epoch = int(float(row.get("epoch", 0)))
|
|
39
|
+
except (ValueError, TypeError):
|
|
40
|
+
continue
|
|
41
|
+
history["epochs"].append(epoch)
|
|
42
|
+
history["lr"].append(float(row.get("lr", 0)))
|
|
43
|
+
history["train_loss"].append(float(row.get("train_loss", 0)))
|
|
44
|
+
|
|
45
|
+
val_loss = row.get("val_loss", "")
|
|
46
|
+
history["val_loss"].append(float(val_loss) if val_loss else None)
|
|
47
|
+
|
|
48
|
+
mAP = row.get("mAP@0.5") or row.get("val_mAP") or ""
|
|
49
|
+
history["mAP50"].append(float(mAP) if mAP else None)
|
|
50
|
+
|
|
51
|
+
for key in ("train_box", "train_cls", "train_l1", "val_box", "val_cls", "val_l1"):
|
|
52
|
+
val = row.get(key, "")
|
|
53
|
+
history[key].append(float(val) if val else None)
|
|
54
|
+
except Exception:
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
# Build paired val_epochs list (epochs where validation actually ran)
|
|
58
|
+
val_epochs = []
|
|
59
|
+
val_loss_clean = []
|
|
60
|
+
mAP50_clean = []
|
|
61
|
+
for i, ep in enumerate(history["epochs"]):
|
|
62
|
+
vl = history["val_loss"][i] if i < len(history["val_loss"]) else None
|
|
63
|
+
mp = history["mAP50"][i] if i < len(history["mAP50"]) else None
|
|
64
|
+
if vl is not None or mp is not None:
|
|
65
|
+
val_epochs.append(ep)
|
|
66
|
+
val_loss_clean.append(vl)
|
|
67
|
+
mAP50_clean.append(mp)
|
|
68
|
+
history["val_epochs"] = val_epochs
|
|
69
|
+
history["val_loss"] = val_loss_clean
|
|
70
|
+
history["mAP50"] = mAP50_clean
|
|
71
|
+
|
|
72
|
+
# Fill metadata from log header FIRST — log has authoritative total_epochs
|
|
73
|
+
log_file = _find_log_file(run_dir)
|
|
74
|
+
if log_file:
|
|
75
|
+
try:
|
|
76
|
+
with open(log_file, "r") as f:
|
|
77
|
+
header_lines = f.readlines()[:30]
|
|
78
|
+
for line in header_lines:
|
|
79
|
+
lc = line.strip()
|
|
80
|
+
if "Model Size:" in lc:
|
|
81
|
+
history["model_info"] = lc.split("Model Size:")[-1].strip()
|
|
82
|
+
if not history["model_info"]:
|
|
83
|
+
model_m = re.search(r"Model:\s*(\w+),\s*Input:\s*\((\d+),\s*(\d+)\)", lc)
|
|
84
|
+
if model_m:
|
|
85
|
+
history["model_info"] = f"{model_m.group(1)} ({model_m.group(2)}x{model_m.group(3)})"
|
|
86
|
+
if "Device:" in lc:
|
|
87
|
+
history["device"] = lc.split("Device:")[-1].strip()
|
|
88
|
+
if "Classes" in lc and ":" in lc:
|
|
89
|
+
m = re.search(r"Classes \((\d+)\):\s*\[(.+)\]", lc)
|
|
90
|
+
if m:
|
|
91
|
+
history["classes"] = [c.strip().strip("'").strip('"') for c in m.group(2).split(",")]
|
|
92
|
+
if "Epochs:" in lc:
|
|
93
|
+
m = re.search(r"Epochs:\s*(\d+)", lc)
|
|
94
|
+
if m:
|
|
95
|
+
history["total_epochs"] = int(m.group(1))
|
|
96
|
+
bm = re.search(r"Batch:\s*(\d+)", lc)
|
|
97
|
+
if bm and not history["batch_size"]:
|
|
98
|
+
history["batch_size"] = int(bm.group(1))
|
|
99
|
+
if "Batch Size:" in lc:
|
|
100
|
+
m = re.search(r"Batch Size:\s*(\d+)", lc)
|
|
101
|
+
if m and not history["batch_size"]:
|
|
102
|
+
history["batch_size"] = int(m.group(1))
|
|
103
|
+
if "Learning Rate:" in lc and not history.get("initial_lr"):
|
|
104
|
+
m = re.search(r"Learning Rate:\s*([\d.e+-]+)", lc)
|
|
105
|
+
if m:
|
|
106
|
+
history["initial_lr"] = float(m.group(1))
|
|
107
|
+
except OSError:
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
# Fallback: if log didn't provide total_epochs, use the number of CSV rows
|
|
111
|
+
if not history["total_epochs"]:
|
|
112
|
+
history["total_epochs"] = len(history["epochs"])
|
|
113
|
+
|
|
114
|
+
return history if history["epochs"] else None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _parse_training_log(log_path: str):
|
|
118
|
+
"""Parse FlashDet training log and extract metrics per epoch.
|
|
119
|
+
|
|
120
|
+
Handles the actual FlashDet log format:
|
|
121
|
+
Epoch N/T (lr=X.XXXXXX)
|
|
122
|
+
Val Loss: V (loss_total: ..., o2m_cls: ..., o2m_box: ..., o2o_cls: ..., o2o_box: ...) | mAP@0.5: M
|
|
123
|
+
"""
|
|
124
|
+
if not log_path or not os.path.isfile(log_path):
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
history = {
|
|
128
|
+
"epochs": [], "lr": [], "train_loss": [],
|
|
129
|
+
"val_loss": [], "mAP50": [],
|
|
130
|
+
"o2m_cls": [], "o2m_box": [], "o2o_cls": [], "o2o_box": [],
|
|
131
|
+
"ema_decay": [], "epoch_time": [],
|
|
132
|
+
"model_info": "", "device": "", "classes": [],
|
|
133
|
+
"total_epochs": 0, "batch_size": 0,
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
with open(log_path, "r", encoding="utf-8", errors="replace") as f:
|
|
137
|
+
lines = f.readlines()
|
|
138
|
+
|
|
139
|
+
current_epoch = 0
|
|
140
|
+
|
|
141
|
+
for line in lines:
|
|
142
|
+
# Parse header: Model: p, Input: (320, 320)
|
|
143
|
+
model_m = re.search(r"Model: (\w+), Input: \((\d+), (\d+)\)", line)
|
|
144
|
+
if model_m:
|
|
145
|
+
history["model_info"] = f"{model_m.group(1)} ({model_m.group(2)}x{model_m.group(3)})"
|
|
146
|
+
|
|
147
|
+
if "Device:" in line:
|
|
148
|
+
dm = re.search(r"Device:\s*(\S+)", line)
|
|
149
|
+
if dm:
|
|
150
|
+
history["device"] = dm.group(1)
|
|
151
|
+
|
|
152
|
+
if "Classes" in line and ":" in line:
|
|
153
|
+
m = re.search(r"Classes \((\d+)\): \[(.+)\]", line)
|
|
154
|
+
if m:
|
|
155
|
+
history["classes"] = [c.strip().strip("'") for c in m.group(2).split(",")]
|
|
156
|
+
|
|
157
|
+
# Epochs: 100, Batch: 16, LR: 0.001
|
|
158
|
+
header_m = re.search(r"Epochs:\s*(\d+),\s*Batch:\s*(\d+),\s*LR:\s*([\d.e+-]+)", line)
|
|
159
|
+
if header_m:
|
|
160
|
+
history["total_epochs"] = int(header_m.group(1))
|
|
161
|
+
history["batch_size"] = int(header_m.group(2))
|
|
162
|
+
|
|
163
|
+
# Epoch N/T (lr=X.XXXXXX) — with or without ema_decay
|
|
164
|
+
epoch_m = re.search(r"Epoch (\d+)/(\d+)\s*\(lr=([\d.e+-]+)", line)
|
|
165
|
+
if epoch_m:
|
|
166
|
+
current_epoch = int(epoch_m.group(1))
|
|
167
|
+
if history["total_epochs"] == 0:
|
|
168
|
+
history["total_epochs"] = int(epoch_m.group(2))
|
|
169
|
+
history["lr"].append(float(epoch_m.group(3)))
|
|
170
|
+
ema_m = re.search(r"ema_decay=([\d.e+-]+)", line)
|
|
171
|
+
if ema_m:
|
|
172
|
+
history["ema_decay"].append(float(ema_m.group(1)))
|
|
173
|
+
|
|
174
|
+
# Val Loss line — two formats:
|
|
175
|
+
# 1. Val Loss: 561.98 (loss_total: ..., o2m_cls: X, ...) | mAP@0.5: M (with sub-losses)
|
|
176
|
+
# 2. Val Loss: 561.98 | mAP@0.5: M (without sub-losses)
|
|
177
|
+
val_m = re.search(r"Val Loss:\s*([\d.]+)\s*\((.+?)\)\s*\|\s*mAP@0\.5:\s*([\d.]+)", line)
|
|
178
|
+
val_simple = None
|
|
179
|
+
if not val_m:
|
|
180
|
+
val_simple = re.search(r"Val Loss:\s*([\d.]+).*?mAP@0\.5:\s*([\d.]+)", line)
|
|
181
|
+
if val_m:
|
|
182
|
+
history["val_loss"].append(float(val_m.group(1)))
|
|
183
|
+
history["mAP50"].append(float(val_m.group(3)))
|
|
184
|
+
|
|
185
|
+
detail = val_m.group(2)
|
|
186
|
+
lt = re.search(r"loss_total:\s*([\d.]+)", detail)
|
|
187
|
+
if lt:
|
|
188
|
+
while len(history["train_loss"]) < current_epoch:
|
|
189
|
+
history["train_loss"].append(None)
|
|
190
|
+
if current_epoch > 0:
|
|
191
|
+
history["train_loss"][current_epoch - 1] = float(lt.group(1))
|
|
192
|
+
|
|
193
|
+
for key, pattern in [
|
|
194
|
+
("o2m_cls", r"o2m_cls:\s*([\d.]+)"),
|
|
195
|
+
("o2m_box", r"o2m_box:\s*([\d.]+)"),
|
|
196
|
+
("o2o_cls", r"o2o_cls:\s*([\d.]+)"),
|
|
197
|
+
("o2o_box", r"o2o_box:\s*([\d.]+)"),
|
|
198
|
+
]:
|
|
199
|
+
sm = re.search(pattern, detail)
|
|
200
|
+
while len(history[key]) < current_epoch:
|
|
201
|
+
history[key].append(None)
|
|
202
|
+
if sm and current_epoch > 0:
|
|
203
|
+
history[key][current_epoch - 1] = float(sm.group(1))
|
|
204
|
+
elif val_simple:
|
|
205
|
+
history["val_loss"].append(float(val_simple.group(1)))
|
|
206
|
+
history["mAP50"].append(float(val_simple.group(2)))
|
|
207
|
+
|
|
208
|
+
# Also try: Validation - Loss: X | mAP@0.5: Y (older format)
|
|
209
|
+
if not val_m and not val_simple:
|
|
210
|
+
alt_val = re.search(r"Validation.*Loss:\s*([\d.]+).*mAP@0\.5:\s*([\d.]+)", line)
|
|
211
|
+
if alt_val:
|
|
212
|
+
history["val_loss"].append(float(alt_val.group(1)))
|
|
213
|
+
history["mAP50"].append(float(alt_val.group(2)))
|
|
214
|
+
|
|
215
|
+
# Batch-level loss: Epoch [N] Batch [B/T] Loss: X (loss_total: Y, ...)
|
|
216
|
+
batch_m = re.search(r"Epoch \[(\d+)\] Batch \[\d+/\d+\] Loss:\s*([\d.]+)", line)
|
|
217
|
+
if batch_m:
|
|
218
|
+
ep = int(batch_m.group(1))
|
|
219
|
+
raw_loss = float(batch_m.group(2))
|
|
220
|
+
# Use loss_total if available (more meaningful), else use raw Loss
|
|
221
|
+
lt = re.search(r"loss_total:\s*([\d.]+)", line)
|
|
222
|
+
loss_val = float(lt.group(1)) if lt else raw_loss
|
|
223
|
+
while len(history["train_loss"]) < ep:
|
|
224
|
+
history["train_loss"].append(None)
|
|
225
|
+
history["train_loss"][ep - 1] = loss_val
|
|
226
|
+
# Extract sub-losses
|
|
227
|
+
for key, pattern in [
|
|
228
|
+
("o2m_cls", r"o2m_cls:\s*([\d.]+)"),
|
|
229
|
+
("o2m_box", r"o2m_box:\s*([\d.]+)"),
|
|
230
|
+
("o2o_cls", r"o2o_cls:\s*([\d.]+)"),
|
|
231
|
+
("o2o_box", r"o2o_box:\s*([\d.]+)"),
|
|
232
|
+
]:
|
|
233
|
+
sm = re.search(pattern, line)
|
|
234
|
+
while len(history[key]) < ep:
|
|
235
|
+
history[key].append(None)
|
|
236
|
+
if sm and ep > 0:
|
|
237
|
+
history[key][ep - 1] = float(sm.group(1))
|
|
238
|
+
|
|
239
|
+
# Epoch time — FlashDet logs "Time: Xs" at end of batch lines
|
|
240
|
+
time_m = re.search(r"Epoch time:\s*([\d.]+)s", line)
|
|
241
|
+
if not time_m:
|
|
242
|
+
time_m = re.search(r"Time:\s*([\d.]+)s\s*$", line)
|
|
243
|
+
if time_m:
|
|
244
|
+
history["epoch_time"].append(float(time_m.group(1)))
|
|
245
|
+
|
|
246
|
+
# Fill in epochs list from the longest metric array
|
|
247
|
+
n = max(len(history["train_loss"]), len(history["lr"]),
|
|
248
|
+
len(history["val_loss"]), len(history["mAP50"]))
|
|
249
|
+
history["epochs"] = list(range(1, n + 1))
|
|
250
|
+
|
|
251
|
+
if not history["epochs"]:
|
|
252
|
+
return None
|
|
253
|
+
return history
|