runnable 0.50.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- extensions/README.md +0 -0
- extensions/__init__.py +0 -0
- extensions/catalog/README.md +0 -0
- extensions/catalog/any_path.py +214 -0
- extensions/catalog/file_system.py +52 -0
- extensions/catalog/minio.py +72 -0
- extensions/catalog/pyproject.toml +14 -0
- extensions/catalog/s3.py +11 -0
- extensions/job_executor/README.md +0 -0
- extensions/job_executor/__init__.py +236 -0
- extensions/job_executor/emulate.py +70 -0
- extensions/job_executor/k8s.py +553 -0
- extensions/job_executor/k8s_job_spec.yaml +37 -0
- extensions/job_executor/local.py +35 -0
- extensions/job_executor/local_container.py +161 -0
- extensions/job_executor/pyproject.toml +16 -0
- extensions/nodes/README.md +0 -0
- extensions/nodes/__init__.py +0 -0
- extensions/nodes/conditional.py +301 -0
- extensions/nodes/fail.py +78 -0
- extensions/nodes/loop.py +394 -0
- extensions/nodes/map.py +477 -0
- extensions/nodes/parallel.py +281 -0
- extensions/nodes/pyproject.toml +15 -0
- extensions/nodes/stub.py +93 -0
- extensions/nodes/success.py +78 -0
- extensions/nodes/task.py +156 -0
- extensions/pipeline_executor/README.md +0 -0
- extensions/pipeline_executor/__init__.py +871 -0
- extensions/pipeline_executor/argo.py +1266 -0
- extensions/pipeline_executor/emulate.py +119 -0
- extensions/pipeline_executor/local.py +226 -0
- extensions/pipeline_executor/local_container.py +369 -0
- extensions/pipeline_executor/mocked.py +159 -0
- extensions/pipeline_executor/pyproject.toml +16 -0
- extensions/run_log_store/README.md +0 -0
- extensions/run_log_store/__init__.py +0 -0
- extensions/run_log_store/any_path.py +100 -0
- extensions/run_log_store/chunked_fs.py +122 -0
- extensions/run_log_store/chunked_minio.py +141 -0
- extensions/run_log_store/file_system.py +91 -0
- extensions/run_log_store/generic_chunked.py +549 -0
- extensions/run_log_store/minio.py +114 -0
- extensions/run_log_store/pyproject.toml +15 -0
- extensions/secrets/README.md +0 -0
- extensions/secrets/dotenv.py +62 -0
- extensions/secrets/pyproject.toml +15 -0
- runnable/__init__.py +108 -0
- runnable/catalog.py +141 -0
- runnable/cli.py +484 -0
- runnable/context.py +730 -0
- runnable/datastore.py +1058 -0
- runnable/defaults.py +159 -0
- runnable/entrypoints.py +390 -0
- runnable/exceptions.py +137 -0
- runnable/executor.py +561 -0
- runnable/gantt.py +1646 -0
- runnable/graph.py +501 -0
- runnable/names.py +546 -0
- runnable/nodes.py +593 -0
- runnable/parameters.py +217 -0
- runnable/pickler.py +96 -0
- runnable/sdk.py +1277 -0
- runnable/secrets.py +92 -0
- runnable/tasks.py +1268 -0
- runnable/telemetry.py +142 -0
- runnable/utils.py +423 -0
- runnable-0.50.0.dist-info/METADATA +189 -0
- runnable-0.50.0.dist-info/RECORD +72 -0
- runnable-0.50.0.dist-info/WHEEL +4 -0
- runnable-0.50.0.dist-info/entry_points.txt +53 -0
- runnable-0.50.0.dist-info/licenses/LICENSE +201 -0
runnable/gantt.py
ADDED
|
@@ -0,0 +1,1646 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Simplified visualization for Runnable pipeline execution.
|
|
3
|
+
|
|
4
|
+
This module provides lightweight, reusable components that understand
|
|
5
|
+
the composite pipeline structure documented in the run logs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ParameterInfo:
|
|
17
|
+
"""Enhanced parameter representation with categorization."""
|
|
18
|
+
|
|
19
|
+
key_inputs: List[str] # Most important inputs for this step
|
|
20
|
+
step_outputs: List[str] # Parameters generated by this step
|
|
21
|
+
context_params: List[str] # Pipeline/inherited parameters
|
|
22
|
+
iteration_vars: List[str] # Map/loop variables (chunk, etc.)
|
|
23
|
+
all_inputs: List[str] # Complete input list
|
|
24
|
+
all_outputs: List[str] # Complete output list
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class StepInfo:
|
|
29
|
+
"""Clean representation of a pipeline step."""
|
|
30
|
+
|
|
31
|
+
name: str
|
|
32
|
+
internal_name: str
|
|
33
|
+
status: str
|
|
34
|
+
step_type: str
|
|
35
|
+
start_time: Optional[datetime]
|
|
36
|
+
end_time: Optional[datetime]
|
|
37
|
+
duration_ms: float
|
|
38
|
+
level: int # 0=top-level, 1=branch, 2=nested
|
|
39
|
+
parent: Optional[str]
|
|
40
|
+
branch: Optional[str]
|
|
41
|
+
command: str
|
|
42
|
+
command_type: str
|
|
43
|
+
input_params: List[str]
|
|
44
|
+
output_params: List[str]
|
|
45
|
+
catalog_ops: Dict[str, List[Dict[str, str]]]
|
|
46
|
+
# Enhanced parameter information
|
|
47
|
+
parameters: Optional["ParameterInfo"] = None
|
|
48
|
+
# Error and attempt information
|
|
49
|
+
error_message: Optional[str] = None
|
|
50
|
+
attempt_count: int = 0
|
|
51
|
+
max_attempts: int = 1
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class StepHierarchyParser:
|
|
55
|
+
"""Parse internal names to understand pipeline hierarchy."""
|
|
56
|
+
|
|
57
|
+
@staticmethod
|
|
58
|
+
def parse_internal_name(internal_name: str) -> Dict[str, str]:
|
|
59
|
+
"""
|
|
60
|
+
Parse internal name into components.
|
|
61
|
+
|
|
62
|
+
Examples:
|
|
63
|
+
- "hello" -> {"step": "hello"}
|
|
64
|
+
- "parallel_step.branch1.hello_stub" -> {
|
|
65
|
+
"composite": "parallel_step",
|
|
66
|
+
"branch": "branch1",
|
|
67
|
+
"step": "hello_stub"
|
|
68
|
+
}
|
|
69
|
+
"""
|
|
70
|
+
parts = internal_name.split(".")
|
|
71
|
+
|
|
72
|
+
if len(parts) == 1:
|
|
73
|
+
return {"step": parts[0]}
|
|
74
|
+
elif len(parts) == 2:
|
|
75
|
+
return {"composite": parts[0], "branch": parts[1]}
|
|
76
|
+
elif len(parts) == 3:
|
|
77
|
+
return {"composite": parts[0], "branch": parts[1], "step": parts[2]}
|
|
78
|
+
else:
|
|
79
|
+
# Handle deeper nesting if needed
|
|
80
|
+
return {
|
|
81
|
+
"composite": parts[0],
|
|
82
|
+
"branch": ".".join(parts[1:-1]),
|
|
83
|
+
"step": parts[-1],
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def get_step_level(internal_name: str) -> int:
|
|
88
|
+
"""Determine hierarchy level from internal name."""
|
|
89
|
+
parts = internal_name.split(".")
|
|
90
|
+
if len(parts) == 1:
|
|
91
|
+
return 0 # Top-level step
|
|
92
|
+
elif len(parts) == 2:
|
|
93
|
+
return 1 # Branch level (for composite step parent)
|
|
94
|
+
else:
|
|
95
|
+
return 2 # Branch step
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class TimelineExtractor:
|
|
99
|
+
"""Extract chronological timeline from run log."""
|
|
100
|
+
|
|
101
|
+
def __init__(self, run_log_data: Dict[str, Any]):
|
|
102
|
+
self.run_log_data = run_log_data
|
|
103
|
+
self.dag_nodes = (
|
|
104
|
+
run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def parse_time(self, time_str: str) -> Optional[datetime]:
|
|
108
|
+
"""Parse ISO timestamp string."""
|
|
109
|
+
try:
|
|
110
|
+
return datetime.fromisoformat(time_str) if time_str else None
|
|
111
|
+
except (ValueError, TypeError):
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
def get_step_timing(
|
|
115
|
+
self, step_data: Dict[str, Any]
|
|
116
|
+
) -> Tuple[Optional[datetime], Optional[datetime], float]:
|
|
117
|
+
"""Extract timing from step attempts."""
|
|
118
|
+
attempts = step_data.get("attempts", [])
|
|
119
|
+
if not attempts:
|
|
120
|
+
return None, None, 0
|
|
121
|
+
|
|
122
|
+
attempt = attempts[0]
|
|
123
|
+
start = self.parse_time(attempt.get("start_time"))
|
|
124
|
+
end = self.parse_time(attempt.get("end_time"))
|
|
125
|
+
|
|
126
|
+
if start and end:
|
|
127
|
+
duration_ms = (end - start).total_seconds() * 1000
|
|
128
|
+
return start, end, duration_ms
|
|
129
|
+
|
|
130
|
+
return None, None, 0
|
|
131
|
+
|
|
132
|
+
def find_dag_node(self, internal_name: str, clean_name: str) -> Dict[str, Any]:
|
|
133
|
+
"""Find DAG node info for command details."""
|
|
134
|
+
# Try direct lookup first
|
|
135
|
+
if clean_name in self.dag_nodes:
|
|
136
|
+
return self.dag_nodes[clean_name]
|
|
137
|
+
|
|
138
|
+
# For composite steps, look in branch structures
|
|
139
|
+
hierarchy = StepHierarchyParser.parse_internal_name(internal_name)
|
|
140
|
+
if "composite" in hierarchy:
|
|
141
|
+
composite_node = self.dag_nodes.get(hierarchy["composite"], {})
|
|
142
|
+
if composite_node.get("is_composite"):
|
|
143
|
+
branches = composite_node.get("branches", {})
|
|
144
|
+
branch_key = hierarchy.get("branch", "")
|
|
145
|
+
|
|
146
|
+
if branch_key in branches:
|
|
147
|
+
branch_nodes = branches[branch_key].get("nodes", {})
|
|
148
|
+
if clean_name in branch_nodes:
|
|
149
|
+
return branch_nodes[clean_name]
|
|
150
|
+
# For map nodes, the branch structure might be different
|
|
151
|
+
elif "branch" in composite_node: # Map node structure
|
|
152
|
+
branch_nodes = composite_node["branch"].get("nodes", {})
|
|
153
|
+
if clean_name in branch_nodes:
|
|
154
|
+
return branch_nodes[clean_name]
|
|
155
|
+
|
|
156
|
+
return {}
|
|
157
|
+
|
|
158
|
+
def _format_parameter_value(self, value: Any, kind: str) -> str:
|
|
159
|
+
"""Format parameter value for display."""
|
|
160
|
+
if kind == "metric":
|
|
161
|
+
if isinstance(value, (int, float)):
|
|
162
|
+
return f"{value:.3g}"
|
|
163
|
+
return str(value)
|
|
164
|
+
|
|
165
|
+
if isinstance(value, str):
|
|
166
|
+
# Truncate long strings
|
|
167
|
+
if len(value) > 50:
|
|
168
|
+
return f'"{value[:47]}..."'
|
|
169
|
+
return f'"{value}"'
|
|
170
|
+
elif isinstance(value, (list, tuple)):
|
|
171
|
+
if len(value) > 3:
|
|
172
|
+
preview = ", ".join(str(v) for v in value[:3])
|
|
173
|
+
return f"[{preview}, ...+{len(value)-3}]"
|
|
174
|
+
return str(value)
|
|
175
|
+
elif isinstance(value, dict):
|
|
176
|
+
if len(value) > 2:
|
|
177
|
+
keys = list(value.keys())[:2]
|
|
178
|
+
preview = ", ".join(f'"{k}": {value[k]}' for k in keys)
|
|
179
|
+
return f"{{{preview}, ...+{len(value)-2}}}"
|
|
180
|
+
return str(value)
|
|
181
|
+
else:
|
|
182
|
+
return str(value)
|
|
183
|
+
|
|
184
|
+
def _categorize_parameters(
|
|
185
|
+
self,
|
|
186
|
+
input_params: List[str],
|
|
187
|
+
output_params: List[str],
|
|
188
|
+
step_name: str,
|
|
189
|
+
internal_name: str,
|
|
190
|
+
) -> ParameterInfo:
|
|
191
|
+
"""Categorize parameters for smarter display."""
|
|
192
|
+
# Common pipeline parameter patterns
|
|
193
|
+
pipeline_params = {"integer", "floater", "stringer", "pydantic_param", "chunks"}
|
|
194
|
+
|
|
195
|
+
# Categorize inputs
|
|
196
|
+
key_inputs = []
|
|
197
|
+
context_params = []
|
|
198
|
+
iteration_vars = []
|
|
199
|
+
|
|
200
|
+
for param in input_params:
|
|
201
|
+
param_name = param.split("=")[0]
|
|
202
|
+
|
|
203
|
+
# Check for iteration variables (chunk, index, etc.)
|
|
204
|
+
if param_name in {"chunk", "index", "iteration", "item"}:
|
|
205
|
+
iteration_vars.append(param)
|
|
206
|
+
# Check for step-specific parameters (not empty and not pipeline defaults)
|
|
207
|
+
elif (
|
|
208
|
+
param_name not in pipeline_params
|
|
209
|
+
and "=" in param
|
|
210
|
+
and param.split("=")[1] not in ['""', "''", "[]", "{}"]
|
|
211
|
+
):
|
|
212
|
+
key_inputs.append(param)
|
|
213
|
+
# Pipeline/context parameters
|
|
214
|
+
elif param_name in pipeline_params:
|
|
215
|
+
context_params.append(param)
|
|
216
|
+
else:
|
|
217
|
+
context_params.append(param)
|
|
218
|
+
|
|
219
|
+
# Limit key inputs to most relevant ones
|
|
220
|
+
if len(key_inputs) > 3:
|
|
221
|
+
key_inputs = key_inputs[:3]
|
|
222
|
+
|
|
223
|
+
# Add iteration vars to key inputs if present
|
|
224
|
+
key_inputs.extend(iteration_vars)
|
|
225
|
+
|
|
226
|
+
# Identify step outputs (parameters generated by this step)
|
|
227
|
+
step_outputs = []
|
|
228
|
+
for param in output_params:
|
|
229
|
+
param_name = param.split("=")[0]
|
|
230
|
+
# Step outputs often have prefixes like "1_processed_python"
|
|
231
|
+
if param_name.startswith(
|
|
232
|
+
f"{step_name.split('.')[-1]}_"
|
|
233
|
+
) or param_name.startswith(f"{internal_name.split('.')[-1]}_"):
|
|
234
|
+
step_outputs.append(param)
|
|
235
|
+
else:
|
|
236
|
+
step_outputs.append(param)
|
|
237
|
+
|
|
238
|
+
return ParameterInfo(
|
|
239
|
+
key_inputs=key_inputs,
|
|
240
|
+
step_outputs=step_outputs,
|
|
241
|
+
context_params=context_params,
|
|
242
|
+
iteration_vars=iteration_vars,
|
|
243
|
+
all_inputs=input_params,
|
|
244
|
+
all_outputs=output_params,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
def extract_timeline(self) -> List[StepInfo]:
|
|
248
|
+
"""Extract all steps in chronological order."""
|
|
249
|
+
steps = []
|
|
250
|
+
|
|
251
|
+
# Process top-level steps
|
|
252
|
+
for step_name, step_data in self.run_log_data.get("steps", {}).items():
|
|
253
|
+
step_info = self._create_step_info(step_name, step_data)
|
|
254
|
+
steps.append(step_info)
|
|
255
|
+
|
|
256
|
+
# Process branches if they exist
|
|
257
|
+
branches = step_data.get("branches", {})
|
|
258
|
+
for branch_name, branch_data in branches.items():
|
|
259
|
+
# Add branch steps
|
|
260
|
+
for sub_step_name, sub_step_data in branch_data.get(
|
|
261
|
+
"steps", {}
|
|
262
|
+
).items():
|
|
263
|
+
sub_step_info = self._create_step_info(
|
|
264
|
+
sub_step_name,
|
|
265
|
+
sub_step_data,
|
|
266
|
+
parent=step_name,
|
|
267
|
+
branch=branch_name,
|
|
268
|
+
)
|
|
269
|
+
steps.append(sub_step_info)
|
|
270
|
+
|
|
271
|
+
# Sort by start time for chronological order
|
|
272
|
+
return sorted(steps, key=lambda x: x.start_time or datetime.min)
|
|
273
|
+
|
|
274
|
+
def _create_step_info(
|
|
275
|
+
self,
|
|
276
|
+
step_name: str,
|
|
277
|
+
step_data: Dict[str, Any],
|
|
278
|
+
parent: Optional[str] = None,
|
|
279
|
+
branch: Optional[str] = None,
|
|
280
|
+
) -> StepInfo:
|
|
281
|
+
"""Create StepInfo from raw step data."""
|
|
282
|
+
internal_name = step_data.get("internal_name", step_name)
|
|
283
|
+
clean_name = step_data.get("name", step_name)
|
|
284
|
+
|
|
285
|
+
# Get timing
|
|
286
|
+
start, end, duration = self.get_step_timing(step_data)
|
|
287
|
+
|
|
288
|
+
# Get command info from DAG
|
|
289
|
+
dag_node = self.find_dag_node(internal_name, clean_name)
|
|
290
|
+
command = dag_node.get("command", "")
|
|
291
|
+
command_type = dag_node.get("command_type", "")
|
|
292
|
+
|
|
293
|
+
# Extract parameters with detailed metadata (exclude pickled/object types)
|
|
294
|
+
input_params = []
|
|
295
|
+
output_params = []
|
|
296
|
+
catalog_ops: Dict[str, List[Dict[str, str]]] = {"put": [], "get": []}
|
|
297
|
+
|
|
298
|
+
attempts = step_data.get("attempts", [])
|
|
299
|
+
if attempts:
|
|
300
|
+
attempt = attempts[0]
|
|
301
|
+
input_param_data = attempt.get("input_parameters", {})
|
|
302
|
+
output_param_data = attempt.get("output_parameters", {})
|
|
303
|
+
|
|
304
|
+
# Process input parameters (exclude object/pickled types)
|
|
305
|
+
for name, param in input_param_data.items():
|
|
306
|
+
if isinstance(param, dict):
|
|
307
|
+
kind = param.get("kind", "")
|
|
308
|
+
if kind in ("json", "metric"):
|
|
309
|
+
value = param.get("value", "")
|
|
310
|
+
# Format value for display
|
|
311
|
+
formatted_value = self._format_parameter_value(value, kind)
|
|
312
|
+
input_params.append(f"{name}={formatted_value}")
|
|
313
|
+
# Skip object/pickled parameters entirely
|
|
314
|
+
|
|
315
|
+
# Process output parameters (exclude object/pickled types)
|
|
316
|
+
for name, param in output_param_data.items():
|
|
317
|
+
if isinstance(param, dict):
|
|
318
|
+
kind = param.get("kind", "")
|
|
319
|
+
if kind in ("json", "metric"):
|
|
320
|
+
value = param.get("value", "")
|
|
321
|
+
# Format value for display
|
|
322
|
+
formatted_value = self._format_parameter_value(value, kind)
|
|
323
|
+
output_params.append(f"{name}={formatted_value}")
|
|
324
|
+
# Skip object/pickled parameters entirely
|
|
325
|
+
|
|
326
|
+
# Extract error information and attempt details
|
|
327
|
+
error_message = None
|
|
328
|
+
attempt_count = len(attempts)
|
|
329
|
+
max_attempts = 1 # Default, will be updated from DAG if available
|
|
330
|
+
|
|
331
|
+
# Get max_attempts from DAG node
|
|
332
|
+
max_attempts = dag_node.get("max_attempts", 1)
|
|
333
|
+
|
|
334
|
+
# Extract error message from failed attempts
|
|
335
|
+
if attempts and step_data.get("status") == "FAIL":
|
|
336
|
+
# Find the failed attempt (usually the last one)
|
|
337
|
+
failed_attempt = None
|
|
338
|
+
for attempt in reversed(attempts):
|
|
339
|
+
if attempt.get("status") == "FAIL":
|
|
340
|
+
failed_attempt = attempt
|
|
341
|
+
break
|
|
342
|
+
|
|
343
|
+
if failed_attempt and failed_attempt.get("message"):
|
|
344
|
+
error_message = failed_attempt["message"].strip()
|
|
345
|
+
|
|
346
|
+
# Extract catalog operations with detailed information
|
|
347
|
+
catalog_data = step_data.get("data_catalog", [])
|
|
348
|
+
for item in catalog_data:
|
|
349
|
+
stage = item.get("stage", "")
|
|
350
|
+
if stage in ("put", "get"):
|
|
351
|
+
catalog_info = {
|
|
352
|
+
"name": item.get("name", ""),
|
|
353
|
+
"data_hash": item.get("data_hash", "")[:8] + "..."
|
|
354
|
+
if item.get("data_hash")
|
|
355
|
+
else "", # Show first 8 chars
|
|
356
|
+
"catalog_path": item.get("catalog_relative_path", ""),
|
|
357
|
+
"stage": stage,
|
|
358
|
+
}
|
|
359
|
+
catalog_ops[stage].append(catalog_info)
|
|
360
|
+
|
|
361
|
+
# Create parameter categorization for better display
|
|
362
|
+
parameters = self._categorize_parameters(
|
|
363
|
+
input_params, output_params, clean_name, internal_name
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
return StepInfo(
|
|
367
|
+
name=clean_name,
|
|
368
|
+
internal_name=internal_name,
|
|
369
|
+
status=step_data.get("status", "UNKNOWN"),
|
|
370
|
+
step_type=step_data.get("step_type", "task"),
|
|
371
|
+
start_time=start,
|
|
372
|
+
end_time=end,
|
|
373
|
+
duration_ms=duration,
|
|
374
|
+
level=StepHierarchyParser.get_step_level(internal_name),
|
|
375
|
+
parent=parent,
|
|
376
|
+
branch=branch,
|
|
377
|
+
command=command,
|
|
378
|
+
command_type=command_type,
|
|
379
|
+
input_params=input_params,
|
|
380
|
+
output_params=output_params,
|
|
381
|
+
catalog_ops=catalog_ops,
|
|
382
|
+
parameters=parameters,
|
|
383
|
+
error_message=error_message,
|
|
384
|
+
attempt_count=attempt_count,
|
|
385
|
+
max_attempts=max_attempts,
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
class SimpleVisualizer:
|
|
390
|
+
"""Simple, lightweight pipeline visualizer."""
|
|
391
|
+
|
|
392
|
+
def __init__(self, run_log_path: Union[str, Path]):
|
|
393
|
+
self.run_log_path = Path(run_log_path)
|
|
394
|
+
self.run_log_data = self._load_run_log()
|
|
395
|
+
self.extractor = TimelineExtractor(self.run_log_data)
|
|
396
|
+
self.timeline = self.extractor.extract_timeline()
|
|
397
|
+
|
|
398
|
+
def _load_run_log(self) -> Dict[str, Any]:
|
|
399
|
+
"""Load run log JSON."""
|
|
400
|
+
if not self.run_log_path.exists():
|
|
401
|
+
raise FileNotFoundError(f"Run log not found: {self.run_log_path}")
|
|
402
|
+
|
|
403
|
+
with open(self.run_log_path, "r") as f:
|
|
404
|
+
return json.load(f)
|
|
405
|
+
|
|
406
|
+
def print_simple_timeline(self) -> None:
|
|
407
|
+
"""Print a clean console timeline."""
|
|
408
|
+
run_id = self.run_log_data.get("run_id", "unknown")
|
|
409
|
+
status = self.run_log_data.get("status", "UNKNOWN")
|
|
410
|
+
|
|
411
|
+
print(f"\n🔄 Pipeline Timeline - {run_id}")
|
|
412
|
+
print(f"Status: {status}")
|
|
413
|
+
print("=" * 80)
|
|
414
|
+
|
|
415
|
+
# Group by composite steps for better display
|
|
416
|
+
current_composite = None
|
|
417
|
+
current_branch = None
|
|
418
|
+
|
|
419
|
+
for step in self.timeline:
|
|
420
|
+
# Skip composite steps themselves (they have no timing)
|
|
421
|
+
if (
|
|
422
|
+
step.step_type in ["parallel", "map", "conditional"]
|
|
423
|
+
and not step.start_time
|
|
424
|
+
):
|
|
425
|
+
continue
|
|
426
|
+
|
|
427
|
+
# Detect composite/branch changes
|
|
428
|
+
hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
|
|
429
|
+
composite = hierarchy.get("composite")
|
|
430
|
+
branch = hierarchy.get("branch")
|
|
431
|
+
|
|
432
|
+
# Show composite header
|
|
433
|
+
if composite and composite != current_composite:
|
|
434
|
+
print(f"\n🔀 {composite} ({self._get_composite_type(composite)})")
|
|
435
|
+
current_composite = composite
|
|
436
|
+
current_branch = None
|
|
437
|
+
|
|
438
|
+
# Show branch header
|
|
439
|
+
if branch and branch != current_branch:
|
|
440
|
+
branch_display = self._format_branch_name(composite or "", branch)
|
|
441
|
+
print(f" ├─ Branch: {branch_display}")
|
|
442
|
+
current_branch = branch
|
|
443
|
+
|
|
444
|
+
# Show step
|
|
445
|
+
indent = (
|
|
446
|
+
" " if step.level == 0 else " " if step.level == 1 else " "
|
|
447
|
+
)
|
|
448
|
+
status_emoji = (
|
|
449
|
+
"✅"
|
|
450
|
+
if step.status == "SUCCESS"
|
|
451
|
+
else "❌"
|
|
452
|
+
if step.status == "FAIL"
|
|
453
|
+
else "⏸️"
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
# Type icon
|
|
457
|
+
type_icons = {
|
|
458
|
+
"task": "⚙️",
|
|
459
|
+
"stub": "📝",
|
|
460
|
+
"success": "✅",
|
|
461
|
+
"fail": "❌",
|
|
462
|
+
"parallel": "🔀",
|
|
463
|
+
"map": "🔁",
|
|
464
|
+
"conditional": "🔀",
|
|
465
|
+
}
|
|
466
|
+
type_icon = type_icons.get(step.step_type, "⚙️")
|
|
467
|
+
|
|
468
|
+
timing = f"({step.duration_ms:.1f}ms)" if step.duration_ms > 0 else ""
|
|
469
|
+
|
|
470
|
+
print(f"{indent}{type_icon} {status_emoji} {step.name} {timing}")
|
|
471
|
+
|
|
472
|
+
# Show error information for failed steps
|
|
473
|
+
if step.status == "FAIL" and step.error_message:
|
|
474
|
+
error_lines = step.error_message.split("\n")
|
|
475
|
+
# Show first line of error, truncated if too long
|
|
476
|
+
error_preview = (
|
|
477
|
+
error_lines[0][:100] + "..."
|
|
478
|
+
if len(error_lines[0]) > 100
|
|
479
|
+
else error_lines[0]
|
|
480
|
+
)
|
|
481
|
+
print(f"{indent} 💥 Error: {error_preview}")
|
|
482
|
+
if step.attempt_count > 1:
|
|
483
|
+
print(
|
|
484
|
+
f"{indent} 🔄 Failed after {step.attempt_count}/{step.max_attempts} attempts"
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
# Show metadata for tasks
|
|
488
|
+
if step.step_type == "task" and (
|
|
489
|
+
step.command
|
|
490
|
+
or step.input_params
|
|
491
|
+
or step.output_params
|
|
492
|
+
or step.catalog_ops["put"]
|
|
493
|
+
or step.catalog_ops["get"]
|
|
494
|
+
):
|
|
495
|
+
if step.command:
|
|
496
|
+
cmd_short = (
|
|
497
|
+
step.command[:50] + "..."
|
|
498
|
+
if len(step.command) > 50
|
|
499
|
+
else step.command
|
|
500
|
+
)
|
|
501
|
+
print(f"{indent} 📝 {step.command_type.upper()}: {cmd_short}")
|
|
502
|
+
|
|
503
|
+
# Show input parameters - compact horizontal display
|
|
504
|
+
if step.input_params:
|
|
505
|
+
params_display = " • ".join(step.input_params)
|
|
506
|
+
print(f"{indent} 📥 {params_display}")
|
|
507
|
+
|
|
508
|
+
# Show output parameters - compact horizontal display
|
|
509
|
+
if step.output_params:
|
|
510
|
+
params_display = " • ".join(step.output_params)
|
|
511
|
+
print(f"{indent} 📤 {params_display}")
|
|
512
|
+
|
|
513
|
+
# Show catalog operations - compact horizontal display
|
|
514
|
+
if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
|
|
515
|
+
catalog_items = []
|
|
516
|
+
if step.catalog_ops.get("put"):
|
|
517
|
+
catalog_items.extend(
|
|
518
|
+
[f"PUT:{item['name']}" for item in step.catalog_ops["put"]]
|
|
519
|
+
)
|
|
520
|
+
if step.catalog_ops.get("get"):
|
|
521
|
+
catalog_items.extend(
|
|
522
|
+
[f"GET:{item['name']}" for item in step.catalog_ops["get"]]
|
|
523
|
+
)
|
|
524
|
+
if catalog_items:
|
|
525
|
+
catalog_display = " • ".join(catalog_items)
|
|
526
|
+
print(f"{indent} 💾 {catalog_display}")
|
|
527
|
+
|
|
528
|
+
print("=" * 80)
|
|
529
|
+
|
|
530
|
+
def _get_composite_type(self, composite_name: str) -> str:
|
|
531
|
+
"""Get composite node type from DAG."""
|
|
532
|
+
dag_nodes = (
|
|
533
|
+
self.run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
|
|
534
|
+
)
|
|
535
|
+
node = dag_nodes.get(composite_name, {})
|
|
536
|
+
return node.get("node_type", "composite")
|
|
537
|
+
|
|
538
|
+
def _format_branch_name(self, composite: str, branch: str) -> str:
|
|
539
|
+
"""Format branch name based on composite type."""
|
|
540
|
+
# Remove composite prefix if present
|
|
541
|
+
if branch.startswith(f"{composite}."):
|
|
542
|
+
branch_clean = branch[len(f"{composite}.") :]
|
|
543
|
+
else:
|
|
544
|
+
branch_clean = branch
|
|
545
|
+
|
|
546
|
+
# Check if it's a map iteration (numeric)
|
|
547
|
+
if branch_clean.isdigit():
|
|
548
|
+
return f"Iteration {branch_clean}"
|
|
549
|
+
|
|
550
|
+
return branch_clean
|
|
551
|
+
|
|
552
|
+
def print_execution_summary(self) -> None:
|
|
553
|
+
"""Print execution summary table."""
|
|
554
|
+
run_id = self.run_log_data.get("run_id", "unknown")
|
|
555
|
+
|
|
556
|
+
print(f"\n📊 Execution Summary - {run_id}")
|
|
557
|
+
print("=" * 80)
|
|
558
|
+
|
|
559
|
+
# Filter to actual executed steps (with timing)
|
|
560
|
+
executed_steps = [step for step in self.timeline if step.start_time]
|
|
561
|
+
|
|
562
|
+
if not executed_steps:
|
|
563
|
+
print("No executed steps found")
|
|
564
|
+
return
|
|
565
|
+
|
|
566
|
+
# Table header
|
|
567
|
+
print(f"{'Step':<30} {'Status':<10} {'Duration':<12} {'Type':<10}")
|
|
568
|
+
print("-" * 80)
|
|
569
|
+
|
|
570
|
+
total_duration = 0
|
|
571
|
+
success_count = 0
|
|
572
|
+
|
|
573
|
+
for step in executed_steps:
|
|
574
|
+
status_emoji = (
|
|
575
|
+
"✅"
|
|
576
|
+
if step.status == "SUCCESS"
|
|
577
|
+
else "❌"
|
|
578
|
+
if step.status == "FAIL"
|
|
579
|
+
else "⏸️"
|
|
580
|
+
)
|
|
581
|
+
duration_text = (
|
|
582
|
+
f"{step.duration_ms:.1f}ms" if step.duration_ms > 0 else "0.0ms"
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
# Truncate long names
|
|
586
|
+
display_name = step.name[:28] + ".." if len(step.name) > 30 else step.name
|
|
587
|
+
|
|
588
|
+
print(
|
|
589
|
+
f"{display_name:<30} {status_emoji}{step.status:<9} {duration_text:<12} {step.step_type:<10}"
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
total_duration += int(step.duration_ms)
|
|
593
|
+
if step.status == "SUCCESS":
|
|
594
|
+
success_count += 1
|
|
595
|
+
|
|
596
|
+
print("-" * 80)
|
|
597
|
+
success_rate = (
|
|
598
|
+
(success_count / len(executed_steps)) * 100 if executed_steps else 0
|
|
599
|
+
)
|
|
600
|
+
overall_status = self.run_log_data.get("status", "UNKNOWN")
|
|
601
|
+
overall_emoji = "✅" if overall_status == "SUCCESS" else "❌"
|
|
602
|
+
|
|
603
|
+
print(
|
|
604
|
+
f"Total Duration: {total_duration:.1f}ms | Success Rate: {success_rate:.1f}% | Status: {overall_emoji} {overall_status}"
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
def generate_html_timeline(
|
|
608
|
+
self, output_path: Optional[Union[str, Path]] = None
|
|
609
|
+
) -> str:
|
|
610
|
+
"""
|
|
611
|
+
Generate an interactive HTML timeline visualization.
|
|
612
|
+
|
|
613
|
+
This creates a lightweight HTML version with:
|
|
614
|
+
- Clean timeline layout
|
|
615
|
+
- Hover tooltips with metadata
|
|
616
|
+
- Expandable composite sections
|
|
617
|
+
- Timing bars proportional to execution duration
|
|
618
|
+
"""
|
|
619
|
+
run_id = self.run_log_data.get("run_id", "unknown")
|
|
620
|
+
status = self.run_log_data.get("status", "UNKNOWN")
|
|
621
|
+
|
|
622
|
+
# Calculate total timeline for proportional bars
|
|
623
|
+
executed_steps = [step for step in self.timeline if step.start_time]
|
|
624
|
+
if executed_steps:
|
|
625
|
+
earliest = min(
|
|
626
|
+
step.start_time for step in executed_steps if step.start_time
|
|
627
|
+
)
|
|
628
|
+
latest = max(step.end_time for step in executed_steps if step.end_time)
|
|
629
|
+
total_duration_ms = (
|
|
630
|
+
(latest - earliest).total_seconds() * 1000 if latest and earliest else 1
|
|
631
|
+
)
|
|
632
|
+
else:
|
|
633
|
+
total_duration_ms = 1
|
|
634
|
+
|
|
635
|
+
html_content = f"""<!DOCTYPE html>
|
|
636
|
+
<html lang="en">
|
|
637
|
+
<head>
|
|
638
|
+
<meta charset="UTF-8">
|
|
639
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
640
|
+
<title>Pipeline Timeline - {run_id}</title>
|
|
641
|
+
<style>
|
|
642
|
+
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
|
643
|
+
|
|
644
|
+
body {{
|
|
645
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
|
|
646
|
+
background: #f8fafc;
|
|
647
|
+
color: #1e293b;
|
|
648
|
+
line-height: 1.6;
|
|
649
|
+
}}
|
|
650
|
+
|
|
651
|
+
.header {{
|
|
652
|
+
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
653
|
+
color: white;
|
|
654
|
+
padding: 2rem 0;
|
|
655
|
+
text-align: center;
|
|
656
|
+
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
|
|
657
|
+
}}
|
|
658
|
+
|
|
659
|
+
.container {{
|
|
660
|
+
max-width: 1600px;
|
|
661
|
+
margin: 2rem auto;
|
|
662
|
+
padding: 0 1rem;
|
|
663
|
+
display: flex;
|
|
664
|
+
gap: 2rem;
|
|
665
|
+
}}
|
|
666
|
+
|
|
667
|
+
.main-content {{
|
|
668
|
+
flex: 1;
|
|
669
|
+
min-width: 0;
|
|
670
|
+
}}
|
|
671
|
+
|
|
672
|
+
.sidebar {{
|
|
673
|
+
width: 400px;
|
|
674
|
+
position: sticky;
|
|
675
|
+
top: 2rem;
|
|
676
|
+
height: fit-content;
|
|
677
|
+
max-height: calc(100vh - 4rem);
|
|
678
|
+
overflow-y: auto;
|
|
679
|
+
background: white;
|
|
680
|
+
border-radius: 12px;
|
|
681
|
+
box-shadow: 0 2px 10px rgba(0,0,0,0.08);
|
|
682
|
+
transition: transform 0.3s ease;
|
|
683
|
+
}}
|
|
684
|
+
|
|
685
|
+
.sidebar.hidden {{
|
|
686
|
+
transform: translateX(100%);
|
|
687
|
+
}}
|
|
688
|
+
|
|
689
|
+
.sidebar-header {{
|
|
690
|
+
background: #f8fafc;
|
|
691
|
+
padding: 1.5rem;
|
|
692
|
+
border-bottom: 1px solid #e2e8f0;
|
|
693
|
+
border-radius: 12px 12px 0 0;
|
|
694
|
+
display: flex;
|
|
695
|
+
justify-content: space-between;
|
|
696
|
+
align-items: center;
|
|
697
|
+
}}
|
|
698
|
+
|
|
699
|
+
.sidebar-content {{
|
|
700
|
+
padding: 1.5rem;
|
|
701
|
+
max-height: 60vh;
|
|
702
|
+
overflow-y: auto;
|
|
703
|
+
}}
|
|
704
|
+
|
|
705
|
+
.sidebar-section {{
|
|
706
|
+
margin-bottom: 1.5rem;
|
|
707
|
+
}}
|
|
708
|
+
|
|
709
|
+
.sidebar-section h4 {{
|
|
710
|
+
color: #374151;
|
|
711
|
+
font-size: 0.875rem;
|
|
712
|
+
font-weight: 600;
|
|
713
|
+
margin-bottom: 0.75rem;
|
|
714
|
+
text-transform: uppercase;
|
|
715
|
+
letter-spacing: 0.05em;
|
|
716
|
+
}}
|
|
717
|
+
|
|
718
|
+
.param-grid {{
|
|
719
|
+
display: grid;
|
|
720
|
+
gap: 0.5rem;
|
|
721
|
+
}}
|
|
722
|
+
|
|
723
|
+
.param-item {{
|
|
724
|
+
padding: 0.5rem 0.75rem;
|
|
725
|
+
border-radius: 6px;
|
|
726
|
+
font-family: monospace;
|
|
727
|
+
font-size: 0.8rem;
|
|
728
|
+
border-left: 3px solid;
|
|
729
|
+
}}
|
|
730
|
+
|
|
731
|
+
.param-item.input {{
|
|
732
|
+
background: #f0fdf4;
|
|
733
|
+
border-left-color: #22c55e;
|
|
734
|
+
color: #166534;
|
|
735
|
+
}}
|
|
736
|
+
|
|
737
|
+
.param-item.output {{
|
|
738
|
+
background: #fef2f2;
|
|
739
|
+
border-left-color: #ef4444;
|
|
740
|
+
color: #991b1b;
|
|
741
|
+
}}
|
|
742
|
+
|
|
743
|
+
.param-item.context {{
|
|
744
|
+
background: #f8fafc;
|
|
745
|
+
border-left-color: #64748b;
|
|
746
|
+
color: #475569;
|
|
747
|
+
}}
|
|
748
|
+
|
|
749
|
+
.param-item.iteration {{
|
|
750
|
+
background: #fefce8;
|
|
751
|
+
border-left-color: #eab308;
|
|
752
|
+
color: #854d0e;
|
|
753
|
+
}}
|
|
754
|
+
|
|
755
|
+
.step-clickable {{
|
|
756
|
+
cursor: pointer;
|
|
757
|
+
transition: background-color 0.2s ease;
|
|
758
|
+
}}
|
|
759
|
+
|
|
760
|
+
.step-clickable:hover {{
|
|
761
|
+
background-color: #f1f5f9 !important;
|
|
762
|
+
}}
|
|
763
|
+
|
|
764
|
+
.step-clickable.selected {{
|
|
765
|
+
background-color: #dbeafe !important;
|
|
766
|
+
border-left: 3px solid #3b82f6;
|
|
767
|
+
}}
|
|
768
|
+
|
|
769
|
+
.close-sidebar {{
|
|
770
|
+
background: none;
|
|
771
|
+
border: none;
|
|
772
|
+
color: #64748b;
|
|
773
|
+
cursor: pointer;
|
|
774
|
+
font-size: 1.2rem;
|
|
775
|
+
padding: 0.25rem;
|
|
776
|
+
}}
|
|
777
|
+
|
|
778
|
+
.close-sidebar:hover {{
|
|
779
|
+
color: #374151;
|
|
780
|
+
}}
|
|
781
|
+
|
|
782
|
+
.timeline-card {{
|
|
783
|
+
background: white;
|
|
784
|
+
border-radius: 12px;
|
|
785
|
+
box-shadow: 0 2px 10px rgba(0,0,0,0.08);
|
|
786
|
+
overflow: hidden;
|
|
787
|
+
margin-bottom: 2rem;
|
|
788
|
+
}}
|
|
789
|
+
|
|
790
|
+
.timeline-header {{
|
|
791
|
+
background: #f8fafc;
|
|
792
|
+
padding: 1.5rem;
|
|
793
|
+
border-bottom: 1px solid #e2e8f0;
|
|
794
|
+
display: flex;
|
|
795
|
+
justify-content: space-between;
|
|
796
|
+
align-items: center;
|
|
797
|
+
}}
|
|
798
|
+
|
|
799
|
+
.timeline-content {{
|
|
800
|
+
padding: 1rem;
|
|
801
|
+
}}
|
|
802
|
+
|
|
803
|
+
.step-row {{
|
|
804
|
+
display: grid;
|
|
805
|
+
grid-template-columns: 300px 1fr 80px;
|
|
806
|
+
align-items: center;
|
|
807
|
+
padding: 0.5rem 0;
|
|
808
|
+
border-bottom: 1px solid #f1f5f9;
|
|
809
|
+
transition: background 0.2s ease;
|
|
810
|
+
gap: 1rem;
|
|
811
|
+
min-height: 40px;
|
|
812
|
+
overflow: visible;
|
|
813
|
+
}}
|
|
814
|
+
|
|
815
|
+
.step-row:hover {{
|
|
816
|
+
background: #f8fafc;
|
|
817
|
+
}}
|
|
818
|
+
|
|
819
|
+
.step-info {{
|
|
820
|
+
display: flex;
|
|
821
|
+
flex-direction: column;
|
|
822
|
+
gap: 0.25rem;
|
|
823
|
+
font-weight: 500;
|
|
824
|
+
min-height: 24px;
|
|
825
|
+
justify-content: flex-start;
|
|
826
|
+
overflow: visible;
|
|
827
|
+
word-wrap: break-word;
|
|
828
|
+
overflow-wrap: break-word;
|
|
829
|
+
}}
|
|
830
|
+
|
|
831
|
+
.step-level-0 {{ padding-left: 0; }}
|
|
832
|
+
.step-level-1 {{ padding-left: 1rem; }}
|
|
833
|
+
.step-level-2 {{ padding-left: 2rem; }}
|
|
834
|
+
|
|
835
|
+
.composite-header {{
|
|
836
|
+
background: #e0f2fe !important;
|
|
837
|
+
border-left: 4px solid #0277bd;
|
|
838
|
+
font-weight: 600;
|
|
839
|
+
color: #01579b;
|
|
840
|
+
}}
|
|
841
|
+
|
|
842
|
+
.branch-header {{
|
|
843
|
+
background: #f3e5f5 !important;
|
|
844
|
+
border-left: 4px solid #7b1fa2;
|
|
845
|
+
font-weight: 600;
|
|
846
|
+
color: #4a148c;
|
|
847
|
+
}}
|
|
848
|
+
|
|
849
|
+
.gantt-container {{
|
|
850
|
+
position: relative;
|
|
851
|
+
height: 30px;
|
|
852
|
+
background: #f8fafc;
|
|
853
|
+
border: 1px solid #e2e8f0;
|
|
854
|
+
border-radius: 4px;
|
|
855
|
+
min-width: 100%;
|
|
856
|
+
overflow: hidden;
|
|
857
|
+
}}
|
|
858
|
+
|
|
859
|
+
.gantt-bar {{
|
|
860
|
+
position: absolute;
|
|
861
|
+
top: 3px;
|
|
862
|
+
height: 24px;
|
|
863
|
+
border-radius: 3px;
|
|
864
|
+
transition: all 0.2s ease;
|
|
865
|
+
cursor: pointer;
|
|
866
|
+
border: 1px solid rgba(255,255,255,0.3);
|
|
867
|
+
}}
|
|
868
|
+
|
|
869
|
+
.gantt-bar:hover {{
|
|
870
|
+
transform: scaleY(1.1);
|
|
871
|
+
z-index: 10;
|
|
872
|
+
box-shadow: 0 2px 8px rgba(0,0,0,0.2);
|
|
873
|
+
}}
|
|
874
|
+
|
|
875
|
+
.time-grid {{
|
|
876
|
+
position: absolute;
|
|
877
|
+
top: 0;
|
|
878
|
+
bottom: 0;
|
|
879
|
+
border-left: 1px solid #e2e8f0;
|
|
880
|
+
opacity: 0.3;
|
|
881
|
+
}}
|
|
882
|
+
|
|
883
|
+
.time-scale {{
|
|
884
|
+
position: relative;
|
|
885
|
+
height: 20px;
|
|
886
|
+
background: #f1f5f9;
|
|
887
|
+
border-bottom: 1px solid #d1d5db;
|
|
888
|
+
font-size: 0.75rem;
|
|
889
|
+
color: #6b7280;
|
|
890
|
+
}}
|
|
891
|
+
|
|
892
|
+
.time-marker {{
|
|
893
|
+
position: absolute;
|
|
894
|
+
top: 0;
|
|
895
|
+
height: 100%;
|
|
896
|
+
display: flex;
|
|
897
|
+
align-items: center;
|
|
898
|
+
padding-left: 4px;
|
|
899
|
+
font-weight: 500;
|
|
900
|
+
}}
|
|
901
|
+
|
|
902
|
+
.timeline-bar:hover {{
|
|
903
|
+
transform: scaleY(1.1);
|
|
904
|
+
z-index: 10;
|
|
905
|
+
}}
|
|
906
|
+
|
|
907
|
+
.bar-success {{ background: linear-gradient(90deg, #22c55e, #16a34a); }}
|
|
908
|
+
.bar-fail {{ background: linear-gradient(90deg, #ef4444, #dc2626); }}
|
|
909
|
+
.bar-unknown {{ background: linear-gradient(90deg, #f59e0b, #d97706); }}
|
|
910
|
+
|
|
911
|
+
.duration-text {{
|
|
912
|
+
font-family: monospace;
|
|
913
|
+
font-size: 0.875rem;
|
|
914
|
+
font-weight: 600;
|
|
915
|
+
}}
|
|
916
|
+
|
|
917
|
+
.duration-fast {{ color: #16a34a; }}
|
|
918
|
+
.duration-medium {{ color: #f59e0b; }}
|
|
919
|
+
.duration-slow {{ color: #dc2626; }}
|
|
920
|
+
|
|
921
|
+
.status-success {{ color: #16a34a; }}
|
|
922
|
+
.status-fail {{ color: #dc2626; }}
|
|
923
|
+
.status-unknown {{ color: #f59e0b; }}
|
|
924
|
+
|
|
925
|
+
.expandable {{
|
|
926
|
+
cursor: pointer;
|
|
927
|
+
user-select: none;
|
|
928
|
+
}}
|
|
929
|
+
|
|
930
|
+
.expandable:hover {{
|
|
931
|
+
background: #e2e8f0 !important;
|
|
932
|
+
}}
|
|
933
|
+
|
|
934
|
+
.step-header.expandable {{
|
|
935
|
+
padding: 0.25rem;
|
|
936
|
+
border-radius: 4px;
|
|
937
|
+
margin: -0.25rem;
|
|
938
|
+
}}
|
|
939
|
+
|
|
940
|
+
.step-header.expandable:hover {{
|
|
941
|
+
background: #f1f5f9 !important;
|
|
942
|
+
}}
|
|
943
|
+
|
|
944
|
+
.collapsible-content {{
|
|
945
|
+
max-height: none;
|
|
946
|
+
overflow: visible;
|
|
947
|
+
transition: max-height 0.3s ease;
|
|
948
|
+
}}
|
|
949
|
+
|
|
950
|
+
.collapsible-content.collapsed {{
|
|
951
|
+
max-height: 0;
|
|
952
|
+
overflow: hidden;
|
|
953
|
+
}}
|
|
954
|
+
|
|
955
|
+
.summary-grid {{
|
|
956
|
+
display: grid;
|
|
957
|
+
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
958
|
+
gap: 1.5rem;
|
|
959
|
+
margin-top: 2rem;
|
|
960
|
+
}}
|
|
961
|
+
|
|
962
|
+
.summary-card {{
|
|
963
|
+
background: white;
|
|
964
|
+
padding: 1.5rem;
|
|
965
|
+
border-radius: 8px;
|
|
966
|
+
box-shadow: 0 2px 10px rgba(0,0,0,0.08);
|
|
967
|
+
text-align: center;
|
|
968
|
+
}}
|
|
969
|
+
|
|
970
|
+
.summary-number {{
|
|
971
|
+
font-size: 2rem;
|
|
972
|
+
font-weight: bold;
|
|
973
|
+
margin-bottom: 0.5rem;
|
|
974
|
+
}}
|
|
975
|
+
|
|
976
|
+
.summary-label {{
|
|
977
|
+
color: #64748b;
|
|
978
|
+
font-size: 0.875rem;
|
|
979
|
+
}}
|
|
980
|
+
</style>
|
|
981
|
+
</head>
|
|
982
|
+
<body>
|
|
983
|
+
<div class="header">
|
|
984
|
+
<h1>🔄 Pipeline Timeline Visualization</h1>
|
|
985
|
+
<p>Interactive execution analysis for {run_id}</p>
|
|
986
|
+
</div>
|
|
987
|
+
|
|
988
|
+
<div class="container">
|
|
989
|
+
<div class="main-content">
|
|
990
|
+
<div class="timeline-card">
|
|
991
|
+
<div class="timeline-header">
|
|
992
|
+
<div>
|
|
993
|
+
<h2>Run ID: {run_id}</h2>
|
|
994
|
+
<p>Status: <span class="status-{status.lower()}">{status}</span></p>
|
|
995
|
+
</div>
|
|
996
|
+
<div>
|
|
997
|
+
<span class="duration-text">Total: {total_duration_ms:.1f}ms</span>
|
|
998
|
+
</div>
|
|
999
|
+
</div>
|
|
1000
|
+
|
|
1001
|
+
<div class="timeline-content">
|
|
1002
|
+
{self._generate_html_timeline_rows(total_duration_ms)}
|
|
1003
|
+
</div>
|
|
1004
|
+
</div>
|
|
1005
|
+
|
|
1006
|
+
{self._generate_html_summary()}
|
|
1007
|
+
</div>
|
|
1008
|
+
|
|
1009
|
+
<div class="sidebar hidden" id="parameterSidebar">
|
|
1010
|
+
<div class="sidebar-header">
|
|
1011
|
+
<div>
|
|
1012
|
+
<h3>📊 Step Details</h3>
|
|
1013
|
+
<p id="sidebarStepName" style="color: #64748b; font-size: 0.875rem; margin: 0;">Select a step to view details</p>
|
|
1014
|
+
</div>
|
|
1015
|
+
<button class="close-sidebar" onclick="closeSidebar()">×</button>
|
|
1016
|
+
</div>
|
|
1017
|
+
<div class="sidebar-content" id="sidebarContent">
|
|
1018
|
+
<div class="sidebar-section">
|
|
1019
|
+
<p style="color: #64748b; text-align: center; padding: 2rem;">Click on any step in the timeline to view its detailed parameters and metadata.</p>
|
|
1020
|
+
</div>
|
|
1021
|
+
</div>
|
|
1022
|
+
</div>
|
|
1023
|
+
</div>
|
|
1024
|
+
|
|
1025
|
+
<script>
|
|
1026
|
+
let currentSelectedStep = null;
|
|
1027
|
+
|
|
1028
|
+
// Collapsible sections
|
|
1029
|
+
document.querySelectorAll('.expandable').forEach(element => {{
|
|
1030
|
+
element.addEventListener('click', (e) => {{
|
|
1031
|
+
e.stopPropagation(); // Prevent triggering step selection
|
|
1032
|
+
let content;
|
|
1033
|
+
|
|
1034
|
+
// Handle step metadata (using data-target)
|
|
1035
|
+
if (element.dataset.target) {{
|
|
1036
|
+
content = document.getElementById(element.dataset.target);
|
|
1037
|
+
}} else {{
|
|
1038
|
+
// Handle composite sections (using nextElementSibling)
|
|
1039
|
+
content = element.nextElementSibling;
|
|
1040
|
+
}}
|
|
1041
|
+
|
|
1042
|
+
if (content && content.classList.contains('collapsible-content')) {{
|
|
1043
|
+
content.classList.toggle('collapsed');
|
|
1044
|
+
|
|
1045
|
+
// Update expand/collapse indicator
|
|
1046
|
+
const indicator = element.querySelector('.expand-indicator');
|
|
1047
|
+
if (indicator) {{
|
|
1048
|
+
indicator.textContent = content.classList.contains('collapsed') ? '▶' : '▼';
|
|
1049
|
+
}}
|
|
1050
|
+
}}
|
|
1051
|
+
}});
|
|
1052
|
+
}});
|
|
1053
|
+
|
|
1054
|
+
function showStepDetails(internalName, startTime) {{
|
|
1055
|
+
const sidebar = document.getElementById('parameterSidebar');
|
|
1056
|
+
const stepNameElement = document.getElementById('sidebarStepName');
|
|
1057
|
+
const contentElement = document.getElementById('sidebarContent');
|
|
1058
|
+
|
|
1059
|
+
// Get step data from embedded JSON
|
|
1060
|
+
const dataId = `step-data-${{internalName.replace(/\\./g, '-')}}-${{startTime}}`;
|
|
1061
|
+
const dataElement = document.getElementById(dataId);
|
|
1062
|
+
|
|
1063
|
+
if (!dataElement) {{
|
|
1064
|
+
console.error('Step data not found for:', dataId);
|
|
1065
|
+
return;
|
|
1066
|
+
}}
|
|
1067
|
+
|
|
1068
|
+
try {{
|
|
1069
|
+
const stepData = JSON.parse(dataElement.textContent);
|
|
1070
|
+
|
|
1071
|
+
// Update step selection visual feedback
|
|
1072
|
+
if (currentSelectedStep) {{
|
|
1073
|
+
currentSelectedStep.classList.remove('selected');
|
|
1074
|
+
}}
|
|
1075
|
+
|
|
1076
|
+
const stepRow = event.currentTarget;
|
|
1077
|
+
stepRow.classList.add('selected');
|
|
1078
|
+
currentSelectedStep = stepRow;
|
|
1079
|
+
|
|
1080
|
+
// Update sidebar content
|
|
1081
|
+
stepNameElement.textContent = `${{stepData.name}} (${{stepData.duration_ms.toFixed(1)}}ms)`;
|
|
1082
|
+
|
|
1083
|
+
const perfEmoji = stepData.performance_class === 'fast' ? '🟢' :
|
|
1084
|
+
stepData.performance_class === 'medium' ? '🟡' : '🔴';
|
|
1085
|
+
|
|
1086
|
+
const statusEmoji = stepData.status === 'SUCCESS' ? '✅' :
|
|
1087
|
+
stepData.status === 'FAIL' ? '❌' : '⏸️';
|
|
1088
|
+
|
|
1089
|
+
contentElement.innerHTML = `
|
|
1090
|
+
<div class="sidebar-section">
|
|
1091
|
+
<h4>📋 Step Information</h4>
|
|
1092
|
+
<div style="display: grid; gap: 0.5rem; font-size: 0.875rem;">
|
|
1093
|
+
<div><strong>Status:</strong> ${{statusEmoji}} ${{stepData.status}}</div>
|
|
1094
|
+
<div><strong>Type:</strong> ${{stepData.step_type}}</div>
|
|
1095
|
+
<div><strong>Duration:</strong> ${{perfEmoji}} ${{stepData.duration_ms.toFixed(1)}}ms</div>
|
|
1096
|
+
<div><strong>Time:</strong> ${{stepData.start_time}} → ${{stepData.end_time}}</div>
|
|
1097
|
+
${{stepData.attempt_count > 1 ? `<div><strong>Attempts:</strong> ${{stepData.attempt_count}}/${{stepData.max_attempts}}</div>` : ''}}
|
|
1098
|
+
</div>
|
|
1099
|
+
</div>
|
|
1100
|
+
|
|
1101
|
+
${{stepData.error_message ? `
|
|
1102
|
+
<div class="sidebar-section">
|
|
1103
|
+
<h4 style="color: #dc2626;">💥 Error Details</h4>
|
|
1104
|
+
<div class="param-item" style="background: #fef2f2; border-left-color: #dc2626; color: #991b1b; white-space: pre-wrap; font-family: monospace; font-size: 0.75rem; line-height: 1.4;">
|
|
1105
|
+
${{stepData.error_message}}
|
|
1106
|
+
</div>
|
|
1107
|
+
${{stepData.attempt_count > 1 ? `
|
|
1108
|
+
<div style="margin-top: 0.5rem; font-size: 0.8rem; color: #6b7280;">
|
|
1109
|
+
⚠️ Step failed after ${{stepData.attempt_count}} attempt(s)
|
|
1110
|
+
</div>
|
|
1111
|
+
` : ''}}
|
|
1112
|
+
</div>
|
|
1113
|
+
` : ''}}
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
${{stepData.command ? `
|
|
1117
|
+
<div class="sidebar-section">
|
|
1118
|
+
<h4>🔧 Command</h4>
|
|
1119
|
+
<div class="param-item context">
|
|
1120
|
+
<strong>${{stepData.command_type.toUpperCase()}}:</strong><br>
|
|
1121
|
+
${{stepData.command}}
|
|
1122
|
+
</div>
|
|
1123
|
+
</div>
|
|
1124
|
+
` : ''}}
|
|
1125
|
+
|
|
1126
|
+
${{stepData.parameters.key_inputs.length > 0 ? `
|
|
1127
|
+
<div class="sidebar-section">
|
|
1128
|
+
<h4>🎯 Key Inputs</h4>
|
|
1129
|
+
<div class="param-grid">
|
|
1130
|
+
${{stepData.parameters.key_inputs.map(param =>
|
|
1131
|
+
`<div class="param-item input">${{param}}</div>`
|
|
1132
|
+
).join('')}}
|
|
1133
|
+
</div>
|
|
1134
|
+
</div>
|
|
1135
|
+
` : ''}}
|
|
1136
|
+
|
|
1137
|
+
${{stepData.parameters.step_outputs.length > 0 ? `
|
|
1138
|
+
<div class="sidebar-section">
|
|
1139
|
+
<h4>📤 Step Outputs</h4>
|
|
1140
|
+
<div class="param-grid">
|
|
1141
|
+
${{stepData.parameters.step_outputs.map(param =>
|
|
1142
|
+
`<div class="param-item output">${{param}}</div>`
|
|
1143
|
+
).join('')}}
|
|
1144
|
+
</div>
|
|
1145
|
+
</div>
|
|
1146
|
+
` : ''}}
|
|
1147
|
+
|
|
1148
|
+
${{stepData.parameters.iteration_vars.length > 0 ? `
|
|
1149
|
+
<div class="sidebar-section">
|
|
1150
|
+
<h4>🔁 Iteration Variables</h4>
|
|
1151
|
+
<div class="param-grid">
|
|
1152
|
+
${{stepData.parameters.iteration_vars.map(param =>
|
|
1153
|
+
`<div class="param-item iteration">${{param}}</div>`
|
|
1154
|
+
).join('')}}
|
|
1155
|
+
</div>
|
|
1156
|
+
</div>
|
|
1157
|
+
` : ''}}
|
|
1158
|
+
|
|
1159
|
+
${{(stepData.catalog_ops.put.length > 0 || stepData.catalog_ops.get.length > 0) ? `
|
|
1160
|
+
<div class="sidebar-section">
|
|
1161
|
+
<h4>💾 Data Operations</h4>
|
|
1162
|
+
${{stepData.catalog_ops.get.length > 0 ? `
|
|
1163
|
+
<div style="margin-bottom: 1rem;">
|
|
1164
|
+
<h5 style="color: #059669; font-size: 0.8rem; font-weight: 600; margin-bottom: 0.5rem;">📥 CATALOG INPUTS</h5>
|
|
1165
|
+
<div class="param-grid">
|
|
1166
|
+
${{stepData.catalog_ops.get.map(item =>
|
|
1167
|
+
`<div class="param-item input" style="font-family: monospace;">
|
|
1168
|
+
<div style="font-weight: 600; margin-bottom: 0.25rem;">${{item.name}}</div>
|
|
1169
|
+
<div style="font-size: 0.7rem; opacity: 0.8;">Hash: ${{item.data_hash}}</div>
|
|
1170
|
+
<div style="font-size: 0.7rem; opacity: 0.8;">Path: ${{item.catalog_path}}</div>
|
|
1171
|
+
</div>`
|
|
1172
|
+
).join('')}}
|
|
1173
|
+
</div>
|
|
1174
|
+
</div>
|
|
1175
|
+
` : ''}}
|
|
1176
|
+
${{stepData.catalog_ops.put.length > 0 ? `
|
|
1177
|
+
<div>
|
|
1178
|
+
<h5 style="color: #dc2626; font-size: 0.8rem; font-weight: 600; margin-bottom: 0.5rem;">📤 CATALOG OUTPUTS</h5>
|
|
1179
|
+
<div class="param-grid">
|
|
1180
|
+
${{stepData.catalog_ops.put.map(item =>
|
|
1181
|
+
`<div class="param-item output" style="font-family: monospace;">
|
|
1182
|
+
<div style="font-weight: 600; margin-bottom: 0.25rem;">${{item.name}}</div>
|
|
1183
|
+
<div style="font-size: 0.7rem; opacity: 0.8;">Hash: ${{item.data_hash}}</div>
|
|
1184
|
+
<div style="font-size: 0.7rem; opacity: 0.8;">Path: ${{item.catalog_path}}</div>
|
|
1185
|
+
</div>`
|
|
1186
|
+
).join('')}}
|
|
1187
|
+
</div>
|
|
1188
|
+
</div>
|
|
1189
|
+
` : ''}}
|
|
1190
|
+
</div>
|
|
1191
|
+
` : ''}}
|
|
1192
|
+
|
|
1193
|
+
${{stepData.parameters.context_params.length > 0 ? `
|
|
1194
|
+
<div class="sidebar-section">
|
|
1195
|
+
<details>
|
|
1196
|
+
<summary style="cursor: pointer; padding: 0.5rem 0; font-weight: 600; color: #374151;">
|
|
1197
|
+
📊 Pipeline Context (${{stepData.parameters.context_params.length}} params)
|
|
1198
|
+
</summary>
|
|
1199
|
+
<div class="param-grid" style="margin-top: 0.5rem;">
|
|
1200
|
+
${{stepData.parameters.context_params.map(param =>
|
|
1201
|
+
`<div class="param-item context">${{param}}</div>`
|
|
1202
|
+
).join('')}}
|
|
1203
|
+
</div>
|
|
1204
|
+
</details>
|
|
1205
|
+
</div>
|
|
1206
|
+
` : ''}}
|
|
1207
|
+
`;
|
|
1208
|
+
|
|
1209
|
+
// Show sidebar
|
|
1210
|
+
sidebar.classList.remove('hidden');
|
|
1211
|
+
|
|
1212
|
+
}} catch (error) {{
|
|
1213
|
+
console.error('Error parsing step data:', error);
|
|
1214
|
+
contentElement.innerHTML = `<div class="sidebar-section"><p style="color: #ef4444;">Error loading step details</p></div>`;
|
|
1215
|
+
sidebar.classList.remove('hidden');
|
|
1216
|
+
}}
|
|
1217
|
+
}}
|
|
1218
|
+
|
|
1219
|
+
function closeSidebar() {{
|
|
1220
|
+
const sidebar = document.getElementById('parameterSidebar');
|
|
1221
|
+
sidebar.classList.add('hidden');
|
|
1222
|
+
|
|
1223
|
+
// Clear step selection
|
|
1224
|
+
if (currentSelectedStep) {{
|
|
1225
|
+
currentSelectedStep.classList.remove('selected');
|
|
1226
|
+
currentSelectedStep = null;
|
|
1227
|
+
}}
|
|
1228
|
+
}}
|
|
1229
|
+
|
|
1230
|
+
// Close sidebar when clicking outside
|
|
1231
|
+
document.addEventListener('click', (e) => {{
|
|
1232
|
+
const sidebar = document.getElementById('parameterSidebar');
|
|
1233
|
+
const isClickInsideSidebar = sidebar.contains(e.target);
|
|
1234
|
+
const isClickOnStep = e.target.closest('.step-clickable');
|
|
1235
|
+
|
|
1236
|
+
if (!isClickInsideSidebar && !isClickOnStep && !sidebar.classList.contains('hidden')) {{
|
|
1237
|
+
closeSidebar();
|
|
1238
|
+
}}
|
|
1239
|
+
}});
|
|
1240
|
+
|
|
1241
|
+
// Keyboard shortcuts
|
|
1242
|
+
document.addEventListener('keydown', (e) => {{
|
|
1243
|
+
if (e.key === 'Escape') {{
|
|
1244
|
+
closeSidebar();
|
|
1245
|
+
}}
|
|
1246
|
+
}});
|
|
1247
|
+
</script>
|
|
1248
|
+
</body>
|
|
1249
|
+
</html>"""
|
|
1250
|
+
|
|
1251
|
+
# Save to file if path provided
|
|
1252
|
+
if output_path:
|
|
1253
|
+
Path(output_path).write_text(html_content)
|
|
1254
|
+
print(f"HTML timeline saved to: {output_path}")
|
|
1255
|
+
|
|
1256
|
+
return html_content
|
|
1257
|
+
|
|
1258
|
+
def _generate_html_timeline_rows(self, total_duration_ms: float) -> str:
|
|
1259
|
+
"""Generate HTML rows for the Gantt chart timeline display."""
|
|
1260
|
+
executed_steps = [
|
|
1261
|
+
step for step in self.timeline if step.start_time and step.end_time
|
|
1262
|
+
]
|
|
1263
|
+
|
|
1264
|
+
if not executed_steps:
|
|
1265
|
+
return "<div>No executed steps found</div>"
|
|
1266
|
+
|
|
1267
|
+
# Calculate the absolute timeline
|
|
1268
|
+
earliest_start = min(
|
|
1269
|
+
step.start_time for step in executed_steps if step.start_time
|
|
1270
|
+
)
|
|
1271
|
+
latest_end = max(step.end_time for step in executed_steps if step.end_time)
|
|
1272
|
+
total_timeline_ms = (
|
|
1273
|
+
(latest_end - earliest_start).total_seconds() * 1000
|
|
1274
|
+
if latest_end and earliest_start
|
|
1275
|
+
else 1
|
|
1276
|
+
)
|
|
1277
|
+
|
|
1278
|
+
# Generate time scale and Gantt rows
|
|
1279
|
+
time_scale_html = self._generate_time_scale(total_timeline_ms)
|
|
1280
|
+
gantt_rows_html = self._generate_gantt_rows(
|
|
1281
|
+
executed_steps, earliest_start, total_timeline_ms
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
return time_scale_html + "\n" + gantt_rows_html
|
|
1285
|
+
|
|
1286
|
+
def _generate_time_scale(self, total_timeline_ms: float) -> str:
|
|
1287
|
+
"""Generate the time scale header for the Gantt chart."""
|
|
1288
|
+
# Create time markers at regular intervals
|
|
1289
|
+
num_markers = 10
|
|
1290
|
+
interval_ms = total_timeline_ms / num_markers
|
|
1291
|
+
|
|
1292
|
+
markers_html = []
|
|
1293
|
+
for i in range(num_markers + 1):
|
|
1294
|
+
time_ms = i * interval_ms
|
|
1295
|
+
position_percent = (time_ms / total_timeline_ms) * 100
|
|
1296
|
+
time_display = (
|
|
1297
|
+
f"{time_ms:.0f}ms" if time_ms < 1000 else f"{time_ms/1000:.1f}s"
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
markers_html.append(f"""
|
|
1301
|
+
<div class="time-marker" style="left: {position_percent:.1f}%;">
|
|
1302
|
+
{time_display}
|
|
1303
|
+
</div>
|
|
1304
|
+
""")
|
|
1305
|
+
|
|
1306
|
+
# Add grid line (except for the first one)
|
|
1307
|
+
if i > 0:
|
|
1308
|
+
markers_html.append(
|
|
1309
|
+
f'<div class="time-grid" style="left: {position_percent:.1f}%;"></div>'
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
return f"""
|
|
1313
|
+
<div class="step-row" style="border-bottom: 2px solid #d1d5db;">
|
|
1314
|
+
<div class="step-info">
|
|
1315
|
+
<strong>Timeline</strong>
|
|
1316
|
+
</div>
|
|
1317
|
+
<div class="time-scale">
|
|
1318
|
+
{"".join(markers_html)}
|
|
1319
|
+
</div>
|
|
1320
|
+
<div></div>
|
|
1321
|
+
</div>
|
|
1322
|
+
"""
|
|
1323
|
+
|
|
1324
|
+
def _generate_gantt_rows(
|
|
1325
|
+
self, executed_steps: List, earliest_start, total_timeline_ms: float
|
|
1326
|
+
) -> str:
|
|
1327
|
+
"""Generate HTML rows for the Gantt chart display."""
|
|
1328
|
+
html_parts = []
|
|
1329
|
+
|
|
1330
|
+
# Group by composite steps for better display
|
|
1331
|
+
current_composite = None
|
|
1332
|
+
current_branch = None
|
|
1333
|
+
|
|
1334
|
+
for step in executed_steps:
|
|
1335
|
+
# Calculate timing positions for Gantt chart
|
|
1336
|
+
start_offset_ms = (step.start_time - earliest_start).total_seconds() * 1000
|
|
1337
|
+
start_percent = (start_offset_ms / total_timeline_ms) * 100
|
|
1338
|
+
width_percent = (step.duration_ms / total_timeline_ms) * 100
|
|
1339
|
+
|
|
1340
|
+
# Detect composite/branch changes
|
|
1341
|
+
hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
|
|
1342
|
+
composite = hierarchy.get("composite")
|
|
1343
|
+
branch = hierarchy.get("branch")
|
|
1344
|
+
|
|
1345
|
+
# Show composite header
|
|
1346
|
+
if composite and composite != current_composite:
|
|
1347
|
+
composite_type = self._get_composite_type(composite)
|
|
1348
|
+
composite_id = f"composite-{composite.replace(' ', '-')}"
|
|
1349
|
+
|
|
1350
|
+
html_parts.append(f"""
|
|
1351
|
+
<div class="step-row composite-header expandable" data-composite="{composite}">
|
|
1352
|
+
<div class="step-info step-level-0">
|
|
1353
|
+
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
1354
|
+
<span class="expand-indicator">▼</span>
|
|
1355
|
+
🔀 <strong>{composite}</strong> ({composite_type})
|
|
1356
|
+
</div>
|
|
1357
|
+
</div>
|
|
1358
|
+
<div class="gantt-container"></div>
|
|
1359
|
+
<div></div>
|
|
1360
|
+
</div>
|
|
1361
|
+
""")
|
|
1362
|
+
|
|
1363
|
+
# Start collapsible content
|
|
1364
|
+
html_parts.append(
|
|
1365
|
+
f'<div class="collapsible-content" id="{composite_id}">'
|
|
1366
|
+
)
|
|
1367
|
+
current_composite = composite
|
|
1368
|
+
current_branch = None
|
|
1369
|
+
|
|
1370
|
+
# Show branch header for parallel/map steps
|
|
1371
|
+
if branch and branch != current_branch:
|
|
1372
|
+
branch_display = self._format_branch_name(composite or "", branch)
|
|
1373
|
+
|
|
1374
|
+
html_parts.append(f"""
|
|
1375
|
+
<div class="step-row branch-header">
|
|
1376
|
+
<div class="step-info step-level-1">
|
|
1377
|
+
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
1378
|
+
🌿 <strong>Branch: {branch_display}</strong>
|
|
1379
|
+
</div>
|
|
1380
|
+
</div>
|
|
1381
|
+
<div class="gantt-container"></div>
|
|
1382
|
+
<div></div>
|
|
1383
|
+
</div>
|
|
1384
|
+
""")
|
|
1385
|
+
current_branch = branch
|
|
1386
|
+
|
|
1387
|
+
# Status styling
|
|
1388
|
+
status_class = step.status.lower()
|
|
1389
|
+
bar_class = (
|
|
1390
|
+
f"bar-{status_class}"
|
|
1391
|
+
if status_class in ["success", "fail"]
|
|
1392
|
+
else "bar-unknown"
|
|
1393
|
+
)
|
|
1394
|
+
|
|
1395
|
+
# Type icon and status emoji
|
|
1396
|
+
type_icons = {
|
|
1397
|
+
"task": "⚙️",
|
|
1398
|
+
"stub": "📝",
|
|
1399
|
+
"success": "✅",
|
|
1400
|
+
"fail": "❌",
|
|
1401
|
+
"parallel": "🔀",
|
|
1402
|
+
"map": "🔁",
|
|
1403
|
+
"conditional": "🔀",
|
|
1404
|
+
}
|
|
1405
|
+
type_icon = type_icons.get(step.step_type, "⚙️")
|
|
1406
|
+
status_emoji = (
|
|
1407
|
+
"✅"
|
|
1408
|
+
if step.status == "SUCCESS"
|
|
1409
|
+
else "❌"
|
|
1410
|
+
if step.status == "FAIL"
|
|
1411
|
+
else "⏸️"
|
|
1412
|
+
)
|
|
1413
|
+
|
|
1414
|
+
# Build parameter display - compact horizontal format
|
|
1415
|
+
param_info = []
|
|
1416
|
+
|
|
1417
|
+
if step.input_params:
|
|
1418
|
+
params_text = " • ".join(step.input_params)
|
|
1419
|
+
param_info.append(
|
|
1420
|
+
f'<div style="color: #059669; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📥 {params_text}</div>'
|
|
1421
|
+
)
|
|
1422
|
+
|
|
1423
|
+
if step.output_params:
|
|
1424
|
+
params_text = " • ".join(step.output_params)
|
|
1425
|
+
param_info.append(
|
|
1426
|
+
f'<div style="color: #dc2626; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📤 {params_text}</div>'
|
|
1427
|
+
)
|
|
1428
|
+
|
|
1429
|
+
if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
|
|
1430
|
+
catalog_items = []
|
|
1431
|
+
if step.catalog_ops.get("put"):
|
|
1432
|
+
catalog_items.extend(
|
|
1433
|
+
[f"PUT:{item['name']}" for item in step.catalog_ops["put"]]
|
|
1434
|
+
)
|
|
1435
|
+
if step.catalog_ops.get("get"):
|
|
1436
|
+
catalog_items.extend(
|
|
1437
|
+
[f"GET:{item['name']}" for item in step.catalog_ops["get"]]
|
|
1438
|
+
)
|
|
1439
|
+
if catalog_items:
|
|
1440
|
+
catalog_text = " • ".join(catalog_items)
|
|
1441
|
+
param_info.append(
|
|
1442
|
+
f'<div style="color: #7c3aed; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">💾 {catalog_text}</div>'
|
|
1443
|
+
)
|
|
1444
|
+
|
|
1445
|
+
# Generate sidebar data for this step
|
|
1446
|
+
sidebar_data = self._generate_step_sidebar_data(step)
|
|
1447
|
+
|
|
1448
|
+
html_parts.append(f"""
|
|
1449
|
+
<div class="step-row step-clickable" onclick="showStepDetails('{step.internal_name}', '{step.start_time.isoformat()}')">
|
|
1450
|
+
<div class="step-info step-level-{step.level}">
|
|
1451
|
+
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
|
1452
|
+
{type_icon} {status_emoji} <strong>{step.name}</strong>
|
|
1453
|
+
<span style="font-size: 0.7rem; color: #64748b;">📊 details</span>
|
|
1454
|
+
</div>
|
|
1455
|
+
{f'<div style="color: #64748b; font-size: 0.75rem; margin-top: 0.25rem;"><strong>{step.command_type.upper()}:</strong> {step.command[:40]}{"..." if len(step.command) > 40 else ""}</div>' if step.command else ''}
|
|
1456
|
+
{'<div style="color: #059669; font-size: 0.65rem; margin-top: 0.25rem;">🎯 ' + (', '.join(step.parameters.key_inputs[:2]) if step.parameters and step.parameters.key_inputs else 'No key inputs') + ('...' if step.parameters and len(step.parameters.key_inputs) > 2 else '') + '</div>' if step.parameters else ''}
|
|
1457
|
+
</div>
|
|
1458
|
+
<div class="gantt-container">
|
|
1459
|
+
<div class="gantt-bar {bar_class}"
|
|
1460
|
+
style="left: {start_percent:.2f}%; width: {max(width_percent, 0.5):.2f}%;"
|
|
1461
|
+
title="{step.name}: {step.duration_ms:.1f}ms">
|
|
1462
|
+
</div>
|
|
1463
|
+
</div>
|
|
1464
|
+
<div style="font-family: monospace; font-size: 0.75rem; color: #6b7280;">
|
|
1465
|
+
{step.duration_ms:.1f}ms
|
|
1466
|
+
</div>
|
|
1467
|
+
</div>
|
|
1468
|
+
|
|
1469
|
+
<!-- Hidden step data for sidebar -->
|
|
1470
|
+
<script type="application/json" id="step-data-{step.internal_name.replace('.', '-')}-{step.start_time.isoformat()}">{sidebar_data}</script>
|
|
1471
|
+
""")
|
|
1472
|
+
|
|
1473
|
+
# Close any open composite sections
|
|
1474
|
+
if current_composite:
|
|
1475
|
+
html_parts.append("</div>") # Close collapsible-content
|
|
1476
|
+
|
|
1477
|
+
return "\n".join(html_parts)
|
|
1478
|
+
|
|
1479
|
+
def _generate_step_sidebar_data(self, step: StepInfo) -> str:
|
|
1480
|
+
"""Generate JSON data for step sidebar display."""
|
|
1481
|
+
import json
|
|
1482
|
+
|
|
1483
|
+
if not step.parameters:
|
|
1484
|
+
return json.dumps({"error": "No parameter data available"})
|
|
1485
|
+
|
|
1486
|
+
# Performance classification
|
|
1487
|
+
perf_class = "fast"
|
|
1488
|
+
if step.duration_ms > 1000:
|
|
1489
|
+
perf_class = "slow"
|
|
1490
|
+
elif step.duration_ms > 100:
|
|
1491
|
+
perf_class = "medium"
|
|
1492
|
+
|
|
1493
|
+
data = {
|
|
1494
|
+
"name": step.name,
|
|
1495
|
+
"internal_name": step.internal_name,
|
|
1496
|
+
"status": step.status,
|
|
1497
|
+
"step_type": step.step_type,
|
|
1498
|
+
"command": step.command,
|
|
1499
|
+
"command_type": step.command_type,
|
|
1500
|
+
"duration_ms": step.duration_ms,
|
|
1501
|
+
"performance_class": perf_class,
|
|
1502
|
+
"start_time": step.start_time.strftime("%H:%M:%S.%f")[:-3]
|
|
1503
|
+
if step.start_time
|
|
1504
|
+
else None,
|
|
1505
|
+
"end_time": step.end_time.strftime("%H:%M:%S.%f")[:-3]
|
|
1506
|
+
if step.end_time
|
|
1507
|
+
else None,
|
|
1508
|
+
"parameters": {
|
|
1509
|
+
"key_inputs": step.parameters.key_inputs,
|
|
1510
|
+
"step_outputs": step.parameters.step_outputs,
|
|
1511
|
+
"context_params": step.parameters.context_params,
|
|
1512
|
+
"iteration_vars": step.parameters.iteration_vars,
|
|
1513
|
+
},
|
|
1514
|
+
"catalog_ops": step.catalog_ops,
|
|
1515
|
+
"error_message": step.error_message,
|
|
1516
|
+
"attempt_count": step.attempt_count,
|
|
1517
|
+
"max_attempts": step.max_attempts,
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
return json.dumps(data)
|
|
1521
|
+
|
|
1522
|
+
def _generate_html_summary(self) -> str:
|
|
1523
|
+
"""Generate HTML summary cards."""
|
|
1524
|
+
executed_steps = [step for step in self.timeline if step.start_time]
|
|
1525
|
+
total_duration = sum(step.duration_ms for step in executed_steps)
|
|
1526
|
+
success_count = sum(1 for step in executed_steps if step.status == "SUCCESS")
|
|
1527
|
+
success_rate = (
|
|
1528
|
+
(success_count / len(executed_steps)) * 100 if executed_steps else 0
|
|
1529
|
+
)
|
|
1530
|
+
|
|
1531
|
+
# Find slowest step
|
|
1532
|
+
slowest_step = (
|
|
1533
|
+
max(executed_steps, key=lambda x: x.duration_ms) if executed_steps else None
|
|
1534
|
+
)
|
|
1535
|
+
|
|
1536
|
+
return f"""
|
|
1537
|
+
<div class="summary-grid">
|
|
1538
|
+
<div class="summary-card">
|
|
1539
|
+
<div class="summary-number status-success">{len(executed_steps)}</div>
|
|
1540
|
+
<div class="summary-label">Total Steps</div>
|
|
1541
|
+
</div>
|
|
1542
|
+
<div class="summary-card">
|
|
1543
|
+
<div class="summary-number duration-medium">{total_duration:.1f}ms</div>
|
|
1544
|
+
<div class="summary-label">Total Duration</div>
|
|
1545
|
+
</div>
|
|
1546
|
+
<div class="summary-card">
|
|
1547
|
+
<div class="summary-number {'status-success' if success_rate == 100 else 'status-fail'}">{success_rate:.1f}%</div>
|
|
1548
|
+
<div class="summary-label">Success Rate</div>
|
|
1549
|
+
</div>
|
|
1550
|
+
<div class="summary-card">
|
|
1551
|
+
<div class="summary-number duration-slow">{'%.1fms' % slowest_step.duration_ms if slowest_step else 'N/A'}</div>
|
|
1552
|
+
<div class="summary-label">Slowest Step<br><small>{slowest_step.name if slowest_step else 'N/A'}</small></div>
|
|
1553
|
+
</div>
|
|
1554
|
+
</div>
|
|
1555
|
+
"""
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
def visualize_simple(
|
|
1559
|
+
run_id: str, show_summary: bool = False, output_html: Optional[str] = None
|
|
1560
|
+
) -> None:
|
|
1561
|
+
"""
|
|
1562
|
+
Simple visualization of a pipeline run.
|
|
1563
|
+
|
|
1564
|
+
Args:
|
|
1565
|
+
run_id: Run ID to visualize
|
|
1566
|
+
show_summary: Whether to show execution summary (deprecated, timeline has enough info)
|
|
1567
|
+
output_html: Optional path to save HTML timeline
|
|
1568
|
+
"""
|
|
1569
|
+
# Find run log file
|
|
1570
|
+
run_log_dir = Path(".run_log_store")
|
|
1571
|
+
log_file = run_log_dir / f"{run_id}.json"
|
|
1572
|
+
|
|
1573
|
+
if not log_file.exists():
|
|
1574
|
+
# Try partial match
|
|
1575
|
+
matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
|
|
1576
|
+
if matching_files:
|
|
1577
|
+
log_file = matching_files[0]
|
|
1578
|
+
else:
|
|
1579
|
+
print(f"❌ Run log not found for: {run_id}")
|
|
1580
|
+
return
|
|
1581
|
+
|
|
1582
|
+
print(f"📊 Visualizing: {log_file.stem}")
|
|
1583
|
+
|
|
1584
|
+
viz = SimpleVisualizer(log_file)
|
|
1585
|
+
viz.print_simple_timeline()
|
|
1586
|
+
|
|
1587
|
+
if show_summary:
|
|
1588
|
+
viz.print_execution_summary()
|
|
1589
|
+
|
|
1590
|
+
# Generate HTML if requested
|
|
1591
|
+
if output_html:
|
|
1592
|
+
viz.generate_html_timeline(output_html)
|
|
1593
|
+
|
|
1594
|
+
|
|
1595
|
+
def generate_html_timeline(
|
|
1596
|
+
run_id: str, output_file: str, open_browser: bool = True
|
|
1597
|
+
) -> None:
|
|
1598
|
+
"""
|
|
1599
|
+
Generate HTML timeline for a specific run ID.
|
|
1600
|
+
|
|
1601
|
+
Args:
|
|
1602
|
+
run_id: The run ID to visualize
|
|
1603
|
+
output_file: Output HTML file path
|
|
1604
|
+
open_browser: Whether to open the result in browser
|
|
1605
|
+
"""
|
|
1606
|
+
from pathlib import Path
|
|
1607
|
+
|
|
1608
|
+
# Find run log file
|
|
1609
|
+
run_log_dir = Path(".run_log_store")
|
|
1610
|
+
log_file = run_log_dir / f"{run_id}.json"
|
|
1611
|
+
|
|
1612
|
+
if not log_file.exists():
|
|
1613
|
+
# Try partial match
|
|
1614
|
+
matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
|
|
1615
|
+
if matching_files:
|
|
1616
|
+
log_file = matching_files[0]
|
|
1617
|
+
else:
|
|
1618
|
+
print(f"❌ Run log not found for: {run_id}")
|
|
1619
|
+
return
|
|
1620
|
+
|
|
1621
|
+
print(f"🌐 Generating HTML timeline for: {log_file.stem}")
|
|
1622
|
+
|
|
1623
|
+
# Create visualizer and generate HTML
|
|
1624
|
+
viz = SimpleVisualizer(log_file)
|
|
1625
|
+
viz.generate_html_timeline(output_file)
|
|
1626
|
+
|
|
1627
|
+
if open_browser:
|
|
1628
|
+
import webbrowser
|
|
1629
|
+
|
|
1630
|
+
file_path = Path(output_file).absolute()
|
|
1631
|
+
print(f"🌐 Opening timeline in browser: {file_path.name}")
|
|
1632
|
+
webbrowser.open(file_path.as_uri())
|
|
1633
|
+
|
|
1634
|
+
|
|
1635
|
+
if __name__ == "__main__":
|
|
1636
|
+
import sys
|
|
1637
|
+
|
|
1638
|
+
if len(sys.argv) > 1:
|
|
1639
|
+
if len(sys.argv) > 2 and sys.argv[2].endswith(".html"):
|
|
1640
|
+
# Generate HTML: python viz_simple.py <run_id> <output.html>
|
|
1641
|
+
generate_html_timeline(sys.argv[1], sys.argv[2])
|
|
1642
|
+
else:
|
|
1643
|
+
# Console visualization: python viz_simple.py <run_id>
|
|
1644
|
+
visualize_simple(sys.argv[1])
|
|
1645
|
+
else:
|
|
1646
|
+
print("Usage: python viz_simple.py <run_id> [output.html]")
|